Repository files, collaboration, releases and project activity.
archforge-external-status / src/monitor.jsmain
▤ src/monitor.js5264 bytes
59 lines · Select a line number for a permalink
1
'use strict';2
const {now,rows,one,run,transaction}=require('./database');3
const {runProbe}=require('./probes');const {notifyQueue,processQueue}=require('./notifications');4
const MAX_SAMPLE_AGE=400*86400;5
function upsertState(db,cfg,id,result,time=now()){6
return transaction(db,()=>{7
const s=one(db,'SELECT * FROM services WHERE id=?',id);if(!s||!s.configured)return null;8
run(db,'INSERT INTO samples(service_id,checked_at,success,latency_ms,warning,detail) VALUES(?,?,?,?,?,?)',id,time,result.success?1:0,result.latency===undefined?null:result.latency,result.warning?1:0,String(result.detail||'').slice(0,120));9
const warning=!!result.warning&&!!result.success;10
const fails=(!result.success||warning)?s.failures+1:0,successes=(result.success&&!warning)?s.successes+1:0;11
let state=s.current_state;12
if(!result.success&&fails>=cfg.failures)state='offline';13
if(warning&&fails>=cfg.failures)state='degraded';14
if(result.success&&!warning&&successes>=cfg.recoveries)state='operational';15
run(db,'UPDATE services SET current_state=?,failures=?,successes=?,last_checked=?,last_latency=? WHERE id=?',state,fails,successes,time,result.latency??null,id);16
if(state!==s.current_state){17
if(state==='offline'||state==='degraded'){18
const title=state==='offline'?`${s.label} is unavailable`:`${s.label} requires attention`;19
const ins=run(db,'INSERT INTO incidents(service_id,title,description,state,severity,automatic,started_at) VALUES(?,?,?,?,?,?,?)',id,title,state==='offline'?'Repeated independent monitoring checks failed.':'Independent monitoring detected a warning.','investigating',state==='offline'?'outage':'degraded',1,time);20
run(db,'INSERT INTO incident_updates(incident_id,state,body,created_at) VALUES(?,?,?,?)',Number(ins.lastInsertRowid),'investigating',state==='offline'?'External monitoring confirmed an outage.':'External monitoring confirmed a warning.',time);21
notifyQueue(db,cfg,'ArchForge: '+title,`External checks detected a problem with ${s.label}. Open the status page for updates.`);22
} else if(state==='operational'){23
const active=rows(db,'SELECT id FROM incidents WHERE service_id=? AND automatic=1 AND resolved_at IS NULL',id);24
for(const inc of active){run(db,"UPDATE incidents SET state='resolved',resolved_at=? WHERE id=?",time,inc.id);run(db,'INSERT INTO incident_updates(incident_id,state,body,created_at) VALUES(?,?,?,?)',inc.id,'resolved','Service recovered after multiple successful checks.',time);}25
if(['offline','degraded'].includes(s.current_state))notifyQueue(db,cfg,'ArchForge: '+s.label+' recovered',`${s.label} is responding successfully again.`);26
}27
}28
return {state,changed:state!==s.current_state};29
});30
}31
function incidentMaintenanceState(db,time=now()){return rows(db,'SELECT * FROM maintenance WHERE starts_at<=? AND ends_at>? ORDER BY starts_at DESC',time,time);}32
function uptimeStats(db,days=30,serviceId=null){if(![7,30,90].includes(Number(days)))days=30;33
const end=now(),since=end-days*86400;34
const svcs=rows(db,`SELECT * FROM services WHERE configured=1 ${serviceId?'AND id=?':''} ORDER BY category,label`,...(serviceId?[serviceId]:[]));35
return svcs.map(s=>{36
const r=rows(db,`SELECT date(checked_at,'unixepoch') AS day, COUNT(*) count, SUM(success) successful, MAX(CASE WHEN success=0 THEN 1 ELSE 0 END) failed, MAX(warning) warned, ROUND(AVG(latency_ms)) latency FROM samples WHERE service_id=? AND checked_at>=? GROUP BY date(checked_at,'unixepoch') ORDER BY day`,s.id,since);37
const map=new Map(r.map(v=>[v.day,v]));const buckets=[];let total=0,good=0,coverage=0;38
for(let i=days-1;i>=0;i--){const date=new Date((Math.floor(end/86400)-i)*86400000).toISOString().slice(0,10),r=map.get(date);if(r){total+=r.count;good+=r.successful;coverage++;}buckets.push({day:date,state:!r?'unknown':r.failed?'outage':r.warned?'degraded':'operational',count:r?.count||0,successful:r?.successful||0,latency:r?.latency??null});}39
return {...s,stats:{total,good,coverage,days,uptime:total?Math.round(good/total*10000)/100:null,buckets}};40
});41
}42
function prune(db){run(db,'DELETE FROM samples WHERE checked_at<?',now()-MAX_SAMPLE_AGE);run(db,'DELETE FROM sessions WHERE expires_at<?',now());run(db,'DELETE FROM messages WHERE delivered_at<?',now()-30*86400);run(db,'DELETE FROM subscriptions WHERE confirmed=0 AND confirm_until<?',now()-86400);}43
function startMonitor(db,cfg,{probe=runProbe,immediate=true}={}){44
let busy=false,stopped=false;45
const tick=async()=>{if(stopped||busy)return;busy=true;46
try{47
for(const s of rows(db,'SELECT * FROM services WHERE configured=1')){48
if(stopped)break;49
let result;try{result=await probe(s.id,cfg);}catch(e){result={success:false,latency:null,detail:String(e.code||e.message||'Probe exception').slice(0,120)};}50
upsertState(db,cfg,s.id,result);51
}52
await processQueue(db,cfg);prune(db);53
}catch(e){console.error('Monitor cycle failure:',e.message);}finally{busy=false;}54
};55
const timer=setInterval(tick,cfg.interval);timer.unref?.();if(immediate)setImmediate(tick);56
return {stop(){stopped=true;clearInterval(timer);},tick};57
}58
module.exports={upsertState,incidentMaintenanceState,uptimeStats,startMonitor,prune};