412 lines
14 KiB
JavaScript
412 lines
14 KiB
JavaScript
|
|
"use strict";
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* 告警记录存储
|
|||
|
|
*
|
|||
|
|
* 告警记录结构:
|
|||
|
|
* {
|
|||
|
|
* id: "alert_xxx",
|
|||
|
|
* rule_id: "rule_cpu_critical",
|
|||
|
|
* rule_name: "CPU 使用率过高",
|
|||
|
|
* device_id: "host_xxx",
|
|||
|
|
* device_name: "核心交换机-01",
|
|||
|
|
* metric: "cpu_usage",
|
|||
|
|
* metric_value: "85",
|
|||
|
|
* threshold: "80",
|
|||
|
|
* level: "critical",
|
|||
|
|
* message: "核心交换机-01 CPU 使用率 85%,超过阈值 80%",
|
|||
|
|
* triggered_at: timestamp,
|
|||
|
|
* status: "active",
|
|||
|
|
* acknowledged_by: null,
|
|||
|
|
* acknowledged_at: null,
|
|||
|
|
* resolved_at: null,
|
|||
|
|
* resolved_reason: null,
|
|||
|
|
* note: null,
|
|||
|
|
* }
|
|||
|
|
*/
|
|||
|
|
|
|||
|
|
const { STORAGE_KEY_ALERT_RECORDS, STORAGE_KEY_ALERT_COUNTERS } = require("./storageKeys.cjs");
|
|||
|
|
const { ALERT_RECORD_STATUS, ALERT_LEVELS } = require("./types.cjs");
|
|||
|
|
|
|||
|
|
const MAX_ALERTS = 2000;
|
|||
|
|
|
|||
|
|
let _alerts = null;
|
|||
|
|
let _counters = null; // 防抖动计数器 { "deviceId_ruleId": count }
|
|||
|
|
|
|||
|
|
function _ensureLoaded(readFn) {
|
|||
|
|
if (_alerts === null) {
|
|||
|
|
const raw = readFn(STORAGE_KEY_ALERT_RECORDS);
|
|||
|
|
_alerts = raw ? _safeParse(raw, []) : [];
|
|||
|
|
}
|
|||
|
|
if (_counters === null) {
|
|||
|
|
const raw = readFn(STORAGE_KEY_ALERT_COUNTERS);
|
|||
|
|
_counters = raw ? _safeParse(raw, {}) : {};
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function _safeParse(text, def) {
|
|||
|
|
try { return JSON.parse(text); }
|
|||
|
|
catch (e) { return def; }
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function _saveAlerts(writeFn) {
|
|||
|
|
if (_alerts.length > MAX_ALERTS) {
|
|||
|
|
// 只保留最近的
|
|||
|
|
_alerts = _alerts.sort((a, b) => b.triggered_at - a.triggered_at).slice(0, MAX_ALERTS);
|
|||
|
|
}
|
|||
|
|
writeFn(STORAGE_KEY_ALERT_RECORDS, JSON.stringify(_alerts));
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function _saveCounters(writeFn) {
|
|||
|
|
writeFn(STORAGE_KEY_ALERT_COUNTERS, JSON.stringify(_counters));
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ---------- 告警触发(带防抖动) ----------
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* 处理告警:检查是否满足持续次数,满足则创建/更新告警记录
|
|||
|
|
* @param {object} device - 设备信息 { device_id, device_name }
|
|||
|
|
* @param {object} rule - 告警规则
|
|||
|
|
* @param {number|string} metricValue - 当前指标值
|
|||
|
|
* @param {boolean} isTriggered - 是否触发(指标值 > 阈值)
|
|||
|
|
* @returns {object|null} 新创建或恢复的告警记录,无变化则返回 null
|
|||
|
|
*/
|
|||
|
|
function processAlert(readFn, writeFn, device, rule, metricValue, isTriggered) {
|
|||
|
|
_ensureLoaded(readFn);
|
|||
|
|
|
|||
|
|
const counterKey = `${device.device_id}_${rule.id}`;
|
|||
|
|
const activeAlert = _findActiveAlert(device.device_id, rule.id);
|
|||
|
|
|
|||
|
|
if (isTriggered) {
|
|||
|
|
// 触发:计数 +1
|
|||
|
|
_counters[counterKey] = (_counters[counterKey] || 0) + 1;
|
|||
|
|
_saveCounters(writeFn);
|
|||
|
|
|
|||
|
|
// 达到持续次数且没有活跃告警 → 创建告警
|
|||
|
|
if (_counters[counterKey] >= rule.duration && !activeAlert) {
|
|||
|
|
const alert = _createAlert(device, rule, metricValue);
|
|||
|
|
_alerts.push(alert);
|
|||
|
|
_saveAlerts(writeFn);
|
|||
|
|
return { ...alert, action: "triggered" };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 已有活跃告警 → 更新指标值 + 刷新触发时间
|
|||
|
|
if (activeAlert) {
|
|||
|
|
// 如果告警之前被手动确认过(acknowledged)但设备继续超限,重置回活跃状态
|
|||
|
|
if (activeAlert.status === ALERT_RECORD_STATUS.ACKNOWLEDGED) {
|
|||
|
|
activeAlert.status = ALERT_RECORD_STATUS.ACTIVE;
|
|||
|
|
activeAlert.acknowledged_by = null;
|
|||
|
|
activeAlert.acknowledged_at = null;
|
|||
|
|
}
|
|||
|
|
activeAlert.metric_value = _stringifyMetric(metricValue);
|
|||
|
|
activeAlert.threshold = _stringifyMetric(rule.threshold);
|
|||
|
|
activeAlert.message = _buildMessage(device.device_name, rule, _stringifyMetric(metricValue), _stringifyMetric(rule.threshold));
|
|||
|
|
activeAlert.triggered_at = Date.now();
|
|||
|
|
_saveAlerts(writeFn);
|
|||
|
|
return { ...activeAlert, action: "updated" };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return null;
|
|||
|
|
} else {
|
|||
|
|
// 未触发:重置计数
|
|||
|
|
if (_counters[counterKey]) {
|
|||
|
|
delete _counters[counterKey];
|
|||
|
|
_saveCounters(writeFn);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 有活跃告警 → 自动恢复(ACTIVE 和 ACKNOWLEDGED 都要恢复)
|
|||
|
|
if (activeAlert && (activeAlert.status === ALERT_RECORD_STATUS.ACTIVE || activeAlert.status === ALERT_RECORD_STATUS.ACKNOWLEDGED)) {
|
|||
|
|
activeAlert.status = ALERT_RECORD_STATUS.RESOLVED;
|
|||
|
|
activeAlert.resolved_at = Date.now();
|
|||
|
|
activeAlert.resolved_reason = "auto_recover";
|
|||
|
|
_saveAlerts(writeFn);
|
|||
|
|
return { ...activeAlert, action: "resolved" };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return null;
|
|||
|
|
}
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function _findActiveAlert(deviceId, ruleId) {
|
|||
|
|
return _alerts.find(
|
|||
|
|
(a) =>
|
|||
|
|
a.device_id === deviceId &&
|
|||
|
|
a.rule_id === ruleId &&
|
|||
|
|
(a.status === ALERT_RECORD_STATUS.ACTIVE ||
|
|||
|
|
a.status === ALERT_RECORD_STATUS.ACKNOWLEDGED)
|
|||
|
|
);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function _stringifyMetric(v) {
|
|||
|
|
if (v === null || v === undefined) return "";
|
|||
|
|
if (typeof v === "number" || typeof v === "string" || typeof v === "boolean") return String(v);
|
|||
|
|
if (Array.isArray(v)) {
|
|||
|
|
// 数组:数字数组 join 逗号,对象数组 JSON 然后截断
|
|||
|
|
if (v.length === 0) return "";
|
|||
|
|
if (v.every(x => typeof x === "number" || typeof x === "string" || typeof x === "boolean")) {
|
|||
|
|
return v.map(x => String(x)).join(", ");
|
|||
|
|
}
|
|||
|
|
try { return JSON.stringify(v).slice(0, 200); } catch { return ""; }
|
|||
|
|
}
|
|||
|
|
if (typeof v === "object") {
|
|||
|
|
// 对象:优先取 value / current / usage 等常见数值字段
|
|||
|
|
if (typeof v.value === "number" || typeof v.value === "string") return String(v.value);
|
|||
|
|
if (typeof v.current === "number" || typeof v.current === "string") return String(v.current);
|
|||
|
|
if (typeof v.usage === "number" || typeof v.usage === "string") return String(v.usage);
|
|||
|
|
if (typeof v.percent === "number" || typeof v.percent === "string") return String(v.percent);
|
|||
|
|
try { return JSON.stringify(v).slice(0, 200); } catch { return ""; }
|
|||
|
|
}
|
|||
|
|
return String(v);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function _createAlert(device, rule, metricValue) {
|
|||
|
|
const mv = _stringifyMetric(metricValue);
|
|||
|
|
const tv = _stringifyMetric(rule.threshold);
|
|||
|
|
return {
|
|||
|
|
id: `alert_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
|
|||
|
|
rule_id: rule.id,
|
|||
|
|
rule_name: rule.name,
|
|||
|
|
device_id: device.device_id,
|
|||
|
|
device_name: device.device_name || device.device_id,
|
|||
|
|
metric: rule.metric,
|
|||
|
|
metric_value: mv,
|
|||
|
|
threshold: tv,
|
|||
|
|
level: rule.level,
|
|||
|
|
message: _buildMessage(device.device_name || device.device_id, rule, mv, tv),
|
|||
|
|
triggered_at: Date.now(),
|
|||
|
|
status: ALERT_RECORD_STATUS.ACTIVE,
|
|||
|
|
acknowledged_by: null,
|
|||
|
|
acknowledged_at: null,
|
|||
|
|
resolved_at: null,
|
|||
|
|
resolved_reason: null,
|
|||
|
|
note: null,
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function _buildMessage(deviceName, rule, metricValueStr, thresholdStr) {
|
|||
|
|
const unit = rule.metric.includes("_usage") || rule.metric.includes("_percent") ? "%" : "";
|
|||
|
|
return `${deviceName} ${rule.name},当前值 ${metricValueStr}${unit},阈值 ${thresholdStr}${unit}`;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
/**
|
|||
|
|
* 创建执行错误告警(timeout / failed 等采集器执行异常)
|
|||
|
|
* 每台设备 + rule_offline 全局唯一:
|
|||
|
|
* - 已存在任何非 RESOLVED 状态 → 复用,强制激活,覆盖 message/triggered_at
|
|||
|
|
* - 同设备多条残留(历史遗留)→ 自动合并:最新一条保留,其余降级为 RESOLVED
|
|||
|
|
* - 全 RESOLVED → 新建一条 ACTIVE
|
|||
|
|
*/
|
|||
|
|
function createExecutionErrorAlert(readFn, writeFn, device, status, errorMessage) {
|
|||
|
|
_ensureLoaded(readFn);
|
|||
|
|
|
|||
|
|
const offlineRuleId = "rule_offline";
|
|||
|
|
const offlineRuleName = "设备离线";
|
|||
|
|
|
|||
|
|
// 1. 强制去重:同 device + rule_offline 的非 RESOLVED 告警,只保留最新的一条
|
|||
|
|
const sameDeviceOffline = _alerts.filter(
|
|||
|
|
(a) => a.device_id === device.device_id && a.rule_id === offlineRuleId
|
|||
|
|
);
|
|||
|
|
const activeList = sameDeviceOffline.filter(
|
|||
|
|
(a) => a.status !== ALERT_RECORD_STATUS.RESOLVED && a.status !== ALERT_RECORD_STATUS.CLOSED
|
|||
|
|
);
|
|||
|
|
if (activeList.length > 1) {
|
|||
|
|
// 多条残留 → triggered_at 最新的保留,其余强制降级为 RESOLVED(清理历史重复)
|
|||
|
|
activeList.sort((a, b) => b.triggered_at - a.triggered_at);
|
|||
|
|
for (let i = 1; i < activeList.length; i++) {
|
|||
|
|
activeList[i].status = ALERT_RECORD_STATUS.RESOLVED;
|
|||
|
|
activeList[i].resolved_at = Date.now();
|
|||
|
|
activeList[i].resolved_reason = "dedup_execution_error";
|
|||
|
|
}
|
|||
|
|
_saveAlerts(writeFn);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 2. 复用唯一剩余的那条非 RESOLVED 告警(ACTIVE 或 ACKNOWLEDGED 都算)
|
|||
|
|
const existing = activeList[0];
|
|||
|
|
if (existing && existing.status !== ALERT_RECORD_STATUS.RESOLVED && existing.status !== ALERT_RECORD_STATUS.CLOSED) {
|
|||
|
|
if (existing.status === ALERT_RECORD_STATUS.ACKNOWLEDGED) {
|
|||
|
|
existing.status = ALERT_RECORD_STATUS.ACTIVE;
|
|||
|
|
existing.acknowledged_by = null;
|
|||
|
|
existing.acknowledged_at = null;
|
|||
|
|
}
|
|||
|
|
existing.metric_value = "false";
|
|||
|
|
existing.message = `${device.device_name || device.device_id} ${offlineRuleName}: ${errorMessage}`;
|
|||
|
|
existing.triggered_at = Date.now();
|
|||
|
|
_saveAlerts(writeFn);
|
|||
|
|
return { ...existing, action: "updated" };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 3. 没有 → 新建
|
|||
|
|
const alert = {
|
|||
|
|
id: `alert_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`,
|
|||
|
|
rule_id: offlineRuleId,
|
|||
|
|
rule_name: offlineRuleName,
|
|||
|
|
device_id: device.device_id,
|
|||
|
|
device_name: device.device_name || device.device_id,
|
|||
|
|
metric: "online",
|
|||
|
|
metric_value: "false",
|
|||
|
|
threshold: "true",
|
|||
|
|
level: "critical",
|
|||
|
|
message: `${device.device_name || device.device_id} ${offlineRuleName}: ${errorMessage}`,
|
|||
|
|
triggered_at: Date.now(),
|
|||
|
|
status: ALERT_RECORD_STATUS.ACTIVE,
|
|||
|
|
acknowledged_by: null,
|
|||
|
|
acknowledged_at: null,
|
|||
|
|
resolved_at: null,
|
|||
|
|
resolved_reason: null,
|
|||
|
|
note: null,
|
|||
|
|
};
|
|||
|
|
|
|||
|
|
_alerts.push(alert);
|
|||
|
|
_saveAlerts(writeFn);
|
|||
|
|
return { ...alert, action: "triggered" };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ---------- 告警查询 ----------
|
|||
|
|
|
|||
|
|
function listAlerts(readFn, options = {}) {
|
|||
|
|
_ensureLoaded(readFn);
|
|||
|
|
let result = [..._alerts];
|
|||
|
|
|
|||
|
|
if (options.status) {
|
|||
|
|
result = result.filter((a) => a.status === options.status);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// 读取时去重:同 device_id + rule_offline + (ACTIVE/ACKNOWLEDGED) 只留最新一条
|
|||
|
|
// 这是对历史遗留重复数据的保护,正常流程已由 createExecutionErrorAlert 保证唯一
|
|||
|
|
const offlineGroups = new Map();
|
|||
|
|
result = result.filter((a) => {
|
|||
|
|
if (a.rule_id === "rule_offline" && (a.status === "active" || a.status === "acknowledged")) {
|
|||
|
|
const key = `${a.device_id}::offline-active`;
|
|||
|
|
const prev = offlineGroups.get(key);
|
|||
|
|
if (prev) {
|
|||
|
|
// 已存在 → 保留触发时间新的那条
|
|||
|
|
if (a.triggered_at > prev.triggered_at) {
|
|||
|
|
offlineGroups.set(key, a);
|
|||
|
|
return true; // 这条更新,过滤掉 prev(但 prev 已经在 result 里了,下面处理)
|
|||
|
|
}
|
|||
|
|
return false; // 这条旧,丢弃
|
|||
|
|
}
|
|||
|
|
offlineGroups.set(key, a);
|
|||
|
|
}
|
|||
|
|
return true;
|
|||
|
|
});
|
|||
|
|
// 二次清理:把每个 group 里最旧的那条也去掉
|
|||
|
|
const bestOffline = new Set(offlineGroups.values().map((a) => a.id));
|
|||
|
|
result = result.filter(
|
|||
|
|
(a) => !(a.rule_id === "rule_offline" && (a.status === "active" || a.status === "acknowledged")) || bestOffline.has(a.id)
|
|||
|
|
);
|
|||
|
|
|
|||
|
|
if (options.level) {
|
|||
|
|
result = result.filter((a) => a.level === options.level);
|
|||
|
|
}
|
|||
|
|
if (options.device_id) {
|
|||
|
|
result = result.filter((a) => a.device_id === options.device_id);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
return result.sort((a, b) => b.triggered_at - a.triggered_at).slice(0, options.limit || MAX_ALERTS);
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function getActiveAlerts(readFn) {
|
|||
|
|
return listAlerts(readFn, { status: ALERT_RECORD_STATUS.ACTIVE, limit: MAX_ALERTS });
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function getAlert(readFn, alertId) {
|
|||
|
|
_ensureLoaded(readFn);
|
|||
|
|
return _alerts.find((a) => a.id === alertId) || null;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ---------- 告警处理 ----------
|
|||
|
|
|
|||
|
|
function acknowledgeAlert(readFn, writeFn, alertId, operator = "system") {
|
|||
|
|
_ensureLoaded(readFn);
|
|||
|
|
const alert = _alerts.find((a) => a.id === alertId);
|
|||
|
|
if (!alert) return null;
|
|||
|
|
|
|||
|
|
alert.status = ALERT_RECORD_STATUS.ACKNOWLEDGED;
|
|||
|
|
alert.acknowledged_by = operator;
|
|||
|
|
alert.acknowledged_at = Date.now();
|
|||
|
|
_saveAlerts(writeFn);
|
|||
|
|
return { ...alert };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function resolveAlert(readFn, writeFn, alertId, reason = "manual", operator = "system") {
|
|||
|
|
_ensureLoaded(readFn);
|
|||
|
|
const alert = _alerts.find((a) => a.id === alertId);
|
|||
|
|
if (!alert) return null;
|
|||
|
|
|
|||
|
|
alert.status = ALERT_RECORD_STATUS.RESOLVED;
|
|||
|
|
alert.resolved_at = Date.now();
|
|||
|
|
alert.resolved_reason = reason;
|
|||
|
|
alert.acknowledged_by = alert.acknowledged_by || operator;
|
|||
|
|
alert.acknowledged_at = alert.acknowledged_at || Date.now();
|
|||
|
|
_saveAlerts(writeFn);
|
|||
|
|
return { ...alert };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function closeAlert(readFn, writeFn, alertId, note = "") {
|
|||
|
|
_ensureLoaded(readFn);
|
|||
|
|
const alert = _alerts.find((a) => a.id === alertId);
|
|||
|
|
if (!alert) return null;
|
|||
|
|
|
|||
|
|
alert.status = ALERT_RECORD_STATUS.CLOSED;
|
|||
|
|
alert.note = note;
|
|||
|
|
if (!alert.resolved_at) {
|
|||
|
|
alert.resolved_at = Date.now();
|
|||
|
|
alert.resolved_reason = "closed";
|
|||
|
|
}
|
|||
|
|
_saveAlerts(writeFn);
|
|||
|
|
return { ...alert };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
function updateNote(readFn, writeFn, alertId, note) {
|
|||
|
|
_ensureLoaded(readFn);
|
|||
|
|
const alert = _alerts.find((a) => a.id === alertId);
|
|||
|
|
if (!alert) return null;
|
|||
|
|
|
|||
|
|
alert.note = note;
|
|||
|
|
_saveAlerts(writeFn);
|
|||
|
|
return { ...alert };
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ---------- 统计 ----------
|
|||
|
|
|
|||
|
|
function getAlertStats(readFn) {
|
|||
|
|
_ensureLoaded(readFn);
|
|||
|
|
|
|||
|
|
const active = _alerts.filter((a) => a.status === ALERT_RECORD_STATUS.ACTIVE);
|
|||
|
|
return {
|
|||
|
|
total: _alerts.length,
|
|||
|
|
active: active.length,
|
|||
|
|
active_critical: active.filter((a) => a.level === ALERT_LEVELS.CRITICAL).length,
|
|||
|
|
active_warning: active.filter((a) => a.level === ALERT_LEVELS.WARNING).length,
|
|||
|
|
active_info: active.filter((a) => a.level === ALERT_LEVELS.INFO).length,
|
|||
|
|
acknowledged: _alerts.filter((a) => a.status === ALERT_RECORD_STATUS.ACKNOWLEDGED).length,
|
|||
|
|
resolved: _alerts.filter((a) => a.status === ALERT_RECORD_STATUS.RESOLVED).length,
|
|||
|
|
closed: _alerts.filter((a) => a.status === ALERT_RECORD_STATUS.CLOSED).length,
|
|||
|
|
};
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
// ---------- 重置(测试用) ----------
|
|||
|
|
function _resetForTest() {
|
|||
|
|
_alerts = null;
|
|||
|
|
_counters = null;
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
module.exports = {
|
|||
|
|
// 告警触发
|
|||
|
|
processAlert,
|
|||
|
|
createExecutionErrorAlert,
|
|||
|
|
// 查询
|
|||
|
|
listAlerts,
|
|||
|
|
getActiveAlerts,
|
|||
|
|
getAlert,
|
|||
|
|
// 处理
|
|||
|
|
acknowledgeAlert,
|
|||
|
|
resolveAlert,
|
|||
|
|
closeAlert,
|
|||
|
|
updateNote,
|
|||
|
|
// 统计
|
|||
|
|
getAlertStats,
|
|||
|
|
// 内部
|
|||
|
|
_resetForTest,
|
|||
|
|
};
|