879 lines
30 KiB
Python
879 lines
30 KiB
Python
|
"""诊断规则实现（15 条）。
|
|||
|
|
|
|||
|
|
规则按类型分四组：
|
|||
|
|
页面必跑（轻量）: R001 / R002 / R003 / R004 / R005
|
|||
|
|
DB 聚合: R006 / R007 / R008
|
|||
|
|
日志匹配: R009-R014（需要 docker.sock）
|
|||
|
|
容器状态: R015（需要 docker.sock）
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import os
|
|||
|
|
import subprocess
|
|||
|
|
import time
|
|||
|
|
from datetime import datetime, timedelta, timezone
|
|||
|
|
from typing import Any, Dict, List, Optional
|
|||
|
|
|
|||
|
|
from sqlalchemy import func
|
|||
|
|
|
|||
|
|
from app.config import get_settings
|
|||
|
|
from app.diagnostic.config import get_diag_config
|
|||
|
|
from app.diagnostic.engine import DiagnosticEngine, RuleResult
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ======================================================================
|
|||
|
|
# 模块级状态缓存（用于检测重启事件）
|
|||
|
|
# key: container_name
|
|||
|
|
# value: {"restart_count": int, "started_at": str, "checked_at": float}
|
|||
|
|
# ======================================================================
|
|||
|
|
_CONTAINER_LAST_STATE: Dict[str, Dict[str, Any]] = {}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _ok(rule_id: str, message: str, hint: str = "",
|
|||
|
|
evidence: Optional[List[str]] = None, run_by: str = "") -> RuleResult:
|
|||
|
|
return RuleResult(
|
|||
|
|
id=rule_id, severity="ok", message=message, hint=hint,
|
|||
|
|
evidence=evidence or [], run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _warn(rule_id: str, message: str, hint: str = "",
|
|||
|
|
evidence: Optional[List[str]] = None, run_by: str = "") -> RuleResult:
|
|||
|
|
return RuleResult(
|
|||
|
|
id=rule_id, severity="medium", message=message, hint=hint,
|
|||
|
|
evidence=evidence or [], run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _crit(rule_id: str, message: str, hint: str = "",
|
|||
|
|
evidence: Optional[List[str]] = None, run_by: str = "") -> RuleResult:
|
|||
|
|
return RuleResult(
|
|||
|
|
id=rule_id, severity="high", message=message, hint=hint,
|
|||
|
|
evidence=evidence or [], run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _skip(rule_id: str, reason: str, run_by: str = "") -> RuleResult:
|
|||
|
|
return RuleResult(
|
|||
|
|
id=rule_id, severity="info", message=f"规则未运行: {reason}",
|
|||
|
|
skipped=True, skip_reason=reason, run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ======================================================================
|
|||
|
|
# 工具函数
|
|||
|
|
# ======================================================================
|
|||
|
|
|
|||
|
|
def _fetch_recent_verdict_counts(engine: DiagnosticEngine, limit: int = 100) -> Dict[str, int]:
|
|||
|
|
"""查最近 N 次扫描的 verdict 分布。
|
|||
|
|
|
|||
|
|
用子查询先 limit 再 group_by，兼容 SQLAlchemy 2.0。
|
|||
|
|
"""
|
|||
|
|
from sqlalchemy import select
|
|||
|
|
from app.models import ScanTask
|
|||
|
|
|
|||
|
|
# 先子查询拿到最近 N 条 verdict
|
|||
|
|
sub_stmt = (
|
|||
|
|
select(ScanTask.verdict)
|
|||
|
|
.order_by(ScanTask.created_at.desc())
|
|||
|
|
.limit(limit)
|
|||
|
|
).subquery()
|
|||
|
|
|
|||
|
|
# 再在外层 group_by 统计
|
|||
|
|
rows = engine.db.execute(
|
|||
|
|
select(sub_stmt.c.verdict, func.count(sub_stmt.c.verdict))
|
|||
|
|
.group_by(sub_stmt.c.verdict)
|
|||
|
|
).all()
|
|||
|
|
|
|||
|
|
return {v or "unknown": int(c) for v, c in rows}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _count_24h(engine: DiagnosticEngine) -> int:
|
|||
|
|
from app.models import ScanTask
|
|||
|
|
cutoff = datetime.now(timezone.utc) - timedelta(hours=24)
|
|||
|
|
return (
|
|||
|
|
engine.db.query(func.count(ScanTask.id))
|
|||
|
|
.filter(ScanTask.created_at >= cutoff)
|
|||
|
|
.scalar() or 0
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _fetch_recent_tasks(engine: DiagnosticEngine,
|
|||
|
|
limit: int = 500) -> List[Any]:
|
|||
|
|
from app.models import ScanTask
|
|||
|
|
return (
|
|||
|
|
engine.db.query(ScanTask)
|
|||
|
|
.order_by(ScanTask.created_at.desc())
|
|||
|
|
.limit(limit)
|
|||
|
|
.all()
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _docker_logs(container: str, tail: int = 500) -> str:
|
|||
|
|
"""用 docker SDK 取容器日志。失败返回空字符串。"""
|
|||
|
|
try:
|
|||
|
|
from app.diagnostic.docker_probe import get_client
|
|||
|
|
client = get_client()
|
|||
|
|
if not client:
|
|||
|
|
return ""
|
|||
|
|
c = client.containers.get(container)
|
|||
|
|
raw = c.logs(tail=tail)
|
|||
|
|
return raw.decode("utf-8", errors="replace") if isinstance(raw, (bytes, bytearray)) else str(raw)
|
|||
|
|
except Exception: # noqa: BLE001
|
|||
|
|
return ""
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _docker_inspect(container: str) -> Optional[Dict[str, Any]]:
|
|||
|
|
"""用 docker SDK 取容器状态元数据。失败返回 None。"""
|
|||
|
|
try:
|
|||
|
|
from app.diagnostic.docker_probe import get_client
|
|||
|
|
client = get_client()
|
|||
|
|
if not client:
|
|||
|
|
return None
|
|||
|
|
c = client.containers.get(container)
|
|||
|
|
attrs = c.attrs or {}
|
|||
|
|
state = attrs.get("State", {})
|
|||
|
|
return {
|
|||
|
|
"status": state.get("Status", "unknown"),
|
|||
|
|
"restart_count": attrs.get("RestartCount", 0),
|
|||
|
|
"started_at": state.get("StartedAt", ""),
|
|||
|
|
"image": attrs.get("Config", {}).get("Image", ""),
|
|||
|
|
}
|
|||
|
|
except Exception: # noqa: BLE001
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _grep_patterns(text: str, patterns: List[str],
|
|||
|
|
max_lines: int = 20) -> List[str]:
|
|||
|
|
"""从日志文本中抓取命中 pattern 的行。"""
|
|||
|
|
hits: List[str] = []
|
|||
|
|
for line in text.splitlines():
|
|||
|
|
low = line.lower()
|
|||
|
|
for p in patterns:
|
|||
|
|
if p.lower() in low:
|
|||
|
|
hits.append(line.strip())
|
|||
|
|
break
|
|||
|
|
if len(hits) >= max_lines:
|
|||
|
|
break
|
|||
|
|
return hits
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ======================================================================
|
|||
|
|
# R001: error 比例
|
|||
|
|
# ======================================================================
|
|||
|
|
|
|||
|
|
def check_r001_error_ratio(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
counts = _fetch_recent_verdict_counts(engine, 100)
|
|||
|
|
total = sum(counts.values())
|
|||
|
|
if total == 0:
|
|||
|
|
return _ok("R001", "暂无扫描记录", run_by=run_by)
|
|||
|
|
|
|||
|
|
error_count = counts.get("error", 0)
|
|||
|
|
error_ratio = error_count / total
|
|||
|
|
threshold = cfg.error_ratio_threshold
|
|||
|
|
|
|||
|
|
if error_ratio > threshold:
|
|||
|
|
return _crit(
|
|||
|
|
"R001",
|
|||
|
|
f"错误比例偏高（{error_ratio:.1%} > 阈值 {threshold:.0%}）",
|
|||
|
|
f"近 100 次扫描 {error_count} 次 error",
|
|||
|
|
hint="建议检查 ClamAV / Redis / Postgres 引擎健康",
|
|||
|
|
evidence=[f"近 100 次 verdict 分布: {counts}"],
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
return _ok(
|
|||
|
|
"R001",
|
|||
|
|
f"错误比例正常（{error_ratio:.1%}，阈值 {threshold:.0%}）",
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
check_r001_error_ratio._rule_id = "R001"
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ======================================================================
|
|||
|
|
# R002: ClamAV 病毒库日期
|
|||
|
|
# ======================================================================
|
|||
|
|
|
|||
|
|
def check_r002_clamav_db(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
try:
|
|||
|
|
clamav_info = engine.scanner.health().get("clamav", {})
|
|||
|
|
except Exception as e: # noqa: BLE001
|
|||
|
|
return _skip("R002", f"ClamAV 健康检查异常: {e}", run_by=run_by)
|
|||
|
|
|
|||
|
|
sig_date_str = clamav_info.get("signature_date")
|
|||
|
|
if not sig_date_str:
|
|||
|
|
available = clamav_info.get("available", clamav_info.get("ok", False))
|
|||
|
|
if not available:
|
|||
|
|
return _crit(
|
|||
|
|
"R002", "ClamAV 引擎不可用",
|
|||
|
|
hint="检查 clamd 容器是否在运行",
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
return _warn(
|
|||
|
|
"R002", "ClamAV 病毒库日期未知",
|
|||
|
|
hint="在线更新失败？U 盘离线导入",
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# signature_date 可能是字符串如 "2026-10-14"
|
|||
|
|
try:
|
|||
|
|
sig_date = datetime.strptime(str(sig_date_str)[:10], "%Y-%m-%d").date()
|
|||
|
|
except Exception: # noqa: BLE001
|
|||
|
|
return _skip("R002", f"无法解析病毒库日期: {sig_date_str}", run_by=run_by)
|
|||
|
|
|
|||
|
|
days_old = (datetime.now().date() - sig_date).days
|
|||
|
|
stale = cfg.clamav_db_stale_days
|
|||
|
|
|
|||
|
|
if days_old > stale:
|
|||
|
|
return _warn(
|
|||
|
|
"R002",
|
|||
|
|
f"ClamAV 病毒库已 {days_old} 天未更新",
|
|||
|
|
hint="在线更新不可用 → U 盘离线导入 main.cvd / daily.cvd",
|
|||
|
|
evidence=[f"signature_date = {sig_date_str}"],
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
return _ok(
|
|||
|
|
"R002",
|
|||
|
|
f"ClamAV 病毒库新鲜（{sig_date_str}，{days_old} 天）",
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
check_r002_clamav_db._rule_id = "R002"
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ======================================================================
|
|||
|
|
# R003: YARA 规则 mtime
|
|||
|
|
# ======================================================================
|
|||
|
|
|
|||
|
|
def check_r003_yara_mtime(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
|||
|
|
settings = get_settings()
|
|||
|
|
rules_dir = settings.rules_dir
|
|||
|
|
|
|||
|
|
if not os.path.isdir(rules_dir):
|
|||
|
|
return _skip("R003", f"规则目录不存在: {rules_dir}", run_by=run_by)
|
|||
|
|
|
|||
|
|
yamls = [f for f in os.listdir(rules_dir) if f.endswith((".yar", ".yara", ".rules"))]
|
|||
|
|
if not yamls:
|
|||
|
|
return _warn(
|
|||
|
|
"R003", "YARA 规则目录为空",
|
|||
|
|
hint="检查 rules_dir 配置是否正确",
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
latest_mtime = max(
|
|||
|
|
os.path.getmtime(os.path.join(rules_dir, f)) for f in yamls
|
|||
|
|
)
|
|||
|
|
latest_date = datetime.fromtimestamp(latest_mtime).date()
|
|||
|
|
days_old = (datetime.now().date() - latest_date).days
|
|||
|
|
|
|||
|
|
# 对比策略版本发布时间（如果策略文件比规则新，说明规则可能没更新）
|
|||
|
|
policy_file = settings.policy_file
|
|||
|
|
policy_mtime = None
|
|||
|
|
if os.path.isfile(policy_file):
|
|||
|
|
policy_mtime = os.path.getmtime(policy_file)
|
|||
|
|
|
|||
|
|
evidence = [f"规则目录: {rules_dir}", f"规则文件数: {len(yamls)}",
|
|||
|
|
f"最新规则 mtime: {latest_date}"]
|
|||
|
|
if policy_mtime and policy_mtime > latest_mtime:
|
|||
|
|
evidence.append(f"策略文件比规则新（策略: {datetime.fromtimestamp(policy_mtime).date()}）")
|
|||
|
|
return _warn(
|
|||
|
|
"R003",
|
|||
|
|
"策略文件比 YARA 规则新，规则可能未随策略版本更新",
|
|||
|
|
hint="同步更新规则文件或确认规则已加载",
|
|||
|
|
evidence=evidence,
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
return _ok(
|
|||
|
|
"R003",
|
|||
|
|
f"YARA 规则正常（{len(yamls)} 条规则，最新 {latest_date}，{days_old} 天）",
|
|||
|
|
evidence=evidence,
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
check_r003_yara_mtime._rule_id = "R003"
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ======================================================================
|
|||
|
|
# R004: 引擎健康
|
|||
|
|
# ======================================================================
|
|||
|
|
|
|||
|
|
def check_r004_engine_health(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
|||
|
|
try:
|
|||
|
|
health = engine.scanner.health()
|
|||
|
|
except Exception as e: # noqa: BLE001
|
|||
|
|
return _crit("R004", f"引擎健康检查异常: {e}", run_by=run_by)
|
|||
|
|
|
|||
|
|
offline: List[str] = []
|
|||
|
|
for name, info in health.items():
|
|||
|
|
ok = info.get("ok", info.get("available", False))
|
|||
|
|
if not ok:
|
|||
|
|
offline.append(name)
|
|||
|
|
|
|||
|
|
if offline:
|
|||
|
|
return _crit(
|
|||
|
|
"R004",
|
|||
|
|
f"以下引擎不可用: {', '.join(offline)}",
|
|||
|
|
hint="检查对应容器状态和网络连通",
|
|||
|
|
evidence=[f"{n}: {health.get(n, {})}" for n in offline],
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
names = list(health.keys())
|
|||
|
|
return _ok(
|
|||
|
|
"R004",
|
|||
|
|
f"所有 {len(names)} 个引擎正常在线",
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
check_r004_engine_health._rule_id = "R004"
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ======================================================================
|
|||
|
|
# R005: 24h 无扫描
|
|||
|
|
# ======================================================================
|
|||
|
|
|
|||
|
|
def check_r005_no_scans(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
|||
|
|
count = _count_24h(engine)
|
|||
|
|
if count == 0:
|
|||
|
|
return _warn(
|
|||
|
|
"R005", "过去 24 小时内没有扫描记录",
|
|||
|
|
hint="摆渡系统可能停了；如果是正常停工可以忽略",
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
return _ok(
|
|||
|
|
"R005", f"过去 24 小时 {count} 次扫描",
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
check_r005_no_scans._rule_id = "R005"
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ======================================================================
|
|||
|
|
# R006: suspicious > malicious
|
|||
|
|
# ======================================================================
|
|||
|
|
|
|||
|
|
def check_r006_suspicious_over_malicious(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
if not cfg.suspicious_exceeds_malicious:
|
|||
|
|
return _skip("R006", "规则已在配置中禁用", run_by=run_by)
|
|||
|
|
|
|||
|
|
counts = _fetch_recent_verdict_counts(engine, 200)
|
|||
|
|
suspicious = counts.get("suspicious", 0)
|
|||
|
|
malicious = counts.get("malicious", 0)
|
|||
|
|
|
|||
|
|
if suspicious > malicious and suspicious > 0:
|
|||
|
|
return _warn(
|
|||
|
|
"R006",
|
|||
|
|
f"可疑数（{suspicious}）超过恶意数（{malicious}）",
|
|||
|
|
hint="suspicious 阈值配置可能过松；建议调紧让更多文件直接判为 malicious",
|
|||
|
|
evidence=[f"近 200 次 verdict 分布: {counts}"],
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
return _ok(
|
|||
|
|
"R006",
|
|||
|
|
f"suspicious={suspicious} vs malicious={malicious}",
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
check_r006_suspicious_over_malicious._rule_id = "R006"
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ======================================================================
|
|||
|
|
# R007: reject 比例
|
|||
|
|
# ======================================================================
|
|||
|
|
|
|||
|
|
def check_r007_reject_ratio(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
counts = _fetch_recent_verdict_counts(engine, 200)
|
|||
|
|
total = sum(counts.values())
|
|||
|
|
if total == 0:
|
|||
|
|
return _ok("R007", "暂无扫描记录", run_by=run_by)
|
|||
|
|
|
|||
|
|
reject = counts.get("reject", 0)
|
|||
|
|
ratio = reject / total
|
|||
|
|
threshold = cfg.reject_ratio_threshold
|
|||
|
|
|
|||
|
|
if ratio > threshold:
|
|||
|
|
return _warn(
|
|||
|
|
"R007",
|
|||
|
|
f"策略拒绝比例偏高（{ratio:.1%} > 阈值 {threshold:.0%}）",
|
|||
|
|
hint="检查拒绝原因 TOP；如果只是正常扩展名策略可以调宽",
|
|||
|
|
evidence=[f"近 200 次 verdict 分布: {counts}"],
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
return _ok(
|
|||
|
|
"R007",
|
|||
|
|
f"策略拒绝比例正常（{ratio:.1%}）",
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
check_r007_reject_ratio._rule_id = "R007"
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ======================================================================
|
|||
|
|
# R008: 同 SHA256 重复扫描
|
|||
|
|
# ======================================================================
|
|||
|
|
|
|||
|
|
def check_r008_duplicate_sha(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
|||
|
|
from app.models import ScanTask
|
|||
|
|
cutoff = datetime.now(timezone.utc) - timedelta(hours=1)
|
|||
|
|
rows = (
|
|||
|
|
engine.db.query(ScanTask.sha256, func.count(ScanTask.id))
|
|||
|
|
.filter(ScanTask.created_at >= cutoff)
|
|||
|
|
.group_by(ScanTask.sha256)
|
|||
|
|
.having(func.count(ScanTask.id) > 1)
|
|||
|
|
.all()
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
if rows:
|
|||
|
|
dups = [(sha[:12], c) for sha, c in rows]
|
|||
|
|
return _crit(
|
|||
|
|
"R008",
|
|||
|
|
f"近 1 小时有 {len(rows)} 个 SHA256 被重复扫描",
|
|||
|
|
hint="幂等机制可能失效；检查 idempotency_records 表写入",
|
|||
|
|
evidence=[f"sha256={sha} count={c}" for sha, c in dups[:10]],
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
return _ok("R008", "幂等机制正常（无重复 SHA256）", run_by=run_by)
|
|||
|
|
|
|||
|
|
check_r008_duplicate_sha._rule_id = "R008"
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ======================================================================
|
|||
|
|
# 日志匹配型规则（需要 docker.sock）
|
|||
|
|
# ======================================================================
|
|||
|
|
|
|||
|
|
def _check_docker_available(engine: DiagnosticEngine) -> Optional[RuleResult]:
|
|||
|
|
"""如果 Docker 不可用，返回 skip RuleResult；否则返回 None（继续执行）。"""
|
|||
|
|
if engine.docker_status == "none":
|
|||
|
|
return _skip(None, "Docker 能力未启用", run_by="")
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def check_r009_clamav_crash(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
|||
|
|
skip = _check_docker_available(engine)
|
|||
|
|
if skip:
|
|||
|
|
skip.id = "R009"
|
|||
|
|
return skip
|
|||
|
|
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
logs = _docker_logs("meshsafe-clamav", cfg.docker_log_tail)
|
|||
|
|
patterns = ["Killed", "exit 137", "segfault", "oom-killed", "Out of memory"]
|
|||
|
|
hits = _grep_patterns(logs, patterns)
|
|||
|
|
|
|||
|
|
# 同时检查 docker inspect 的 restart_count
|
|||
|
|
info = _docker_inspect("meshsafe-clamav")
|
|||
|
|
restart_count = info.get("restart_count", 0) if info else 0
|
|||
|
|
|
|||
|
|
if hits or restart_count > 3:
|
|||
|
|
ev = hits[:5] if hits else []
|
|||
|
|
if restart_count > 3:
|
|||
|
|
ev.append(f"docker inspect restart_count={restart_count}")
|
|||
|
|
return _crit(
|
|||
|
|
"R009",
|
|||
|
|
f"ClamAV 可能崩溃（restart_count={restart_count}，日志命中 {len(hits)} 行）",
|
|||
|
|
hint="OOM kill → 给 clamav 容器加内存限制 512m → 1g；\nsegfault → 检查 ClamAV 版本兼容性",
|
|||
|
|
evidence=ev,
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
return _ok(
|
|||
|
|
"R009",
|
|||
|
|
f"ClamAV 容器稳定（restart_count={restart_count}，无崩溃日志）",
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
check_r009_clamav_crash._rule_id = "R009"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def check_r010_postgres_fatal(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
|||
|
|
skip = _check_docker_available(engine)
|
|||
|
|
if skip:
|
|||
|
|
skip.id = "R010"
|
|||
|
|
return skip
|
|||
|
|
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
logs = _docker_logs("meshsafe-postgres", cfg.docker_log_tail)
|
|||
|
|
patterns = ["FATAL", "PANIC", "too many connections", "database is in recovery"]
|
|||
|
|
hits = _grep_patterns(logs, patterns)
|
|||
|
|
|
|||
|
|
if hits:
|
|||
|
|
return _crit(
|
|||
|
|
"R010",
|
|||
|
|
f"Postgres 出现严重错误（命中 {len(hits)} 行）",
|
|||
|
|
hint="too many connections → 调大 max_connections；\nFATAL → 检查磁盘空间和 WAL",
|
|||
|
|
evidence=hits[:8],
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
return _ok("R010", "Postgres 日志无 FATAL/PANIC", run_by=run_by)
|
|||
|
|
|
|||
|
|
check_r010_postgres_fatal._rule_id = "R010"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def check_r011_redis_oom(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
|||
|
|
skip = _check_docker_available(engine)
|
|||
|
|
if skip:
|
|||
|
|
skip.id = "R011"
|
|||
|
|
return skip
|
|||
|
|
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
logs = _docker_logs("meshsafe-redis", cfg.docker_log_tail)
|
|||
|
|
# 注意：不能用 "LOADING" — Redis 正常启动也会打 "Loading RDB / Done loading"
|
|||
|
|
# 只匹配真正有问题的：加载卡死循环、OOM、panic、assertion
|
|||
|
|
patterns = [
|
|||
|
|
"OOM command not allowed",
|
|||
|
|
"out of memory",
|
|||
|
|
"LOADING loop",
|
|||
|
|
"still loading",
|
|||
|
|
"LOADING state",
|
|||
|
|
"panic",
|
|||
|
|
"Assertion",
|
|||
|
|
]
|
|||
|
|
hits = _grep_patterns(logs, patterns)
|
|||
|
|
|
|||
|
|
if hits:
|
|||
|
|
return _crit(
|
|||
|
|
"R011",
|
|||
|
|
f"Redis 可能 OOM 或 panic（命中 {len(hits)} 行）",
|
|||
|
|
hint="OOM → 调大 maxmemory 或清缓存；\nLOADING 循环 → 检查 RDB/AOF 恢复",
|
|||
|
|
evidence=hits[:5],
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
return _ok("R011", "Redis 日志无 OOM/panic", run_by=run_by)
|
|||
|
|
|
|||
|
|
check_r011_redis_oom._rule_id = "R011"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def check_r012_no_space(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
|||
|
|
skip = _check_docker_available(engine)
|
|||
|
|
if skip:
|
|||
|
|
skip.id = "R012"
|
|||
|
|
return skip
|
|||
|
|
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
all_patterns = ["No space left on device", "ENOSPC", "disk quota exceeded"]
|
|||
|
|
containers = [
|
|||
|
|
"meshsafe-api", "meshsafe-clamav", "meshsafe-postgres",
|
|||
|
|
"meshsafe-redis", "meshsafe-nginx",
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
all_hits: List[str] = []
|
|||
|
|
for c in containers:
|
|||
|
|
logs = _docker_logs(c, cfg.docker_log_tail)
|
|||
|
|
hits = _grep_patterns(logs, all_patterns, max_lines=5)
|
|||
|
|
for h in hits:
|
|||
|
|
all_hits.append(f"[{c}] {h}")
|
|||
|
|
if len(all_hits) >= 10:
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
if all_hits:
|
|||
|
|
return _crit(
|
|||
|
|
"R012",
|
|||
|
|
"容器日志出现磁盘空间不足（严重！）",
|
|||
|
|
hint="立即清理磁盘，否则扫描会持续失败",
|
|||
|
|
evidence=all_hits[:10],
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
return _ok("R012", "所有容器日志无磁盘空间错误", run_by=run_by)
|
|||
|
|
|
|||
|
|
check_r012_no_space._rule_id = "R012"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def check_r013_nginx_upstream(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
|||
|
|
skip = _check_docker_available(engine)
|
|||
|
|
if skip:
|
|||
|
|
skip.id = "R013"
|
|||
|
|
return skip
|
|||
|
|
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
logs = _docker_logs("meshsafe-nginx", cfg.docker_log_tail)
|
|||
|
|
# 只匹配真正的 upstream 故障。
|
|||
|
|
# 注意 nginx 正常日志里有 "upstream response is buffered to a temporary file"，
|
|||
|
|
# 那是响应太大被缓冲的正常行为，绝不能在这里误伤。
|
|||
|
|
patterns = [
|
|||
|
|
"Connection refused",
|
|||
|
|
"upstream timed out",
|
|||
|
|
"upstream prematurely closed",
|
|||
|
|
"no live upstreams",
|
|||
|
|
"connect() failed",
|
|||
|
|
"502 Bad Gateway",
|
|||
|
|
"upstream server temporarily disabled",
|
|||
|
|
]
|
|||
|
|
hits = _grep_patterns(logs, patterns)
|
|||
|
|
|
|||
|
|
if hits:
|
|||
|
|
return _warn(
|
|||
|
|
"R013",
|
|||
|
|
f"Nginx upstream 异常（命中 {len(hits)} 行）",
|
|||
|
|
hint="api 容器可能临时不可用；检查 api 健康状态",
|
|||
|
|
evidence=hits[:5],
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
return _ok("R013", "Nginx 无 upstream 异常", run_by=run_by)
|
|||
|
|
|
|||
|
|
check_r013_nginx_upstream._rule_id = "R013"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def check_r014_api_connection(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
|||
|
|
skip = _check_docker_available(engine)
|
|||
|
|
if skip:
|
|||
|
|
skip.id = "R014"
|
|||
|
|
return skip
|
|||
|
|
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
logs = _docker_logs("meshsafe-api", cfg.docker_log_tail)
|
|||
|
|
patterns = ["Redis connection refused", "Postgres connection refused",
|
|||
|
|
"ConnectionError", "OperationalError", "can't connect"]
|
|||
|
|
hits = _grep_patterns(logs, patterns)
|
|||
|
|
|
|||
|
|
if hits:
|
|||
|
|
return _crit(
|
|||
|
|
"R014",
|
|||
|
|
f"API 与依赖服务连接异常（命中 {len(hits)} 行）",
|
|||
|
|
hint="Redis/Postgres 网络断了？检查 docker 网络和依赖容器健康",
|
|||
|
|
evidence=hits[:5],
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
return _ok("R014", "API 与依赖连接正常", run_by=run_by)
|
|||
|
|
|
|||
|
|
check_r014_api_connection._rule_id = "R014"
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ======================================================================
|
|||
|
|
# R015: 容器重启次数
|
|||
|
|
# ======================================================================
|
|||
|
|
|
|||
|
|
def check_r015_container_restart(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
|||
|
|
skip = _check_docker_available(engine)
|
|||
|
|
if skip:
|
|||
|
|
skip.id = "R015"
|
|||
|
|
return skip
|
|||
|
|
|
|||
|
|
containers = [
|
|||
|
|
"meshsafe-api", "meshsafe-clamav", "meshsafe-postgres",
|
|||
|
|
"meshsafe-redis", "meshsafe-nginx",
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
now_ts = time.time()
|
|||
|
|
bj = datetime.now(timezone(timedelta(hours=8)))
|
|||
|
|
restart_events_now: List[str] = [] # 本轮检测到的新事件（写进 evidence）
|
|||
|
|
new_restart_containers: List[str] = [] # 有新重启事件的容器名
|
|||
|
|
|
|||
|
|
for c in containers:
|
|||
|
|
info = _docker_inspect(c)
|
|||
|
|
if not info:
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
cur_restart = info.get("restart_count", 0)
|
|||
|
|
cur_started = info.get("started_at", "") or ""
|
|||
|
|
|
|||
|
|
old = _CONTAINER_LAST_STATE.get(c)
|
|||
|
|
restart_changed = False
|
|||
|
|
started_changed = False
|
|||
|
|
|
|||
|
|
if old is not None:
|
|||
|
|
# 第一次跑没有 old，不算事件，只建缓存
|
|||
|
|
restart_changed = cur_restart > old.get("restart_count", -1)
|
|||
|
|
started_changed = cur_started != old.get("started_at", "")
|
|||
|
|
|
|||
|
|
if restart_changed or started_changed:
|
|||
|
|
trigger = "restart_count_increased" if restart_changed else "started_at_changed"
|
|||
|
|
uptime_before_s = 0
|
|||
|
|
try:
|
|||
|
|
uptime_before_s = int(now_ts - old.get("checked_at", now_ts))
|
|||
|
|
except Exception:
|
|||
|
|
uptime_before_s = 0
|
|||
|
|
m, uptime_before_s = divmod(uptime_before_s, 60)
|
|||
|
|
h, m = divmod(m, 60)
|
|||
|
|
uptime_before_str = (
|
|||
|
|
f"{h}h {m}m" if h > 0 else f"{m}m" if m > 0 else "刚启动"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
event_line = (
|
|||
|
|
f"{bj.strftime('%H:%M:%S')} {c} "
|
|||
|
|
f"restart_count {old.get('restart_count','?')} → {cur_restart} "
|
|||
|
|
f"uptime_before={uptime_before_str} "
|
|||
|
|
f"trigger={trigger}"
|
|||
|
|
)
|
|||
|
|
restart_events_now.append(event_line)
|
|||
|
|
new_restart_containers.append(c)
|
|||
|
|
|
|||
|
|
# 持久化重启事件
|
|||
|
|
try:
|
|||
|
|
from app.diagnostic.storage import (
|
|||
|
|
append_restart_event, snapshot_on_restart,
|
|||
|
|
)
|
|||
|
|
append_restart_event(
|
|||
|
|
container=c,
|
|||
|
|
restart_before=old.get("restart_count", 0),
|
|||
|
|
restart_after=cur_restart,
|
|||
|
|
uptime_before=uptime_before_str,
|
|||
|
|
uptime_before_seconds=now_ts - old.get("checked_at", now_ts),
|
|||
|
|
trigger=trigger,
|
|||
|
|
)
|
|||
|
|
# 抓重启时的日志片段
|
|||
|
|
try:
|
|||
|
|
snapshot_on_restart(c, tail=300)
|
|||
|
|
except Exception:
|
|||
|
|
pass
|
|||
|
|
except Exception:
|
|||
|
|
pass
|
|||
|
|
|
|||
|
|
# 更新缓存为当前状态
|
|||
|
|
_CONTAINER_LAST_STATE[c] = {
|
|||
|
|
"restart_count": cur_restart,
|
|||
|
|
"started_at": cur_started,
|
|||
|
|
"checked_at": now_ts,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# ---------- 判定 severity ----------
|
|||
|
|
# 1. 有新重启事件 → medium（哪怕 restart_count 还没超 3）
|
|||
|
|
# 2. restart_count > 3 → crit
|
|||
|
|
# 3. 否则 → ok
|
|||
|
|
crit_list: List[str] = []
|
|||
|
|
all_status: List[str] = []
|
|||
|
|
for c in containers:
|
|||
|
|
info = _docker_inspect(c)
|
|||
|
|
if info:
|
|||
|
|
r = info.get("restart_count", 0)
|
|||
|
|
all_status.append(f"{c}: restart={r}")
|
|||
|
|
if r > 3:
|
|||
|
|
crit_list.append(f"{c} (restart_count={r})")
|
|||
|
|
|
|||
|
|
if crit_list:
|
|||
|
|
# 严重：累积重启次数过多
|
|||
|
|
combined_evidence = list(restart_events_now) + [f"⚠️ {x}" for x in crit_list]
|
|||
|
|
return _crit(
|
|||
|
|
"R015",
|
|||
|
|
f"以下容器不稳定（restart_count > 3）: {', '.join(crit_list)}",
|
|||
|
|
hint="检查对应容器日志找 crash 根因（OOM/segmentation/配置错误）",
|
|||
|
|
evidence=combined_evidence + all_status,
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
if new_restart_containers:
|
|||
|
|
# 中等：本轮检测到新重启事件（但累积次数还没超 3）
|
|||
|
|
return _warn(
|
|||
|
|
"R015",
|
|||
|
|
f"本轮检测到容器重启事件: {', '.join(new_restart_containers)}",
|
|||
|
|
hint="开发/运维可能执行了 docker restart，或容器健康检查触发了自动重启。"
|
|||
|
|
"日志片段已自动归档到 diagnostics_logs/*-restart-*.log",
|
|||
|
|
evidence=restart_events_now + all_status,
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
return _ok(
|
|||
|
|
"R015",
|
|||
|
|
"所有容器稳定（restart_count 均 ≤ 3，本轮无新重启事件）",
|
|||
|
|
evidence=all_status,
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
check_r015_container_restart._rule_id = "R015"
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ======================================================================
|
|||
|
|
# R016: 容器重启历史汇总（读持久化 restart_events.jsonl）
|
|||
|
|
# ======================================================================
|
|||
|
|
|
|||
|
|
def check_r016_restart_history(engine: DiagnosticEngine, run_by: str = "") -> RuleResult:
|
|||
|
|
"""读 restart_events.jsonl + compose_state，给出过去 N 天每个容器重启次数汇总。
|
|||
|
|
|
|||
|
|
和 R015 的区别：
|
|||
|
|
R015 = 当前状态检测（restart_count 阈值 + 本轮变化）
|
|||
|
|
R016 = 历史回顾（持久化事件日志聚合）
|
|||
|
|
"""
|
|||
|
|
skip = _check_docker_available(engine)
|
|||
|
|
if skip:
|
|||
|
|
skip.id = "R016"
|
|||
|
|
return skip
|
|||
|
|
|
|||
|
|
cfg = get_diag_config()
|
|||
|
|
|
|||
|
|
# 1. 读持久化重启事件
|
|||
|
|
events: List[Dict[str, Any]] = []
|
|||
|
|
try:
|
|||
|
|
from app.diagnostic.storage import load_restart_events
|
|||
|
|
if cfg.storage_enabled:
|
|||
|
|
events = load_restart_events(cfg.history_retention_days)
|
|||
|
|
except Exception:
|
|||
|
|
events = []
|
|||
|
|
|
|||
|
|
# 2. 按容器聚合
|
|||
|
|
by_container: Dict[str, List[Dict[str, Any]]] = {}
|
|||
|
|
for ev in events:
|
|||
|
|
c = ev.get("container", "?")
|
|||
|
|
by_container.setdefault(c, []).append(ev)
|
|||
|
|
|
|||
|
|
# 3. 取当前 restart_count（compose_state）
|
|||
|
|
containers = [
|
|||
|
|
"meshsafe-api", "meshsafe-clamav", "meshsafe-postgres",
|
|||
|
|
"meshsafe-redis", "meshsafe-nginx",
|
|||
|
|
]
|
|||
|
|
current_counts: Dict[str, int] = {}
|
|||
|
|
for c in containers:
|
|||
|
|
info = _docker_inspect(c)
|
|||
|
|
if info:
|
|||
|
|
current_counts[c] = info.get("restart_count", 0)
|
|||
|
|
|
|||
|
|
# 4. 构建 evidence（每个容器一行汇总）
|
|||
|
|
evidence: List[str] = []
|
|||
|
|
total_events = 0
|
|||
|
|
problem_containers: List[str] = []
|
|||
|
|
for c in containers:
|
|||
|
|
hist_count = len(by_container.get(c, []))
|
|||
|
|
cur_count = current_counts.get(c, 0)
|
|||
|
|
total_events += hist_count
|
|||
|
|
|
|||
|
|
line = f"{c}: 历史重启事件={hist_count} 当前restart_count={cur_count}"
|
|||
|
|
if hist_count > 0:
|
|||
|
|
last_ev = by_container[c][-1]
|
|||
|
|
line += f" 最近事件={last_ev.get('ts')}"
|
|||
|
|
evidence.append(line)
|
|||
|
|
|
|||
|
|
if hist_count >= 3:
|
|||
|
|
problem_containers.append(f"{c}({hist_count}次)")
|
|||
|
|
|
|||
|
|
# 5. 判定
|
|||
|
|
if total_events == 0:
|
|||
|
|
return _ok(
|
|||
|
|
"R016",
|
|||
|
|
f"过去 {cfg.history_retention_days} 天内，无容器重启事件记录",
|
|||
|
|
evidence=evidence,
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
if problem_containers:
|
|||
|
|
return _warn(
|
|||
|
|
"R016",
|
|||
|
|
f"过去 {cfg.history_retention_days} 天内，以下容器重启较频繁: "
|
|||
|
|
f"{', '.join(problem_containers)}",
|
|||
|
|
hint="开发/运维可能执行了多次 docker restart，"
|
|||
|
|
"或容器健康检查/初始化阶段触发重启。"
|
|||
|
|
"详见 restart_events.json 和 diagnostics_logs/*-restart-*.log",
|
|||
|
|
evidence=evidence,
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# 有事件但不频繁
|
|||
|
|
return _ok(
|
|||
|
|
"R016",
|
|||
|
|
f"过去 {cfg.history_retention_days} 天内共 {total_events} 次重启事件（均不频繁）",
|
|||
|
|
evidence=evidence,
|
|||
|
|
run_by=run_by,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
check_r016_restart_history._rule_id = "R016"
|