"""一键导出诊断包(diagnostics-YYYYMMDD-HHMMSS.tar.gz)。 调用方式(HTTP 接口): GET /api/v1/telemetry/dump → 返回文件下载 打包内容见方案文档 §3.4。 """ from __future__ import annotations import io import json import os import subprocess import tarfile import time from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, Dict, List, Optional from app.config import get_settings from app.db import SessionLocal from app.diagnostic.config import get_diag_config from app.diagnostic.engine import DiagnosticEngine from app.diagnostic.docker_probe import detect_docker_access def _sanitize_filename(name: str) -> str: """文件名脱敏:保留扩展名 + SHA256 前 8 位。""" if not name: return "" ext = os.path.splitext(name)[1].lower() # 截取前 20 字符 + hash-like 后缀 return f"file_{hash(name) % 999999:06d}{ext}" def _sanitize_ip(ip: str) -> str: if not ip: return "" parts = ip.split(".") if len(parts) == 4: return f"{'.'.join(parts[:3])}.*" return ip def _bj_time(dt) -> Optional[str]: """把 DB 里的 UTC datetime(naive 或 aware)转北京时间字符串。""" if dt is None: return None bj = timezone(timedelta(hours=8)) try: # naive UTC → 加上 tz 再转 if dt.tzinfo is None: dt = dt.replace(tzinfo=timezone.utc) bj_dt = dt.astimezone(bj) return bj_dt.strftime("%Y-%m-%d %H:%M:%S") except Exception: return str(dt) def _collect_scan_tasks(days: int = 7, max_rows: int = 10000) -> List[Dict[str, Any]]: """拉过去 N 天的 ScanTask,脱敏后返回 dict 列表。有 max_rows 兜底。""" from app.models import ScanTask db = SessionLocal() try: cutoff = datetime.now(timezone.utc) - timedelta(days=days) rows = ( db.query(ScanTask) .filter(ScanTask.created_at >= cutoff) .order_by(ScanTask.created_at.desc()) .limit(max_rows) .all() ) return [ { "id": r.id, "filename_sanitized": _sanitize_filename(r.filename), "declared_ext": r.declared_ext, "size": r.size, "sha256": r.sha256[:16], "channel": r.channel, "status": r.status, "mode": r.mode, "verdict": r.verdict, "action": r.action, "score": r.score, "reason": r.reason[:500] if r.reason else "", "policy_version": r.policy_version, "created_at": _bj_time(r.created_at), "finished_at": _bj_time(r.finished_at), "duration_ms": r.duration_ms, } for r in rows ] finally: db.close() def _collect_scan_findings(days: int = 7, max_rows: int = 50000) -> List[Dict[str, Any]]: from app.models import ScanFinding db = SessionLocal() try: # ScanFinding 没有 created_at 字段?查 ScanTask 关联时间 from app.models import ScanTask cutoff = datetime.now(timezone.utc) - timedelta(days=days) rows = ( db.query(ScanFinding) .join(ScanTask, ScanFinding.task_id == ScanTask.id) .filter(ScanTask.created_at >= cutoff) .order_by(ScanFinding.id.desc()) .limit(max_rows) .all() ) return [ { "task_id": r.task_id, "engine": r.engine, "layer": r.layer, "severity": r.severity, "rule": r.rule, "weight": r.weight, "detail": (r.detail[:2000] if r.detail else ""), "extracted_path": r.extracted_path, } for r in rows ] finally: db.close() def _collect_engine_errors(days: int = 7, max_rows: int = 5000) -> List[Dict[str, Any]]: from app.models import ScanTask db = SessionLocal() try: cutoff = datetime.now(timezone.utc) - timedelta(days=days) rows = ( db.query(ScanTask) .filter(ScanTask.verdict == "error") .filter(ScanTask.created_at >= cutoff) .order_by(ScanTask.created_at.desc()) .limit(max_rows) .all() ) return [ { "id": r.id, "filename_sanitized": _sanitize_filename(r.filename), "reason": r.reason[:500] if r.reason else "", "created_at": _bj_time(r.created_at), } for r in rows ] finally: db.close() def _collect_policy_snapshot() -> Dict[str, Any]: """策略版本 + YARA 规则列表(不含规则原文)。""" from app.main import policy_store policy = policy_store.current() settings = get_settings() rules_dir = settings.rules_dir yara_rules: List[str] = [] if os.path.isdir(rules_dir): yara_rules = sorted( f for f in os.listdir(rules_dir) if f.endswith((".yar", ".yara", ".rules")) ) return { "version": policy.version, "sha256": policy.sha256, "yara_rules_dir": rules_dir, "yara_rules_count": len(yara_rules), "yara_rules_list": yara_rules[:50], # 最多 50 个文件名 "global": policy.raw.get("global", {}), } def _collect_container_logs(containers: List[str], tail: int) -> Dict[str, str]: """用 docker SDK 取日志。""" from app.diagnostic.docker_probe import get_client client = get_client() result: Dict[str, str] = {} for c in containers: try: container = client.containers.get(c) raw = container.logs(tail=tail) result[c] = raw.decode("utf-8", errors="replace") if isinstance(raw, (bytes, bytearray)) else str(raw) except Exception as e: # noqa: BLE001 result[c] = f"[导出失败: {e}]" return result def _collect_container_meta(containers: List[str]) -> Dict[str, Dict[str, Any]]: """用 docker SDK 取容器状态元数据(含 uptime / OOM)。""" from app.diagnostic.docker_probe import get_client client = get_client() meta: Dict[str, Dict[str, Any]] = {} now = datetime.now(timezone.utc) for c in containers: try: container = client.containers.get(c) attrs = container.attrs or {} state = attrs.get("State", {}) started_raw = state.get("StartedAt", "") started_at = _docker_time(started_raw) uptime_s = (now - started_at).total_seconds() if started_at else -1 meta[c] = { "status": state.get("Status", "unknown"), "health": state.get("Health", {}).get("Status", ""), "restart_count": attrs.get("RestartCount", -1), "oom_killed": state.get("OOMKilled", False), "exit_code": state.get("ExitCode", 0), "started_at": _bj_time(started_at), "uptime": _fmt_duration(uptime_s), "uptime_seconds": int(uptime_s) if uptime_s >= 0 else -1, "image": attrs.get("Config", {}).get("Image", ""), } except Exception as e: # noqa: BLE001 meta[c] = {"error": str(e)[:200]} return meta def _collect_compose_state() -> Dict[str, Any]: """用 docker SDK 列容器(替代 docker compose ps,输出结构化 JSON)。""" from app.diagnostic.docker_probe import get_client client = get_client() now = datetime.now(timezone.utc) items: List[Dict[str, Any]] = [] try: for c in client.containers.list(all=True): attrs = c.attrs or {} state = attrs.get("State", {}) started_raw = state.get("StartedAt", "") started_at = _docker_time(started_raw) uptime_s = (now - started_at).total_seconds() if started_at else -1 items.append({ "name": c.name, "status": state.get("Status", "unknown"), "health": state.get("Health", {}).get("Status", ""), "restart_count": attrs.get("RestartCount", -1), "oom_killed": state.get("OOMKilled", False), "exit_code": state.get("ExitCode", 0), "image": c.image.tags[0] if c.image.tags else "unknown", "started_at": _bj_time(started_at), "uptime": _fmt_duration(uptime_s), }) except Exception as e: # noqa: BLE001 return {"error": str(e)[:300], "containers": []} return {"containers": items, "count": len(items)} def _docker_time(raw: str) -> Optional[datetime]: """把 docker 返回的 ISO 时间串转 aware UTC datetime。""" if not raw: return None try: # docker 返回: "2026-10-28T05:00:12.123456789Z" 或 ".123Z" s = raw.replace("Z", "+00:00") # 截断纳秒到微秒(Python fromisoformat 不接受纳秒) if "." in s: head, frac = s.split(".", 1) tz_idx = frac.rfind("+") if tz_idx >= 0: frac_ns = frac[:tz_idx] tz = frac[tz_idx:] frac_ns = (frac_ns + "000000")[:6] s = head + "." + frac_ns + tz return datetime.fromisoformat(s) except Exception: return None def _fmt_duration(seconds: float) -> str: """把秒数格式化成人类可读的时长。""" if seconds < 0: return "未知" s = int(seconds) days, rem = divmod(s, 86400) hours, rem = divmod(rem, 3600) mins, secs = divmod(rem, 60) if days > 0: return f"{days}d {hours}h {mins}m" if hours > 0: return f"{hours}h {mins}m" if mins > 0: return f"{mins}m {secs}s" return f"{secs}s" # ====================================================================== # 主入口:build_diagnostics_package # ====================================================================== def build_diagnostics_package() -> bytes: """生成完整诊断包,返回 tar.gz 字节。""" started = time.time() now = datetime.now(timezone.utc) cfg = get_diag_config() settings = get_settings() containers = [ "meshsafe-api", "meshsafe-postgres", "meshsafe-redis", "meshsafe-clamav", "meshsafe-nginx", ] # ---- 1. docker 能力 ---- docker_info = detect_docker_access(mode=cfg.docker_mode) # ---- 2. 导出前先存当前 docker 日志快照(持久化目录里)---- snapshot_files: List[Path] = [] try: from app.diagnostic.storage import snapshot_container_logs if cfg.storage_enabled and docker_info["status"] != "none": snapshot_files = snapshot_container_logs( containers, cfg.export_log_tail, filename_prefix="export", ) except Exception: # noqa: BLE001 snapshot_files = [] # ---- 3. 规则引擎(全量跑一遍)---- db = SessionLocal() from app.main import scanner, policy_store eng = DiagnosticEngine( db=db, scanner=scanner, policy_store=policy_store, docker_status=docker_info["status"], docker_log_tail=cfg.docker_log_tail, ) rule_results = eng.run_all(run_by="export") db.close() # ---- 4. 加载规则历史(持久化 JSONL)---- rule_history: List[Dict[str, Any]] = [] restart_events: List[Dict[str, Any]] = [] try: from app.diagnostic.storage import load_rule_history, load_restart_events if cfg.storage_enabled: rule_history = load_rule_history(cfg.history_retention_days) restart_events = load_restart_events(cfg.history_retention_days) except Exception: # noqa: BLE001 rule_history = [] restart_events = [] # 按容器聚合重启事件数(用于 rules_report 的 summary) restart_by_container: Dict[str, int] = {} for ev in restart_events: c = ev.get("container", "?") restart_by_container[c] = restart_by_container.get(c, 0) + 1 # ---- 5. DB 数据(默认过去 7 天,各有上限兜底)---- span_days = getattr(cfg, "export_db_span_days", 7) scan_tasks = _collect_scan_tasks( days=span_days, max_rows=getattr(cfg, "export_scan_tasks_max", 10000), ) scan_findings = _collect_scan_findings( days=span_days, max_rows=getattr(cfg, "export_scan_findings_max", 50000), ) engine_errors = _collect_engine_errors( days=span_days, max_rows=getattr(cfg, "export_engine_errors_max", 5000), ) policy_snapshot = _collect_policy_snapshot() # ---- 6. 容器数据(如果 Docker 可用)---- docker_logs_out: Dict[str, str] = {} container_meta: Dict[str, Dict[str, Any]] = {} compose_state: Dict[str, Any] = {"containers": [], "error": "Docker 不可用"} if docker_info["status"] != "none": docker_logs_out = _collect_container_logs(containers, cfg.export_log_tail) container_meta = _collect_container_meta(containers) compose_state = _collect_compose_state() # ---- 7. 组装 tar.gz ---- buf = io.BytesIO() with tarfile.open(fileobj=buf, mode="w:gz") as tar: def _add_json(name: str, obj: Any) -> None: data = json.dumps(obj, ensure_ascii=False, indent=2).encode("utf-8") info = tarfile.TarInfo(name) info.size = len(data) tar.addfile(info, io.BytesIO(data)) def _add_text(name: str, text: str) -> None: data = text.encode("utf-8") info = tarfile.TarInfo(name) info.size = len(data) tar.addfile(info, io.BytesIO(data)) # meta.json _add_json("meta.json", { "exported_at": now.isoformat(), "generated_by": "MeshSafe Diagnostics Kit v1.0", "meshsafe_version": "1.1.0", "docker_image": container_meta.get("meshsafe-api", {}).get("image", "unknown"), "policy_version": policy_snapshot["version"], "storage_enabled": cfg.storage_enabled, "rules_history_count": len(rule_history), "log_snapshots_included": len(snapshot_files), "export_duration_ms": int((time.time() - started) * 1000), # 数据范围 "db_span_days": getattr(cfg, "export_db_span_days", 7), "scan_tasks_count": len(scan_tasks), "scan_findings_count": len(scan_findings), "engine_errors_count": len(engine_errors), }) # docker_detect.json _add_json("docker_detect.json", docker_info) # compose_state.json(结构化 JSON) _add_json("compose_state.json", compose_state) # rules_report.json(实时全量 + 持久化历史 + 重启事件汇总) _add_json("rules_report.json", { "exported_at": now.isoformat(), "docker_status": docker_info["status"], "current_run": [r.to_dict() for r in rule_results], "history_retention_days": cfg.history_retention_days, "history_count": len(rule_history), "history": rule_history, "restart_events_count": len(restart_events), "restart_events_summary": restart_by_container, }) # restart_events.json(完整重启事件历史) _add_json("restart_events.json", { "exported_at": now.isoformat(), "retention_days": cfg.history_retention_days, "count": len(restart_events), "events": restart_events, }) # scan_tasks.json / scan_findings.json / engine_errors.json _add_json("scan_tasks.json", scan_tasks) _add_json("scan_findings.json", scan_findings) _add_json("engine_errors.json", engine_errors) # policy_snapshot.json _add_json("policy_snapshot.json", policy_snapshot) # engine_health.json _add_json("engine_health.json", scanner.health()) # containers/*.log / *.meta.json(当前快照) for c in containers: if c in docker_logs_out: _add_text(f"containers/{c}.log", docker_logs_out[c]) if c in container_meta: _add_json(f"containers/{c}.meta.json", container_meta[c]) # diagnostics_logs/(持久化目录下所有 .log — timer 快照 + export 快照 + restart 快照) try: from app.diagnostic.storage import _logs_dir logs_dir = _logs_dir() if cfg.storage_enabled and logs_dir.exists(): for fpath in logs_dir.glob("*.log"): try: tar.add( str(fpath), arcname=f"diagnostics_logs/{fpath.name}", ) except Exception: # noqa: BLE001 pass except Exception: # noqa: BLE001 # 回退:只用 export 生成的快照 for fpath in snapshot_files: try: tar.add(str(fpath), arcname=f"diagnostics_logs/{fpath.name}") except Exception: pass return buf.getvalue()