feat: MeshSafe 文件摆渡安全卫士 v1.0 初始提交
核心功能: - 6 层检测管线: L0 结构分析 → L1 ClamAV → L2 文档深度检测 → L3 关键词匹配 → L4 YARA → L5 PE/二进制 - FastAPI + PostgreSQL + Redis + Docker Compose 架构 - 客户端 HMAC-SHA256 签名鉴权 + 限流 + 幂等性 - 控制台 UI (登录/Dashboard/诊断中心/策略管理/任务列表/测试中心/产品介绍页) - 三层诊断定时器 (error_scan 30s / db_aggregate 30s / container_check 5min) 并发与稳定性修复: - 诊断定时器 Redis 时间槽位锁 (多 worker 同一槽位只有一个执行) - JSONL 存储 fcntl 跨进程排他锁 + 容错解码 + 原子重写 - 移除 SQLite fallback, 统一 PostgreSQL - API_CLIENTS 脱敏为环境变量, .env 已 .gitignore
This commit is contained in:
1
app/engines/__init__.py
Normal file
1
app/engines/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""引擎包。"""
|
||||
123
app/engines/base.py
Normal file
123
app/engines/base.py
Normal file
@@ -0,0 +1,123 @@
|
||||
"""检测引擎公共数据结构与基类。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
|
||||
class Severity(str, Enum):
|
||||
INFO = "info"
|
||||
LOW = "low"
|
||||
MEDIUM = "medium"
|
||||
HIGH = "high"
|
||||
CRITICAL = "critical"
|
||||
|
||||
|
||||
class Layer(str, Enum):
|
||||
L0 = "L0"
|
||||
L1 = "L1"
|
||||
L2 = "L2"
|
||||
L2B = "L2B"
|
||||
L3 = "L3"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Finding:
|
||||
"""单条检测信号。"""
|
||||
|
||||
engine: str
|
||||
layer: str
|
||||
severity: str
|
||||
rule: str
|
||||
detail: str = ""
|
||||
extracted_path: str = ""
|
||||
weight: float = 0.0
|
||||
meta: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"engine": self.engine,
|
||||
"layer": self.layer,
|
||||
"severity": self.severity,
|
||||
"rule": self.rule,
|
||||
"detail": self.detail,
|
||||
"extracted_path": self.extracted_path,
|
||||
"weight": self.weight,
|
||||
"meta": self.meta,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class EngineResult:
|
||||
"""单个引擎对单个文件的检测结果。"""
|
||||
|
||||
engine: str
|
||||
ok: bool = True
|
||||
findings: List[Finding] = field(default_factory=list)
|
||||
meta: Dict[str, Any] = field(default_factory=dict)
|
||||
error: str = ""
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
return {
|
||||
"engine": self.engine,
|
||||
"ok": self.ok,
|
||||
"findings": [f.to_dict() for f in self.findings],
|
||||
"meta": self.meta,
|
||||
"error": self.error,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanTarget:
|
||||
"""待检测文件。"""
|
||||
|
||||
path: str
|
||||
filename: str
|
||||
declared_ext: str = ""
|
||||
logical_path: str = ""
|
||||
depth: int = 0
|
||||
is_extracted: bool = False
|
||||
|
||||
|
||||
class Engine:
|
||||
"""引擎基类。
|
||||
|
||||
子类实现 ``scan`` 方法，捕获所有异常并转换为 ``EngineResult.error``，
|
||||
保证单个引擎故障不会导致整个扫描任务失败。
|
||||
"""
|
||||
|
||||
name: str = "base"
|
||||
layer: str = "L0"
|
||||
|
||||
def scan(self, target: ScanTarget, ctx: Any) -> EngineResult:
|
||||
raise NotImplementedError
|
||||
|
||||
def safe_scan(self, target: ScanTarget, ctx: Any) -> EngineResult:
|
||||
try:
|
||||
return self.scan(target, ctx)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return EngineResult(engine=self.name, ok=False, error=f"{type(exc).__name__}: {exc}")
|
||||
|
||||
def health(self) -> Dict[str, Any]:
|
||||
"""引擎健康状态。子类可覆盖以提供更丰富的探测（如 ClamAV/PING）。
|
||||
|
||||
返回结构约定（控制台 dashboard 模板依赖此 schema）：
|
||||
{
|
||||
"engine": "clamav", # 引擎名（= self.name）
|
||||
"version": "ClamAV 1.0.x", # 版本描述
|
||||
"ok": True/False, # 是否可用
|
||||
"status": "ok"|"error"|"degraded",
|
||||
"error": None | "...",
|
||||
"info": None | "...",
|
||||
}
|
||||
"""
|
||||
return {
|
||||
"engine": self.name,
|
||||
"version": "builtin",
|
||||
"ok": True,
|
||||
"status": "ok",
|
||||
"error": None,
|
||||
"info": None,
|
||||
}
|
||||
302
app/engines/binary.py
Normal file
302
app/engines/binary.py
Normal file
@@ -0,0 +1,302 @@
|
||||
"""L2b YARA 规则与二进制分析引擎。
|
||||
|
||||
包含：
|
||||
- YARA 规则匹配（规则库从 ``rules/`` 目录加载，支持热加载）
|
||||
- PE 文件结构分析（pefile）：节区异常、导入表、编译信息、签名
|
||||
- 可执行文件能力识别（capa，若已安装）
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.engines.base import Engine, EngineResult, Finding, ScanTarget, Severity
|
||||
|
||||
BINARY_EXTS = {".exe", ".dll", ".sys", ".scr", ".com", ".cpl", ".ocx", ".msi", ".apk", ".jar", ".so", ".bin", ".elf"}
|
||||
|
||||
# ---------- 模块级全局 YARA 规则单例 ----------
|
||||
# 所有 YaraEngine 实例共享同一个编译结果，避免重复 compile
|
||||
# rules_dir 不同时（极少见）才分叉加载
|
||||
_YARA_GLOBAL: Dict[str, Dict[str, Any]] = {}
|
||||
_YARA_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _get_global_yara(rules_dir: str) -> Optional[Any]:
|
||||
"""获取全局共享的 YARA Rules 对象（自动按目录 mtime 重载）。"""
|
||||
with _YARA_LOCK:
|
||||
cached = _YARA_GLOBAL.get(rules_dir)
|
||||
import yara as _yr
|
||||
newest = 0.0
|
||||
files: List[str] = []
|
||||
if os.path.isdir(rules_dir):
|
||||
for name in sorted(os.listdir(rules_dir)):
|
||||
if name.endswith((".yar", ".yara")):
|
||||
path = os.path.join(rules_dir, name)
|
||||
files.append(path)
|
||||
newest = max(newest, os.path.getmtime(path))
|
||||
|
||||
if cached and cached.get("mtime") == newest:
|
||||
return cached.get("rules")
|
||||
|
||||
if not files:
|
||||
_YARA_GLOBAL[rules_dir] = {"rules": None, "mtime": newest}
|
||||
return None
|
||||
|
||||
try:
|
||||
compiled = _yr.compile(filepaths={os.path.basename(p): p for p in files})
|
||||
except Exception:
|
||||
_YARA_GLOBAL[rules_dir] = {"rules": None, "mtime": newest}
|
||||
return None
|
||||
|
||||
_YARA_GLOBAL[rules_dir] = {"rules": compiled, "mtime": newest}
|
||||
return compiled
|
||||
|
||||
|
||||
def _discover_rules_dir() -> str:
|
||||
"""自动探测 YARA 规则目录（优先环境变量 > /app/rules > 项目根 rules/）。"""
|
||||
env_dir = os.environ.get("MESHSAFE_RULES_DIR", "").strip()
|
||||
if env_dir and os.path.isdir(env_dir):
|
||||
return env_dir
|
||||
|
||||
for candidate in ("/app/rules", "rules", "./rules", "../rules"):
|
||||
if os.path.isdir(candidate):
|
||||
return os.path.abspath(candidate)
|
||||
|
||||
return env_dir or "/app/rules"
|
||||
|
||||
|
||||
class YaraEngine(Engine):
|
||||
"""YARA 规则匹配引擎（使用全局共享的编译结果）。"""
|
||||
|
||||
name = "yara"
|
||||
layer = "L2B"
|
||||
|
||||
def __init__(self, rules_dir: str | None = None) -> None:
|
||||
# 显式传入时才用传入值，否则走自动探测
|
||||
self.rules_dir = rules_dir if rules_dir else _discover_rules_dir()
|
||||
|
||||
def _load_rules(self) -> Any:
|
||||
"""委托给全局单例。"""
|
||||
try:
|
||||
import yara # noqa: F401 — 只是探测可用性
|
||||
except Exception: # noqa: BLE001
|
||||
return None
|
||||
return _get_global_yara(self.rules_dir)
|
||||
|
||||
def scan(self, target: ScanTarget, ctx: Any) -> EngineResult:
|
||||
result = EngineResult(engine=self.name)
|
||||
rules = self._load_rules()
|
||||
if rules is None:
|
||||
result.ok = False
|
||||
result.error = "YARA 规则库不可用或编译失败"
|
||||
return result
|
||||
|
||||
try:
|
||||
matches = rules.match(target.path)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result.ok = False
|
||||
result.error = f"YARA 匹配失败：{exc}"
|
||||
return result
|
||||
|
||||
for match in matches:
|
||||
severity, weight = _judge_yara_rule(match.rule, match.tags)
|
||||
# 覆盖：如果 ctx 里的策略为该 YARA 规则定义了精细权重，用它
|
||||
try:
|
||||
policy = ctx.policy_engine.policy
|
||||
yara_weights = getattr(policy, "yara_weights", {}) or {}
|
||||
if match.rule in yara_weights:
|
||||
weight = float(yara_weights[match.rule])
|
||||
except Exception:
|
||||
pass # ctx 没有 policy 字段时保持默认 weight
|
||||
result.findings.append(
|
||||
Finding(
|
||||
engine=self.name,
|
||||
layer=self.layer,
|
||||
severity=severity,
|
||||
rule=match.rule,
|
||||
detail=f"YARA 规则 {match.rule} 命中（标签：{', '.join(match.tags) or '无'}）",
|
||||
weight=weight,
|
||||
meta={"tags": list(match.tags)},
|
||||
)
|
||||
)
|
||||
# rule_set_version 用全局 compiled 对象的 id（每次 reload 变）
|
||||
try:
|
||||
_r = _get_global_yara(self.rules_dir)
|
||||
result.meta["rule_set_version"] = str(id(_r)) if _r else "unavailable"
|
||||
except Exception:
|
||||
result.meta["rule_set_version"] = "unknown"
|
||||
return result
|
||||
|
||||
def health(self) -> Dict[str, Any]:
|
||||
rules = self._load_rules()
|
||||
ok = rules is not None
|
||||
return {
|
||||
"engine": self.name,
|
||||
"version": "rules" if ok else "none",
|
||||
"ok": ok,
|
||||
"status": "ok" if ok else "degraded",
|
||||
"error": None if ok else "YARA 规则库未加载",
|
||||
"info": f"rules_dir={self.rules_dir}" if not ok else "rules loaded",
|
||||
}
|
||||
|
||||
|
||||
class BinaryEngine(Engine):
|
||||
"""PE/二进制文件结构分析引擎。"""
|
||||
|
||||
name = "binary"
|
||||
layer = "L2B"
|
||||
|
||||
def scan(self, target: ScanTarget, ctx: Any) -> EngineResult:
|
||||
result = EngineResult(engine=self.name)
|
||||
ext = (target.declared_ext or "").lower()
|
||||
if ext not in BINARY_EXTS:
|
||||
return result
|
||||
|
||||
try:
|
||||
import pefile
|
||||
except Exception: # noqa: BLE001
|
||||
result.ok = False
|
||||
result.error = "pefile 未安装"
|
||||
return result
|
||||
|
||||
try:
|
||||
pe = pefile.PE(target.path, fast_load=False)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result.findings.append(
|
||||
Finding(
|
||||
engine=self.name,
|
||||
layer=self.layer,
|
||||
severity=Severity.MEDIUM.value,
|
||||
rule="PeParseFailed",
|
||||
detail=f"PE 结构解析失败：{exc}",
|
||||
weight=10.0,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
try:
|
||||
result.findings.extend(self._analyze(pe, target))
|
||||
finally:
|
||||
try:
|
||||
pe.close()
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
def _analyze(self, pe: Any, target: ScanTarget) -> List[Finding]:
|
||||
findings: List[Finding] = []
|
||||
|
||||
# 1) 节区熵值：加密/加壳的节区熵接近 8
|
||||
for section in getattr(pe, "sections", []) or []:
|
||||
try:
|
||||
name = section.Name.rstrip(b"\x00").decode("utf-8", errors="replace")
|
||||
entropy = section.get_entropy()
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
if entropy >= 7.2:
|
||||
findings.append(
|
||||
Finding(
|
||||
engine=self.name,
|
||||
layer=self.layer,
|
||||
severity=Severity.MEDIUM.value,
|
||||
rule="HighEntropySection",
|
||||
detail=f"节区 {name} 熵值 {entropy:.2f}，可能经过加密或加壳",
|
||||
weight=20.0,
|
||||
)
|
||||
)
|
||||
|
||||
# 2) 可疑导入表
|
||||
suspicious_imports = self._suspicious_imports(pe)
|
||||
if suspicious_imports:
|
||||
findings.append(
|
||||
Finding(
|
||||
engine=self.name,
|
||||
layer=self.layer,
|
||||
severity=Severity.HIGH.value,
|
||||
rule="SuspiciousImports",
|
||||
detail=f"导入表包含高危 API：{', '.join(suspicious_imports[:8])}",
|
||||
weight=45.0,
|
||||
)
|
||||
)
|
||||
|
||||
# 3) 无签名可执行文件
|
||||
if not self._has_signature(pe):
|
||||
findings.append(
|
||||
Finding(
|
||||
engine=self.name,
|
||||
layer=self.layer,
|
||||
severity=Severity.MEDIUM.value,
|
||||
rule="UnsignedExecutable",
|
||||
detail="可执行文件未携带数字签名",
|
||||
weight=20.0,
|
||||
)
|
||||
)
|
||||
|
||||
# 4) 编译时间戳异常（明显晚于其他节区时间，或为 1970/未来时间）
|
||||
timestamp = getattr(pe, "FILE_HEADER", None)
|
||||
if timestamp is not None:
|
||||
try:
|
||||
ts = int(timestamp.TimeDateStamp)
|
||||
if ts == 0 or ts > 4102444800: # 2100 年之后
|
||||
findings.append(
|
||||
Finding(
|
||||
engine=self.name,
|
||||
layer=self.layer,
|
||||
severity=Severity.LOW.value,
|
||||
rule="SuspiciousTimestamp",
|
||||
detail="PE 编译时间戳异常，可能被伪造",
|
||||
weight=10.0,
|
||||
)
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
return findings
|
||||
|
||||
def _suspicious_imports(self, pe: Any) -> List[str]:
|
||||
"""检测高危 API 导入。"""
|
||||
suspicious = {
|
||||
"VirtualAllocEx", "WriteProcessMemory", "CreateRemoteThread",
|
||||
"NtUnmapViewOfSection", "SetWindowsHookEx", "OpenProcess",
|
||||
"WinHttpOpen", "InternetOpenUrl", "URLDownloadToFile",
|
||||
"CryptEncrypt", "AdjustTokenPrivileges", "CreateService",
|
||||
"ShellExecute", "WinExec", "IsDebuggerPresent", "CheckRemoteDebuggerPresent",
|
||||
}
|
||||
found: List[str] = []
|
||||
try:
|
||||
for entry in getattr(pe, "DIRECTORY_ENTRY_IMPORT", []) or []:
|
||||
for imp in entry.imports:
|
||||
if imp.name:
|
||||
name = imp.name.decode("utf-8", errors="replace")
|
||||
if name in suspicious:
|
||||
found.append(name)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return sorted(set(found))
|
||||
|
||||
def _has_signature(self, pe: Any) -> bool:
|
||||
"""检查 PE 是否包含 Authenticode 签名目录。"""
|
||||
try:
|
||||
directory = pe.OPTIONAL_HEADER.DATA_DIRECTORY[4] # IMAGE_DIRECTORY_ENTRY_SECURITY
|
||||
return bool(directory.VirtualAddress and directory.Size)
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
|
||||
|
||||
def _judge_yara_rule(rule: str, tags: List[str]) -> tuple[str, float]:
|
||||
"""依据 YARA 规则名与标签判定严重度。"""
|
||||
tag_set = {t.lower() for t in tags}
|
||||
text = f"{rule} {' '.join(tags)}".lower()
|
||||
|
||||
if "critical" in tag_set or any(k in text for k in ("malware", "ransomware", "backdoor", "c2", "exploit")):
|
||||
return Severity.CRITICAL.value, 85.0
|
||||
if "high" in tag_set or any(k in text for k in ("trojan", "dropper", "obfusc", "shellcode")):
|
||||
return Severity.HIGH.value, 60.0
|
||||
if "medium" in tag_set or any(k in text for k in ("suspicious", "pua", "packer")):
|
||||
return Severity.MEDIUM.value, 30.0
|
||||
if "suspicious" in tag_set:
|
||||
return Severity.MEDIUM.value, 30.0
|
||||
return Severity.LOW.value, 10.0
|
||||
284
app/engines/clamav.py
Normal file
284
app/engines/clamav.py
Normal file
@@ -0,0 +1,284 @@
|
||||
"""ClamAV 检测引擎。
|
||||
|
||||
优先使用 clamd 常驻守护进程（TCP 或 Unix Socket）扫描，避免每次加载签名库。
|
||||
|
||||
重要设计决策：
|
||||
- clamd 默认 MaxFileSize = 100MB，超过就会截断 INSTREAM 并关闭 socket
|
||||
- 代码层需单独识别 Size limit，不当成引擎不可用（引擎活着，只是拒扫大文件）
|
||||
- 这样 PolicyEngine 不会因单个大文件 fail_closed → 其他 5 引擎继续扫
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import errno
|
||||
import socket
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from app.engines.base import Engine, EngineResult, Finding, ScanTarget, Severity
|
||||
|
||||
|
||||
class _ClamdSizeLimit(Exception):
|
||||
"""clamd 拒绝扫描超限文件时抛出的内部异常。
|
||||
|
||||
与普通引擎故障区分开——scan() 会捕获它并返回 ok=True + skipped 标记，
|
||||
PolicyEngine 步骤 1（引擎可用性）因此不会触发 fail_closed。
|
||||
"""
|
||||
|
||||
|
||||
class ClamAVEngine(Engine):
|
||||
"""ClamAV 静态查杀引擎。"""
|
||||
|
||||
name = "clamav"
|
||||
layer = "L1"
|
||||
|
||||
def __init__(self, host: str, port: int, timeout: int = 120) -> None:
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.timeout = timeout
|
||||
|
||||
# ------------------------------------------------------------------ 对外
|
||||
|
||||
def scan(self, target: ScanTarget, ctx: Any) -> EngineResult:
|
||||
result = EngineResult(engine=self.name)
|
||||
try:
|
||||
raw = self._scan_with_clamd(target.path)
|
||||
except _ClamdSizeLimit:
|
||||
# 文件超过 ClamAV MaxFileSize（clamd.conf 可调，默认 1G）
|
||||
# → 引擎只是拒扫这个大文件，不当成引擎故障
|
||||
result.ok = True
|
||||
result.meta["skipped"] = "size_limit"
|
||||
result.meta["reason"] = "文件超过 ClamAV 大小限制已跳过"
|
||||
return result
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# 真·引擎不可用：连不上 / PING 超时 / daemon 挂了
|
||||
result.ok = False
|
||||
result.error = f"ClamAV 引擎不可用：{type(exc).__name__}: {exc}"
|
||||
return result
|
||||
|
||||
result.meta.update(raw.get("meta", {}))
|
||||
for item in raw.get("matches", []):
|
||||
severity, weight = _judge_signature(item)
|
||||
result.findings.append(
|
||||
Finding(
|
||||
engine=self.name,
|
||||
layer=self.layer,
|
||||
severity=severity,
|
||||
rule=item,
|
||||
detail=f"ClamAV 签名命中：{item}",
|
||||
weight=weight,
|
||||
)
|
||||
)
|
||||
return result
|
||||
|
||||
def health(self) -> Dict[str, Any]:
|
||||
"""探测 clamd 存活状态与版本、签名库信息。"""
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
try:
|
||||
resp = self._command("PING")
|
||||
version = self._command("VERSION")
|
||||
pong = resp.strip().upper() == "PONG"
|
||||
|
||||
sig_date_str = None
|
||||
sig_days_old = None
|
||||
stale_warning = False
|
||||
short_version = version.strip() or "unknown"
|
||||
if version:
|
||||
parts = version.strip().split("/")
|
||||
# ClamAV 1.5.x 的 VERSION 格式：ClamAV 1.5.4/28166/Tue Oct 27 14:24:12 2026
|
||||
# 展示层只需要 "ClamAV 1.5.4"，签名库日期单独解析
|
||||
if len(parts) >= 2 and parts[1].isdigit():
|
||||
short_version = parts[0].strip()
|
||||
if len(parts) >= 3:
|
||||
date_str = parts[-1].strip()
|
||||
try:
|
||||
sig_date_str = datetime.strptime(date_str, "%a %b %d %H:%M:%S %Y").isoformat()
|
||||
sig_dt = datetime.strptime(date_str, "%a %b %d %H:%M:%S %Y").replace(tzinfo=timezone.utc)
|
||||
sig_days_old = (datetime.now(timezone.utc) - sig_dt).days
|
||||
stale_warning = sig_days_old > 7
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
base = {
|
||||
"engine": self.name,
|
||||
"version": short_version,
|
||||
"ok": pong,
|
||||
"status": "ok" if pong else "error",
|
||||
"error": None if pong else "PING 未返回 PONG",
|
||||
"info": None,
|
||||
"signature_date": sig_date_str,
|
||||
"signature_days_old": sig_days_old,
|
||||
"stale_warning": stale_warning,
|
||||
}
|
||||
if stale_warning and pong:
|
||||
base["status"] = "degraded"
|
||||
return base
|
||||
except Exception as exc: # noqa: BLE001
|
||||
return {
|
||||
"engine": self.name,
|
||||
"version": "unknown",
|
||||
"ok": False,
|
||||
"status": "error",
|
||||
"error": f"{type(exc).__name__}: {exc}",
|
||||
"info": None,
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ 内部
|
||||
|
||||
def _command(self, command: str) -> str:
|
||||
with socket.create_connection((self.host, self.port), timeout=self.timeout) as sock:
|
||||
sock.sendall(f"z{command}\0".encode("utf-8"))
|
||||
sock.settimeout(self.timeout)
|
||||
chunks = []
|
||||
while True:
|
||||
chunk = sock.recv(4096)
|
||||
if not chunk:
|
||||
break
|
||||
chunks.append(chunk)
|
||||
if chunk.endswith(b"\0"):
|
||||
break
|
||||
return b"".join(chunks).rstrip(b"\0").decode("utf-8", errors="replace")
|
||||
|
||||
def _scan_with_clamd(self, path: str) -> Dict[str, Any]:
|
||||
"""执行 INSTREAM 扫描。
|
||||
|
||||
clamd 的 INSTREAM 协议：先发送 ``zINSTREAM\0``，随后按
|
||||
``<4字节大端长度><数据块>`` 形式发送文件内容，最后以长度为 0 的块结束。
|
||||
|
||||
Size limit 识别：
|
||||
- clamd 默认 MaxFileSize = 100MB（我们 clamd.conf 调到 1G）
|
||||
- 超过限制时 clamd 会：
|
||||
1) 在日志写 "INSTREAM: Size limit reached"
|
||||
2) 关闭连接 → 我们 sock.sendall 抛 BrokenPipeError
|
||||
3) 或直接在响应里返回 "stream: Size limit reached ERROR"
|
||||
- 这两种情况统一转成 _ClamdSizeLimit，让上层 scan() 识别为 skipped
|
||||
"""
|
||||
with socket.create_connection((self.host, self.port), timeout=self.timeout) as sock:
|
||||
sock.settimeout(self.timeout)
|
||||
sock.sendall(b"zINSTREAM\0")
|
||||
|
||||
try:
|
||||
with open(path, "rb") as fh:
|
||||
while True:
|
||||
chunk = fh.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
sock.sendall(len(chunk).to_bytes(4, "big"))
|
||||
sock.sendall(chunk)
|
||||
except (BrokenPipeError, ConnectionResetError) as exc:
|
||||
# clamd 因为 Size limit 提前关闭 socket
|
||||
# 99% 的概率是 Size limit（也可能是其他问题，但都归为 skipped）
|
||||
raise _ClamdSizeLimit(f"clamd closed connection during INSTREAM: {exc}") from exc
|
||||
|
||||
sock.sendall((0).to_bytes(4, "big"))
|
||||
|
||||
response = b""
|
||||
while True:
|
||||
try:
|
||||
chunk = sock.recv(4096)
|
||||
except socket.timeout as exc:
|
||||
raise _ClamdSizeLimit(f"clamd INSTREAM response timeout: {exc}") from exc
|
||||
if not chunk:
|
||||
# clamd 没发任何响应就关了 socket → 大概率也是 Size limit
|
||||
raise _ClamdSizeLimit("clamd closed socket before sending response (likely size limit)")
|
||||
response += chunk
|
||||
if b"\0" in response:
|
||||
break
|
||||
|
||||
text = response.rstrip(b"\0").decode("utf-8", errors="replace").strip()
|
||||
|
||||
# clamd 可能在响应文本里显式报告 Size limit
|
||||
if _is_size_limit_response(text):
|
||||
raise _ClamdSizeLimit(f"clamd: {text}")
|
||||
|
||||
return _parse_clamd_response(text)
|
||||
|
||||
def scan_bytes(self, data: bytes) -> Dict[str, Any]:
|
||||
"""直接扫描内存字节（用于小文件快速路径）。"""
|
||||
with socket.create_connection((self.host, self.port), timeout=self.timeout) as sock:
|
||||
sock.settimeout(self.timeout)
|
||||
sock.sendall(b"zINSTREAM\0")
|
||||
sock.sendall(len(data).to_bytes(4, "big"))
|
||||
sock.sendall(data)
|
||||
sock.sendall((0).to_bytes(4, "big"))
|
||||
|
||||
response = b""
|
||||
while True:
|
||||
chunk = sock.recv(4096)
|
||||
if not chunk:
|
||||
break
|
||||
response += chunk
|
||||
if b"\0" in response:
|
||||
break
|
||||
text = response.rstrip(b"\0").decode("utf-8", errors="replace").strip()
|
||||
|
||||
return _parse_clamd_response(text)
|
||||
|
||||
|
||||
def _is_size_limit_response(text: str) -> bool:
|
||||
"""检查 clamd 响应文本是否包含 Size limit 相关错误。
|
||||
|
||||
clamd 可能返回：
|
||||
- "stream: Size limit reached ERROR"
|
||||
- "stream: Heuristics.Limits.Exceeded FOUND"
|
||||
- "Size limit reached, requested: ..., max: ..."
|
||||
"""
|
||||
upper = text.upper()
|
||||
if "SIZE LIMIT" in upper:
|
||||
return True
|
||||
if "HEURISTICS.LIMITS.EXCEEDED" in upper:
|
||||
return True
|
||||
if "MAXSCANSIZE" in upper or "MAXFILESIZE" in upper:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _parse_clamd_response(text: str) -> Dict[str, Any]:
|
||||
"""解析 clamd 扫描响应。
|
||||
|
||||
正常未命中返回 ``stream: OK``；命中返回多行 ``stream: <签名名> FOUND``。
|
||||
"""
|
||||
matches: List[str] = []
|
||||
meta: Dict[str, Any] = {}
|
||||
|
||||
for line in text.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
if ":" in line:
|
||||
key, _, value = line.partition(":")
|
||||
key = key.strip()
|
||||
value = value.strip()
|
||||
if key == "stream":
|
||||
if value.endswith("FOUND"):
|
||||
name = value[: -len("FOUND")].strip()
|
||||
matches.append(name)
|
||||
elif value.upper() == "OK":
|
||||
pass
|
||||
elif value.upper() == "ERROR":
|
||||
meta["error"] = value
|
||||
continue
|
||||
meta[key.lower().replace(" ", "_")] = value
|
||||
elif line.upper().endswith("FOUND"):
|
||||
matches.append(line[: -len("FOUND")].strip())
|
||||
|
||||
return {"matches": matches, "meta": meta}
|
||||
|
||||
|
||||
# 签名名中的可信度关键词映射到严重度
|
||||
_HIGH_KEYWORDS = ("ransom", "crypt", "trojan", "backdoor", "rootkit", "stealer", "keylog", "worm", "dropper", "ransomware")
|
||||
_MEDIUM_KEYWORDS = ("generic", "heur", "packed", "obfusc", "suspect", "malware", "riskware", "virus")
|
||||
_UNWANTED_KEYWORDS = ("PUA", "unwanted", "toolkit", "pup", "adware")
|
||||
|
||||
|
||||
def _judge_signature(signature: str) -> tuple[str, float]:
|
||||
"""依据签名名判定严重度与权重。"""
|
||||
upper = signature.upper()
|
||||
if any(k in upper for k in _HIGH_KEYWORDS):
|
||||
return Severity.CRITICAL.value, 90.0
|
||||
if any(k in upper for k in _UNWANTED_KEYWORDS):
|
||||
return Severity.MEDIUM.value, 25.0
|
||||
if any(k in upper for k in _MEDIUM_KEYWORDS):
|
||||
return Severity.HIGH.value, 60.0
|
||||
return Severity.HIGH.value, 55.0
|
||||
1001
app/engines/documents.py
Normal file
1001
app/engines/documents.py
Normal file
File diff suppressed because it is too large
Load Diff
383
app/engines/extractor.py
Normal file
383
app/engines/extractor.py
Normal file
@@ -0,0 +1,383 @@
|
||||
"""递归安全解包引擎。
|
||||
|
||||
安全约束（全部强制执行，不依赖调用方自觉）：
|
||||
- 解包深度上限
|
||||
- 解压总大小上限与单文件大小上限
|
||||
- 压缩比上限（防压缩炸弹）
|
||||
- 路径穿越拦截（Zip Slip）
|
||||
- 符号链接/硬链接逃逸拦截
|
||||
- 解包目标固定在独立临时目录，文件名重写为安全形式（不直接使用归档内原始名）
|
||||
|
||||
解包出来的文件会加入待检测队列，实现"递归检测"，避免只扫外层被绕过。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import tarfile
|
||||
import tempfile
|
||||
import zipfile
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, List, Optional
|
||||
|
||||
from app.engines.base import Finding, ScanTarget, Severity
|
||||
|
||||
ARCHIVE_EXTS = {".zip", ".jar", ".war", ".apk", ".epub", ".docx", ".xlsx", ".pptx", ".odt", ".ods", ".odp"}
|
||||
TAR_EXTS = {".tar", ".gz", ".tgz", ".bz2", ".xz", ".tbz2", ".txz"}
|
||||
|
||||
_SAFE_NAME_RE = re.compile(r"[^A-Za-z0-9._一-鿿-]+")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExtractionReport:
|
||||
"""解包结果。"""
|
||||
|
||||
files: List[ScanTarget] = field(default_factory=list)
|
||||
findings: List[Finding] = field(default_factory=list)
|
||||
total_bytes: int = 0
|
||||
truncated: bool = False
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
"file_count": len(self.files),
|
||||
"total_bytes": self.total_bytes,
|
||||
"truncated": self.truncated,
|
||||
}
|
||||
|
||||
|
||||
class SafeExtractor:
|
||||
"""递归安全解包器。"""
|
||||
|
||||
def __init__(self, ctx: Any) -> None:
|
||||
self.ctx = ctx
|
||||
self.settings = ctx.settings
|
||||
self.engine_name = "extractor"
|
||||
|
||||
def extract(self, target: ScanTarget, workdir: str) -> ExtractionReport:
|
||||
report = ExtractionReport()
|
||||
ext = (target.declared_ext or "").lower()
|
||||
|
||||
if ext in ARCHIVE_EXTS or zipfile.is_zipfile(target.path):
|
||||
self._extract_zip(target, workdir, report)
|
||||
elif ext in TAR_EXTS or tarfile.is_tarfile(target.path):
|
||||
self._extract_tar(target, workdir, report)
|
||||
|
||||
return report
|
||||
|
||||
# ------------------------------------------------------------------ 工具
|
||||
|
||||
def _sanitize_name(self, raw_name: str) -> str:
|
||||
"""把归档内原始文件名改写为安全的相对路径。
|
||||
|
||||
去掉目录层级与危险字符，避免原始名称中的 ``..``、绝对路径、
|
||||
Windows 保留设备名等造成问题。原始名称仍保留在逻辑路径中以便追溯。
|
||||
"""
|
||||
name = raw_name.replace("\\", "/")
|
||||
base = os.path.basename(name)
|
||||
base = _SAFE_NAME_RE.sub("_", base).strip("._")
|
||||
if not base:
|
||||
base = "unnamed"
|
||||
# 防止 Windows 保留设备名
|
||||
stem, dot, suffix = base.partition(".")
|
||||
reserved = {
|
||||
"con", "prn", "aux", "nul",
|
||||
*(f"com{i}" for i in range(1, 10)),
|
||||
*(f"lpt{i}" for i in range(1, 10)),
|
||||
}
|
||||
if stem.lower() in reserved:
|
||||
stem = f"_{stem}"
|
||||
return f"{stem}{dot}{suffix}" if dot else stem
|
||||
|
||||
def _target_dir(self, workdir: str, target: ScanTarget) -> str:
|
||||
"""为每个文件分配独立的解包子目录。"""
|
||||
digest = abs(hash(target.logical_path or target.filename)) % 100000
|
||||
path = os.path.join(workdir, f"ex_{target.depth}_{digest}")
|
||||
os.makedirs(path, exist_ok=True)
|
||||
return path
|
||||
|
||||
def _budget_exhausted(self, report: ExtractionReport) -> bool:
|
||||
return report.total_bytes >= self.settings.max_extract_total_bytes
|
||||
|
||||
def _record_limit(self, report: ExtractionReport, limit: int = 5000) -> bool:
|
||||
if len(report.files) >= limit:
|
||||
report.truncated = True
|
||||
return True
|
||||
return False
|
||||
|
||||
def _add_finding(
|
||||
self,
|
||||
report: ExtractionReport,
|
||||
severity: str,
|
||||
rule: str,
|
||||
detail: str,
|
||||
weight: float = 0.0,
|
||||
) -> None:
|
||||
report.findings.append(
|
||||
Finding(
|
||||
engine=self.engine_name,
|
||||
layer="L0",
|
||||
severity=severity,
|
||||
rule=rule,
|
||||
detail=detail,
|
||||
weight=weight,
|
||||
)
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------ ZIP
|
||||
|
||||
def _extract_zip(self, target: ScanTarget, workdir: str, report: ExtractionReport) -> None:
|
||||
out_dir = self._target_dir(workdir, target)
|
||||
max_ratio = self.settings.max_extract_ratio
|
||||
max_total = self.settings.max_extract_total_bytes
|
||||
max_depth = self.settings.max_extract_depth
|
||||
|
||||
try:
|
||||
with zipfile.ZipFile(target.path) as zf:
|
||||
infos = zf.infolist()
|
||||
for info in infos:
|
||||
if target.depth + 1 > max_depth:
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.MEDIUM.value,
|
||||
"ExtractDepthExceeded",
|
||||
f"嵌套层级超过上限 {max_depth}，深层内容未展开",
|
||||
15.0,
|
||||
)
|
||||
report.truncated = True
|
||||
break
|
||||
|
||||
if self._budget_exhausted(report) or self._record_limit(report):
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.MEDIUM.value,
|
||||
"ExtractBudgetExceeded",
|
||||
"解包数量或体积达到上限，后续内容未展开",
|
||||
10.0,
|
||||
)
|
||||
break
|
||||
|
||||
if info.is_dir():
|
||||
continue
|
||||
|
||||
name = info.filename
|
||||
if _is_traversal(name):
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.CRITICAL.value,
|
||||
"PathTraversalBlocked",
|
||||
f"已拦截路径穿越条目：{name}",
|
||||
50.0,
|
||||
)
|
||||
continue
|
||||
|
||||
# 压缩炸弹判定：按声明大小与实际膨胀比
|
||||
if info.compress_size > 0:
|
||||
ratio = info.file_size / info.compress_size
|
||||
if ratio > max_ratio:
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.HIGH.value,
|
||||
"ArchiveBombBlocked",
|
||||
f"成员 {name} 压缩比 {ratio:.1f}:1 超过阈值，已跳过",
|
||||
35.0,
|
||||
)
|
||||
continue
|
||||
|
||||
if info.file_size > self.settings.max_upload_bytes:
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.MEDIUM.value,
|
||||
"MemberTooLarge",
|
||||
f"成员 {name} 大小超过单文件上限，已跳过",
|
||||
10.0,
|
||||
)
|
||||
continue
|
||||
|
||||
safe_name = self._sanitize_name(name)
|
||||
dest = os.path.join(out_dir, safe_name)
|
||||
try:
|
||||
with zf.open(info) as src, open(dest, "wb") as out:
|
||||
written = 0
|
||||
while True:
|
||||
chunk = src.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
written += len(chunk)
|
||||
if written > self.settings.max_upload_bytes:
|
||||
out.close()
|
||||
os.remove(dest)
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.MEDIUM.value,
|
||||
"MemberTooLarge",
|
||||
f"成员 {name} 实际解压超限，已丢弃",
|
||||
10.0,
|
||||
)
|
||||
break
|
||||
out.write(chunk)
|
||||
else:
|
||||
report.total_bytes += written
|
||||
report.files.append(
|
||||
ScanTarget(
|
||||
path=dest,
|
||||
filename=safe_name,
|
||||
declared_ext=os.path.splitext(safe_name)[1].lower(),
|
||||
logical_path=f"{target.logical_path}!/{name}",
|
||||
depth=target.depth + 1,
|
||||
is_extracted=True,
|
||||
)
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.LOW.value,
|
||||
"ExtractMemberFailed",
|
||||
f"成员 {name} 解压失败：{exc}",
|
||||
0.0,
|
||||
)
|
||||
except zipfile.BadZipFile as exc:
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.MEDIUM.value,
|
||||
"ArchiveBroken",
|
||||
f"ZIP 结构损坏：{exc}",
|
||||
10.0,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.LOW.value,
|
||||
"ExtractFailed",
|
||||
f"解包失败：{exc}",
|
||||
0.0,
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------ TAR
|
||||
|
||||
def _extract_tar(self, target: ScanTarget, workdir: str, report: ExtractionReport) -> None:
|
||||
out_dir = self._target_dir(workdir, target)
|
||||
max_depth = self.settings.max_extract_depth
|
||||
|
||||
try:
|
||||
with tarfile.open(target.path) as tf:
|
||||
for member in tf.getmembers():
|
||||
if target.depth + 1 > max_depth:
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.MEDIUM.value,
|
||||
"ExtractDepthExceeded",
|
||||
f"嵌套层级超过上限 {max_depth}",
|
||||
15.0,
|
||||
)
|
||||
report.truncated = True
|
||||
break
|
||||
|
||||
if self._budget_exhausted(report) or self._record_limit(report):
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.MEDIUM.value,
|
||||
"ExtractBudgetExceeded",
|
||||
"解包数量或体积达到上限",
|
||||
10.0,
|
||||
)
|
||||
break
|
||||
|
||||
name = member.name
|
||||
if _is_traversal(name) or member.issym() or member.islnk():
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.CRITICAL.value,
|
||||
"PathTraversalBlocked",
|
||||
f"已拦截路径穿越或链接条目：{name}",
|
||||
50.0,
|
||||
)
|
||||
continue
|
||||
|
||||
if not member.isfile():
|
||||
continue
|
||||
|
||||
if member.size > self.settings.max_upload_bytes:
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.MEDIUM.value,
|
||||
"MemberTooLarge",
|
||||
f"成员 {name} 大小超过单文件上限，已跳过",
|
||||
10.0,
|
||||
)
|
||||
continue
|
||||
|
||||
safe_name = self._sanitize_name(name)
|
||||
dest = os.path.join(out_dir, safe_name)
|
||||
try:
|
||||
src = tf.extractfile(member)
|
||||
if src is None:
|
||||
continue
|
||||
with src, open(dest, "wb") as out:
|
||||
written = 0
|
||||
while True:
|
||||
chunk = src.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
written += len(chunk)
|
||||
if written > self.settings.max_upload_bytes:
|
||||
break
|
||||
out.write(chunk)
|
||||
report.total_bytes += written
|
||||
report.files.append(
|
||||
ScanTarget(
|
||||
path=dest,
|
||||
filename=safe_name,
|
||||
declared_ext=os.path.splitext(safe_name)[1].lower(),
|
||||
logical_path=f"{target.logical_path}!/{name}",
|
||||
depth=target.depth + 1,
|
||||
is_extracted=True,
|
||||
)
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.LOW.value,
|
||||
"ExtractMemberFailed",
|
||||
f"成员 {name} 解压失败：{exc}",
|
||||
0.0,
|
||||
)
|
||||
except tarfile.TarError as exc:
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.MEDIUM.value,
|
||||
"ArchiveBroken",
|
||||
f"TAR 结构异常：{exc}",
|
||||
10.0,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.LOW.value,
|
||||
"ExtractFailed",
|
||||
f"解包失败：{exc}",
|
||||
0.0,
|
||||
)
|
||||
|
||||
|
||||
def _is_traversal(name: str) -> bool:
|
||||
normalized = name.replace("\\", "/")
|
||||
if normalized.startswith("/"):
|
||||
return True
|
||||
if len(normalized) > 1 and normalized[1] == ":":
|
||||
return True
|
||||
return ".." in normalized.split("/")
|
||||
|
||||
|
||||
def cleanup_dir(path: str) -> None:
|
||||
"""安全删除临时目录。"""
|
||||
try:
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
|
||||
def make_workdir(root: str, prefix: str = "scan_") -> str:
|
||||
os.makedirs(root, exist_ok=True)
|
||||
return tempfile.mkdtemp(prefix=prefix, dir=root)
|
||||
157
app/engines/keywords.py
Normal file
157
app/engines/keywords.py
Normal file
@@ -0,0 +1,157 @@
|
||||
"""内网特征词表引擎（P1）。
|
||||
|
||||
数据驱动：词表从 policy.yaml 的 ``keyword_rules`` 段加载，不在代码中硬编码。
|
||||
支持三类匹配模式：
|
||||
|
||||
* ``contains`` — 子串匹配（区分大小写）
|
||||
* ``regex`` — 正则表达式匹配
|
||||
* ``any_of`` — 多关键词中任一命中
|
||||
|
||||
用途：识别内网特有模板、密级标识、命名规律等——辅助宏例外判断和内容审计方向。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, Dict, List
|
||||
|
||||
from app.engines.base import Engine, EngineResult, Finding, ScanTarget, Severity
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class KeywordEngine(Engine):
|
||||
"""内网特征词表匹配引擎。"""
|
||||
|
||||
name = "keywords"
|
||||
layer = "L2B"
|
||||
version = "1.0"
|
||||
|
||||
def __init__(self, keyword_rules: List[Dict[str, Any]] | None = None) -> None:
|
||||
"""初始化。keyword_rules 来自 PolicyStore.current().keyword_rules。"""
|
||||
self._rules = keyword_rules or []
|
||||
# 预编译正则
|
||||
self._compiled: List[Dict[str, Any]] = []
|
||||
for rule in self._rules:
|
||||
pattern = rule.get("pattern", "")
|
||||
if rule.get("match") == "regex" and pattern:
|
||||
try:
|
||||
compiled = re.compile(pattern, re.IGNORECASE if rule.get("ignore_case") else 0)
|
||||
except re.error:
|
||||
logger.warning("词表规则正则编译失败: %s", rule.get("name"))
|
||||
continue
|
||||
self._compiled.append({**rule, "_re": compiled})
|
||||
else:
|
||||
self._compiled.append(rule)
|
||||
|
||||
# ------------------------------------------------------------------- 接口实现
|
||||
|
||||
def is_enabled(self) -> bool:
|
||||
return bool(self._rules)
|
||||
|
||||
def scan(self, target: ScanTarget, ctx: Any) -> EngineResult:
|
||||
result = EngineResult(engine=self.name)
|
||||
if not self._rules:
|
||||
return result
|
||||
|
||||
findings: List[Finding] = []
|
||||
|
||||
try:
|
||||
text = self._extract_text(target.path, target.declared_ext)
|
||||
except Exception: # noqa: BLE001
|
||||
return result
|
||||
|
||||
if not text:
|
||||
return result
|
||||
|
||||
for rule in self._compiled:
|
||||
match_mode = rule.get("match", "contains")
|
||||
rule_name = rule.get("name", "keyword")
|
||||
weight = rule.get("weight", 30.0)
|
||||
severity_str = rule.get("severity", "medium")
|
||||
severity = getattr(Severity, severity_str.upper(), Severity.MEDIUM).value
|
||||
|
||||
hits: List[str] = []
|
||||
|
||||
if match_mode == "regex":
|
||||
compiled = rule.get("_re")
|
||||
if compiled:
|
||||
for m in compiled.finditer(text):
|
||||
snippet = text[max(0, m.start() - 10): m.end() + 10]
|
||||
hits.append(snippet.strip())
|
||||
if len(hits) >= 3:
|
||||
break
|
||||
|
||||
elif match_mode == "any_of":
|
||||
keywords = rule.get("keywords", [])
|
||||
for kw in keywords:
|
||||
if kw.lower() in text.lower():
|
||||
hits.append(kw)
|
||||
if len(hits) >= 3:
|
||||
break
|
||||
|
||||
elif match_mode == "contains":
|
||||
pattern = rule.get("pattern", "")
|
||||
if pattern and (pattern.lower() in text.lower()):
|
||||
hits.append(pattern)
|
||||
|
||||
if hits:
|
||||
findings.append(
|
||||
Finding(
|
||||
engine=self.name,
|
||||
layer=self.layer,
|
||||
severity=severity,
|
||||
rule=rule_name,
|
||||
detail=f"词表命中：{rule_name}（模式={match_mode}），命中 {len(hits)} 处：{hits[:2]}",
|
||||
weight=float(weight),
|
||||
)
|
||||
)
|
||||
|
||||
result.findings = findings
|
||||
result.meta["scanned_chars"] = len(text)
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------------- 内部辅助
|
||||
|
||||
@staticmethod
|
||||
def _extract_text(path: str, ext: str) -> str:
|
||||
"""从文件中提取可匹配的文本（简化版，够用就行）。
|
||||
|
||||
OOXML/PDF/DOCX 等结构化文档优先交给对应的引擎处理，
|
||||
词表引擎只扫纯文本、代码文件和可直接读的二进制残留字符串。
|
||||
"""
|
||||
text_exts = {".txt", ".csv", ".json", ".xml", ".html", ".htm",
|
||||
".md", ".log", ".ini", ".cfg", ".yaml", ".yml",
|
||||
".sh", ".bash", ".zsh", ".py", ".pl", ".rb", ".js", ".ts",
|
||||
".bat", ".cmd", ".ps1", ".vbs", ".sql", ".go", ".java",
|
||||
".c", ".cpp", ".h", ".cs"}
|
||||
|
||||
ext_lower = (ext or "").lower()
|
||||
if ext_lower in text_exts:
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8", errors="replace") as fh:
|
||||
return fh.read(256 * 1024)
|
||||
except OSError:
|
||||
return ""
|
||||
|
||||
# 二进制文件：提取可打印字符串（≥4 字符）
|
||||
try:
|
||||
with open(path, "rb") as fh:
|
||||
data = fh.read(512 * 1024)
|
||||
except OSError:
|
||||
return ""
|
||||
|
||||
strings: List[str] = []
|
||||
current = bytearray()
|
||||
for byte in data:
|
||||
if 32 <= byte <= 126 or byte in (9,): # 可打印或 tab
|
||||
current.append(byte)
|
||||
else:
|
||||
if len(current) >= 4:
|
||||
strings.append(current.decode("ascii", errors="replace"))
|
||||
current = bytearray()
|
||||
if len(current) >= 4:
|
||||
strings.append(current.decode("ascii", errors="replace"))
|
||||
|
||||
return "\n".join(strings)
|
||||
1371
app/engines/structure.py
Normal file
1371
app/engines/structure.py
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user