feat: MeshSafe 文件摆渡安全卫士 v1.0 初始提交

核心功能:
- 6 层检测管线: L0 结构分析 → L1 ClamAV → L2 文档深度检测 → L3 关键词匹配 → L4 YARA → L5 PE/二进制
- FastAPI + PostgreSQL + Redis + Docker Compose 架构
- 客户端 HMAC-SHA256 签名鉴权 + 限流 + 幂等性
- 控制台 UI (登录/Dashboard/诊断中心/策略管理/任务列表/测试中心/产品介绍页)
- 三层诊断定时器 (error_scan 30s / db_aggregate 30s / container_check 5min)

并发与稳定性修复:
- 诊断定时器 Redis 时间槽位锁 (多 worker 同一槽位只有一个执行)
- JSONL 存储 fcntl 跨进程排他锁 + 容错解码 + 原子重写
- 移除 SQLite fallback, 统一 PostgreSQL
- API_CLIENTS 脱敏为环境变量, .env 已 .gitignore
This commit is contained in:
2026-10-28 19:07:11 +08:00
commit 16f134e328
113 changed files with 24604 additions and 0 deletions

157
app/engines/keywords.py Normal file
View File

@@ -0,0 +1,157 @@
"""内网特征词表引擎(P1)。
数据驱动:词表从 policy.yaml 的 ``keyword_rules`` 段加载,不在代码中硬编码。
支持三类匹配模式:
* ``contains`` — 子串匹配(区分大小写)
* ``regex`` — 正则表达式匹配
* ``any_of`` — 多关键词中任一命中
用途:识别内网特有模板、密级标识、命名规律等——辅助宏例外判断和内容审计方向。
"""
from __future__ import annotations
import logging
import re
from typing import Any, Dict, List
from app.engines.base import Engine, EngineResult, Finding, ScanTarget, Severity
logger = logging.getLogger(__name__)
class KeywordEngine(Engine):
"""内网特征词表匹配引擎。"""
name = "keywords"
layer = "L2B"
version = "1.0"
def __init__(self, keyword_rules: List[Dict[str, Any]] | None = None) -> None:
"""初始化。keyword_rules 来自 PolicyStore.current().keyword_rules。"""
self._rules = keyword_rules or []
# 预编译正则
self._compiled: List[Dict[str, Any]] = []
for rule in self._rules:
pattern = rule.get("pattern", "")
if rule.get("match") == "regex" and pattern:
try:
compiled = re.compile(pattern, re.IGNORECASE if rule.get("ignore_case") else 0)
except re.error:
logger.warning("词表规则正则编译失败: %s", rule.get("name"))
continue
self._compiled.append({**rule, "_re": compiled})
else:
self._compiled.append(rule)
# ------------------------------------------------------------------- 接口实现
def is_enabled(self) -> bool:
return bool(self._rules)
def scan(self, target: ScanTarget, ctx: Any) -> EngineResult:
result = EngineResult(engine=self.name)
if not self._rules:
return result
findings: List[Finding] = []
try:
text = self._extract_text(target.path, target.declared_ext)
except Exception: # noqa: BLE001
return result
if not text:
return result
for rule in self._compiled:
match_mode = rule.get("match", "contains")
rule_name = rule.get("name", "keyword")
weight = rule.get("weight", 30.0)
severity_str = rule.get("severity", "medium")
severity = getattr(Severity, severity_str.upper(), Severity.MEDIUM).value
hits: List[str] = []
if match_mode == "regex":
compiled = rule.get("_re")
if compiled:
for m in compiled.finditer(text):
snippet = text[max(0, m.start() - 10): m.end() + 10]
hits.append(snippet.strip())
if len(hits) >= 3:
break
elif match_mode == "any_of":
keywords = rule.get("keywords", [])
for kw in keywords:
if kw.lower() in text.lower():
hits.append(kw)
if len(hits) >= 3:
break
elif match_mode == "contains":
pattern = rule.get("pattern", "")
if pattern and (pattern.lower() in text.lower()):
hits.append(pattern)
if hits:
findings.append(
Finding(
engine=self.name,
layer=self.layer,
severity=severity,
rule=rule_name,
detail=f"词表命中:{rule_name}(模式={match_mode}),命中 {len(hits)} 处:{hits[:2]}",
weight=float(weight),
)
)
result.findings = findings
result.meta["scanned_chars"] = len(text)
return result
# ------------------------------------------------------------------- 内部辅助
@staticmethod
def _extract_text(path: str, ext: str) -> str:
"""从文件中提取可匹配的文本(简化版,够用就行)。
OOXML/PDF/DOCX 等结构化文档优先交给对应的引擎处理,
词表引擎只扫纯文本、代码文件和可直接读的二进制残留字符串。
"""
text_exts = {".txt", ".csv", ".json", ".xml", ".html", ".htm",
".md", ".log", ".ini", ".cfg", ".yaml", ".yml",
".sh", ".bash", ".zsh", ".py", ".pl", ".rb", ".js", ".ts",
".bat", ".cmd", ".ps1", ".vbs", ".sql", ".go", ".java",
".c", ".cpp", ".h", ".cs"}
ext_lower = (ext or "").lower()
if ext_lower in text_exts:
try:
with open(path, "r", encoding="utf-8", errors="replace") as fh:
return fh.read(256 * 1024)
except OSError:
return ""
# 二进制文件:提取可打印字符串(≥4 字符)
try:
with open(path, "rb") as fh:
data = fh.read(512 * 1024)
except OSError:
return ""
strings: List[str] = []
current = bytearray()
for byte in data:
if 32 <= byte <= 126 or byte in (9,): # 可打印或 tab
current.append(byte)
else:
if len(current) >= 4:
strings.append(current.decode("ascii", errors="replace"))
current = bytearray()
if len(current) >= 4:
strings.append(current.decode("ascii", errors="replace"))
return "\n".join(strings)