fix: 六引擎误报根因修复 — 解包递归、YARA 恒真规则、签名分级、real_type 溢出
检测范围与证据链修复,不改裁决口径(口径改动在下一条提交)。 - extractor:zip/tar 成员在超限分支外漏加回扫描队列,解包后内容永远不参与检测; zip 目录条目被当成内嵌对象;tar 超限成员先落盘不删。递归真正生效后深扫耗时可测。 - clamav:_HIGH/_MEDIUM/_UNWANTED 关键词是小写,而签名名是 "Win.Trojan..." 大小写混排, 分级判定恒不命中全走默认档;改统一大写匹配。超过 clamd MaxFileSize 的裸奔区间 (1G~1.5G) 新增 ClamAVSkippedSizeLimit(30) — 按"不可扫描不放行"进复核。 - yara:pe.yar C1 系列改用导入表判定,不再靠裸子串命中字符串表; linux.yar C7 多条规则条件恒真(裸 nc/fetch 之类)已收紧;C8/C3 相应收窄。 - structure:DangerousExecutableExtension 是载体信号而非恶意行为,critical 权重 45→35, 不再作为把中低置信度信号一起顶过 block_score 的底座。 - scanner/documents:real_type 拼进 libmagic 对 MSI/OLE 的元数据串实测 334+ 字符, 超列宽导致整条 scan_tasks INSERT 连 findings 一起丢(被拒文件不留痕)→ 列宽 256→512,migrate.sql 补 ALTER。
This commit is contained in:
@@ -23,6 +23,32 @@ BINARY_EXTS = {".exe", ".dll", ".sys", ".scr", ".com", ".cpl", ".ocx", ".msi", "
|
||||
_PE_SCN_MEM_EXECUTE = 0x20000000
|
||||
_PE_SCN_MEM_WRITE = 0x80000000
|
||||
|
||||
# 声明扩展名 → 该格式自己的魔数。这些容器本就不是 PE,用 pefile 解析必然抛错;
|
||||
# 把"格式完全正常的安装包/共享库"记成 PeParseFailed 会给每个 .msi/.jar/.so 白送 10 分。
|
||||
_NATIVE_MAGIC: Dict[str, Any] = {
|
||||
".msi": (b"\xd0\xcf\x11\xe0",),
|
||||
".msp": (b"\xd0\xcf\x11\xe0",),
|
||||
".mst": (b"\xd0\xcf\x11\xe0",),
|
||||
".jar": (b"PK\x03\x04", b"PK\x05\x06"),
|
||||
".war": (b"PK\x03\x04", b"PK\x05\x06"),
|
||||
".apk": (b"PK\x03\x04", b"PK\x05\x06"),
|
||||
".so": (b"\x7fELF",),
|
||||
".elf": (b"\x7fELF",),
|
||||
}
|
||||
|
||||
|
||||
def _is_native_container(path: str, ext: str) -> bool:
|
||||
"""文件内容是否就是其声明格式(非 PE)。"""
|
||||
expected = _NATIVE_MAGIC.get(ext)
|
||||
if not expected:
|
||||
return False
|
||||
try:
|
||||
with open(path, "rb") as fh:
|
||||
head = fh.read(4)
|
||||
except OSError:
|
||||
return False
|
||||
return any(head.startswith(m) for m in expected)
|
||||
|
||||
|
||||
def _manual_pe_analysis(path: str) -> List[Finding]:
|
||||
"""pefile 解析失败时,手工搜索 PE\\0\\0 签名并解析节表。
|
||||
@@ -172,6 +198,8 @@ class YaraEngine(Engine):
|
||||
return result
|
||||
|
||||
try:
|
||||
# 本环境的 yara-python 4.5.4 在 Match 对象上默认就带 meta,
|
||||
# 且 rules.match() 不接受 with_meta 关键字(传了会整引擎报错)。
|
||||
matches = rules.match(target.path)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
result.ok = False
|
||||
@@ -179,7 +207,8 @@ class YaraEngine(Engine):
|
||||
return result
|
||||
|
||||
for match in matches:
|
||||
severity, weight = _judge_yara_rule(match.rule, match.tags)
|
||||
meta = getattr(match, "meta", None) or {}
|
||||
severity, weight = _judge_yara_rule(match.rule, match.tags, meta)
|
||||
# 覆盖:如果 ctx 里的策略为该 YARA 规则定义了精细权重,用它
|
||||
try:
|
||||
policy = ctx.policy_engine.policy
|
||||
@@ -188,13 +217,18 @@ class YaraEngine(Engine):
|
||||
weight = float(yara_weights[match.rule])
|
||||
except Exception:
|
||||
pass # ctx 没有 policy 字段时保持默认 weight
|
||||
desc = str(meta.get("description", "")).strip()
|
||||
tags_text = ", ".join(match.tags)
|
||||
result.findings.append(
|
||||
Finding(
|
||||
engine=self.name,
|
||||
layer=self.layer,
|
||||
severity=severity,
|
||||
rule=match.rule,
|
||||
detail=f"YARA 规则 {match.rule} 命中(标签:{', '.join(match.tags) or '无'})",
|
||||
detail=(
|
||||
f"YARA 规则 {match.rule} 命中"
|
||||
+ (f"({desc})" if desc else f"(标签:{tags_text or '无'})")
|
||||
),
|
||||
weight=weight,
|
||||
meta={"tags": list(match.tags)},
|
||||
)
|
||||
@@ -232,6 +266,10 @@ class BinaryEngine(Engine):
|
||||
if ext not in BINARY_EXTS:
|
||||
return result
|
||||
|
||||
# 内容确实是声明的那个非 PE 容器(MSI=CFB、JAR/APK=ZIP、SO/ELF=ELF)→ 不做 PE 分析
|
||||
if _is_native_container(target.path, ext):
|
||||
return result
|
||||
|
||||
try:
|
||||
import pefile
|
||||
except Exception: # noqa: BLE001
|
||||
@@ -290,17 +328,54 @@ class BinaryEngine(Engine):
|
||||
)
|
||||
)
|
||||
|
||||
# 2) 可疑导入表
|
||||
suspicious_imports = self._suspicious_imports(pe)
|
||||
if suspicious_imports:
|
||||
# 2) 导入表能力信号:只做佐证,不做独立定罪
|
||||
# Windows 自身的内核/外壳组件(kernel32、shell32、wer)必然导入
|
||||
# VirtualAllocEx/WriteProcessMemory 这类 API——它们就是这些 API 的实现方。
|
||||
# 真正的注入判定由 YARA C1_PE_ProcessInjection(导入名邻接窗口)负责,
|
||||
# 这里按「命中个数」而不是「命中即高危」计分,避免正常软件被判恶意。
|
||||
tier1, tier2, tier3 = self._suspicious_imports(pe)
|
||||
if len(tier1) >= 2:
|
||||
findings.append(
|
||||
Finding(
|
||||
engine=self.name,
|
||||
layer=self.layer,
|
||||
severity=Severity.HIGH.value,
|
||||
rule="SuspiciousImports",
|
||||
detail=f"导入表包含高危 API:{', '.join(suspicious_imports[:8])}",
|
||||
weight=25.0,
|
||||
detail=f"导入表同时包含多个注入/后门级 API:{', '.join(tier1[:8])}",
|
||||
weight=20.0,
|
||||
)
|
||||
)
|
||||
elif len(tier1) == 1:
|
||||
findings.append(
|
||||
Finding(
|
||||
engine=self.name,
|
||||
layer=self.layer,
|
||||
severity=Severity.LOW.value,
|
||||
rule="SuspiciousImports",
|
||||
detail=f"导入表含单个注入/后门级 API(系统组件常态,需其他信号佐证):{tier1[0]}",
|
||||
weight=5.0,
|
||||
)
|
||||
)
|
||||
if len(tier2) >= 3:
|
||||
findings.append(
|
||||
Finding(
|
||||
engine=self.name,
|
||||
layer=self.layer,
|
||||
severity=Severity.LOW.value,
|
||||
rule="CapabilityImports",
|
||||
detail=f"导入表同时具备多项危险能力:{', '.join(tier2[:8])}",
|
||||
weight=10.0,
|
||||
)
|
||||
)
|
||||
elif tier3:
|
||||
findings.append(
|
||||
Finding(
|
||||
engine=self.name,
|
||||
layer=self.layer,
|
||||
severity=Severity.INFO.value,
|
||||
rule="AntiDebugImports",
|
||||
detail=f"导入表含反调试/环境探测 API(正常软件普遍使用):{', '.join(tier3[:8])}",
|
||||
weight=0.0,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -312,8 +387,8 @@ class BinaryEngine(Engine):
|
||||
layer=self.layer,
|
||||
severity=Severity.MEDIUM.value,
|
||||
rule="UnsignedExecutable",
|
||||
detail="可执行文件未携带数字签名",
|
||||
weight=20.0,
|
||||
detail="可执行文件未携带数字签名(内网自研工具常态,仅辅助人工判断)",
|
||||
weight=10.0,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -330,7 +405,7 @@ class BinaryEngine(Engine):
|
||||
severity=Severity.LOW.value,
|
||||
rule="SuspiciousTimestamp",
|
||||
detail="PE 编译时间戳异常,可能被伪造",
|
||||
weight=10.0,
|
||||
weight=0.0,
|
||||
)
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
@@ -433,32 +508,62 @@ class BinaryEngine(Engine):
|
||||
severity=Severity.LOW.value,
|
||||
rule="DebugInformationPresent",
|
||||
detail="PE 包含调试信息(PDB 路径或 .debug 节),可用于逆向定位",
|
||||
weight=10.0,
|
||||
weight=0.0,
|
||||
)
|
||||
)
|
||||
|
||||
return findings
|
||||
|
||||
def _suspicious_imports(self, pe: Any) -> List[str]:
|
||||
"""检测高危 API 导入。"""
|
||||
suspicious = {
|
||||
def _suspicious_imports(self, pe: Any) -> "tuple[List[str], List[str], List[str]]":
|
||||
"""导入 API 按恶意指向分档。
|
||||
|
||||
返回 (tier1, tier2, tier3):
|
||||
tier1 注入/后门级 API —— 单独出现即可疑
|
||||
tier2 危险能力 API —— 正常软件也常用,需多项共存才计分
|
||||
tier3 反调试/探测 API —— 信息项,不计分(大量合法商业软件带加壳自保会用)
|
||||
"""
|
||||
tier1_set = {
|
||||
"VirtualAllocEx", "WriteProcessMemory", "CreateRemoteThread",
|
||||
"NtUnmapViewOfSection", "SetWindowsHookEx", "OpenProcess",
|
||||
"WinHttpOpen", "InternetOpenUrl", "URLDownloadToFile",
|
||||
"CryptEncrypt", "AdjustTokenPrivileges", "CreateService",
|
||||
"ShellExecute", "WinExec", "IsDebuggerPresent", "CheckRemoteDebuggerPresent",
|
||||
"NtUnmapViewOfSection", "SetWindowsHookEx", "SetWindowsHookExA",
|
||||
"SetWindowsHookExW", "URLDownloadToFileA", "URLDownloadToFileW",
|
||||
"QueueUserAPC", "NtQueueApcThread", "RtlCreateUserThread",
|
||||
}
|
||||
found: List[str] = []
|
||||
tier2_set = {
|
||||
"OpenProcess", "WinHttpOpen", "WinHttpOpenRequest", "WinHttpSendRequest",
|
||||
"CryptEncrypt", "CryptDeriveKey", "AdjustTokenPrivileges",
|
||||
"LookupPrivilegeValueA", "LookupPrivilegeValueW", "OpenProcessToken",
|
||||
"VirtualProtectEx", "ReadProcessMemory", "SuspendThread", "GetAsyncKeyState",
|
||||
"SetWinEventHook", "HttpSendRequestA", "InternetWriteFile",
|
||||
"RegSetValueExA", "RegSetValueExW",
|
||||
# 以下 API 恶意软件常用,但任何"要装服务/要启动进程/要联网"的正常
|
||||
# 软件都必然导入它们,不足以单独定罪
|
||||
"WinExec", "ShellExecuteExA", "ShellExecuteExW", "CreateServiceA",
|
||||
"CreateServiceW", "InternetOpenUrlA", "InternetOpenUrlW",
|
||||
"CreateNamedPipeA", "DefineDosDeviceW",
|
||||
}
|
||||
tier3_set = {
|
||||
"IsDebuggerPresent", "CheckRemoteDebuggerPresent", "OutputDebugStringA",
|
||||
"GetTickCount", "GetTickCount64", "QueryPerformanceCounter",
|
||||
"GetProcessHeap", "FindWindowA", "FindWindowW",
|
||||
}
|
||||
found1: List[str] = []
|
||||
found2: List[str] = []
|
||||
found3: List[str] = []
|
||||
try:
|
||||
for entry in getattr(pe, "DIRECTORY_ENTRY_IMPORT", []) or []:
|
||||
for imp in entry.imports:
|
||||
if imp.name:
|
||||
name = imp.name.decode("utf-8", errors="replace")
|
||||
if name in suspicious:
|
||||
found.append(name)
|
||||
if not imp.name:
|
||||
continue
|
||||
name = imp.name.decode("utf-8", errors="replace")
|
||||
if name in tier1_set:
|
||||
found1.append(name)
|
||||
elif name in tier2_set:
|
||||
found2.append(name)
|
||||
elif name in tier3_set:
|
||||
found3.append(name)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return sorted(set(found))
|
||||
return sorted(set(found1)), sorted(set(found2)), sorted(set(found3))
|
||||
|
||||
def _has_signature(self, pe: Any) -> bool:
|
||||
"""检查 PE 是否包含 Authenticode 签名目录。"""
|
||||
@@ -469,8 +574,25 @@ class BinaryEngine(Engine):
|
||||
return False
|
||||
|
||||
|
||||
def _judge_yara_rule(rule: str, tags: List[str]) -> tuple[str, float]:
|
||||
"""依据 YARA 规则名与标签判定严重度。"""
|
||||
_YARA_RISK_SEVERITY = {
|
||||
"critical": Severity.CRITICAL.value,
|
||||
"high": Severity.HIGH.value,
|
||||
"medium": Severity.MEDIUM.value,
|
||||
"low": Severity.LOW.value,
|
||||
}
|
||||
_YARA_RISK_WEIGHT = {"critical": 85.0, "high": 60.0, "medium": 30.0, "low": 10.0}
|
||||
|
||||
|
||||
def _judge_yara_rule(rule: str, tags: List[str], meta: Dict[str, Any] | None = None) -> tuple[str, float]:
|
||||
"""依据规则 meta.risk_level 判定严重度(其次才是标签/规则名启发式)。
|
||||
|
||||
v0.2.3 只看标签与规则名,规则库里的 risk_level 被完全忽略,
|
||||
结果 65 分的 C1_PE_RAT 显示成"低危",权重与严重度两套口径。
|
||||
"""
|
||||
risk = str((meta or {}).get("risk_level", "")).strip().lower()
|
||||
if risk in _YARA_RISK_SEVERITY:
|
||||
return _YARA_RISK_SEVERITY[risk], _YARA_RISK_WEIGHT[risk]
|
||||
|
||||
tag_set = {t.lower() for t in tags}
|
||||
text = f"{rule} {' '.join(tags)}".lower()
|
||||
|
||||
|
||||
@@ -45,9 +45,21 @@ class ClamAVEngine(Engine):
|
||||
except _ClamdSizeLimit:
|
||||
# 文件超过 ClamAV MaxFileSize(clamd.conf 可调,默认 1G)
|
||||
# → 引擎只是拒扫这个大文件,不当成引擎故障
|
||||
# 但"没查过"必须留下信号:策略里 max_file_size 是 1.5G,1G~1.5G 的文件
|
||||
# 原本在 L1 完全裸奔却仍可能 clean/pass。按"不可扫描不放行"落人工复核。
|
||||
result.ok = True
|
||||
result.meta["skipped"] = "size_limit"
|
||||
result.meta["reason"] = "文件超过 ClamAV 大小限制已跳过"
|
||||
result.findings.append(
|
||||
Finding(
|
||||
engine=self.name,
|
||||
layer=self.layer,
|
||||
severity=Severity.MEDIUM.value,
|
||||
rule="ClamAVSkippedSizeLimit",
|
||||
detail="文件超过 clamd MaxFileSize,L1 签名查杀未执行,按不可扫描内容处置",
|
||||
weight=30.0,
|
||||
)
|
||||
)
|
||||
return result
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# 真·引擎不可用:连不上 / PING 超时 / daemon 挂了
|
||||
@@ -267,9 +279,9 @@ def _parse_clamd_response(text: str) -> Dict[str, Any]:
|
||||
|
||||
|
||||
# 签名名中的可信度关键词映射到严重度
|
||||
_HIGH_KEYWORDS = ("ransom", "crypt", "trojan", "backdoor", "rootkit", "stealer", "keylog", "worm", "dropper", "ransomware")
|
||||
_MEDIUM_KEYWORDS = ("generic", "heur", "packed", "obfusc", "suspect", "malware", "riskware", "virus")
|
||||
_UNWANTED_KEYWORDS = ("PUA", "unwanted", "toolkit", "pup", "adware")
|
||||
_HIGH_KEYWORDS = ("RANSOM", "CRYPT", "TROJAN", "BACKDOOR", "ROOTKIT", "STEALER", "KEYLOG", "WORM", "DROPPER")
|
||||
_MEDIUM_KEYWORDS = ("GENERIC", "HEUR", "PACKED", "OBFUSC", "SUSPECT", "MALWARE", "RISKWARE", "VIRUS")
|
||||
_UNWANTED_KEYWORDS = ("PUA", "UNWANTED", "TOOLKIT", "PUP", "ADWARE")
|
||||
|
||||
# EICAR 测试病毒签名 —— 明确标为 critical(等保要求:检测到即阻断)
|
||||
_EICAR_SIGNATURES = ("Eicar-Test-Signature", "EICAR-AV-Test", "Eicar-Test-Signature.UNOFFICIAL")
|
||||
|
||||
@@ -56,6 +56,19 @@ EML_EXTS = {".eml", ".msg"}
|
||||
# 带 VBA 宏的 Office 扩展名(用于 VBA 纯文本扫描触发)
|
||||
VBA_MACRO_OFFICE_EXTS = OLE_EXTS | OOXML_EXTS | {".bas", ".vba", ".cls", ".frm"}
|
||||
|
||||
# OOXML 包内的内嵌 OLE 部件目录
|
||||
EMBED_PREFIX_RE = re.compile(r"^(?:word|xl|ppt)/embeddings/.+")
|
||||
|
||||
# 内嵌对象中的办公载体:文档互嵌(子表格/图表/PDF)是正常用法,不等于藏可执行
|
||||
OFFICE_PART_EXTS = {
|
||||
".docx", ".docm", ".xlsx", ".xlsm", ".pptx", ".pptm",
|
||||
".doc", ".xls", ".ppt", ".pdf", ".rtf", ".xml", ".txt", ".csv",
|
||||
".htm", ".html", ".wmf", ".emf", ".bin",
|
||||
}
|
||||
|
||||
# 内嵌部件里直接判定为可执行对象的扩展名
|
||||
EXEC_PART_EXTS = {".exe", ".dll", ".scr", ".js", ".jse", ".vbs", ".vbe", ".bat", ".cmd", ".ps1", ".hta", ".com", ".msi"}
|
||||
|
||||
# VBA 自动执行入口关键词
|
||||
VBA_AUTOEXEC_PATTERNS = [
|
||||
r"Sub\s+AutoOpen\s*\(", r"Sub\s+AutoExec\s*\(", r"Sub\s+AutoNew\s*\(",
|
||||
@@ -211,14 +224,16 @@ class DocumentEngine(Engine):
|
||||
else:
|
||||
parser.close()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# 宏流解析不了 = 内容不可判定。这类畸形 OLE 常被用来探测解析器异常,
|
||||
# 给 0 分等于"看不懂就放行",落人工复核。
|
||||
findings.append(
|
||||
Finding(
|
||||
engine=self.name,
|
||||
layer=self.layer,
|
||||
severity=Severity.LOW.value,
|
||||
severity=Severity.MEDIUM.value,
|
||||
rule="OleToolsUnavailable",
|
||||
detail=f"VBA 宏解析失败:{exc}",
|
||||
weight=0.0,
|
||||
weight=30.0,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -407,6 +422,24 @@ class DocumentEngine(Engine):
|
||||
"excerpt": excerpt,
|
||||
}
|
||||
|
||||
def _looks_like_executable(self, zf: Any, name: str) -> bool:
|
||||
"""判断内嵌部件是否为可执行载荷(扩展名 + 文件头魔数)。
|
||||
|
||||
oleObject1.bin 之类的外壳里可能装着真正的 PE,光看扩展名会漏;
|
||||
反之把每个 .bin 都判成可执行又会拦掉正常的图表/子文档。
|
||||
"""
|
||||
if os.path.splitext(name)[1].lower() in EXEC_PART_EXTS:
|
||||
return True
|
||||
try:
|
||||
with zf.open(name) as fp:
|
||||
head = fp.read(1 << 20)
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
if head[:2] == b"MZ" or head[:4] in (b"\x7fELF", b"\xca\xfe\xba\xbe"):
|
||||
return True
|
||||
# OLE 复合容器(Package/Ole10Native)内嵌 PE:找 MZ..PE 头
|
||||
return bool(re.search(rb"MZ[\x00-\xff]{0,64}PE\x00\x00", head))
|
||||
|
||||
def _scan_ooxml(self, target: ScanTarget) -> List[Finding]:
|
||||
"""检测 OOXML(docx/xlsx/pptx)中的外部引用与外部链接。"""
|
||||
findings: List[Finding] = []
|
||||
@@ -429,9 +462,11 @@ class DocumentEngine(Engine):
|
||||
weight=45.0,
|
||||
)
|
||||
)
|
||||
if name.startswith("word/embeddings/") or name.startswith("xl/embeddings/") or name.startswith("ppt/embeddings/"):
|
||||
# 内嵌 OLE 部件。zip 目录条目(以 / 结尾、长度为 0)不是部件——
|
||||
# PowerPoint 导出的包常带空的 ppt/embeddings/ 目录,按文件处理会凭空报内嵌对象。
|
||||
if EMBED_PREFIX_RE.match(name) and not name.endswith("/"):
|
||||
embedded_ext = os.path.splitext(name)[1].lower()
|
||||
if embedded_ext in {".exe", ".dll", ".scr", ".js", ".vbs", ".bat", ".bin"}:
|
||||
if self._looks_like_executable(zf, name):
|
||||
findings.append(
|
||||
Finding(
|
||||
engine=self.name,
|
||||
@@ -442,6 +477,17 @@ class DocumentEngine(Engine):
|
||||
weight=75.0,
|
||||
)
|
||||
)
|
||||
elif embedded_ext in OFFICE_PART_EXTS:
|
||||
findings.append(
|
||||
Finding(
|
||||
engine=self.name,
|
||||
layer=self.layer,
|
||||
severity=Severity.LOW.value,
|
||||
rule="EmbeddedObject",
|
||||
detail=f"文档内嵌办公对象:{name}",
|
||||
weight=5.0,
|
||||
)
|
||||
)
|
||||
else:
|
||||
findings.append(
|
||||
Finding(
|
||||
|
||||
@@ -205,8 +205,9 @@ class SafeExtractor:
|
||||
safe_name = self._sanitize_name(name)
|
||||
dest = os.path.join(out_dir, safe_name)
|
||||
try:
|
||||
written = 0
|
||||
overflow = False
|
||||
with zf.open(info) as src, open(dest, "wb") as out:
|
||||
written = 0
|
||||
while True:
|
||||
chunk = src.read(1024 * 1024)
|
||||
if not chunk:
|
||||
@@ -222,20 +223,22 @@ class SafeExtractor:
|
||||
f"成员 {name} 实际解压超限,已丢弃",
|
||||
10.0,
|
||||
)
|
||||
overflow = True
|
||||
break
|
||||
out.write(chunk)
|
||||
else:
|
||||
report.total_bytes += written
|
||||
report.files.append(
|
||||
ScanTarget(
|
||||
path=dest,
|
||||
filename=safe_name,
|
||||
declared_ext=os.path.splitext(safe_name)[1].lower(),
|
||||
logical_path=f"{target.logical_path}!/{name}",
|
||||
depth=target.depth + 1,
|
||||
is_extracted=True,
|
||||
)
|
||||
# 成员必须回到扫描队列,否则压缩包/OOXML 里的内容永远不参与检测
|
||||
if not overflow:
|
||||
report.total_bytes += written
|
||||
report.files.append(
|
||||
ScanTarget(
|
||||
path=dest,
|
||||
filename=safe_name,
|
||||
declared_ext=os.path.splitext(safe_name)[1].lower(),
|
||||
logical_path=f"{target.logical_path}!/{name}",
|
||||
depth=target.depth + 1,
|
||||
is_extracted=True,
|
||||
)
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._add_finding(
|
||||
report,
|
||||
@@ -323,14 +326,26 @@ class SafeExtractor:
|
||||
continue
|
||||
with src, open(dest, "wb") as out:
|
||||
written = 0
|
||||
overflow = False
|
||||
while True:
|
||||
chunk = src.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
written += len(chunk)
|
||||
if written > self.settings.max_upload_bytes:
|
||||
out.close()
|
||||
os.remove(dest)
|
||||
self._add_finding(
|
||||
report,
|
||||
Severity.MEDIUM.value,
|
||||
"MemberTooLarge",
|
||||
f"成员 {name} 实际解压超限,已丢弃",
|
||||
10.0,
|
||||
)
|
||||
overflow = True
|
||||
break
|
||||
out.write(chunk)
|
||||
if not overflow:
|
||||
report.total_bytes += written
|
||||
report.files.append(
|
||||
ScanTarget(
|
||||
|
||||
@@ -21,6 +21,9 @@ from app.engines.base import Engine, EngineResult, Finding, ScanTarget, Severity
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# cross_line 规则用的续行归一:换行(及其两侧空白)压成一个空格
|
||||
NEWLINE_RUN_RE = re.compile(r"\s*[\r\n]+\s*")
|
||||
|
||||
|
||||
class KeywordEngine(Engine):
|
||||
"""内网特征词表匹配引擎。"""
|
||||
@@ -94,12 +97,13 @@ class KeywordEngine(Engine):
|
||||
if not text:
|
||||
return result
|
||||
|
||||
# ---- 预处理:合并换行符以支持跨行关键词 ----
|
||||
# 攻击者会把 Invoke-WebRequest 这种关键字用 CRLF 拆成
|
||||
# "Invoke-Web\r\nRequest" 来绕过基于单行的匹配。
|
||||
# 这里把所有 \r\n / \r / \n 替换为空,让跨行关键词重新拼接。
|
||||
# 注意:不影响纯文本/多行语义,我们匹配的都是单行关键词或跨行命令。
|
||||
text = text.replace("\r\n", "").replace("\r", "").replace("\n", "")
|
||||
# ---- 预处理 ----
|
||||
# 默认按原始行匹配:把整篇文本的换行全部删除会让上一行行尾和下一行行首
|
||||
# 黏成一句话(Ruby 源码 "…breed\n+fetch()…" 曾被当成 curl 特征命中)。
|
||||
# cross_line: true 的规则再跑一遍"换行压成空格"的文本,用于兜住
|
||||
# wget \ + 换行 + -O payload 这类续行绕过;保留分隔符,不产生黏连。
|
||||
text_norm = text.replace("\r\n", "\n").replace("\r", "\n")
|
||||
text_joined = NEWLINE_RUN_RE.sub(" ", text_norm)
|
||||
|
||||
for rule in self._compiled:
|
||||
match_mode = rule.get("match", "contains")
|
||||
@@ -113,23 +117,29 @@ class KeywordEngine(Engine):
|
||||
if match_mode == "regex":
|
||||
compiled = rule.get("_re")
|
||||
if compiled:
|
||||
for m in compiled.finditer(text):
|
||||
snippet = text[max(0, m.start() - 10): m.end() + 10]
|
||||
hits.append(snippet.strip())
|
||||
scan_targets = [text_norm]
|
||||
if rule.get("cross_line"):
|
||||
scan_targets.append(text_joined)
|
||||
for body in scan_targets:
|
||||
for m in compiled.finditer(body):
|
||||
snippet = body[max(0, m.start() - 10): m.end() + 10]
|
||||
hits.append(" ".join(snippet.split()))
|
||||
if len(hits) >= 3:
|
||||
break
|
||||
if len(hits) >= 3:
|
||||
break
|
||||
|
||||
elif match_mode == "any_of":
|
||||
keywords = rule.get("keywords", [])
|
||||
for kw in keywords:
|
||||
if kw.lower() in text.lower():
|
||||
if kw.lower() in text_norm.lower():
|
||||
hits.append(kw)
|
||||
if len(hits) >= 3:
|
||||
break
|
||||
|
||||
elif match_mode == "contains":
|
||||
pattern = rule.get("pattern", "")
|
||||
if pattern and (pattern.lower() in text.lower()):
|
||||
if pattern and (pattern.lower() in text_norm.lower()):
|
||||
hits.append(pattern)
|
||||
|
||||
if hits:
|
||||
|
||||
@@ -108,6 +108,9 @@ class StructureEngine(Engine):
|
||||
)
|
||||
|
||||
# 3) 危险可执行扩展名(顶层文件)
|
||||
# 这是「载体」信号而不是恶意行为:单独命中应当进人工复核(≥review_score),
|
||||
# 但不能成为把后续中/低置信度检测一起顶过 block_score 的底座,
|
||||
# 否则正常签名的 exe/dll 会被判恶意。
|
||||
if ext in DANGEROUS_EXTS and not target.is_extracted:
|
||||
severity = Severity.CRITICAL.value if ext in {".exe", ".scr", ".dll", ".com", ".pif"} else Severity.HIGH.value
|
||||
findings.append(
|
||||
@@ -117,7 +120,7 @@ class StructureEngine(Engine):
|
||||
severity=severity,
|
||||
rule="DangerousExecutableExtension",
|
||||
detail=f"文件类型为可执行/脚本格式:{ext}",
|
||||
weight=45.0 if severity == Severity.CRITICAL.value else 30.0,
|
||||
weight=35.0 if severity == Severity.CRITICAL.value else 30.0,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ class ScanTask(Base):
|
||||
size: Mapped[int] = mapped_column(Integer, default=0)
|
||||
sha256: Mapped[str] = mapped_column(String(64), index=True)
|
||||
md5: Mapped[str] = mapped_column(String(32), default="")
|
||||
real_type: Mapped[str] = mapped_column(String(256), default="")
|
||||
real_type: Mapped[str] = mapped_column(String(512), default="")
|
||||
|
||||
status: Mapped[str] = mapped_column(String(16), default="pending", index=True)
|
||||
mode: Mapped[str] = mapped_column(String(16), default="sync")
|
||||
|
||||
@@ -34,6 +34,17 @@ _settings = get_settings()
|
||||
TEMP_ROOT = os.environ.get("MESHSAFE_TEMP_ROOT", "/tmp/meshsafe")
|
||||
|
||||
|
||||
def _clip(value: Any, limit: int) -> str:
|
||||
"""按列宽截断字符串。
|
||||
|
||||
外部可控的长度(libmagic 描述串、归档成员路径、用户文件名)一旦超出列宽,
|
||||
PostgreSQL 会拒绝整条 INSERT,连带同事务里的 scan_findings 一起丢失——
|
||||
被拒绝的文件反而查不到记录。宁可截断,也不能整条不落库。
|
||||
"""
|
||||
text = str(value or "")
|
||||
return text[:limit]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanContext:
|
||||
"""一次扫描任务的上下文。"""
|
||||
@@ -95,12 +106,12 @@ class ScanOutcome:
|
||||
id=self.task_id,
|
||||
channel=self.channel,
|
||||
client_id=self.client_id,
|
||||
filename=self.file_info.get("filename", ""),
|
||||
declared_ext=self.file_info.get("declared_ext", ""),
|
||||
filename=_clip(self.file_info.get("filename", ""), 512),
|
||||
declared_ext=_clip(self.file_info.get("declared_ext", ""), 32),
|
||||
size=self.file_info.get("size", 0),
|
||||
sha256=self.file_info.get("sha256", ""),
|
||||
md5=self.file_info.get("md5", ""),
|
||||
real_type=self.file_info.get("real_type", ""),
|
||||
real_type=_clip(self.file_info.get("real_type", ""), 512),
|
||||
status="completed",
|
||||
mode=self.mode,
|
||||
verdict=self.verdict,
|
||||
@@ -122,7 +133,7 @@ class ScanOutcome:
|
||||
severity=f.severity,
|
||||
rule=f.rule,
|
||||
detail=f.detail,
|
||||
extracted_path=f.extracted_path,
|
||||
extracted_path=_clip(f.extracted_path, 1024),
|
||||
weight=f.weight,
|
||||
))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user