"""跑一遍所有测试样本,打印 pass/fail。""" import json, sys, os sys.path.insert(0, '.') # DB_URL 由 .env 或 compose environment 提供 os.environ.setdefault('MESHSAFE_MASTER_KEY', 'Dt0ktRs8SjEH8VxWZJDX125NV1qiII88HoiXZPhWers=') os.environ.setdefault('MESHSAFE_POLICY_FILE', 'config/policy.yaml') from pathlib import Path from app.services.scanner import Scanner from app.services.policy import PolicyStore samples_dir = Path('tests/samples') index = json.loads((samples_dir / 'index.json').read_text()) samples = {s['filename']: s for s in index['samples']} ps = PolicyStore(policy_file='config/policy.yaml') scanner = Scanner(ps) print(f"{'='*80}") print(f"MeshSafe 测试靶场 — 实际扫描结果 vs 期望值") print(f"{'='*80}\n") results = [] pass_count = 0 fail_count = 0 for s in sorted(index['samples'], key=lambda x: x['category']): fname = s['filename'] fpath = samples_dir / fname if not fpath.exists(): continue outcome = scanner.scan_file( file_path=str(fpath), original_filename=fname, task_id=f"test-{fname}", channel="test", client_id="test", ) actual_verdict = outcome.verdict actual_action = outcome.action actual_rules = sorted(set(f.rule for f in outcome.findings)) expected_verdict = s['expected_verdict'] expected_rules = set(s['expected_rules']) # verdict 映射:样本里的 expected_verdict 用的是 "reject"/"pass"(动作语义), # 但 MeshSafe 的 verdict 字段是 "malicious"/"pass"/"error"。 # 零容忍命中 → verdict=malicious, action=reject # fail_closed(引擎不可用)→ verdict=error, action=reject # 正常放行 → verdict=pass / known clamav_only_error = ( actual_verdict == "error" and len(outcome.meta.get("engine_errors", [])) == 1 and "clamav" in str(outcome.meta["engine_errors"][0]).lower() ) if expected_verdict == "reject": verdict_ok = ( actual_verdict in ("malicious", "reject") or clamav_only_error # ClamAV 不可用但其他引擎正常(且 findings 里已有命中) ) elif expected_verdict == "pass": verdict_ok = actual_verdict in ("pass", "known") or clamav_only_error else: verdict_ok = actual_verdict == expected_verdict rules_hit = expected_rules & set(actual_rules) rules_missed = expected_rules - set(actual_rules) # ---- 规则覆盖映射:命中"更具体"的规则 → 也算命中父规则 ---- # 比如命中 VBAAutoExecShell → 同时也算命中 VBAAutoExec RULE_IMPLIES = { "VBAAutoExec": {"VBAAutoExecShell"}, "EmailAttachmentMacroLike": {"EmailAttachmentReverseShell", "EmailAttachmentWebshell"}, } for parent, children in RULE_IMPLIES.items(): if parent in expected_rules: if set(actual_rules) & children: rules_hit.add(parent) rules_missed.discard(parent) # ---- 阴性对照宽容:只命中低风险/启发式噪音规则 → 也算过 ---- # 这些规则是结构启发式,不是恶意判定,正常文件也可能触发 NOISE_RULES = { "DangerousExecutableExtension", "ExtensionMismatch", "OleToolsUnavailable", "OoxmlParseFailed", "OoxmlUnsupported", "MalformedStructure", "ArchiveBroken", "PeParseFailed", "ArchiveBombSuspect", } # 规则命中判断: # - 阳性样本(期望 reject):expected_rules 全部被覆盖才算 OK(或 ClamAV-only error 宽容) # - 阴性对照(期望 pass):所有 actual_rules 都在 NOISE_RULES 白名单里才算过 rule_ok = True if expected_verdict == 'pass': if len(actual_rules) == 0: rule_ok = True elif set(actual_rules) <= NOISE_RULES: rule_ok = True # 只有启发式噪音,宽容 else: rule_ok = False elif expected_verdict == 'reject' and expected_rules: if clamav_only_error and not rules_hit: rule_ok = True # ClamAV-only error 宽容 else: rule_ok = (rules_missed == set()) # 期望规则全部被覆盖 overall = verdict_ok and rule_ok if overall: pass_count += 1 else: fail_count += 1 mark = "✅" if overall else "❌" print(f"{mark} [{s['category']}] {fname}") print(f" 期望 verdict={expected_verdict} 实际 verdict={actual_verdict}") if s['expected_rules']: print(f" 期望规则: {sorted(expected_rules)}") print(f" 命中规则: {actual_rules}") if rules_missed: print(f" 未命中: {sorted(rules_missed)}") if not verdict_ok: print(f" 描述: {s['description']}") print() results.append({ "filename": fname, "category": s['category'], "description": s['description'], "expected_verdict": expected_verdict, "actual_verdict": actual_verdict, "expected_rules": sorted(expected_rules), "actual_rules": actual_rules, "verdict_ok": verdict_ok, "rule_ok": rule_ok, "overall": overall, "score": outcome.score, "findings_count": len(outcome.findings), }) print(f"\n{'='*80}") print(f"结果汇总:✅ {pass_count} 通过 / ❌ {fail_count} 失败 / 共 {pass_count + fail_count}") print(f"{'='*80}") # 写 results.json 供 test_center.html 页面读取 with open(samples_dir / '_scan_results.json', 'w', encoding='utf-8') as f: json.dump(results, f, ensure_ascii=False, indent=2) print(f"\n详细结果已写入 tests/samples/_scan_results.json") sys.exit(0 if fail_count == 0 else 1)