Tokenizer 与预处理配置为何也要纳入模型保护

#!/usr/bin/env python3
"""Diagnose integrity of model companion assets via hash manifest."""
import hashlib
import json
import os
import sys

MANIFEST = "assets-integrity.json"

def file_digest(path, algo="sha256"):
    h = hashlib.new(algo)
    with open(path, "rb") as f:
        for chunk in iter(lambda: f.read(65536), b""):
            h.update(chunk)
    return h.hexdigest()

def verify(manifest_path):
    if not os.path.exists(manifest_path):
        print(f"Manifest not found: {manifest_path}", file=sys.stderr)
        sys.exit(2)
    
    with open(manifest_path, "r", encoding="utf-8") as fh:
        plan = json.load(fh)
    
    errors = []
    base_dir = plan.get("base_dir", ".")
    
    for entry in plan.get("assets", []):
        rel = entry.get("path")
        expected = entry.get("expected_hash")
        algo = entry.get("algo", "sha256")
        
        if not rel or not expected:
            errors.append(f"Missing path/hash in entry: {entry}")
            continue
        
        full = os.path.join(base_dir, rel)
        if not os.path.exists(full):
            errors.append(f"Missing asset: {full}")
            continue
        
        actual = file_digest(full, algo)
        if actual != expected:
            errors.append(f"Hash mismatch: {full} expected={expected} actual={actual}")
    
    if errors:
        for e in errors:
            print(f"[FAIL] {e}", file=sys.stderr)
        sys.exit(1)
    
    print("[PASS] All companion assets match integrity manifest.")
    sys.exit(0)

if __name__ == "__main__":
    verify(MANIFEST)