Organize project tools and local workflows: c/tools, c/scripts, c/tests, root Makefile (flat C core untouched) (#22)
This commit is contained in:
ZacharyZcR
@@ -0,0 +1,16 @@
|
||||
# Tools
|
||||
|
||||
These scripts support model preparation and offline engineering work. They are
|
||||
not runtime dependencies of the C engine.
|
||||
|
||||
- `convert_fp8_to_int4.py`, `download_glm52.py`: model preparation
|
||||
- `make_glm_oracle.py`, `make_glm_bench_model.py`: deterministic fixtures
|
||||
- `benchmark_cuda_fixture.py`, `eval_glm.py`, `fetch_benchmarks.py`: benchmarks
|
||||
- `gen_unicode.py`: tokenizer table generation
|
||||
|
||||
Run them from `c/`, for example:
|
||||
|
||||
```sh
|
||||
python3 tools/convert_fp8_to_int4.py --selftest
|
||||
python3 tools/make_glm_bench_model.py --output /tmp/colibri-bench
|
||||
```
|
||||
@@ -0,0 +1 @@
|
||||
"""Offline conversion, fixture, benchmark, and evaluation utilities."""
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Reproducible CPU/CUDA A/B benchmark for tools/make_glm_bench_model.py output."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import statistics
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SPEED_RE = re.compile(r"REPLAY decode:.*\| ([0-9.]+) tok/s")
|
||||
PROFILE_RE = re.compile(
|
||||
r"PROFILO: expert-disk ([0-9.]+)s \| expert-matmul ([0-9.]+)s "
|
||||
r"\| attention ([0-9.]+)s .* lm_head ([0-9.]+)s \| altro ([0-9.-]+)s"
|
||||
)
|
||||
PROFILE_KEYS = ("disk", "expert_matmul", "attention", "lm_head", "other")
|
||||
|
||||
|
||||
def parse_output(stdout: str, stderr: str = "") -> tuple[float, list[float]]:
|
||||
"""Extract throughput and profile timings from one engine run."""
|
||||
speed = SPEED_RE.search(stdout)
|
||||
profile = PROFILE_RE.search(stdout)
|
||||
if not speed or not profile:
|
||||
raise RuntimeError(f"benchmark output missing\nstdout:\n{stdout}\nstderr:\n{stderr}")
|
||||
return float(speed.group(1)), [float(value) for value in profile.groups()]
|
||||
|
||||
|
||||
def execute(engine: str, env: dict[str, str]) -> tuple[float, list[float]]:
|
||||
run = subprocess.run(
|
||||
[engine, "4", "4", "4"], env=env, text=True, capture_output=True, check=True
|
||||
)
|
||||
return parse_output(run.stdout, run.stderr)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--model", required=True)
|
||||
parser.add_argument("--engine", default="./glm")
|
||||
parser.add_argument("--gpu", default="0")
|
||||
parser.add_argument("--runs", type=int, default=7)
|
||||
parser.add_argument("--threads", type=int, default=os.cpu_count() or 1)
|
||||
parser.add_argument("--pin-gb", default="1")
|
||||
parser.add_argument("--cuda-expert-gb", default="2")
|
||||
args = parser.parse_args()
|
||||
|
||||
model = Path(args.model).resolve()
|
||||
stats = model / "bench_stats.txt"
|
||||
base = os.environ.copy()
|
||||
for key in (
|
||||
"COLI_CUDA", "COLI_GPU", "COLI_GPUS", "CUDA_EXPERT_GB",
|
||||
"PIN", "PIN_GB", "STATS", "TF", "REPLAY", "CUDA_DENSE",
|
||||
):
|
||||
base.pop(key, None)
|
||||
base.update(
|
||||
SNAP=str(model),
|
||||
REF=str(model / "ref_glm.json"),
|
||||
REPLAY="1",
|
||||
OMP_NUM_THREADS=str(args.threads),
|
||||
OMP_PROC_BIND="spread",
|
||||
OMP_PLACES="cores",
|
||||
)
|
||||
|
||||
execute(args.engine, base | {"STATS": str(stats)})
|
||||
modes = {
|
||||
"cpu_stream": {},
|
||||
"dense_cuda": {"COLI_CUDA": "1", "COLI_GPU": args.gpu, "CUDA_DENSE": "1"},
|
||||
"cpu_pin": {"PIN": str(stats), "PIN_GB": args.pin_gb},
|
||||
"cuda_pin": {
|
||||
"COLI_CUDA": "1", "COLI_GPU": args.gpu,
|
||||
"PIN": str(stats), "PIN_GB": args.pin_gb,
|
||||
"CUDA_EXPERT_GB": args.cuda_expert_gb,
|
||||
},
|
||||
"cuda_pin_dense": {
|
||||
"COLI_CUDA": "1", "COLI_GPU": args.gpu, "CUDA_DENSE": "1",
|
||||
"PIN": str(stats), "PIN_GB": args.pin_gb,
|
||||
"CUDA_EXPERT_GB": args.cuda_expert_gb,
|
||||
},
|
||||
}
|
||||
|
||||
for extra in modes.values():
|
||||
execute(args.engine, base | extra) # warm-up
|
||||
speeds = {name: [] for name in modes}
|
||||
profiles = {name: [] for name in modes}
|
||||
names = list(modes)
|
||||
for run_index in range(args.runs):
|
||||
order = names[run_index % len(names):] + names[:run_index % len(names)]
|
||||
for name in order:
|
||||
speed, profile = execute(args.engine, base | modes[name])
|
||||
speeds[name].append(speed)
|
||||
profiles[name].append(profile)
|
||||
|
||||
result = {}
|
||||
for name in names:
|
||||
result[name] = {
|
||||
"runs_tok_s": speeds[name],
|
||||
"median_tok_s": statistics.median(speeds[name]),
|
||||
"median_profile_s": {
|
||||
key: statistics.median(row[index] for row in profiles[name])
|
||||
for index, key in enumerate(PROFILE_KEYS)
|
||||
},
|
||||
}
|
||||
print(json.dumps(result, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,445 @@
|
||||
"""
|
||||
Convertitore GLM-5.2-FP8 -> nostro container int4 (STADIO B).
|
||||
|
||||
Strategia DISK-SAFE (richiesta dell'utente): scarica UNO shard (~5 GB), lo converte in
|
||||
int4, lo CANCELLA, passa al prossimo. Il disco non si riempie mai: picco = 1 shard + l'output
|
||||
int4 che cresce fino a ~372 GB. Controllo di spazio che si ferma se manca margine.
|
||||
|
||||
Cosa fa per ogni tensore:
|
||||
- pesi FP8 (e4m3) con `*.weight_scale_inv` -> dequant a blocchi 128x128 -> f32
|
||||
- pesi BF16 (norme/embed/lm_head/...) -> f32
|
||||
poi:
|
||||
- attn/mlp/shared/expert/embed/lm_head -> QUANTIZZATO int4 (o int8) con la STESSA matematica
|
||||
del motore C (np.rint = lrintf, stesse soglie, stesso packing dei nibble) -> token identici
|
||||
- norme / router (mlp.gate.weight) / bias / e_score_correction_bias -> tenuti F32
|
||||
- indexer DSA / layer MTP (78) / shared_head / eh_proj / *norm dell'indexer -> SALTATI
|
||||
|
||||
Output: una dir di safetensors leggibile dal motore C (per ogni peso quantizzato: `nome` U8 =
|
||||
dati impacchettati, `nome.qs` F32 = scale per riga).
|
||||
|
||||
USO:
|
||||
# test locale (oracolo tiny, niente download): converte una dir gia' presente
|
||||
python3 tools/convert_fp8_to_int4.py --indir glm_tiny --outdir glm_tiny_i4 --ebits 4 --io-bits 4
|
||||
# selftest del dequant fp8 (richiede torch)
|
||||
python3 tools/convert_fp8_to_int4.py --selftest
|
||||
# reale: scarica+converte+cancella shard per shard
|
||||
python3 tools/convert_fp8_to_int4.py --repo zai-org/GLM-5.2-FP8 --outdir /home/vincenzo/glm52_i4
|
||||
"""
|
||||
import os, sys, glob, json, shutil, argparse
|
||||
import numpy as np
|
||||
|
||||
# ---------- quantizzazione: identica al C (glm.c) ----------
|
||||
def quant_int8(w, bits): # w: [O,I] f32 -> (qbytes U8 [O*I], scale f32 [O])
|
||||
qmax = (1 << (bits - 1)) - 1
|
||||
amax = np.abs(w).max(axis=1, keepdims=True)
|
||||
s = np.maximum(amax / qmax, 1e-8)
|
||||
q = np.clip(np.rint(w / s), -qmax - 1, qmax).astype(np.int8)
|
||||
return q.reshape(-1).view(np.uint8).copy(), s[:, 0].astype(np.float32)
|
||||
|
||||
def quant_int4(w, bits): # -> (qbytes U8 [O*ceil(I/2)], scale f32 [O])
|
||||
O, I = w.shape
|
||||
qmax = (1 << (bits - 1)) - 1
|
||||
amax = np.abs(w).max(axis=1, keepdims=True)
|
||||
s = np.maximum(amax / qmax, 1e-8)
|
||||
q = np.clip(np.rint(w / s), -8, qmax).astype(np.int32) # nibble [-8,7]
|
||||
rb = (I + 1) // 2
|
||||
out = np.zeros((O, rb), np.uint8)
|
||||
v0 = (q[:, 0::2] + 8).astype(np.uint8)
|
||||
out[:, :v0.shape[1]] = v0
|
||||
if I > 1:
|
||||
v1 = (q[:, 1::2] + 8).astype(np.uint8)
|
||||
out[:, :v1.shape[1]] |= (v1 << 4)
|
||||
return out.reshape(-1), s[:, 0].astype(np.float32)
|
||||
|
||||
def quant_int2(w, bits): # -> (qbytes U8 [O*ceil(I/4)], scale f32 [O]); 4/byte
|
||||
O, I = w.shape
|
||||
qmax = (1 << (bits - 1)) - 1 # bits=2 -> qmax=1, valori [-2,1]
|
||||
amax = np.abs(w).max(axis=1, keepdims=True)
|
||||
s = np.maximum(amax / qmax, 1e-8)
|
||||
q = np.clip(np.rint(w / s), -2, qmax).astype(np.int32)
|
||||
rb = (I + 3) // 4
|
||||
out = np.zeros((O, rb), np.uint8)
|
||||
for k in range(4): # impacchetta 4 valori per byte (identico a pack_int2 in C)
|
||||
vk = q[:, k::4]
|
||||
out[:, :vk.shape[1]] |= ((vk + 2).astype(np.uint8) << (k * 2))
|
||||
return out.reshape(-1), s[:, 0].astype(np.float32)
|
||||
|
||||
# ---------- classificazione dei tensori ----------
|
||||
def layer_idx(name):
|
||||
p = name.split(".")
|
||||
if len(p) > 2 and p[0] == "model" and p[1] == "layers":
|
||||
try: return int(p[2])
|
||||
except ValueError: return -1
|
||||
return -1
|
||||
|
||||
def classify(name, n_layers, keep_mtp=False, keep_idx=False):
|
||||
if name.endswith("_scale_inv"): return "consumed" # gestito col suo peso
|
||||
li = layer_idx(name)
|
||||
if keep_idx:
|
||||
# modalita' --indexer: SOLO i pesi del DSA lightning indexer dei layer principali
|
||||
if li < 0 or li >= n_layers or "indexer" not in name: return "skip"
|
||||
if name.endswith("norm.weight"): return "f32"
|
||||
return "q" # int8 consigliato (--ebits 8): pesi di scoring
|
||||
if keep_mtp:
|
||||
if li != n_layers: return "skip" # solo il layer MTP
|
||||
if "indexer" in name: return "skip" # il DSA indexer resta un no-op
|
||||
else:
|
||||
if li >= n_layers: return "skip" # layer MTP (78)
|
||||
if any(k in name for k in ["indexer", "indexers_proj", "eh_proj",
|
||||
"enorm", "hnorm", "shared_head"]): return "skip"
|
||||
if name.endswith("e_score_correction_bias"): return "f32"
|
||||
if name.endswith("mlp.gate.weight"): return "f32" # router (NON gate_proj)
|
||||
if name.endswith("norm.weight") or name == "model.norm.weight": return "f32"
|
||||
if name in ("model.embed_tokens.weight", "lm_head.weight"): return "io"
|
||||
if ".mlp.experts." in name and name.endswith(".weight"): return "x" # expert ROUTED (streaming)
|
||||
if name.endswith(".weight"): return "q" # attn/dense-mlp/shared (residente)
|
||||
return "f32"
|
||||
|
||||
# ---------- dequant di un tensore (fp8+scale a blocchi / bf16 / f32) ----------
|
||||
def dequant(f, name):
|
||||
import torch
|
||||
sl = f.get_slice(name); dt = sl.get_dtype()
|
||||
if dt in ("F8_E4M3", "float8_e4m3fn"):
|
||||
w = f.get_tensor(name).to(torch.float32)
|
||||
sc = f.get_tensor(name + "_scale_inv").to(torch.float32) # [ceil(O/128),ceil(I/128)]
|
||||
O, I = w.shape
|
||||
sc = sc.repeat_interleave(128, 0).repeat_interleave(128, 1)[:O, :I]
|
||||
return (w * sc).numpy()
|
||||
return f.get_tensor(name).to(torch.float32).numpy()
|
||||
|
||||
def convert_shard(path, out_dict, n_layers, ebits, io_bits, xbits, keep_mtp=False, keep_idx=False):
|
||||
from safetensors import safe_open
|
||||
with safe_open(path, framework="pt") as f:
|
||||
for name in f.keys():
|
||||
kind = classify(name, n_layers, keep_mtp, keep_idx)
|
||||
if kind in ("skip", "consumed"): continue
|
||||
w = dequant(f, name)
|
||||
if kind == "f32":
|
||||
out_dict[name] = w.astype(np.float32)
|
||||
else:
|
||||
bits = io_bits if kind == "io" else xbits if kind == "x" else ebits
|
||||
if w.ndim != 2: # es. bias 1D non previsto come 'q' -> tienilo f32
|
||||
out_dict[name] = w.astype(np.float32); continue
|
||||
q, s = (quant_int2(w, bits) if bits <= 2 else
|
||||
quant_int4(w, bits) if bits <= 4 else quant_int8(w, bits))
|
||||
out_dict[name] = q
|
||||
out_dict[name + ".qs"] = s
|
||||
|
||||
def free_gb(p): return shutil.disk_usage(p).free / 1e9
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--repo", default=None)
|
||||
ap.add_argument("--indir", default=None)
|
||||
ap.add_argument("--outdir", required=False)
|
||||
ap.add_argument("--ebits", type=int, default=None) # bit residenti (default 4; 8 per --mtp/--indexer)
|
||||
ap.add_argument("--io-bits", type=int, default=8) # bit di embed/lm_head
|
||||
ap.add_argument("--xbits", type=int, default=None) # bit degli expert ROUTED (streaming); default=ebits
|
||||
ap.add_argument("--n-layers", type=int, default=78)
|
||||
ap.add_argument("--min-free-gb", type=float, default=20.0)
|
||||
ap.add_argument("--selftest", action="store_true")
|
||||
ap.add_argument("--mtp", action="store_true",
|
||||
help="scarica/converte SOLO la testa MTP (model.layers.<n_layers>.*) -> out-mtp-*.safetensors")
|
||||
ap.add_argument("--indexer", action="store_true",
|
||||
help="estrae SOLO i pesi del DSA lightning indexer -> out-idx-*.safetensors. ATTENZIONE: "
|
||||
"i tensori indexer sono sparsi su ~tutti gli shard: ri-scarica l'intero repo (~756 GB "
|
||||
"di traffico) per tenerne pochi GB. Resumabile shard per shard. Consigliato --ebits 8.")
|
||||
a = ap.parse_args()
|
||||
if a.ebits is None:
|
||||
# testa MTP a int4 = acceptance ~0-4% (misurato, issue #8): il draft sbaglia sempre
|
||||
# e la speculazione non parte mai. A int8: 39-59%, 2.2-2.8 token/forward.
|
||||
a.ebits = 8 if (a.mtp or a.indexer) else 4
|
||||
if a.xbits is None: a.xbits = a.ebits
|
||||
|
||||
if a.selftest:
|
||||
import torch
|
||||
w = (torch.randn(256, 256) * 0.3)
|
||||
O, I = w.shape; bs = 128
|
||||
sc = torch.zeros(O // bs, I // bs)
|
||||
for bi in range(O // bs):
|
||||
for bj in range(I // bs):
|
||||
blk = w[bi*bs:(bi+1)*bs, bj*bs:(bj+1)*bs]
|
||||
sc[bi, bj] = blk.abs().max() / 448.0
|
||||
q = (w / sc.repeat_interleave(bs,0).repeat_interleave(bs,1)).to(torch.float8_e4m3fn)
|
||||
deq = (q.to(torch.float32) * sc.repeat_interleave(bs,0).repeat_interleave(bs,1))
|
||||
rel = (deq - w).abs().mean() / w.abs().mean()
|
||||
print(f"[selftest fp8 block-dequant] errore relativo medio = {rel:.4f} "
|
||||
f"({'OK' if rel < 0.05 else 'ALTO'})")
|
||||
return
|
||||
|
||||
os.makedirs(a.outdir, exist_ok=True)
|
||||
if a.indir: # conversione locale (test)
|
||||
shards = sorted(glob.glob(os.path.join(a.indir, "*.safetensors")))
|
||||
from safetensors.numpy import save_file
|
||||
for i, sp in enumerate(shards):
|
||||
out = {}; convert_shard(sp, out, a.n_layers, a.ebits, a.io_bits, a.xbits)
|
||||
save_file(out, os.path.join(a.outdir, f"out-{i:05d}.safetensors"))
|
||||
# copia config + tokenizer
|
||||
for fn in ["config.json"]:
|
||||
src = os.path.join(a.indir, fn)
|
||||
if os.path.exists(src): shutil.copy(src, a.outdir)
|
||||
print(f"convertito {len(shards)} shard -> {a.outdir}")
|
||||
return
|
||||
|
||||
# reale: scarica shard per shard, converte, cancella
|
||||
# EN: real: download shard by shard, convert, delete
|
||||
#
|
||||
# ROBUSTEZZA RETE: timeout brevi sulle read cosi' un download appeso FALLISCE invece
|
||||
# di restare fermo per sempre. 8s, non 30: "timeout" = ZERO byte ricevuti in quella
|
||||
# finestra; su un transfer vivo i chunk arrivano di continuo, quindi 8s e' sicuro e
|
||||
# uno stallo costa 8s invece di 30.
|
||||
# EN: NETWORK ROBUSTNESS: short read timeouts so a hung download FAILS instead of
|
||||
# EN: sitting there forever. 8s, not 30: a "timeout" means ZERO bytes received in that
|
||||
# EN: window; a live transfer delivers chunks constantly, so 8s is safe and a stall
|
||||
# EN: costs 8s instead of 30.
|
||||
os.environ.setdefault("HF_HUB_DOWNLOAD_TIMEOUT", "8")
|
||||
os.environ.setdefault("HF_HUB_ETAG_TIMEOUT", "15")
|
||||
# log con timestamp: i messaggi "Trying to resume" di hf_hub diventano databili.
|
||||
# EN: timestamped logs: hf_hub's "Trying to resume" messages become datable.
|
||||
import logging
|
||||
logging.basicConfig(format="%(asctime)s %(name)s: %(message)s", datefmt="%H:%M:%S")
|
||||
# hf_xet si blocca quando la rete si riavvia (connessioni zombie senza timeout):
|
||||
# forza la via HTTP classica, che curl ha dimostrato funzionare. (misurato 2026-07-02)
|
||||
# EN: hf_xet hangs when the network restarts (zombie connections with no timeout):
|
||||
# EN: force the classic HTTP path, which curl proved works (measured 2026-07-02).
|
||||
os.environ.setdefault("HF_HUB_DISABLE_XET", "1") # =0 per riabilitare xet / to re-enable xet
|
||||
from huggingface_hub import HfApi, hf_hub_download
|
||||
|
||||
# lock anti-doppione: DUE convertitori sulla stessa outdir si corrompono a vicenda.
|
||||
# EN: anti-duplicate lock: TWO converters on the same outdir corrupt each other.
|
||||
import fcntl
|
||||
lock = open(os.path.join(a.outdir, ".convert.lock"), "w")
|
||||
try: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError:
|
||||
print("ERRORE: un altro convertitore sta gia' lavorando su questa outdir. Esco."); return
|
||||
|
||||
# dimensioni note dei file, riempite dopo repo_info: il downloader multi-stream le usa
|
||||
# per calcolare i confini dei segmenti e per sapere quando un file e' completo.
|
||||
# EN: known file sizes, filled after repo_info: the multi-stream downloader uses them
|
||||
# EN: to compute segment boundaries and to know when a file is complete.
|
||||
SIZES = {}
|
||||
|
||||
def download_retry(repo, fn, dest, tries=999):
|
||||
"""Downloader multi-stream con resume via Range. Apre N segmenti concorrenti
|
||||
(default 2, COLI_DL_STREAMS per cambiarli) e salva lo stato per-segmento in un
|
||||
sidecar .seg -> NESSUN byte perso comunque muoia la connessione. Un singolo stream
|
||||
HF e' limitato a ~2 MB/s (misurato); 2 stream ~ raddoppiano il throughput senza
|
||||
saturare una linea domestica. File piccoli, COLI_DL_STREAMS=1 o un vecchio .part
|
||||
legacy -> percorso a stream singolo (_download_single).
|
||||
EN: multi-stream Range-resume downloader. Opens N concurrent segments (default 2,
|
||||
EN: COLI_DL_STREAMS to change) and saves per-segment state in a .seg sidecar -> NO
|
||||
EN: byte is lost however the connection dies. A single HF stream is paced at
|
||||
EN: ~2 MB/s (measured); 2 streams roughly double throughput without saturating a
|
||||
EN: home line. Small files, COLI_DL_STREAMS=1 or a legacy .part -> single-stream
|
||||
EN: path (_download_single)."""
|
||||
import time as _t, threading, urllib.request, urllib.error
|
||||
url = f"https://huggingface.co/{repo}/resolve/main/{fn}"
|
||||
out = os.path.join(dest, fn); part = out + ".part"; side = part + ".seg"
|
||||
os.makedirs(dest, exist_ok=True)
|
||||
expected = SIZES.get(fn)
|
||||
if os.path.exists(out) and (expected is None or os.path.getsize(out) == expected):
|
||||
return out
|
||||
NS = max(1, min(8, int(os.environ.get("COLI_DL_STREAMS", "2"))))
|
||||
# un .part senza sidecar l'ha scritto una versione precedente a stream singolo.
|
||||
# EN: a .part without a sidecar was written by an older single-stream version.
|
||||
legacy = os.path.exists(part) and not os.path.exists(side)
|
||||
if expected is None or expected < (256 << 20) or NS == 1 or legacy:
|
||||
return _download_single(url, fn, out, part, expected)
|
||||
# ---- multi-stream ----
|
||||
segs = [(expected * t // NS, expected * (t + 1) // NS) for t in range(NS)]
|
||||
done = [0] * NS
|
||||
# riprendi lo stato dei segmenti se il sidecar combacia (stesso N, stessa size).
|
||||
# EN: resume per-segment progress if the sidecar matches (same N, same size).
|
||||
if os.path.exists(side):
|
||||
try:
|
||||
st = json.loads(open(side).read())
|
||||
if st.get("n") == NS and st.get("size") == expected: done = st["done"]
|
||||
except Exception: pass
|
||||
if not os.path.exists(part):
|
||||
with open(part, "wb") as f: f.truncate(expected) # file sparse / sparse file
|
||||
fd = os.open(part, os.O_WRONLY)
|
||||
t0 = _t.time(); nres = [0]; log_lock = threading.Lock(); stopfail = []
|
||||
def worker(t):
|
||||
s0, s1 = segs[t]
|
||||
while done[t] < s1 - s0 and not stopfail:
|
||||
pos = s0 + done[t]
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "colibri-convert",
|
||||
"Range": f"bytes={pos}-{s1-1}"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=8) as r:
|
||||
if r.status != 206: # Range ignorato: multi-stream impossibile
|
||||
stopfail.append(t); return # EN: Range ignored: multi-stream impossible
|
||||
while done[t] < s1 - s0:
|
||||
chunk = r.read(1 << 20)
|
||||
if not chunk: break
|
||||
rem = (s1 - s0) - done[t] # mai oltre il segmento / never past the segment
|
||||
if len(chunk) > rem: chunk = chunk[:rem]
|
||||
os.pwrite(fd, chunk, s0 + done[t])
|
||||
done[t] += len(chunk)
|
||||
except KeyboardInterrupt: raise
|
||||
except Exception as ex:
|
||||
with log_lock:
|
||||
nres[0] += 1
|
||||
print(f" [dl] s{t}: {type(ex).__name__} a/at {(s0+done[t])/1e9:.2f} GB: "
|
||||
f"riprendo/resuming (#{nres[0]})", flush=True)
|
||||
_t.sleep(min(15, 1 + nres[0] // NS))
|
||||
th = [threading.Thread(target=worker, args=(t,), daemon=True) for t in range(NS)]
|
||||
for x in th: x.start()
|
||||
print(f" [dl {_t.strftime('%H:%M:%S')}] connesso/connected: {NS} stream, "
|
||||
f"{sum(done)/1e9:.2f} di/of {expected/1e9:.2f} GB", flush=True)
|
||||
mark = sum(done); tmark = t0
|
||||
while any(x.is_alive() for x in th):
|
||||
_t.sleep(5)
|
||||
have = sum(done)
|
||||
tmpside = side + ".tmp" # checkpoint atomico / atomic checkpoint
|
||||
open(tmpside, "w").write(json.dumps({"n": NS, "size": expected, "done": list(done)}))
|
||||
os.replace(tmpside, side)
|
||||
now = _t.time()
|
||||
if now - tmark >= 30:
|
||||
print(f" [dl {_t.strftime('%H:%M:%S')}] {have/1e9:5.2f} GB "
|
||||
f"({(have-mark)/max(now-tmark,1e-9)/1e6:5.1f} MB/s, {NS} stream)", flush=True)
|
||||
mark = have; tmark = now
|
||||
os.close(fd)
|
||||
if stopfail: # il server non onora il Range: fallback
|
||||
for f2 in (part, side): # EN: server won't honor Range: fall back
|
||||
if os.path.exists(f2): os.remove(f2)
|
||||
return _download_single(url, fn, out, part, expected)
|
||||
assert sum(done) == expected
|
||||
if os.path.exists(side): os.remove(side)
|
||||
os.replace(part, out)
|
||||
dt = max(_t.time() - t0, 1e-9)
|
||||
print(f" [dl] {fn}: {expected/1e9:.2f} GB in {dt/60:.1f} min "
|
||||
f"({expected/dt/1e6:.1f} MB/s medi/avg, {NS} stream, {nres[0]} riprese/resumes)", flush=True)
|
||||
return out
|
||||
|
||||
def _download_single(url, fn, out, part, expected):
|
||||
"""Percorso a stream singolo con resume via Range (file piccoli / .part legacy /
|
||||
COLI_DL_STREAMS=1). Un EOF corto ma pulito conta come ripresa; se non arriva
|
||||
NESSUN byte nuovo, backoff invece di girare a vuoto.
|
||||
EN: single-stream path with Range resume (small files / legacy .part /
|
||||
EN: COLI_DL_STREAMS=1). A clean short EOF counts as a resume; if NO new byte
|
||||
EN: arrives, back off instead of spinning."""
|
||||
import time as _t, urllib.request, urllib.error
|
||||
t0 = _t.time(); nres = 0; mark = 0; tmark = t0
|
||||
while True:
|
||||
have = os.path.getsize(part) if os.path.exists(part) else 0
|
||||
if expected is not None and have >= expected: break
|
||||
have0 = have
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "colibri-convert"})
|
||||
if have: req.add_header("Range", f"bytes={have}-")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=8) as r:
|
||||
if have and r.status == 200: # server ha ignorato il Range: riparti pulito
|
||||
have = 0 # EN: server ignored Range: restart clean
|
||||
if expected is None:
|
||||
cl = r.headers.get("Content-Length")
|
||||
if cl: expected = have + int(cl)
|
||||
if have == 0 or nres: # segnale di vita subito / immediate sign of life
|
||||
print(f" [dl {_t.strftime('%H:%M:%S')}] connesso/connected"
|
||||
f"{f' @ {have/1e9:.2f} GB' if have else ''}"
|
||||
f"{f' di/of {expected/1e9:.2f} GB' if expected else ''}", flush=True)
|
||||
with open(part, "ab" if have else "wb") as f:
|
||||
if not have: f.truncate(0)
|
||||
while True:
|
||||
chunk = r.read(1 << 20)
|
||||
if not chunk: break
|
||||
f.write(chunk); have += len(chunk)
|
||||
if have - mark >= 512 * 1024 * 1024 or _t.time() - tmark >= 30:
|
||||
now = _t.time()
|
||||
print(f" [dl {_t.strftime('%H:%M:%S')}] {have/1e9:5.2f} GB "
|
||||
f"({(have-mark)/max(now-tmark,1e-9)/1e6:5.1f} MB/s)", flush=True)
|
||||
mark = have; tmark = now
|
||||
if expected is None: break # lunghezza ignota: passata singola / unknown length
|
||||
if have < expected: # EOF corto ma pulito: conta come ripresa
|
||||
nres += 1 # EN: clean short EOF: counts as a resume
|
||||
if have == have0: _t.sleep(min(15, 1 + nres)) # zero progresso -> backoff / zero progress -> back off
|
||||
except KeyboardInterrupt: raise
|
||||
except urllib.error.HTTPError as ex:
|
||||
if ex.code == 416: break # gia' completo / already complete
|
||||
nres += 1
|
||||
print(f" [dl] HTTP {ex.code} a/at {have/1e9:.2f} GB: riprendo/resuming (#{nres})", flush=True)
|
||||
_t.sleep(min(15, 1 + nres))
|
||||
except Exception as ex:
|
||||
nres += 1
|
||||
print(f" [dl] {type(ex).__name__} a/at {have/1e9:.2f} GB: riprendo/resuming (#{nres})", flush=True)
|
||||
_t.sleep(min(15, 1 + nres))
|
||||
os.replace(part, out)
|
||||
dt = max(_t.time() - t0, 1e-9); sz = os.path.getsize(out)
|
||||
print(f" [dl] {fn}: {sz/1e9:.2f} GB in {dt/60:.1f} min "
|
||||
f"({sz/dt/1e6:.1f} MB/s medi/avg, {nres} riprese/resumes)", flush=True)
|
||||
return out
|
||||
|
||||
from safetensors.numpy import save_file
|
||||
import time as _t
|
||||
for att in range(999):
|
||||
try:
|
||||
info = HfApi().repo_info(a.repo, files_metadata=True)
|
||||
# dimensioni note dallo store: abilitano il download multi-stream a segmenti.
|
||||
# EN: sizes known from the store: enable segmented multi-stream download.
|
||||
SIZES.update({s.rfilename: s.size for s in info.siblings if s.size})
|
||||
break
|
||||
except KeyboardInterrupt: raise
|
||||
except Exception as ex:
|
||||
w = min(60, 5*(att+1)); print(f"repo_info KO ({type(ex).__name__}): riprovo tra {w}s", flush=True); _t.sleep(w)
|
||||
shards = sorted(s.rfilename for s in info.siblings if s.rfilename.endswith(".safetensors"))
|
||||
for fn in ["config.json", "tokenizer.json", "tokenizer_config.json", "generation_config.json"]:
|
||||
try: shutil.copy(hf_hub_download(a.repo, fn, local_dir=a.outdir+"/_meta"), a.outdir)
|
||||
except Exception: pass
|
||||
tmp = os.path.join(a.outdir, "_inflight"); os.makedirs(tmp, exist_ok=True)
|
||||
if a.mtp:
|
||||
import urllib.request
|
||||
idx = json.loads(urllib.request.urlopen(
|
||||
f"https://huggingface.co/{a.repo}/resolve/main/model.safetensors.index.json", timeout=30).read())["weight_map"]
|
||||
pref = f"model.layers.{a.n_layers}."
|
||||
mtp_shards = sorted(set(v for k, v in idx.items() if k.startswith(pref)))
|
||||
print(f"[MTP] testa nel layer {a.n_layers}: {len(mtp_shards)} shard da processare: {mtp_shards}")
|
||||
for i, sh in enumerate(mtp_shards):
|
||||
outp = os.path.join(a.outdir, f"out-mtp-{i:05d}.safetensors")
|
||||
if os.path.exists(outp): print(f"[MTP] {outp} gia' fatto"); continue
|
||||
print(f"[MTP {i+1}/{len(mtp_shards)}] scarico {sh}...", flush=True)
|
||||
p = download_retry(a.repo, sh, tmp)
|
||||
out = {}; convert_shard(p, out, a.n_layers, a.ebits, a.io_bits, a.xbits, keep_mtp=True)
|
||||
save_file(out, outp)
|
||||
os.remove(p)
|
||||
for blob in glob.glob(os.path.join(tmp, "**", "*"), recursive=True):
|
||||
if os.path.isfile(blob): os.remove(blob)
|
||||
print(f" -> {os.path.basename(outp)} ({os.path.getsize(outp)/1e9:.2f} GB, {len(out)} tensori)", flush=True)
|
||||
shutil.rmtree(tmp, ignore_errors=True); print("[MTP] FATTO."); return
|
||||
if a.indexer:
|
||||
import urllib.request
|
||||
idx = json.loads(urllib.request.urlopen(
|
||||
f"https://huggingface.co/{a.repo}/resolve/main/model.safetensors.index.json", timeout=30).read())["weight_map"]
|
||||
idx_shards = sorted(set(v for k, v in idx.items()
|
||||
if "indexer" in k and 0 <= layer_idx(k) < a.n_layers))
|
||||
tot_gb = len(idx_shards) * 5.4
|
||||
print(f"[IDX] pesi indexer su {len(idx_shards)} shard (~{tot_gb:.0f} GB di download totale, resumabile)")
|
||||
for i, sh in enumerate(idx_shards):
|
||||
outp = os.path.join(a.outdir, f"out-idx-{i:05d}.safetensors")
|
||||
if os.path.exists(outp): continue # gia' fatto -> ripartibile
|
||||
print(f"[IDX {i+1}/{len(idx_shards)}] scarico {sh}...", flush=True)
|
||||
p = download_retry(a.repo, sh, tmp)
|
||||
out = {}; convert_shard(p, out, a.n_layers, a.ebits, a.io_bits, a.xbits, keep_idx=True)
|
||||
if out: save_file(out, outp)
|
||||
os.remove(p)
|
||||
for blob in glob.glob(os.path.join(tmp, "**", "*"), recursive=True):
|
||||
if os.path.isfile(blob): os.remove(blob)
|
||||
print(f" -> {os.path.basename(outp)} ({len(out)} tensori)", flush=True)
|
||||
shutil.rmtree(tmp, ignore_errors=True); print("[IDX] FATTO."); return
|
||||
for i, sh in enumerate(shards):
|
||||
if free_gb(a.outdir) < a.min_free_gb:
|
||||
print(f"STOP: spazio libero < {a.min_free_gb} GB. Libera spazio e rilancia (riprende)."); break
|
||||
outp = os.path.join(a.outdir, f"out-{i:05d}.safetensors")
|
||||
if os.path.exists(outp): continue # gia' fatto -> ripartibile
|
||||
print(f"[{i+1}/{len(shards)}] scarico {sh} (libero {free_gb(a.outdir):.0f} GB)...", flush=True)
|
||||
p = download_retry(a.repo, sh, tmp)
|
||||
out = {}; convert_shard(p, out, a.n_layers, a.ebits, a.io_bits, a.xbits)
|
||||
save_file(out, outp)
|
||||
os.remove(p) # <-- cancella subito lo shard fp8
|
||||
for blob in glob.glob(os.path.join(tmp, "**", "*"), recursive=True):
|
||||
if os.path.isfile(blob): os.remove(blob)
|
||||
print(f" -> {os.path.basename(outp)} ({os.path.getsize(outp)/1e9:.2f} GB)", flush=True)
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
print("FATTO." if i == len(shards)-1 else "INTERROTTO (rilancia per riprendere).")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,53 @@
|
||||
"""
|
||||
Download dei pesi reali di GLM-5.2 per il motore C — STADIO B.
|
||||
|
||||
Target: zai-org/GLM-5.2-FP8 (FP8 e4m3, 141 shard, ~756 GB) -> ENTRA nei 926 GB di ext4.
|
||||
(La variante bf16 zai-org/GLM-5.2 e' 1.5 TB e NON entra.)
|
||||
|
||||
Il motore C leggera' questi safetensors in streaming e li (ri)quantizzera' a int4/int8.
|
||||
NB: i pesi sono F8_E4M3 + tensori `*.weight_scale_inv` (blocchi 128x128). Il loader st.h
|
||||
deve supportare fp8+block-scale prima di poterli usare (vedi memoria glm52-specs).
|
||||
|
||||
USO:
|
||||
python3 tools/download_glm52.py # scarica tutto in /home/vincenzo/glm52 (ripartibile)
|
||||
python3 tools/download_glm52.py --check # solo stima spazio e conteggio file, niente download
|
||||
|
||||
Lo scaricamento e' di centinaia di GB e ore: lancialo tu quando il resto e' pronto.
|
||||
"""
|
||||
import os, sys, shutil
|
||||
from huggingface_hub import snapshot_download, HfApi
|
||||
|
||||
REPO = "zai-org/GLM-5.2-FP8"
|
||||
DEST = os.environ.get("GLM_DIR", "/home/vincenzo/glm52") # su ext4 (/dev/sdd), MAI su /mnt/c
|
||||
|
||||
def human(n): return f"{n/1e9:.0f} GB"
|
||||
|
||||
def check():
|
||||
info = HfApi().repo_info(REPO, files_metadata=True)
|
||||
tot = sum((s.size or 0) for s in info.siblings)
|
||||
sts = [s for s in info.siblings if s.rfilename.endswith(".safetensors")]
|
||||
free = shutil.disk_usage(os.path.dirname(DEST) or "/").free
|
||||
print(f"repo: {REPO}")
|
||||
print(f" file totali: {len(info.siblings)} ({len(sts)} shard safetensors)")
|
||||
print(f" dimensione totale: {human(tot)}")
|
||||
print(f" spazio libero in {DEST}: {human(free)}")
|
||||
print(f" {'OK: ci sta' if free > tot*1.05 else 'ATTENZIONE: spazio insufficiente'}")
|
||||
|
||||
def download():
|
||||
os.makedirs(DEST, exist_ok=True)
|
||||
free = shutil.disk_usage(DEST).free
|
||||
print(f"Scarico {REPO} -> {DEST} (libero: {human(free)})")
|
||||
# resume_download e' implicito; in caso di interruzione, rilancia e riprende.
|
||||
snapshot_download(
|
||||
repo_id=REPO,
|
||||
local_dir=DEST,
|
||||
allow_patterns=["*.safetensors", "*.json", "*.txt", "*.model"],
|
||||
max_workers=8,
|
||||
)
|
||||
print("FATTO. Pesi in:", DEST)
|
||||
|
||||
if __name__ == "__main__":
|
||||
if "--check" in sys.argv:
|
||||
check()
|
||||
else:
|
||||
check(); print("---"); download()
|
||||
@@ -0,0 +1,145 @@
|
||||
"""
|
||||
Harness di validazione qualita' per il motore C GLM-5.2 (int4 streaming).
|
||||
Fa passare IL NOSTRO modello sugli stessi benchmark LLM standard (stile EleutherAI
|
||||
lm-evaluation-harness) usando la **log-likelihood** delle risposte multiple: un solo
|
||||
forward per opzione (niente generazione) -> fattibile anche a bassa velocita'.
|
||||
Serve a capire se la quantizzazione int4 ha lasciato il modello "tale" rispetto ai
|
||||
punteggi PUBBLICATI di GLM-5.2 (e, per contesto, Claude/GPT).
|
||||
|
||||
Dipendenze: solo `tokenizers` + il binario ./glm. I dataset si leggono da JSONL locali
|
||||
(uno per task) prodotti da `tools/fetch_benchmarks.py`. Formato di ogni riga JSONL:
|
||||
{"ctx": "...", "choices": ["...","..."], "gold": 0}
|
||||
Cosi' la harness e' offline e deterministica.
|
||||
|
||||
USO:
|
||||
# 1) (una volta, quando hai rete) scarica i benchmark in ./bench/*.jsonl
|
||||
python3 tools/fetch_benchmarks.py --out ./bench --tasks hellaswag,arc_challenge,mmlu --limit 200
|
||||
# 2) plumbing test della meccanica (senza motore):
|
||||
python3 tools/eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench --tasks smoke --dry
|
||||
# 3) validazione vera quando il modello e' pronto:
|
||||
python3 tools/eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench \
|
||||
--tasks hellaswag,arc_challenge,mmlu --limit 40 --ram 15
|
||||
# leve di ricerca: passate al motore via env
|
||||
TOPP=0.9 python3 tools/eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench --tasks mmlu --ram 15
|
||||
"""
|
||||
import os, sys, subprocess, argparse, random, json, tempfile, time
|
||||
|
||||
# mini-set OFFLINE per testare la meccanica (NON misura qualita': domande banali)
|
||||
SMOKE = [
|
||||
{"ctx": "The capital of France is", "choices": [" Paris", " Berlin", " Rome"], "gold": 0},
|
||||
{"ctx": "2 + 2 =", "choices": [" 4", " 5", " 7"], "gold": 0},
|
||||
{"ctx": "The sun rises in the", "choices": [" east", " west", " north"], "gold": 0},
|
||||
]
|
||||
|
||||
# punteggi PUBBLICATI (accuracy %), SOLO PER CONTESTO — DA VERIFICARE/AGGIORNARE dalla model card.
|
||||
REFERENCE = {
|
||||
"mmlu": {"GLM-5.2 (pubbl.)": None, "Claude (rif.)": None, "GPT (rif.)": None},
|
||||
"hellaswag": {"GLM-5.2 (pubbl.)": None},
|
||||
"arc_challenge": {"GLM-5.2 (pubbl.)": None},
|
||||
}
|
||||
|
||||
def load_docs(task, data_dir, limit, seed):
|
||||
if task == "smoke":
|
||||
return SMOKE[:limit] if limit else SMOKE
|
||||
path = os.path.join(data_dir, task + ".jsonl")
|
||||
if not os.path.exists(path):
|
||||
sys.exit(f"manca {path} — generalo con: python3 tools/fetch_benchmarks.py --out {data_dir} --tasks {task}")
|
||||
docs = [json.loads(l) for l in open(path) if l.strip()]
|
||||
random.Random(seed).shuffle(docs)
|
||||
return docs[:limit] if limit else docs
|
||||
|
||||
def build_requests(tk, docs_by_task):
|
||||
reqs, meta, perq = [], [], {}
|
||||
for t, docs in docs_by_task.items():
|
||||
for qi, d in enumerate(docs):
|
||||
ctx, conts, gold = d["ctx"], d["choices"], int(d["gold"])
|
||||
ctx_ids = tk.encode(ctx).ids
|
||||
for oi, cont in enumerate(conts):
|
||||
full = tk.encode(ctx + cont).ids
|
||||
cl = len(ctx_ids)
|
||||
while cl > 0 and (cl > len(full) or full[:cl] != ctx_ids[:cl]): cl -= 1
|
||||
cont_ids = full[cl:]
|
||||
if not cont_ids: # boundary degenere: forza split esplicito
|
||||
full = ctx_ids + tk.encode(cont).ids; cl = len(ctx_ids); cont_ids = full[cl:]
|
||||
if cl < 1: cl = 1 # serve almeno 1 token di contesto
|
||||
reqs.append(f"{cl} {len(full)-cl} " + " ".join(map(str, full)))
|
||||
meta.append((t, qi, oi, len(full) - cl, max(1, len(cont)), gold))
|
||||
perq.setdefault((t, qi), []).append(len(meta) - 1)
|
||||
return reqs, meta, perq
|
||||
|
||||
def score_accuracy(tasks, meta, perq, lp):
|
||||
print(f"\n{'task':<18} {'n':>4} {'acc':>7} {'acc_norm':>9}")
|
||||
overall = []
|
||||
for t in tasks:
|
||||
qs = [k for k in perq if k[0] == t]
|
||||
acc = accn = 0
|
||||
for k in qs:
|
||||
ridx = perq[k]; gold = meta[ridx[0]][5]
|
||||
best = max(ridx, key=lambda r: lp[r])
|
||||
bestn = max(ridx, key=lambda r: lp[r] / meta[r][4]) # acc_norm: per carattere
|
||||
acc += (meta[best][2] == gold)
|
||||
accn += (meta[bestn][2] == gold)
|
||||
n = len(qs)
|
||||
if not n: continue
|
||||
print(f"{t:<18} {n:>4} {100*acc/n:>6.1f}% {100*accn/n:>8.1f}%")
|
||||
overall.append(100 * accn / n)
|
||||
for mdl, sc in REFERENCE.get(t, {}).items():
|
||||
if sc is not None: print(f"{' rif '+mdl:<18} {'':>4} {'':>7} {sc:>8.1f}%")
|
||||
if overall:
|
||||
print(f"\nMEDIA acc_norm: {sum(overall)/len(overall):.1f}% su {len(overall)} task")
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--snap", required=True)
|
||||
ap.add_argument("--glm", default="./glm")
|
||||
ap.add_argument("--data", default="./bench")
|
||||
ap.add_argument("--tasks", default="smoke")
|
||||
ap.add_argument("--limit", type=int, default=40)
|
||||
ap.add_argument("--ram", type=int, default=0)
|
||||
ap.add_argument("--cap", type=int, default=64)
|
||||
ap.add_argument("--bits", default="")
|
||||
ap.add_argument("--seed", type=int, default=1234)
|
||||
ap.add_argument("--dry", action="store_true", help="costruisci le richieste e fermati (no motore)")
|
||||
ap.add_argument("--selftest", action="store_true", help="verifica la matematica dello scoring")
|
||||
a = ap.parse_args()
|
||||
|
||||
if a.selftest: # acc/acc_norm con logprob sintetici
|
||||
meta = [("t",0,0,1,4,1),("t",0,1,1,2,1),("t",0,2,1,8,1)]; perq = {("t",0):[0,1,2]}
|
||||
lp = [-3.0, -2.0, -5.0] # opt1 ha lp piu' alto -> acc sceglie 1 (=gold) OK
|
||||
score_accuracy(["t"], meta, perq, lp)
|
||||
print("selftest OK" if True else ""); return
|
||||
|
||||
from tokenizers import Tokenizer
|
||||
tk = Tokenizer.from_file(os.path.join(a.snap, "tokenizer.json"))
|
||||
tasks = [t.strip() for t in a.tasks.split(",") if t.strip()]
|
||||
docs_by_task = {t: load_docs(t, a.data, a.limit, a.seed) for t in tasks}
|
||||
for t, d in docs_by_task.items(): print(f"[{t}] {len(d)} domande", file=sys.stderr)
|
||||
|
||||
reqs, meta, perq = build_requests(tk, docs_by_task)
|
||||
print(f"richieste totali: {len(reqs)} (opzioni)", file=sys.stderr)
|
||||
if a.dry:
|
||||
for r in reqs[:3]: print(" esempio req:", r[:80], "...", file=sys.stderr)
|
||||
print("DRY: meccanica ok (tokenizzazione+richieste). Niente motore.", file=sys.stderr); return
|
||||
|
||||
req_path = tempfile.mktemp(suffix=".txt")
|
||||
open(req_path, "w").write("\n".join(reqs) + "\n")
|
||||
env = dict(os.environ, SNAP=a.snap, SCORE=req_path)
|
||||
if a.ram: env["RAM_GB"] = str(a.ram)
|
||||
cmd = [a.glm, str(a.cap)] + a.bits.split()
|
||||
print("eseguo:", " ".join(cmd), file=sys.stderr)
|
||||
t0 = time.time()
|
||||
proc = subprocess.run(cmd, env=env, capture_output=True, text=True)
|
||||
if proc.returncode != 0:
|
||||
print("ERRORE motore:\n", proc.stderr[-2000:], file=sys.stderr); sys.exit(1)
|
||||
lines = [l for l in proc.stdout.strip().splitlines() if l and l[0] in "-0123456789"]
|
||||
if len(lines) != len(reqs):
|
||||
print(f"ATTENZIONE: {len(lines)} output vs {len(reqs)} richieste", file=sys.stderr)
|
||||
lp = [float(l.split()[0]) for l in lines]
|
||||
print(f"(motore: {time.time()-t0:.0f}s){proc.stderr.strip().splitlines()[-1] if proc.stderr.strip() else ''}", file=sys.stderr)
|
||||
score_accuracy(tasks, meta, perq, lp)
|
||||
print("\nNB: confronta acc_norm col punteggio PUBBLICATO di GLM-5.2 (model card). Se vicino,"
|
||||
"\n la quantizzazione int4 ha preservato il modello. (riempi REFERENCE in tools/eval_glm.py)")
|
||||
os.remove(req_path)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,69 @@
|
||||
"""
|
||||
Scarica i benchmark LLM standard e li converte nel formato JSONL della harness
|
||||
({"ctx","choices","gold"} per riga). Da eseguire UNA volta, quando hai rete.
|
||||
Richiede `datasets`: pip install --break-system-packages datasets (o in una venv)
|
||||
|
||||
USO:
|
||||
python3 tools/fetch_benchmarks.py --out ./bench --tasks hellaswag,arc_challenge,arc_easy,mmlu,winogrande,piqa,openbookqa --limit 300
|
||||
Poi:
|
||||
python3 tools/eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench --tasks mmlu --limit 40 --ram 15
|
||||
"""
|
||||
import os, json, argparse, random
|
||||
|
||||
def f_hellaswag(d):
|
||||
ctx = (d["activity_label"] + ": " + d["ctx_a"] + " " + d["ctx_b"].capitalize()).strip()
|
||||
return ctx, [" " + e.strip() for e in d["endings"]], int(d["label"])
|
||||
def f_arc(d):
|
||||
letters, texts = d["choices"]["label"], d["choices"]["text"]
|
||||
return ("Question: " + d["question"].strip() + "\nAnswer:",
|
||||
[" " + t.strip() for t in texts], letters.index(d["answerKey"]))
|
||||
def f_mmlu(d):
|
||||
ctx = d["question"].strip() + "\n" + "\n".join(f"{c}. {t}" for c, t in zip("ABCD", d["choices"])) + "\nAnswer:"
|
||||
return ctx, [f" {c}" for c in "ABCD"], int(d["answer"])
|
||||
def f_winogrande(d):
|
||||
pre, post = d["sentence"].split("_")
|
||||
return pre.strip(), [(" " + o + post).rstrip() for o in (d["option1"], d["option2"])], int(d["answer"]) - 1
|
||||
def f_piqa(d):
|
||||
return "Question: " + d["goal"].strip() + "\nAnswer:", [" " + d["sol1"], " " + d["sol2"]], int(d["label"])
|
||||
def f_openbookqa(d):
|
||||
return d["question_stem"].strip(), [" " + t for t in d["choices"]["text"]], d["choices"]["label"].index(d["answerKey"])
|
||||
|
||||
TASKS = { # task: (path, config, split, formatter)
|
||||
"hellaswag": ("Rowan/hellaswag", None, "validation", f_hellaswag),
|
||||
"arc_easy": ("allenai/ai2_arc", "ARC-Easy", "validation", f_arc),
|
||||
"arc_challenge": ("allenai/ai2_arc", "ARC-Challenge", "validation", f_arc),
|
||||
"mmlu": ("cais/mmlu", "all", "test", f_mmlu),
|
||||
"winogrande": ("allenai/winogrande", "winogrande_xl", "validation", f_winogrande),
|
||||
"piqa": ("ybisk/piqa", None, "validation", f_piqa),
|
||||
"openbookqa": ("allenai/openbookqa", "main", "validation", f_openbookqa),
|
||||
}
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", default="./bench")
|
||||
ap.add_argument("--tasks", default="hellaswag,arc_challenge,mmlu")
|
||||
ap.add_argument("--limit", type=int, default=300)
|
||||
ap.add_argument("--seed", type=int, default=1234)
|
||||
a = ap.parse_args()
|
||||
from datasets import load_dataset
|
||||
os.makedirs(a.out, exist_ok=True)
|
||||
for t in [x.strip() for x in a.tasks.split(",") if x.strip()]:
|
||||
if t not in TASKS: print("task ignoto:", t); continue
|
||||
path, cfg, split, fn = TASKS[t]
|
||||
ds = load_dataset(path, cfg, split=split)
|
||||
idx = list(range(len(ds))); random.Random(a.seed).shuffle(idx)
|
||||
rows, n = [], 0
|
||||
for i in idx:
|
||||
try:
|
||||
ctx, choices, gold = fn(ds[i])
|
||||
if ctx and choices and 0 <= gold < len(choices):
|
||||
rows.append({"ctx": ctx, "choices": choices, "gold": gold}); n += 1
|
||||
except Exception: continue
|
||||
if n >= a.limit: break
|
||||
outp = os.path.join(a.out, t + ".jsonl")
|
||||
with open(outp, "w") as f:
|
||||
for r in rows: f.write(json.dumps(r) + "\n")
|
||||
print(f"{t}: {len(rows)} -> {outp}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Genera tok_unicode.h: tabelle di range per le classi Unicode usate dal
|
||||
pre-tokenizer cl100k (regex del tokenizer GLM-5.2):
|
||||
- \\p{L} lettere (categoria Unicode che inizia per 'L')
|
||||
- \\p{N} numeri (categoria che inizia per 'N')
|
||||
- \\s whitespace (proprieta' Unicode White_Space)
|
||||
Ogni classe diventa un array ordinato di range [lo,hi] inclusivi; il C fa ricerca
|
||||
binaria. Eseguire una volta: python3 tools/gen_unicode.py > tok_unicode.h
|
||||
"""
|
||||
import sys, unicodedata
|
||||
|
||||
WHITE_SPACE = {0x09,0x0A,0x0B,0x0C,0x0D,0x20,0x85,0xA0,0x1680,
|
||||
0x2000,0x2001,0x2002,0x2003,0x2004,0x2005,0x2006,0x2007,0x2008,0x2009,0x200A,
|
||||
0x2028,0x2029,0x202F,0x205F,0x3000}
|
||||
|
||||
def ranges(pred):
|
||||
out=[]; lo=None
|
||||
for cp in range(0x110000):
|
||||
if 0xD800<=cp<=0xDFFF: # surrogati: mai
|
||||
if lo is not None: out.append((lo,cp-1)); lo=None
|
||||
continue
|
||||
if pred(cp):
|
||||
if lo is None: lo=cp
|
||||
else:
|
||||
if lo is not None: out.append((lo,cp-1)); lo=None
|
||||
if lo is not None: out.append((lo,0x10FFFF))
|
||||
return out
|
||||
|
||||
def cat(cp):
|
||||
try: return unicodedata.category(chr(cp))
|
||||
except ValueError: return "Cn"
|
||||
|
||||
L = ranges(lambda c: cat(c).startswith("L"))
|
||||
N = ranges(lambda c: cat(c).startswith("N"))
|
||||
S = ranges(lambda c: c in WHITE_SPACE)
|
||||
|
||||
def emit(name, rs):
|
||||
print(f"static const uint32_t {name}[][2] = {{")
|
||||
for i in range(0,len(rs),6):
|
||||
chunk="".join(f"{{0x{lo:X},0x{hi:X}}}," for lo,hi in rs[i:i+6])
|
||||
print(" "+chunk)
|
||||
print("};")
|
||||
print(f"static const int {name}_n = {len(rs)};\n")
|
||||
|
||||
print("/* GENERATO da tools/gen_unicode.py — non modificare a mano. */")
|
||||
print("#ifndef TOK_UNICODE_H\n#define TOK_UNICODE_H\n#include <stdint.h>\n")
|
||||
emit("uni_L", L); emit("uni_N", N); emit("uni_S", S)
|
||||
print("""static int uni_in(const uint32_t t[][2], int n, uint32_t cp){
|
||||
int lo=0, hi=n-1;
|
||||
while(lo<=hi){ int m=(lo+hi)>>1;
|
||||
if(cp<t[m][0]) hi=m-1; else if(cp>t[m][1]) lo=m+1; else return 1; }
|
||||
return 0;
|
||||
}
|
||||
static inline int is_L(uint32_t c){ return uni_in(uni_L,uni_L_n,c); }
|
||||
static inline int is_N(uint32_t c){ return uni_in(uni_N,uni_N_n,c); }
|
||||
static inline int is_S(uint32_t c){ return uni_in(uni_S,uni_S_n,c); }
|
||||
#endif""")
|
||||
@@ -0,0 +1,99 @@
|
||||
"""Build a deterministic, medium-size GLM-MoE fixture for backend benchmarks.
|
||||
|
||||
This is not a useful language model. It preserves the real glm_moe_dsa data
|
||||
flow while remaining small enough to generate locally and run repeated CPU/CUDA
|
||||
A/B tests without downloading the 379 GB checkpoint.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from transformers import GlmMoeDsaConfig, GlmMoeDsaForCausalLM
|
||||
|
||||
|
||||
def build_config() -> GlmMoeDsaConfig:
|
||||
return GlmMoeDsaConfig(
|
||||
vocab_size=8192,
|
||||
hidden_size=1024,
|
||||
intermediate_size=2048,
|
||||
moe_intermediate_size=512,
|
||||
num_hidden_layers=8,
|
||||
first_k_dense_replace=3,
|
||||
num_attention_heads=16,
|
||||
num_key_value_heads=16,
|
||||
n_routed_experts=32,
|
||||
num_experts_per_tok=8,
|
||||
n_shared_experts=1,
|
||||
q_lora_rank=256,
|
||||
kv_lora_rank=128,
|
||||
qk_nope_head_dim=64,
|
||||
qk_rope_head_dim=32,
|
||||
v_head_dim=64,
|
||||
index_topk=4096,
|
||||
index_head_dim=32,
|
||||
index_n_heads=4,
|
||||
n_group=1,
|
||||
topk_group=1,
|
||||
norm_topk_prob=True,
|
||||
routed_scaling_factor=2.5,
|
||||
rope_parameters={"rope_type": "default", "rope_theta": 10000.0},
|
||||
tie_word_embeddings=False,
|
||||
rms_norm_eps=1e-5,
|
||||
attention_bias=False,
|
||||
max_position_embeddings=4096,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="glm_bench_medium")
|
||||
parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu")
|
||||
parser.add_argument("--seed", type=int, default=1234)
|
||||
args = parser.parse_args()
|
||||
|
||||
torch.manual_seed(args.seed)
|
||||
cfg = build_config()
|
||||
cfg._attn_implementation = "eager"
|
||||
model = GlmMoeDsaForCausalLM(cfg).eval()
|
||||
with torch.no_grad():
|
||||
for param in model.parameters():
|
||||
if param.dim() >= 2:
|
||||
param.normal_(0, 0.02)
|
||||
for layer in model.model.layers:
|
||||
if hasattr(layer.mlp, "gate"):
|
||||
layer.mlp.gate.e_score_correction_bias.copy_(
|
||||
torch.linspace(-0.1, 0.1, cfg.n_routed_experts)
|
||||
)
|
||||
|
||||
output = Path(args.output)
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
params = sum(p.numel() for p in model.parameters())
|
||||
model.save_pretrained(output, safe_serialization=True, max_shard_size="4GB")
|
||||
|
||||
model.to(args.device)
|
||||
prompt = [3, 14, 159, 26, 53, 58, 200, 11, 77, 240, 5, 99]
|
||||
ids = torch.tensor([prompt], device=args.device)
|
||||
with torch.inference_mode():
|
||||
full = model.generate(ids, max_new_tokens=8, do_sample=False, use_cache=True)[0]
|
||||
logits = model(full.unsqueeze(0), use_cache=False).logits[0]
|
||||
|
||||
ref = {
|
||||
"prompt_ids": prompt,
|
||||
"full_ids": full.cpu().tolist(),
|
||||
"tf_pred": logits.argmax(-1).cpu().tolist(),
|
||||
}
|
||||
(output / "ref_glm.json").write_text(json.dumps(ref))
|
||||
manifest = {
|
||||
"seed": args.seed,
|
||||
"parameters": params,
|
||||
"parameters_billions": round(params / 1e9, 4),
|
||||
"purpose": "backend benchmark fixture; random weights, not a language model",
|
||||
}
|
||||
(output / "bench_manifest.json").write_text(json.dumps(manifest, indent=2))
|
||||
print(json.dumps(manifest, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,79 @@
|
||||
"""Costruisce un GLM-5.2 (glm_moe_dsa) MINUSCOLO a pesi random come ORACOLO.
|
||||
Architettura vera (MLA + DSA indexer + router sigmoid/noaux_tc + shared expert),
|
||||
dimensioni minuscole. Salva pesi+config in c/glm_tiny/ e un riferimento greedy in
|
||||
c/ref_glm.json. seq corta (<= index_topk) cosi' il DSA seleziona tutte le key e
|
||||
l'attenzione coincide con la MLA densa: il motore C puo' validare senza implementare
|
||||
l'indexer sparso."""
|
||||
import json, torch
|
||||
from transformers import GlmMoeDsaConfig, GlmMoeDsaForCausalLM
|
||||
|
||||
torch.manual_seed(1234)
|
||||
|
||||
cfg = GlmMoeDsaConfig(
|
||||
vocab_size=256,
|
||||
hidden_size=128,
|
||||
intermediate_size=64, # MLP densa (primi 3 layer)
|
||||
moe_intermediate_size=32, # expert
|
||||
num_hidden_layers=5, # 3 densi + 2 sparse
|
||||
first_k_dense_replace=3,
|
||||
num_attention_heads=4,
|
||||
num_key_value_heads=4,
|
||||
n_routed_experts=8,
|
||||
num_experts_per_tok=2,
|
||||
n_shared_experts=1,
|
||||
q_lora_rank=64,
|
||||
kv_lora_rank=32,
|
||||
qk_nope_head_dim=24,
|
||||
qk_rope_head_dim=8, # pari -> interleave ok; head_dim diventa 8
|
||||
v_head_dim=32,
|
||||
index_topk=4096, # >> seq_len -> DSA seleziona tutto (no-op)
|
||||
index_head_dim=16,
|
||||
index_n_heads=2,
|
||||
n_group=1,
|
||||
topk_group=1,
|
||||
norm_topk_prob=True,
|
||||
routed_scaling_factor=2.5,
|
||||
rope_parameters={"rope_type": "default", "rope_theta": 10000.0},
|
||||
tie_word_embeddings=False,
|
||||
rms_norm_eps=1e-5,
|
||||
attention_bias=False,
|
||||
max_position_embeddings=4096,
|
||||
)
|
||||
cfg._attn_implementation = "eager"
|
||||
|
||||
model = GlmMoeDsaForCausalLM(cfg).eval()
|
||||
# rende i pesi non banali (default init e' molto piccolo): scala router/bias per topk vario
|
||||
with torch.no_grad():
|
||||
for n, p in model.named_parameters():
|
||||
if p.dim() >= 2:
|
||||
p.normal_(0, 0.05)
|
||||
# bias di correzione del router: valori distinti cosi' la selezione e' sensata
|
||||
for i, layer in enumerate(model.model.layers):
|
||||
if hasattr(layer.mlp, "gate"):
|
||||
layer.mlp.gate.e_score_correction_bias.copy_(
|
||||
torch.linspace(-0.1, 0.1, cfg.n_routed_experts))
|
||||
|
||||
print("=== tensori dello state_dict (nomi per il loader C) ===")
|
||||
for n, p in model.state_dict().items():
|
||||
print(f" {n:60s} {tuple(p.shape)}")
|
||||
|
||||
prompt = [3, 14, 159, 26, 53, 58, 200, 11, 77, 240, 5, 99] # token id arbitrari, seq corta
|
||||
ids = torch.tensor([prompt])
|
||||
with torch.no_grad():
|
||||
out = model.generate(ids, max_new_tokens=20, do_sample=False, use_cache=True)
|
||||
full = out[0].tolist()
|
||||
print("\nprompt:", prompt)
|
||||
print("full :", full)
|
||||
|
||||
# teacher-forcing: un singolo forward su tutta la sequenza -> argmax per posizione.
|
||||
# Per il greedy vale tf_pred[i] == full[i+1] per i >= len(prompt)-1; serve a validare
|
||||
# il PREFILL del motore C separandolo dal decode.
|
||||
with torch.no_grad():
|
||||
lg = model(torch.tensor([full]), use_cache=False).logits[0] # [seq, vocab]
|
||||
tf_pred = lg.argmax(-1).tolist()
|
||||
print("tf_pred:", tf_pred)
|
||||
|
||||
model.save_pretrained("glm_tiny", safe_serialization=True)
|
||||
json.dump(cfg.to_dict(), open("glm_tiny/config.json", "w"))
|
||||
json.dump({"prompt_ids": prompt, "full_ids": full, "tf_pred": tf_pred}, open("ref_glm.json", "w"))
|
||||
print("\nsalvato: glm_tiny/ (pesi+config) e ref_glm.json")
|
||||
Reference in New Issue
Block a user