diff --git a/README.md b/README.md index f901484..8dff2d2 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,19 @@ PIN=stats.txt PIN_GB=20 ./coli chat # scale PIN_GB to your free RAM These are estimates, not measurements — if you run colibrì on serious hardware, **please open an issue with your numbers**: real datapoints from better machines are exactly what this project needs next. +## Quality benchmark — help wanted + +We have never measured how much the int4 quantization costs in accuracy — the harness is built and wired, but scoring is one forward per answer option, and on the dev box's ~1 GB/s disk a full run takes the better part of a day. **This is the single most valuable thing a faster machine can contribute.** The code is here and ready; one command runs it end to end (it auto-downloads the datasets on first use): + +```bash +cd c +./coli bench # hellaswag, arc_challenge, mmlu — 40 questions each +./coli bench hellaswag --limit 200 # one task, more questions +./coli bench mmlu arc_challenge --ram 100 # pick tasks, set a RAM budget +``` + +It prints per-task accuracy (log-likelihood scoring, EleutherAI-harness style). Published full-precision GLM-5.2 scores on these tasks sit around 85–95%; if our int4 container lands within a few points, the quantization is validated — if it doesn't, we know to invest in mixed / grouped-scale quantization. **If you have the hardware to run this, please open an issue with the numbers** — it's the measurement the project is missing. + ## Supporting the project colibrì is a one-person project, written and tested entirely on a 12-core laptop with 25 GB of RAM — the numbers above are the ceiling of what I can measure at home. If this project is useful or interesting to you and you'd like to support its development (better test hardware translates *directly* into a faster engine for everyone: real NVMe scaling data, bigger pinned caches, int2/int3 quality sweeps on real benchmarks), you can: diff --git a/c/coli b/c/coli index ee3e171..d56c810 100644 --- a/c/coli +++ b/c/coli @@ -288,13 +288,23 @@ def cmd_chat(a): def cmd_bench(a): need_model(a.model) banner("bench") - cmd=[sys.executable, os.path.join(HERE,"eval_glm.py"), "--snap",a.model, - "--tasks", ",".join(a.tasks) if a.tasks else "hellaswag,arc_challenge,mmlu", - "--limit", str(a.limit), "--data", a.data] + # python con `tokenizers`: l'ambiente del progetto se c'e', altrimenti quello corrente + venv_py=os.path.join(HERE,"mio_env","bin","python3") + py = venv_py if os.path.exists(venv_py) else sys.executable + tasks = ",".join(a.tasks) if a.tasks else "hellaswag,arc_challenge,mmlu" + # dataset mancanti -> li scarica una volta (fetch_benchmarks.py li mette in --data come JSONL) + missing=[t for t in tasks.split(",") if not os.path.exists(os.path.join(a.data,f"{t}.jsonl"))] + if missing: + print(f" {C.dim}scarico i dataset mancanti: {', '.join(missing)}{C.r}") + subprocess.call([py, os.path.join(HERE,"fetch_benchmarks.py"), + "--out", a.data, "--tasks", ",".join(missing), "--limit", str(max(a.limit,200))]) + cmd=[py, os.path.join(HERE,"eval_glm.py"), "--snap",a.model, + "--tasks", tasks, "--limit", str(a.limit), "--data", a.data] if a.ram: cmd+=["--ram",str(a.ram)] e=dict(os.environ) if a.topp: e["TOPP"]=str(a.topp) if a.topk: e["TOPK"]=str(a.topk) + print(f" {C.dim}decode disk-bound: su hardware lento questo richiede ORE. Alza --limit su macchine veloci.{C.r}\n") sys.exit(subprocess.call(cmd, env=e)) def cmd_convert(a):