From b2855c0da1e7d3da9777d1bd0e0ad3cbdf636267 Mon Sep 17 00:00:00 2001 From: JustVugg Date: Mon, 6 Jul 2026 16:59:27 +0200 Subject: [PATCH] coli convert: one command does model + MTP head; README documents the full zero-to-chat path Co-Authored-By: Claude Fable 5 --- README.md | 17 ++++++++++++----- c/coli | 20 +++++++++++++++----- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7472bf0..ed5fd53 100644 --- a/README.md +++ b/README.md @@ -56,14 +56,21 @@ Cold starts are heavy on random reads (~11 GB/token). Reads themselves are safe, cd c ./setup.sh # checks gcc/OpenMP, builds, self-tests -# convert the model (resumable, needs ~400 GB free on a real ext4/NVMe path): -./coli convert # from zai-org/GLM-5.2-FP8 +# ONE command does everything model-side: downloads GLM-5.2-FP8 shard by shard +# (never needs the full 756 GB at once), converts to the int4 container, then +# converts the MTP head for speculative decoding. Resumable at any point. +# Conversion (only) needs python with: pip install torch safetensors huggingface_hub numpy +./coli convert --model /nvme/glm52_i4 # ~400 GB free on a real ext4/NVMe path -# chat (RAM budget and expert cache size itself automatically): -COLI_MODEL=/path/to/glm52_i4 ./coli chat +# chat — RAM budget, expert cache and MTP are all detected automatically: +COLI_MODEL=/nvme/glm52_i4 ./coli chat ``` -Useful knobs (env or flags): `--topp 0.7` adaptive expert top-p (30–40% less disk), `--ngen N` max tokens, `STATS=f`/`PIN=f PIN_GB=g` record expert usage then pin the hottest in RAM, `THINK=1` enable GLM-5.2's reasoning block, `DRAFT=n` MTP draft depth, `TF=1` teacher-forcing validation. +The engine at runtime is pure C — python is only used by the one-time converter. + +Useful knobs (env or flags): `--temp 0.7` token sampling temperature (0 = greedy, default 1.0 + nucleus 0.95), `--topp 0.7` adaptive expert top-p (30–40% less disk), `--ngen N` max tokens per answer (`:piu` in chat continues a truncated one), `AUTOPIN=0` disable the learning cache's auto-pin, `THINK=1` enable GLM-5.2's reasoning block, `DRAFT=n` MTP draft depth, `TF=1` teacher-forcing validation. + +**The learning cache**: the engine records which experts your usage actually routes to (`.coli_usage` next to the model, updated every turn) and at startup automatically pins the hottest ones in spare RAM. colibrì literally gets faster the more you use it. ## Got a better machine? Try it — here's what to expect diff --git a/c/coli b/c/coli index b4c64ba..320af9f 100644 --- a/c/coli +++ b/c/coli @@ -269,11 +269,20 @@ def cmd_bench(a): def cmd_convert(a): banner("convert") - cmd=[sys.executable, os.path.join(HERE,"convert_fp8_to_int4.py"), - "--repo", a.repo, "--outdir", a.model, "--ebits", str(a.ebits), "--io-bits", str(a.io_bits)] - if a.xbits: cmd+=["--xbits",str(a.xbits)] - print(f" {C.dim}{' '.join(cmd)}{C.r}") - sys.exit(subprocess.call(cmd)) + # python con torch/safetensors: l'ambiente del progetto se c'e', altrimenti quello corrente + venv_py=os.path.join(HERE,"mio_env","bin","python3") + py = venv_py if os.path.exists(venv_py) else sys.executable + base=[py, os.path.join(HERE,"convert_fp8_to_int4.py"), + "--repo", a.repo, "--outdir", a.model, "--ebits", str(a.ebits), "--io-bits", str(a.io_bits)] + if a.xbits: base+=["--xbits",str(a.xbits)] + # passo 1: modello principale (78 layer). Resumabile: riparte dagli shard mancanti. + print(f" {C.dim}[1/2] modello: {' '.join(base)}{C.r}") + rc=subprocess.call(base) + if rc!=0: sys.exit(rc) + if a.no_mtp: sys.exit(0) + # passo 2: testa MTP (layer 78) -> abilita la decodifica speculativa nativa (piu' veloce) + print(f" {C.dim}[2/2] testa MTP (draft speculativi){C.r}") + sys.exit(subprocess.call(base+["--mtp"])) def main(): common=argparse.ArgumentParser(add_help=False) @@ -290,6 +299,7 @@ def main(): pb.add_argument("--limit",type=int,default=40); pb.add_argument("--data",default=os.path.join(HERE,"bench")) pc=sub.add_parser("convert", parents=[common]); pc.add_argument("--repo",default="zai-org/GLM-5.2-FP8") pc.add_argument("--ebits",type=int,default=4); pc.add_argument("--io-bits",type=int,default=8); pc.add_argument("--xbits",type=int,default=0) + pc.add_argument("--no-mtp",action="store_true",help="salta la testa MTP (niente draft speculativi)") a=ap.parse_args() {"build":cmd_build,"info":cmd_info,"run":cmd_run,"chat":cmd_chat,"bench":cmd_bench, "convert":cmd_convert}.get(a.cmd, lambda _:(banner(),print(__doc__)))(a)