Organize project tools and local workflows: c/tools, c/scripts, c/tests, root Makefile (flat C core untouched) (#22)

2026-07-10 16:08:39 +08:00
parent a2942b2172
commit 13e8f09ffc
23 changed files with 87 additions and 52 deletions
@@ -7,9 +7,12 @@ Keep changes focused and preserve Colibri's dependency-free default CPU path.
 Run the lightweight checks locally:

 ```sh
-make -C c check
+make check
 ```

+`make -C c check` remains available for scripts that already run from the
+engine directory.
+
 This performs one portable CPU build, C unit tests, and Python standard-library
 tests. It does not download a model or require CUDA.

@@ -0,0 +1,4 @@
+.PHONY: all glm portable test check cuda-test clean
+
+all glm portable test check cuda-test clean:
+	$(MAKE) -C c $@
@@ -36,7 +36,7 @@ The engine is a single C file (`c/glm.c`, ~1,300 lines) plus small headers. No B
 - **Batch-union MoE**: in prefill (and MTP verification), each unique expert of the batch is read once and applied to every position that routes to it.
 - **Byte-level BPE tokenizer in C** (GPT-2-style with Unicode-property regex, 320k merges).
 - **RAM safety**: the expert cache is auto-sized from `MemAvailable` at startup — an honest peak projection (working set, KV, MTP row, reconstruction buffers) so the kernel OOM-killer never fires.
- **Offline FP8→int4 converter** (`c/convert_fp8_to_int4.py`): downloads one shard at a time (~5 GB), dequants (128×128 block scales), requantizes to the engine's container, deletes the shard — the 756 GB FP8 checkpoint never needs to exist on disk at once. Resumable.
+- **Offline FP8→int4 converter** (`c/tools/convert_fp8_to_int4.py`): downloads one shard at a time (~5 GB), dequants (128×128 block scales), requantizes to the engine's container, deletes the shard — the 756 GB FP8 checkpoint never needs to exist on disk at once. Resumable.

 ## Honest numbers (WSL2, 12 cores, 25 GB RAM, NVMe via VHDX)

@@ -147,8 +147,8 @@ deterministic 313M-parameter `glm_moe_dsa` fixture and run fixed-token replay:

 ```bash
 cd c
-python make_glm_bench_model.py --output /nvme/colibri-bench-medium --device cuda
-python benchmark_cuda_fixture.py --model /nvme/colibri-bench-medium --gpu 0
+python tools/make_glm_bench_model.py --output /nvme/colibri-bench-medium --device cuda
+python tools/benchmark_cuda_fixture.py --model /nvme/colibri-bench-medium --gpu 0
 ```

 The fixture has random weights and is not a language model. It exists only to
@@ -242,16 +242,26 @@ Every contribution, from a datapoint to a disk, moves the ceiling.
 ## Repo layout

 ```
-c/glm.c          the engine (GLM-5.2 forward, streaming MoE, MTP, serve mode)
-c/st.h           safetensors reader: pread + fadvise, no mmap (RSS stays flat)
-c/tok.h          byte-level BPE tokenizer in C
-c/coli           CLI: chat / run / bench / convert / info
-c/iobench.c      parallel disk microbenchmark (measures what the engine feels)
-c/convert_fp8_to_int4.py   disk-safe FP8 → int4 converter
-c/make_glm_oracle.py       tiny-random oracle generator for validation
-c/olmoe.c        stage-A engine (OLMoE), first validation target
+Makefile                  root build/check entry point
+c/
+├── glm.c                 single-file GLM engine
+├── st.h, tok.h, json.h   runtime headers
+├── backend_cuda.*        optional CUDA tier
+├── Makefile              build and local checks
+├── coli                  user-facing CLI
+├── setup.sh              one-command local setup
+├── tools/                offline conversion, fixtures and benchmarks
+├── scripts/              long-running conversion helpers
+└── tests/                dependency-free C and Python tests
 ```

+The runtime path intentionally stays flat and readable: `glm.c` plus its small
+headers. Auxiliary Python and shell tooling is grouped separately and is never a
+runtime dependency of the engine.
+
+From the repository root, `make`, `make check`, and `make clean` delegate to the
+engine Makefile. Existing commands run from `c/` continue to work unchanged.
+
 ## Why "colibrì"

 The hummingbird weighs a few grams, hovers in place, and visits a thousand flowers a day. This engine keeps a 744-billion-parameter giant alive on hummingbird rations: 25 GB of RAM, twelve CPU cores, and a lot of disk patience.
@@ -56,9 +56,9 @@ backend_cuda.o: backend_cuda.cu backend_cuda.h
 	@command -v $(NVCC) >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
 	$(NVCC) $(NVCCFLAGS) -c backend_cuda.cu -o $@

-cuda-test: backend_cuda.cu backend_cuda.h backend_cuda_test.cu
+cuda-test: backend_cuda.cu backend_cuda.h tests/test_backend_cuda.cu
 	@command -v $(NVCC) >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; }
-	$(NVCC) $(NVCCFLAGS) backend_cuda.cu backend_cuda_test.cu -o backend_cuda_test
+	$(NVCC) $(NVCCFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test
 	./backend_cuda_test

 olmoe: olmoe.c st.h json.h
@@ -82,7 +82,7 @@ test-python:

 test: test-c test-python

-# CI-friendly validation: one portable CPU build and dependency-free tests.
+# Local validation: one portable CPU build and dependency-free tests.
 check:
 	$(MAKE) clean
 	$(MAKE) portable
@@ -19,6 +19,7 @@ Config via env o flag (validi anche dopo il sottocomando):
 import os, sys, subprocess, argparse, json, time, signal, shutil, threading, re, codecs, tempfile, textwrap

 HERE = os.path.dirname(os.path.abspath(__file__))
+TOOLS = os.path.join(HERE, "tools")
 GLM  = os.path.join(HERE, "glm")
 DEF_MODEL = os.environ.get("COLI_MODEL", "/home/vincenzo/glm52_i4")
 END   = b"\x01\x01END\x01\x01\n"
@@ -385,9 +386,9 @@ def cmd_bench(a):
    missing=[t for t in tasks.split(",") if not os.path.exists(os.path.join(a.data,f"{t}.jsonl"))]
    if missing:
        print(f"  {C.dim}scarico i dataset mancanti: {', '.join(missing)}{C.r}")
-        subprocess.call([py, os.path.join(HERE,"fetch_benchmarks.py"),
+        subprocess.call([py, os.path.join(TOOLS,"fetch_benchmarks.py"),
                         "--out", a.data, "--tasks", ",".join(missing), "--limit", str(max(a.limit,200))])
-    cmd=[py, os.path.join(HERE,"eval_glm.py"), "--snap",a.model,
+    cmd=[py, os.path.join(TOOLS,"eval_glm.py"), "--snap",a.model,
         "--tasks", tasks, "--limit", str(a.limit), "--data", a.data]
    if a.ram: cmd+=["--ram",str(a.ram)]
    e=dict(os.environ)
@@ -401,7 +402,7 @@ def cmd_convert(a):
    # python con torch/safetensors: l'ambiente del progetto se c'e', altrimenti quello corrente
    venv_py=os.path.join(HERE,"mio_env","bin","python3")
    py = venv_py if os.path.exists(venv_py) else sys.executable
-    base=[py, os.path.join(HERE,"convert_fp8_to_int4.py"),
+    base=[py, os.path.join(TOOLS,"convert_fp8_to_int4.py"),
          "--repo", a.repo, "--outdir", a.model, "--ebits", str(a.ebits), "--io-bits", str(a.io_bits)]
    if a.xbits: base+=["--xbits",str(a.xbits)]
    # passo 1: modello principale (78 layer). Resumabile: riparte dagli shard mancanti.
@@ -12,7 +12,7 @@
 * E' cio' che fa entrare GLM-5.2 nei 15 GB: ~17B param residenti a int4 ~= 8.7 GB.
 * Norme/router/bias restano f32 (piccoli e sensibili).
 *
- * Validazione: stessi token id di ref_glm.json (oracolo transformers, c/make_glm_oracle.py).
+ * Validazione: stessi token id di ref_glm.json (oracolo transformers, c/tools/make_glm_oracle.py).
 *   build: make glm   run: SNAP=./glm_tiny ./glm <cap> <expert_bits> <dense_bits>
 *   TF=1 -> teacher-forcing (valida il prefill su tutta la sequenza)
 */
@@ -1,14 +1,14 @@
 #!/usr/bin/env bash
 # Pipeline GLM-5.2 (int4, streaming, 15 GB RAM) — tutto in WSL, modello su ext4.
-#   uso:  ./run.sh ["prompt"] [n_token]
+#   uso da c/:  scripts/run.sh ["prompt"] [n_token]
 # Fa: (1) attende lo spostamento su ext4, (2) riprende la conversione fino a completarla,
 #     (3) compila il motore, (4) genera testo restando nel budget RAM.
 set -euo pipefail

-DIR=/home/vincenzo/glm52_i4          # modello int4 su ext4 (NON /mnt/c!)
-REPO=zai-org/GLM-5.2-FP8
-CODE=/mnt/c/Users/User/Desktop/moe-stream/c
-RAM_GB=15
+DIR="${COLI_MODEL:-/home/vincenzo/glm52_i4}"          # modello int4 su ext4 (NON /mnt/c!)
+REPO="${COLI_REPO:-zai-org/GLM-5.2-FP8}"
+CODE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+RAM_GB="${RAM_GB:-15}"
 PROMPT="${1:-Ciao, chi sei?}"
 NGEN="${2:-64}"

@@ -25,7 +25,7 @@ echo "[1/4] spostamento completato: $(du -sh "$DIR" | cut -f1), shard $(ls "$DIR

 # 2) riprende+completa la conversione (ripartibile: salta gli shard gia' fatti)
 echo "[2/4] conversione (riprende da dove era): output -> $DIR"
-python3 convert_fp8_to_int4.py --repo "$REPO" --outdir "$DIR" --ebits 4 --io-bits 8
+python3 tools/convert_fp8_to_int4.py --repo "$REPO" --outdir "$DIR" --ebits 4 --io-bits 8

 # 3) il motore richiede tokenizer.json + config.json nella dir del modello
 for f in config.json tokenizer.json; do
@@ -4,11 +4,11 @@
 #  - se un download resta FERMO >180s (connessione zombie), lo ammazza e lo rilancia:
 #    hf_hub riprende il .incomplete dal punto esatto, non si perde nulla
 #  - esce da solo quando tutti i 141 shard sono fatti
-# uso:  nohup ./supervisor.sh > supervisor.log 2>&1 &
+# uso da c/:  nohup scripts/supervisor.sh > supervisor.log 2>&1 &
 set -u
-DIR=/home/vincenzo/glm52_i4
-CODE=/mnt/c/Users/User/Desktop/moe-stream/c
-TOTAL=141
+DIR="${COLI_MODEL:-/home/vincenzo/glm52_i4}"
+CODE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+TOTAL="${TOTAL_SHARDS:-141}"
 STALL_S=180          # secondi senza crescita del download -> riavvio
 CONVLOG=/tmp/convert_supervised.log

@@ -19,7 +19,7 @@ log(){ echo "[$(date +%H:%M:%S)] $*"; }

 start_conv(){
    cd "$CODE"
-    nohup python3 convert_fp8_to_int4.py --repo zai-org/GLM-5.2-FP8 \
+    nohup python3 tools/convert_fp8_to_int4.py --repo zai-org/GLM-5.2-FP8 \
        --outdir "$DIR" --ebits 4 --io-bits 8 >> "$CONVLOG" 2>&1 &
    log "convertitore avviato (PID $!)"
 }
@@ -1,4 +1,4 @@
-#include "backend_cuda.h"
+#include "../backend_cuda.h"

 #include <cmath>
 #include <cstdio>
@@ -1,6 +1,6 @@
 import unittest

-from benchmark_cuda_fixture import parse_output
+from tools.benchmark_cuda_fixture import parse_output


 SAMPLE = """
@@ -1,8 +1,8 @@
 /* Validazione del tokenizer C contro l'oracolo HF.
- * build: gcc -O2 tok_test.c -o tok_test
+ * build da c/: gcc -O2 tests/test_tok.c -o tok_test
 * uso:  ./tok_test <tokenizer.json>   (legge righe "TEXT\tID,ID,.." da stdin) */
 #define _GNU_SOURCE
-#include "tok.h"
+#include "../tok.h"

 int main(int argc, char **argv){
    if(argc<2){ fprintf(stderr,"uso: %s tokenizer.json < casi\n",argv[0]); return 1; }
@@ -1,4 +1,4 @@
-/* GENERATO da gen_unicode.py — non modificare a mano. */
+/* GENERATO da tools/gen_unicode.py — non modificare a mano. */
 #ifndef TOK_UNICODE_H
 #define TOK_UNICODE_H
 #include <stdint.h>
@@ -0,0 +1,16 @@
+# Tools
+
+These scripts support model preparation and offline engineering work. They are
+not runtime dependencies of the C engine.
+
+- `convert_fp8_to_int4.py`, `download_glm52.py`: model preparation
+- `make_glm_oracle.py`, `make_glm_bench_model.py`: deterministic fixtures
+- `benchmark_cuda_fixture.py`, `eval_glm.py`, `fetch_benchmarks.py`: benchmarks
+- `gen_unicode.py`: tokenizer table generation
+
+Run them from `c/`, for example:
+
+```sh
+python3 tools/convert_fp8_to_int4.py --selftest
+python3 tools/make_glm_bench_model.py --output /tmp/colibri-bench
+```
@@ -0,0 +1 @@
+"""Offline conversion, fixture, benchmark, and evaluation utilities."""
@@ -1,4 +1,4 @@
-"""Reproducible CPU/CUDA A/B benchmark for make_glm_bench_model.py output."""
+"""Reproducible CPU/CUDA A/B benchmark for tools/make_glm_bench_model.py output."""

 import argparse
 import json
@@ -19,11 +19,11 @@ dati impacchettati, `nome.qs` F32 = scale per riga).

 USO:
  # test locale (oracolo tiny, niente download): converte una dir gia' presente
-  python3 convert_fp8_to_int4.py --indir glm_tiny --outdir glm_tiny_i4 --ebits 4 --io-bits 4
+  python3 tools/convert_fp8_to_int4.py --indir glm_tiny --outdir glm_tiny_i4 --ebits 4 --io-bits 4
  # selftest del dequant fp8 (richiede torch)
-  python3 convert_fp8_to_int4.py --selftest
+  python3 tools/convert_fp8_to_int4.py --selftest
  # reale: scarica+converte+cancella shard per shard
-  python3 convert_fp8_to_int4.py --repo zai-org/GLM-5.2-FP8 --outdir /home/vincenzo/glm52_i4
+  python3 tools/convert_fp8_to_int4.py --repo zai-org/GLM-5.2-FP8 --outdir /home/vincenzo/glm52_i4
 """
 import os, sys, glob, json, shutil, argparse
 import numpy as np
@@ -9,8 +9,8 @@ NB: i pesi sono F8_E4M3 + tensori `*.weight_scale_inv` (blocchi 128x128). Il loa
 deve supportare fp8+block-scale prima di poterli usare (vedi memoria glm52-specs).

 USO:
-    python3 download_glm52.py            # scarica tutto in /home/vincenzo/glm52  (ripartibile)
-    python3 download_glm52.py --check    # solo stima spazio e conteggio file, niente download
+    python3 tools/download_glm52.py            # scarica tutto in /home/vincenzo/glm52  (ripartibile)
+    python3 tools/download_glm52.py --check    # solo stima spazio e conteggio file, niente download

 Lo scaricamento e' di centinaia di GB e ore: lancialo tu quando il resto e' pronto.
 """
@@ -7,20 +7,20 @@ Serve a capire se la quantizzazione int4 ha lasciato il modello "tale" rispetto
 punteggi PUBBLICATI di GLM-5.2 (e, per contesto, Claude/GPT).

 Dipendenze: solo `tokenizers` + il binario ./glm. I dataset si leggono da JSONL locali
-(uno per task) prodotti da `fetch_benchmarks.py`. Formato di ogni riga JSONL:
+(uno per task) prodotti da `tools/fetch_benchmarks.py`. Formato di ogni riga JSONL:
    {"ctx": "...", "choices": ["...","..."], "gold": 0}
 Cosi' la harness e' offline e deterministica.

 USO:
  # 1) (una volta, quando hai rete) scarica i benchmark in ./bench/*.jsonl
-  python3 fetch_benchmarks.py --out ./bench --tasks hellaswag,arc_challenge,mmlu --limit 200
+  python3 tools/fetch_benchmarks.py --out ./bench --tasks hellaswag,arc_challenge,mmlu --limit 200
  # 2) plumbing test della meccanica (senza motore):
-  python3 eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench --tasks smoke --dry
+  python3 tools/eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench --tasks smoke --dry
  # 3) validazione vera quando il modello e' pronto:
-  python3 eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench \
+  python3 tools/eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench \
                      --tasks hellaswag,arc_challenge,mmlu --limit 40 --ram 15
  # leve di ricerca: passate al motore via env
-  TOPP=0.9 python3 eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench --tasks mmlu --ram 15
+  TOPP=0.9 python3 tools/eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench --tasks mmlu --ram 15
 """
 import os, sys, subprocess, argparse, random, json, tempfile, time

@@ -43,7 +43,7 @@ def load_docs(task, data_dir, limit, seed):
        return SMOKE[:limit] if limit else SMOKE
    path = os.path.join(data_dir, task + ".jsonl")
    if not os.path.exists(path):
-        sys.exit(f"manca {path} — generalo con: python3 fetch_benchmarks.py --out {data_dir} --tasks {task}")
+        sys.exit(f"manca {path} — generalo con: python3 tools/fetch_benchmarks.py --out {data_dir} --tasks {task}")
    docs = [json.loads(l) for l in open(path) if l.strip()]
    random.Random(seed).shuffle(docs)
    return docs[:limit] if limit else docs
@@ -138,7 +138,7 @@ def main():
    print(f"(motore: {time.time()-t0:.0f}s){proc.stderr.strip().splitlines()[-1] if proc.stderr.strip() else ''}", file=sys.stderr)
    score_accuracy(tasks, meta, perq, lp)
    print("\nNB: confronta acc_norm col punteggio PUBBLICATO di GLM-5.2 (model card). Se vicino,"
-          "\n    la quantizzazione int4 ha preservato il modello. (riempi REFERENCE in eval_glm.py)")
+          "\n    la quantizzazione int4 ha preservato il modello. (riempi REFERENCE in tools/eval_glm.py)")
    os.remove(req_path)

 if __name__ == "__main__":
@@ -4,9 +4,9 @@ Scarica i benchmark LLM standard e li converte nel formato JSONL della harness
 Richiede `datasets`:  pip install --break-system-packages datasets   (o in una venv)

 USO:
-  python3 fetch_benchmarks.py --out ./bench --tasks hellaswag,arc_challenge,arc_easy,mmlu,winogrande,piqa,openbookqa --limit 300
+  python3 tools/fetch_benchmarks.py --out ./bench --tasks hellaswag,arc_challenge,arc_easy,mmlu,winogrande,piqa,openbookqa --limit 300
 Poi:
-  python3 eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench --tasks mmlu --limit 40 --ram 15
+  python3 tools/eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench --tasks mmlu --limit 40 --ram 15
 """
 import os, json, argparse, random

@@ -4,7 +4,7 @@ pre-tokenizer cl100k (regex del tokenizer GLM-5.2):
  - \\p{N}  numeri    (categoria che inizia per 'N')
  - \\s     whitespace (proprieta' Unicode White_Space)
 Ogni classe diventa un array ordinato di range [lo,hi] inclusivi; il C fa ricerca
-binaria. Eseguire una volta: python3 gen_unicode.py > tok_unicode.h
+binaria. Eseguire una volta: python3 tools/gen_unicode.py > tok_unicode.h
 """
 import sys, unicodedata

@@ -41,7 +41,7 @@ def emit(name, rs):
    print("};")
    print(f"static const int {name}_n = {len(rs)};\n")

-print("/* GENERATO da gen_unicode.py — non modificare a mano. */")
+print("/* GENERATO da tools/gen_unicode.py — non modificare a mano. */")
 print("#ifndef TOK_UNICODE_H\n#define TOK_UNICODE_H\n#include <stdint.h>\n")
 emit("uni_L", L); emit("uni_N", N); emit("uni_S", S)
 print("""static int uni_in(const uint32_t t[][2], int n, uint32_t cp){
				`@@ -0,0 +1 @@`
				`"""Offline conversion, fixture, benchmark, and evaluation utilities."""`