From 13e8f09ffc09b889ef11a5b695d3f66e1c675ef3 Mon Sep 17 00:00:00 2001 From: ZacharyZcR Date: Fri, 10 Jul 2026 16:08:39 +0800 Subject: [PATCH] Organize project tools and local workflows: c/tools, c/scripts, c/tests, root Makefile (flat C core untouched) (#22) --- CONTRIBUTING.md | 5 ++- Makefile | 4 +++ README.md | 32 ++++++++++++------- c/Makefile | 6 ++-- c/coli | 7 ++-- c/glm.c | 2 +- c/{ => scripts}/run.sh | 12 +++---- c/{ => scripts}/supervisor.sh | 10 +++--- c/setup.sh | 0 .../test_backend_cuda.cu} | 2 +- c/tests/test_benchmark_cuda_fixture.py | 2 +- c/{tok_test.c => tests/test_tok.c} | 4 +-- c/tok_unicode.h | 2 +- c/tools/README.md | 16 ++++++++++ c/tools/__init__.py | 1 + c/{ => tools}/benchmark_cuda_fixture.py | 2 +- c/{ => tools}/convert_fp8_to_int4.py | 6 ++-- c/{ => tools}/download_glm52.py | 4 +-- c/{ => tools}/eval_glm.py | 14 ++++---- c/{ => tools}/fetch_benchmarks.py | 4 +-- c/{ => tools}/gen_unicode.py | 4 +-- c/{ => tools}/make_glm_bench_model.py | 0 c/{ => tools}/make_glm_oracle.py | 0 23 files changed, 87 insertions(+), 52 deletions(-) create mode 100644 Makefile mode change 100644 => 100755 c/coli rename c/{ => scripts}/run.sh (80%) mode change 100644 => 100755 rename c/{ => scripts}/supervisor.sh (87%) mode change 100644 => 100755 mode change 100644 => 100755 c/setup.sh rename c/{backend_cuda_test.cu => tests/test_backend_cuda.cu} (99%) rename c/{tok_test.c => tests/test_tok.c} (96%) create mode 100644 c/tools/README.md create mode 100644 c/tools/__init__.py rename c/{ => tools}/benchmark_cuda_fixture.py (97%) rename c/{ => tools}/convert_fp8_to_int4.py (99%) rename c/{ => tools}/download_glm52.py (90%) rename c/{ => tools}/eval_glm.py (91%) rename c/{ => tools}/fetch_benchmarks.py (92%) rename c/{ => tools}/gen_unicode.py (93%) rename c/{ => tools}/make_glm_bench_model.py (100%) rename c/{ => tools}/make_glm_oracle.py (100%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e215c76..a85b860 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -7,9 +7,12 @@ Keep changes focused and preserve Colibri's dependency-free default CPU path. Run the lightweight checks locally: ```sh -make -C c check +make check ``` +`make -C c check` remains available for scripts that already run from the +engine directory. + This performs one portable CPU build, C unit tests, and Python standard-library tests. It does not download a model or require CUDA. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4da2ce1 --- /dev/null +++ b/Makefile @@ -0,0 +1,4 @@ +.PHONY: all glm portable test check cuda-test clean + +all glm portable test check cuda-test clean: + $(MAKE) -C c $@ diff --git a/README.md b/README.md index 5114ce9..c0e1b75 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ The engine is a single C file (`c/glm.c`, ~1,300 lines) plus small headers. No B - **Batch-union MoE**: in prefill (and MTP verification), each unique expert of the batch is read once and applied to every position that routes to it. - **Byte-level BPE tokenizer in C** (GPT-2-style with Unicode-property regex, 320k merges). - **RAM safety**: the expert cache is auto-sized from `MemAvailable` at startup — an honest peak projection (working set, KV, MTP row, reconstruction buffers) so the kernel OOM-killer never fires. -- **Offline FP8→int4 converter** (`c/convert_fp8_to_int4.py`): downloads one shard at a time (~5 GB), dequants (128×128 block scales), requantizes to the engine's container, deletes the shard — the 756 GB FP8 checkpoint never needs to exist on disk at once. Resumable. +- **Offline FP8→int4 converter** (`c/tools/convert_fp8_to_int4.py`): downloads one shard at a time (~5 GB), dequants (128×128 block scales), requantizes to the engine's container, deletes the shard — the 756 GB FP8 checkpoint never needs to exist on disk at once. Resumable. ## Honest numbers (WSL2, 12 cores, 25 GB RAM, NVMe via VHDX) @@ -147,8 +147,8 @@ deterministic 313M-parameter `glm_moe_dsa` fixture and run fixed-token replay: ```bash cd c -python make_glm_bench_model.py --output /nvme/colibri-bench-medium --device cuda -python benchmark_cuda_fixture.py --model /nvme/colibri-bench-medium --gpu 0 +python tools/make_glm_bench_model.py --output /nvme/colibri-bench-medium --device cuda +python tools/benchmark_cuda_fixture.py --model /nvme/colibri-bench-medium --gpu 0 ``` The fixture has random weights and is not a language model. It exists only to @@ -242,16 +242,26 @@ Every contribution, from a datapoint to a disk, moves the ceiling. ## Repo layout ``` -c/glm.c the engine (GLM-5.2 forward, streaming MoE, MTP, serve mode) -c/st.h safetensors reader: pread + fadvise, no mmap (RSS stays flat) -c/tok.h byte-level BPE tokenizer in C -c/coli CLI: chat / run / bench / convert / info -c/iobench.c parallel disk microbenchmark (measures what the engine feels) -c/convert_fp8_to_int4.py disk-safe FP8 → int4 converter -c/make_glm_oracle.py tiny-random oracle generator for validation -c/olmoe.c stage-A engine (OLMoE), first validation target +Makefile root build/check entry point +c/ +├── glm.c single-file GLM engine +├── st.h, tok.h, json.h runtime headers +├── backend_cuda.* optional CUDA tier +├── Makefile build and local checks +├── coli user-facing CLI +├── setup.sh one-command local setup +├── tools/ offline conversion, fixtures and benchmarks +├── scripts/ long-running conversion helpers +└── tests/ dependency-free C and Python tests ``` +The runtime path intentionally stays flat and readable: `glm.c` plus its small +headers. Auxiliary Python and shell tooling is grouped separately and is never a +runtime dependency of the engine. + +From the repository root, `make`, `make check`, and `make clean` delegate to the +engine Makefile. Existing commands run from `c/` continue to work unchanged. + ## Why "colibrì" The hummingbird weighs a few grams, hovers in place, and visits a thousand flowers a day. This engine keeps a 744-billion-parameter giant alive on hummingbird rations: 25 GB of RAM, twelve CPU cores, and a lot of disk patience. diff --git a/c/Makefile b/c/Makefile index 04101df..5921c5c 100644 --- a/c/Makefile +++ b/c/Makefile @@ -56,9 +56,9 @@ backend_cuda.o: backend_cuda.cu backend_cuda.h @command -v $(NVCC) >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } $(NVCC) $(NVCCFLAGS) -c backend_cuda.cu -o $@ -cuda-test: backend_cuda.cu backend_cuda.h backend_cuda_test.cu +cuda-test: backend_cuda.cu backend_cuda.h tests/test_backend_cuda.cu @command -v $(NVCC) >/dev/null 2>&1 || { echo "nvcc not found: set CUDA_HOME or NVCC" >&2; exit 1; } - $(NVCC) $(NVCCFLAGS) backend_cuda.cu backend_cuda_test.cu -o backend_cuda_test + $(NVCC) $(NVCCFLAGS) backend_cuda.cu tests/test_backend_cuda.cu -o backend_cuda_test ./backend_cuda_test olmoe: olmoe.c st.h json.h @@ -82,7 +82,7 @@ test-python: test: test-c test-python -# CI-friendly validation: one portable CPU build and dependency-free tests. +# Local validation: one portable CPU build and dependency-free tests. check: $(MAKE) clean $(MAKE) portable diff --git a/c/coli b/c/coli old mode 100644 new mode 100755 index f1022ec..5e304b7 --- a/c/coli +++ b/c/coli @@ -19,6 +19,7 @@ Config via env o flag (validi anche dopo il sottocomando): import os, sys, subprocess, argparse, json, time, signal, shutil, threading, re, codecs, tempfile, textwrap HERE = os.path.dirname(os.path.abspath(__file__)) +TOOLS = os.path.join(HERE, "tools") GLM = os.path.join(HERE, "glm") DEF_MODEL = os.environ.get("COLI_MODEL", "/home/vincenzo/glm52_i4") END = b"\x01\x01END\x01\x01\n" @@ -385,9 +386,9 @@ def cmd_bench(a): missing=[t for t in tasks.split(",") if not os.path.exists(os.path.join(a.data,f"{t}.jsonl"))] if missing: print(f" {C.dim}scarico i dataset mancanti: {', '.join(missing)}{C.r}") - subprocess.call([py, os.path.join(HERE,"fetch_benchmarks.py"), + subprocess.call([py, os.path.join(TOOLS,"fetch_benchmarks.py"), "--out", a.data, "--tasks", ",".join(missing), "--limit", str(max(a.limit,200))]) - cmd=[py, os.path.join(HERE,"eval_glm.py"), "--snap",a.model, + cmd=[py, os.path.join(TOOLS,"eval_glm.py"), "--snap",a.model, "--tasks", tasks, "--limit", str(a.limit), "--data", a.data] if a.ram: cmd+=["--ram",str(a.ram)] e=dict(os.environ) @@ -401,7 +402,7 @@ def cmd_convert(a): # python con torch/safetensors: l'ambiente del progetto se c'e', altrimenti quello corrente venv_py=os.path.join(HERE,"mio_env","bin","python3") py = venv_py if os.path.exists(venv_py) else sys.executable - base=[py, os.path.join(HERE,"convert_fp8_to_int4.py"), + base=[py, os.path.join(TOOLS,"convert_fp8_to_int4.py"), "--repo", a.repo, "--outdir", a.model, "--ebits", str(a.ebits), "--io-bits", str(a.io_bits)] if a.xbits: base+=["--xbits",str(a.xbits)] # passo 1: modello principale (78 layer). Resumabile: riparte dagli shard mancanti. diff --git a/c/glm.c b/c/glm.c index 6d0604b..292ae23 100644 --- a/c/glm.c +++ b/c/glm.c @@ -12,7 +12,7 @@ * E' cio' che fa entrare GLM-5.2 nei 15 GB: ~17B param residenti a int4 ~= 8.7 GB. * Norme/router/bias restano f32 (piccoli e sensibili). * - * Validazione: stessi token id di ref_glm.json (oracolo transformers, c/make_glm_oracle.py). + * Validazione: stessi token id di ref_glm.json (oracolo transformers, c/tools/make_glm_oracle.py). * build: make glm run: SNAP=./glm_tiny ./glm * TF=1 -> teacher-forcing (valida il prefill su tutta la sequenza) */ diff --git a/c/run.sh b/c/scripts/run.sh old mode 100644 new mode 100755 similarity index 80% rename from c/run.sh rename to c/scripts/run.sh index 1cbcd35..6c797ca --- a/c/run.sh +++ b/c/scripts/run.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash # Pipeline GLM-5.2 (int4, streaming, 15 GB RAM) — tutto in WSL, modello su ext4. -# uso: ./run.sh ["prompt"] [n_token] +# uso da c/: scripts/run.sh ["prompt"] [n_token] # Fa: (1) attende lo spostamento su ext4, (2) riprende la conversione fino a completarla, # (3) compila il motore, (4) genera testo restando nel budget RAM. set -euo pipefail -DIR=/home/vincenzo/glm52_i4 # modello int4 su ext4 (NON /mnt/c!) -REPO=zai-org/GLM-5.2-FP8 -CODE=/mnt/c/Users/User/Desktop/moe-stream/c -RAM_GB=15 +DIR="${COLI_MODEL:-/home/vincenzo/glm52_i4}" # modello int4 su ext4 (NON /mnt/c!) +REPO="${COLI_REPO:-zai-org/GLM-5.2-FP8}" +CODE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +RAM_GB="${RAM_GB:-15}" PROMPT="${1:-Ciao, chi sei?}" NGEN="${2:-64}" @@ -25,7 +25,7 @@ echo "[1/4] spostamento completato: $(du -sh "$DIR" | cut -f1), shard $(ls "$DIR # 2) riprende+completa la conversione (ripartibile: salta gli shard gia' fatti) echo "[2/4] conversione (riprende da dove era): output -> $DIR" -python3 convert_fp8_to_int4.py --repo "$REPO" --outdir "$DIR" --ebits 4 --io-bits 8 +python3 tools/convert_fp8_to_int4.py --repo "$REPO" --outdir "$DIR" --ebits 4 --io-bits 8 # 3) il motore richiede tokenizer.json + config.json nella dir del modello for f in config.json tokenizer.json; do diff --git a/c/supervisor.sh b/c/scripts/supervisor.sh old mode 100644 new mode 100755 similarity index 87% rename from c/supervisor.sh rename to c/scripts/supervisor.sh index 144aa32..bba4184 --- a/c/supervisor.sh +++ b/c/scripts/supervisor.sh @@ -4,11 +4,11 @@ # - se un download resta FERMO >180s (connessione zombie), lo ammazza e lo rilancia: # hf_hub riprende il .incomplete dal punto esatto, non si perde nulla # - esce da solo quando tutti i 141 shard sono fatti -# uso: nohup ./supervisor.sh > supervisor.log 2>&1 & +# uso da c/: nohup scripts/supervisor.sh > supervisor.log 2>&1 & set -u -DIR=/home/vincenzo/glm52_i4 -CODE=/mnt/c/Users/User/Desktop/moe-stream/c -TOTAL=141 +DIR="${COLI_MODEL:-/home/vincenzo/glm52_i4}" +CODE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TOTAL="${TOTAL_SHARDS:-141}" STALL_S=180 # secondi senza crescita del download -> riavvio CONVLOG=/tmp/convert_supervised.log @@ -19,7 +19,7 @@ log(){ echo "[$(date +%H:%M:%S)] $*"; } start_conv(){ cd "$CODE" - nohup python3 convert_fp8_to_int4.py --repo zai-org/GLM-5.2-FP8 \ + nohup python3 tools/convert_fp8_to_int4.py --repo zai-org/GLM-5.2-FP8 \ --outdir "$DIR" --ebits 4 --io-bits 8 >> "$CONVLOG" 2>&1 & log "convertitore avviato (PID $!)" } diff --git a/c/setup.sh b/c/setup.sh old mode 100644 new mode 100755 diff --git a/c/backend_cuda_test.cu b/c/tests/test_backend_cuda.cu similarity index 99% rename from c/backend_cuda_test.cu rename to c/tests/test_backend_cuda.cu index cc28ef6..f72b4fd 100644 --- a/c/backend_cuda_test.cu +++ b/c/tests/test_backend_cuda.cu @@ -1,4 +1,4 @@ -#include "backend_cuda.h" +#include "../backend_cuda.h" #include #include diff --git a/c/tests/test_benchmark_cuda_fixture.py b/c/tests/test_benchmark_cuda_fixture.py index 4ff34dd..542a7da 100644 --- a/c/tests/test_benchmark_cuda_fixture.py +++ b/c/tests/test_benchmark_cuda_fixture.py @@ -1,6 +1,6 @@ import unittest -from benchmark_cuda_fixture import parse_output +from tools.benchmark_cuda_fixture import parse_output SAMPLE = """ diff --git a/c/tok_test.c b/c/tests/test_tok.c similarity index 96% rename from c/tok_test.c rename to c/tests/test_tok.c index 986e50f..10562f8 100644 --- a/c/tok_test.c +++ b/c/tests/test_tok.c @@ -1,8 +1,8 @@ /* Validazione del tokenizer C contro l'oracolo HF. - * build: gcc -O2 tok_test.c -o tok_test + * build da c/: gcc -O2 tests/test_tok.c -o tok_test * uso: ./tok_test (legge righe "TEXT\tID,ID,.." da stdin) */ #define _GNU_SOURCE -#include "tok.h" +#include "../tok.h" int main(int argc, char **argv){ if(argc<2){ fprintf(stderr,"uso: %s tokenizer.json < casi\n",argv[0]); return 1; } diff --git a/c/tok_unicode.h b/c/tok_unicode.h index c0839c2..62959de 100644 --- a/c/tok_unicode.h +++ b/c/tok_unicode.h @@ -1,4 +1,4 @@ -/* GENERATO da gen_unicode.py — non modificare a mano. */ +/* GENERATO da tools/gen_unicode.py — non modificare a mano. */ #ifndef TOK_UNICODE_H #define TOK_UNICODE_H #include diff --git a/c/tools/README.md b/c/tools/README.md new file mode 100644 index 0000000..2093788 --- /dev/null +++ b/c/tools/README.md @@ -0,0 +1,16 @@ +# Tools + +These scripts support model preparation and offline engineering work. They are +not runtime dependencies of the C engine. + +- `convert_fp8_to_int4.py`, `download_glm52.py`: model preparation +- `make_glm_oracle.py`, `make_glm_bench_model.py`: deterministic fixtures +- `benchmark_cuda_fixture.py`, `eval_glm.py`, `fetch_benchmarks.py`: benchmarks +- `gen_unicode.py`: tokenizer table generation + +Run them from `c/`, for example: + +```sh +python3 tools/convert_fp8_to_int4.py --selftest +python3 tools/make_glm_bench_model.py --output /tmp/colibri-bench +``` diff --git a/c/tools/__init__.py b/c/tools/__init__.py new file mode 100644 index 0000000..aaa0579 --- /dev/null +++ b/c/tools/__init__.py @@ -0,0 +1 @@ +"""Offline conversion, fixture, benchmark, and evaluation utilities.""" diff --git a/c/benchmark_cuda_fixture.py b/c/tools/benchmark_cuda_fixture.py similarity index 97% rename from c/benchmark_cuda_fixture.py rename to c/tools/benchmark_cuda_fixture.py index 73a30db..7352ed4 100644 --- a/c/benchmark_cuda_fixture.py +++ b/c/tools/benchmark_cuda_fixture.py @@ -1,4 +1,4 @@ -"""Reproducible CPU/CUDA A/B benchmark for make_glm_bench_model.py output.""" +"""Reproducible CPU/CUDA A/B benchmark for tools/make_glm_bench_model.py output.""" import argparse import json diff --git a/c/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.py similarity index 99% rename from c/convert_fp8_to_int4.py rename to c/tools/convert_fp8_to_int4.py index 5785ee9..4ba1812 100644 --- a/c/convert_fp8_to_int4.py +++ b/c/tools/convert_fp8_to_int4.py @@ -19,11 +19,11 @@ dati impacchettati, `nome.qs` F32 = scale per riga). USO: # test locale (oracolo tiny, niente download): converte una dir gia' presente - python3 convert_fp8_to_int4.py --indir glm_tiny --outdir glm_tiny_i4 --ebits 4 --io-bits 4 + python3 tools/convert_fp8_to_int4.py --indir glm_tiny --outdir glm_tiny_i4 --ebits 4 --io-bits 4 # selftest del dequant fp8 (richiede torch) - python3 convert_fp8_to_int4.py --selftest + python3 tools/convert_fp8_to_int4.py --selftest # reale: scarica+converte+cancella shard per shard - python3 convert_fp8_to_int4.py --repo zai-org/GLM-5.2-FP8 --outdir /home/vincenzo/glm52_i4 + python3 tools/convert_fp8_to_int4.py --repo zai-org/GLM-5.2-FP8 --outdir /home/vincenzo/glm52_i4 """ import os, sys, glob, json, shutil, argparse import numpy as np diff --git a/c/download_glm52.py b/c/tools/download_glm52.py similarity index 90% rename from c/download_glm52.py rename to c/tools/download_glm52.py index 5688967..544a3f6 100644 --- a/c/download_glm52.py +++ b/c/tools/download_glm52.py @@ -9,8 +9,8 @@ NB: i pesi sono F8_E4M3 + tensori `*.weight_scale_inv` (blocchi 128x128). Il loa deve supportare fp8+block-scale prima di poterli usare (vedi memoria glm52-specs). USO: - python3 download_glm52.py # scarica tutto in /home/vincenzo/glm52 (ripartibile) - python3 download_glm52.py --check # solo stima spazio e conteggio file, niente download + python3 tools/download_glm52.py # scarica tutto in /home/vincenzo/glm52 (ripartibile) + python3 tools/download_glm52.py --check # solo stima spazio e conteggio file, niente download Lo scaricamento e' di centinaia di GB e ore: lancialo tu quando il resto e' pronto. """ diff --git a/c/eval_glm.py b/c/tools/eval_glm.py similarity index 91% rename from c/eval_glm.py rename to c/tools/eval_glm.py index 7de1c00..83778b1 100644 --- a/c/eval_glm.py +++ b/c/tools/eval_glm.py @@ -7,20 +7,20 @@ Serve a capire se la quantizzazione int4 ha lasciato il modello "tale" rispetto punteggi PUBBLICATI di GLM-5.2 (e, per contesto, Claude/GPT). Dipendenze: solo `tokenizers` + il binario ./glm. I dataset si leggono da JSONL locali -(uno per task) prodotti da `fetch_benchmarks.py`. Formato di ogni riga JSONL: +(uno per task) prodotti da `tools/fetch_benchmarks.py`. Formato di ogni riga JSONL: {"ctx": "...", "choices": ["...","..."], "gold": 0} Cosi' la harness e' offline e deterministica. USO: # 1) (una volta, quando hai rete) scarica i benchmark in ./bench/*.jsonl - python3 fetch_benchmarks.py --out ./bench --tasks hellaswag,arc_challenge,mmlu --limit 200 + python3 tools/fetch_benchmarks.py --out ./bench --tasks hellaswag,arc_challenge,mmlu --limit 200 # 2) plumbing test della meccanica (senza motore): - python3 eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench --tasks smoke --dry + python3 tools/eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench --tasks smoke --dry # 3) validazione vera quando il modello e' pronto: - python3 eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench \ + python3 tools/eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench \ --tasks hellaswag,arc_challenge,mmlu --limit 40 --ram 15 # leve di ricerca: passate al motore via env - TOPP=0.9 python3 eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench --tasks mmlu --ram 15 + TOPP=0.9 python3 tools/eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench --tasks mmlu --ram 15 """ import os, sys, subprocess, argparse, random, json, tempfile, time @@ -43,7 +43,7 @@ def load_docs(task, data_dir, limit, seed): return SMOKE[:limit] if limit else SMOKE path = os.path.join(data_dir, task + ".jsonl") if not os.path.exists(path): - sys.exit(f"manca {path} — generalo con: python3 fetch_benchmarks.py --out {data_dir} --tasks {task}") + sys.exit(f"manca {path} — generalo con: python3 tools/fetch_benchmarks.py --out {data_dir} --tasks {task}") docs = [json.loads(l) for l in open(path) if l.strip()] random.Random(seed).shuffle(docs) return docs[:limit] if limit else docs @@ -138,7 +138,7 @@ def main(): print(f"(motore: {time.time()-t0:.0f}s){proc.stderr.strip().splitlines()[-1] if proc.stderr.strip() else ''}", file=sys.stderr) score_accuracy(tasks, meta, perq, lp) print("\nNB: confronta acc_norm col punteggio PUBBLICATO di GLM-5.2 (model card). Se vicino," - "\n la quantizzazione int4 ha preservato il modello. (riempi REFERENCE in eval_glm.py)") + "\n la quantizzazione int4 ha preservato il modello. (riempi REFERENCE in tools/eval_glm.py)") os.remove(req_path) if __name__ == "__main__": diff --git a/c/fetch_benchmarks.py b/c/tools/fetch_benchmarks.py similarity index 92% rename from c/fetch_benchmarks.py rename to c/tools/fetch_benchmarks.py index b0200c9..d0069dc 100644 --- a/c/fetch_benchmarks.py +++ b/c/tools/fetch_benchmarks.py @@ -4,9 +4,9 @@ Scarica i benchmark LLM standard e li converte nel formato JSONL della harness Richiede `datasets`: pip install --break-system-packages datasets (o in una venv) USO: - python3 fetch_benchmarks.py --out ./bench --tasks hellaswag,arc_challenge,arc_easy,mmlu,winogrande,piqa,openbookqa --limit 300 + python3 tools/fetch_benchmarks.py --out ./bench --tasks hellaswag,arc_challenge,arc_easy,mmlu,winogrande,piqa,openbookqa --limit 300 Poi: - python3 eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench --tasks mmlu --limit 40 --ram 15 + python3 tools/eval_glm.py --snap /home/vincenzo/glm52_i4 --data ./bench --tasks mmlu --limit 40 --ram 15 """ import os, json, argparse, random diff --git a/c/gen_unicode.py b/c/tools/gen_unicode.py similarity index 93% rename from c/gen_unicode.py rename to c/tools/gen_unicode.py index 1414488..780baa2 100644 --- a/c/gen_unicode.py +++ b/c/tools/gen_unicode.py @@ -4,7 +4,7 @@ pre-tokenizer cl100k (regex del tokenizer GLM-5.2): - \\p{N} numeri (categoria che inizia per 'N') - \\s whitespace (proprieta' Unicode White_Space) Ogni classe diventa un array ordinato di range [lo,hi] inclusivi; il C fa ricerca -binaria. Eseguire una volta: python3 gen_unicode.py > tok_unicode.h +binaria. Eseguire una volta: python3 tools/gen_unicode.py > tok_unicode.h """ import sys, unicodedata @@ -41,7 +41,7 @@ def emit(name, rs): print("};") print(f"static const int {name}_n = {len(rs)};\n") -print("/* GENERATO da gen_unicode.py — non modificare a mano. */") +print("/* GENERATO da tools/gen_unicode.py — non modificare a mano. */") print("#ifndef TOK_UNICODE_H\n#define TOK_UNICODE_H\n#include \n") emit("uni_L", L); emit("uni_N", N); emit("uni_S", S) print("""static int uni_in(const uint32_t t[][2], int n, uint32_t cp){ diff --git a/c/make_glm_bench_model.py b/c/tools/make_glm_bench_model.py similarity index 100% rename from c/make_glm_bench_model.py rename to c/tools/make_glm_bench_model.py diff --git a/c/make_glm_oracle.py b/c/tools/make_glm_oracle.py similarity index 100% rename from c/make_glm_oracle.py rename to c/tools/make_glm_oracle.py