danielkrause/colibri-strix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

colibrì: pure-C GLM-5.2 (744B MoE) engine with disk-streamed experts

Browse Source 
Engine (c/glm.c): MLA attention with compressed KV, sigmoid noaux_tc router,
int8/int4/int2 quant kernels (AVX2), per-layer LRU expert cache + pinned
hot-store, batch-union MoE, native MTP speculative decoding (lossless),
multi-stop + official chat template, RAM auto-budget from MemAvailable.
Tokenizer: byte-level BPE in C. Tooling: coli CLI, disk-safe FP8→int4
converter, tiny-random oracle validation (TF 32/32, greedy 20/20).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
JustVugg
2026-07-05 20:52:05 +02:00
parent a6deef8e44
commit 1ae22a6135
32 changed files with 4440 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
c/tok_test.c
+46
View File
@@ -0,0 +1,46 @@
/* Validazione del tokenizer C contro l'oracolo HF.
* build: gcc -O2 tok_test.c -o tok_test
* uso: ./tok_test <tokenizer.json> (legge righe "TEXT\tID,ID,.." da stdin) */
#define _GNU_SOURCE
#include "tok.h"
int main(int argc, char **argv){
if(argc<2){ fprintf(stderr,"uso: %s tokenizer.json < casi\n",argv[0]); return 1; }
Tok T;
tok_load(&T, argv[1]);
fprintf(stderr,"caricato: vocab_ids=%d specials=%d\n", T.n_ids, T.nsp);
char *line=NULL; size_t cap=0; ssize_t nr;
int pass=0, tot=0, dpass=0;
while((nr=getline(&line,&cap,stdin))>=0){
if(nr>0 && line[nr-1]=='\n'){ line[--nr]=0; }
if(nr==0) continue;
char *tab=strchr(line,'\t'); if(!tab) continue;
*tab=0; const char *text=line; const char *idstr=tab+1;
/* il testo puo' contenere \n e \t codificati come \\n \\t */
char tbuf[4096]; int tn=0;
for(const char *q=text; *q && tn<4095; q++){
if(q[0]=='\\' && q[1]=='n'){ tbuf[tn++]='\n'; q++; }
else if(q[0]=='\\' && q[1]=='t'){ tbuf[tn++]='\t'; q++; }
else if(q[0]=='\\' && q[1]=='r'){ tbuf[tn++]='\r'; q++; }
else if(q[0]=='\\' && q[1]=='\\'){ tbuf[tn++]='\\'; q++; }
else tbuf[tn++]=*q;
}
tbuf[tn]=0;
int exp[4096], ne=0;
for(const char *q=idstr; *q; ){ while(*q==','||*q==' ')q++; if(!*q)break; exp[ne++]=atoi(q); while(*q&&*q!=',')q++; }
int got[4096]; int ng=tok_encode(&T,tbuf,tn,got,4096);
int ok = (ng==ne); for(int i=0;i<ng&&ok;i++) ok = (got[i]==exp[i]);
tot++; if(ok) pass++;
/* round-trip decode */
char dec[8192]; int dn=tok_decode(&T,got,ng,dec,8191);
int drt = (dn==tn) && !memcmp(dec,tbuf,tn);
if(drt) dpass++;
if(!ok || !drt){
fprintf(stderr,"MISMATCH text=%s\n exp(%d):",text,ne); for(int i=0;i<ne;i++)fprintf(stderr," %d",exp[i]);
fprintf(stderr,"\n got(%d):",ng); for(int i=0;i<ng;i++)fprintf(stderr," %d",got[i]);
fprintf(stderr,"\n decode_ok=%d\n", drt);
}
}
printf("ENCODE: %d/%d DECODE(round-trip): %d/%d\n", pass,tot, dpass,tot);
return pass==tot ? 0 : 2;
}