From 077894210e1a5f6d20e8f30a4e1285cdcd54cadd Mon Sep 17 00:00:00 2001 From: JustVugg Date: Fri, 10 Jul 2026 07:38:47 +0200 Subject: [PATCH] LOOKA: routing predictability counters; PILOT: router-lookahead disk prefetch (async I/O thread); cap auto-raise to RAM budget (#12) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured on GLM-5.2 (48 tok, greedy): next-layer routing is 71.6% predictable one full layer ahead (79.4% skipping attention only; 41.3% previous-token). PILOT=1 issues next-layer expert WILLNEED from a dedicated I/O thread while the current layer computes — inline fadvise BLOCKS ~0.5ms/call on a saturated disk queue (+92s/48 tok, measured), hence the lock-free ring + worker. Neutral-in-noise on this dev box (disk already ~80% duty); expected to pay on balanced machines (#12: 43% disk / 46% matmul) — opt-in, default off. cap_for_ram now RAISES the LRU cap up to the RAM budget (ceiling n_experts, CAP_RAISE=0 opt-out): big-RAM machines were silently running with cap=8 (#12: 128GB box using 22GB of a 110GB budget; #13: 92GB box, same). DRAFT=3 on cold cache measured locally: 1399s vs 880s baseline for the same 48 tokens (acceptance 16%, experts/token 1809 vs 800) — confirms #8; DRAFT re-evaluation belongs to warm-cache serve sessions. Co-Authored-By: Claude Fable 5 --- README.md | 8 +++- c/glm.c | 140 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 144 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 53fa86d..ea52356 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ The engine at runtime is pure C — python is only used by the one-time converte ### Experimental resident CUDA backend -This fork includes an opt-in CUDA backend for model-resident tensors. Streaming +colibrì includes an opt-in CUDA backend for model-resident tensors. Streaming experts deliberately remain on the original CPU path for now: copying an expert from NVMe to the GPU on every use would only replace the disk bottleneck with a PCIe bottleneck. Resident quantized tensors are uploaded lazily once and reused. @@ -153,7 +153,11 @@ The fixture has random weights and is not a language model. It exists only to preserve the real MLA/MoE/streaming shapes and compare CPU streaming, dense-only CUDA, CPU hot-store, and CUDA hot-expert execution with identical replay tokens. -Useful knobs (env or flags): `--temp T` token sampling temperature (default 0.7 + nucleus 0.90 — tuned for int4; 0 = greedy), `--topp 0.7` adaptive expert top-p (30–40% less disk), `--ngen N` max tokens per answer (`:piu` in chat continues a truncated one), `AUTOPIN=0` disable the learning cache's auto-pin, `THINK=1` enable GLM-5.2's reasoning block, `DRAFT=n` MTP draft depth, `TF=1` teacher-forcing validation. +Useful knobs (env or flags): `--temp T` token sampling temperature (default 0.7 + nucleus 0.90 — tuned for int4; 0 = greedy), `--topp 0.7` adaptive expert top-p (30–40% less disk), `--ngen N` max tokens per answer (`:piu` in chat continues a truncated one), `AUTOPIN=0` disable the learning cache's auto-pin, `THINK=1` enable GLM-5.2's reasoning block, `DRAFT=n` MTP draft depth, `TF=1` teacher-forcing validation, `PILOT=1` router-lookahead disk prefetch (experimental — see below), `CAP_RAISE=0` don't auto-grow the expert cache. + +**The expert cache auto-sizes to your RAM** (since 2026-07-10): the engine now *raises* the LRU cap to fill your `--ram` budget instead of only lowering it. Before this fix a 128 GB machine ran with the same 8-experts/layer cache as a 16 GB one (issue #12) — **if you benchmarked colibrì before this date, rerun: your numbers were capped.** + +**Router-lookahead prefetch** (`PILOT=1`, experimental): GLM-5.2's expert routing is measurably predictable *ahead of time* — applying layer L+1's router to layer L's post-attention state recalls **71.6%** of the true top-8 (vs 41.3% for "same experts as last token"). `PILOT=1` uses this to issue next-layer expert readahead from a dedicated I/O thread while the current layer computes. On our dev box the disk is already ~80% saturated, so it measures neutral; on machines where compute and disk are balanced (like the Ryzen AI 9 in issue #12: 43% disk / 46% matmul) it should overlap real work — measurements welcome. **The learning cache**: the engine records which experts your usage actually routes to (`.coli_usage` next to the model, updated every turn) and at startup automatically pins the hottest ones in spare RAM. colibrì literally gets faster the more you use it. diff --git a/c/glm.c b/c/glm.c index c4ef863..3eefe36 100644 --- a/c/glm.c +++ b/c/glm.c @@ -23,6 +23,8 @@ #include #include #include +#include /* thread I/O del PILOTA */ +#include #include #if defined(__APPLE__) || defined(__linux__) #include /* mlock: inchioda le pagine in RAM / wire pages into RAM */ @@ -537,6 +539,18 @@ static int g_draft=0; /* metodo E: DRAFT=n token auto-speculati per forward v * misurato sul run reale (2026-07-03) acceptance ~5% -> ogni draft * rifiutato paga comunque i suoi expert dal disco = ~3x piu' lento. * Opt-in (DRAFT=4) per testi ripetitivi dove l'acceptance e' alta. */ +static int g_looka=0; /* LOOKA=1: misura (solo contatori, zero effetti) quanto il routing MoE + * e' predicibile IN ANTICIPO — la domanda che decide se un prefetch + * pilotato dal router puo' riempire i tempi morti del disco. + * [0] token precedente, stesso layer (cio' che usa gia' SPEC/PREFETCH) + * [1] ingresso del layer -> routing dello STESSO layer (salta l'attention) + * [2] post-attention del layer L -> routing di L+1 (un residuo MoE e + * un'attention di anticipo: il punto dove il prefetch avrebbe + * un intero giro di disco per lavorare in ombra). */ +static int64_t la_hit[3], la_tot[3]; +static int la_pred[2][130][16]; static signed char la_val[2][130]; +static int g_pilot=0; /* PILOT=1: prefetch pilotato dal router (vedi pilot_prefetch) */ +static int g_pilot_k=8; /* PILOT_K=k: prefetcha solo le prime k predizioni per posizione */ /* sceglie il formato da `bits`: >=16 f32, 5..8 int8, <=4 int4-packed */ static void qt_alloc(QT *t, int O, int I, int bits){ t->O=O; t->I=I; t->qf=NULL; t->q8=NULL; t->q4=NULL; t->s=NULL; @@ -1130,6 +1144,19 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out){ for(int kk=0;kkrouted_scale; for(int d=0;dn_layers){ + int Ke=keff[0]; + if(m->enr[layer]>0){ /* [0] vs routing del token precedente */ + for(int kk=0;kkenr[layer];z++) + if(m->eroute[layer][z]==idxs[kk]){ la_hit[0]++; break; } + la_tot[0]+=Ke; + } + for(int kind=0;kind<2;kind++) if(la_val[kind][layer]){ /* [1]/[2] vs predizioni */ + for(int kk=0;kkenr[layer]=keff[S-1]; for(int kk=0;kkeroute[layer][kk]=idxs[(int64_t)(S-1)*K+kk]; /* ---- FASE B: union degli expert del batch ---- */ int *uniq=malloc((size_t)E*sizeof(int)); int nu=0; @@ -1212,14 +1239,89 @@ static void dense_mlp(Layer *l, float *x, int S, int D, int I, float *out){ free(g); free(u); } +/* LOOKA: predice il top-K del router del layer `target` dallo stato h (residual stream), + * usando la STESSA pipeline del routing vero (post_ln -> router -> sigmoid+bias, top-K). + * kind 0 = stesso layer saltando l'attention, kind 1 = layer successivo. */ +static void la_predict(Model *m, int target, const float *h, int kind){ + Cfg *c=&m->c; Layer *l=&m->L[target]; int D=c->hidden, E=c->n_experts, K=c->topk; + float *nrm=falloc(D), *ch=falloc(E); + rmsnorm(nrm,h,l->post_ln,D,c->eps); + matmul(ch,nrm,l->router,1,D,E); + for(int e=0;erouter_bias[e]; + int *pred=la_pred[kind][target]; + for(int kk=0;kkbv){bv=ch[e];best=e;} } + pred[kk]=best; } + la_val[kind][target]=1; + free(nrm); free(ch); +} + +/* PILOTA: prefetch guidato dal router. Predice il top-K del layer L+1 dallo stato + * post-attention di L (recall misurato 71.6% su GLM-5.2, vs 41.3% del token precedente) + * e lancia il WILLNEED degli expert mancanti MENTRE il MoE di L legge i suoi: il disco + * lavora nei tempi morti del calcolo invece di aspettare il routing vero. Con MTP attiva + * predice per TUTTE le posizioni del draft: la speculazione pilota anche l'I/O. + * PILOT_K limita alle prime k predizioni (la testa del ranking e' piu' affidabile + * della coda: meno banda sprecata sulle predizioni sbagliate). + * + * I WILLNEED partono da un THREAD I/O dedicato: con la coda disco satura la submit + * del fadvise BLOCCA (~0.5ms x 169k chiamate = +92s/48 token, misurato) — inline + * il pilota costava piu' di quanto rendesse. Ring lock-free 1P/1C; pieno = scarta + * (un hint perso non e' un errore). */ +static struct { int l,e; } pilot_q[4096]; +static volatile unsigned pilot_w=0, pilot_r=0; +static Model *pilot_m=NULL; +static void *pilot_worker(void *arg){ + (void)arg; + for(;;){ + unsigned r=__atomic_load_n(&pilot_r,__ATOMIC_ACQUIRE); + unsigned w=__atomic_load_n(&pilot_w,__ATOMIC_ACQUIRE); + if(r==w){ usleep(200); continue; } + expert_prefetch(pilot_m, pilot_q[r&4095].l, pilot_q[r&4095].e); + __atomic_store_n(&pilot_r,r+1,__ATOMIC_RELEASE); + } + return NULL; +} +static void pilot_prefetch(Model *m, int lnext, const float *x, int S){ + Cfg *c=&m->c; Layer *l=&m->L[lnext]; int D=c->hidden, E=c->n_experts; + int K = g_pilot_ktopk ? g_pilot_k : c->topk; + if(!pilot_m){ pilot_m=m; pthread_t t; pthread_create(&t,NULL,pilot_worker,NULL); } + float *nrm=falloc(D), *ch=falloc(E); + for(int s=0;spost_ln, D, c->eps); + matmul(ch, nrm, l->router, 1, D, E); + for(int e=0;erouter_bias[e]; + for(int kk=0;kkch[best]) best=e; + ch[best]=-2e30f; + int found=0; ESlot *P=m->pin[lnext]; + for(int z=0;znpin[lnext] && !found;z++) if(P[z].eid==best) found=1; + ESlot *Sl=m->ecache[lnext]; + for(int z=0;zecn[lnext] && !found;z++) if(Sl[z].eid==best) found=1; + if(!found){ + unsigned w=__atomic_load_n(&pilot_w,__ATOMIC_RELAXED); + if(w-__atomic_load_n(&pilot_r,__ATOMIC_ACQUIRE)<4096){ + pilot_q[w&4095].l=lnext; pilot_q[w&4095].e=best; + __atomic_store_n(&pilot_w,w+1,__ATOMIC_RELEASE); + } + } + } + } + free(nrm); free(ch); +} + /* forward di UN layer (usato dai 78 principali e dal layer MTP) */ static void layer_forward(Model *m, Layer *l, int li, float *x, int S, int pos_base, float *nrm, float *tmp){ Cfg *c=&m->c; int D=c->hidden; if(g_spec && g_prefetch && l->sparse && m->enr[li]>0) for(int z=0;zenr[li];z++) expert_prefetch(m,li,m->eroute[li][z]); + if(g_looka && S==1 && lin_layers && l->sparse) la_predict(m,li,x,0); for(int s=0;sin_ln, D, c->eps); attention(m,l,li,nrm,S,pos_base,tmp); for(int64_t j=0;j<(int64_t)S*D;j++) x[j]+=tmp[j]; + if(g_pilot && S<=8 && li+1n_layers && m->L[li+1].sparse) pilot_prefetch(m,li+1,x,S); + if(g_looka && S==1 && li+1n_layers && m->L[li+1].sparse) la_predict(m,li+1,x,1); for(int s=0;spost_ln, D, c->eps); if(l->sparse) moe(m,l,li,nrm,S,tmp); else dense_mlp(l,nrm,S,D,c->dense_inter,tmp); for(int64_t j=0;j<(int64_t)S*D;j++) x[j]+=tmp[j]; @@ -1628,6 +1730,12 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ if(g_cuda_enabled) cuda_stats_print(); #endif profile_print(m,dt); + if(g_looka){ + const char *nm[3]={"token precedente (=SPEC prefetch)","ingresso layer, salto attention","layer successivo (1 giro di anticipo)"}; + printf("LOOKAHEAD routing — recall degli expert veri nel top-8 predetto:\n"); + for(int i=0;i<3;i++) printf(" %-38s %5.1f%% (%lld/%lld)\n", nm[i], + la_tot[i]?100.0*la_hit[i]/la_tot[i]:0.0, (long long)la_hit[i], (long long)la_tot[i]); + } free(pids); free(all); usage_save(m); } @@ -2012,8 +2120,26 @@ static void cap_for_ram(Model *m, double ram_gb, int ebits, int max_ctx){ (m->resident_bytes + (double)capmax*nsp*eb + slack)/1e9); m->ecap=capmax; } else { - fprintf(stderr,"[RAM_GB=%.1f%s] cap=%d ok (proiezione picco %.1f GB)\n", ram_gb, auto_b?" auto":"", m->ecap, - (m->resident_bytes + (double)m->ecap*nsp*eb + slack)/1e9); + /* AUTO-RAISE (issue #12): il budget consente PIU' cache di quella chiesta. + * Senza questo, una macchina da 128 GB girava con la LRU di una da 16 + * (cap=8 di default in coli): hit 23-28% con decine di GB inutilizzati. + * Tetto a n_experts: oltre, ogni layer avrebbe slot che non puo' riempire. + * CAP_RAISE=0 ripristina il comportamento fisso. */ + int raise_on = getenv("CAP_RAISE")?atoi(getenv("CAP_RAISE")):1; + int newcap = capmax>c->n_experts ? c->n_experts : capmax; + if(raise_on && newcap>m->ecap){ + for(int i=0;i<=c->n_layers;i++) if(m->ecache[i]){ + m->ecache[i]=realloc(m->ecache[i],(size_t)newcap*sizeof(ESlot)); + memset(m->ecache[i]+m->ecap,0,(size_t)(newcap-m->ecap)*sizeof(ESlot)); + } + fprintf(stderr,"[RAM_GB=%.1f%s] cap ALZATO %d->%d: il budget lo consente " + "(proiezione picco %.1f GB; CAP_RAISE=0 per disattivare)\n", + ram_gb, auto_b?" auto":"", m->ecap, newcap, + (m->resident_bytes + (double)newcap*nsp*eb + slack)/1e9); + m->ecap=newcap; + } else + fprintf(stderr,"[RAM_GB=%.1f%s] cap=%d ok (proiezione picco %.1f GB)\n", ram_gb, auto_b?" auto":"", m->ecap, + (m->resident_bytes + (double)m->ecap*nsp*eb + slack)/1e9); } } @@ -2029,6 +2155,10 @@ int main(int argc, char **argv){ g_mlock = getenv("MLOCK")?atoi(getenv("MLOCK")):-1; /* -1 auto (ON macOS), 0 off, 1 force / auto (ON macOS), 0 off, 1 force */ g_spec = getenv("SPEC")?atoi(getenv("SPEC")):1; g_draft = getenv("DRAFT")?atoi(getenv("DRAFT")):-1; /* -1 = auto: 3 se MTP, 0 senza */ + g_looka = getenv("LOOKA")?atoi(getenv("LOOKA")):0; /* 1 = misura predicibilita' routing */ + g_pilot = getenv("PILOT")?atoi(getenv("PILOT")):0; /* 1 = prefetch pilotato dal router */ + g_pilot_k = getenv("PILOT_K")?atoi(getenv("PILOT_K")):8; + if(g_pilot_k<1) g_pilot_k=1; g_direct = getenv("DIRECT")?atoi(getenv("DIRECT")):0; g_idot = getenv("IDOT")?atoi(getenv("IDOT")):1; /* 0 = kernel f32 esatti (A/B) */ g_repin = getenv("REPIN")?atoi(getenv("REPIN")):0; /* RFC: re-pin ogni n token emessi (0=off) / live re-pin every n emitted tokens (0=off) */ @@ -2165,6 +2295,12 @@ int main(int argc, char **argv){ m.gpu_expert_count,m.gpu_expert_bytes/1e9,(unsigned long long)m.gpu_expert_calls); if(g_cuda_enabled) cuda_stats_print(); #endif + if(g_looka){ + const char *nm[3]={"token precedente (=SPEC prefetch)","ingresso layer, salto attention","layer successivo (1 giro di anticipo)"}; + printf("LOOKAHEAD routing — recall degli expert veri nel top-8 predetto:\n"); + for(int i=0;i<3;i++) printf(" %-38s %5.1f%% (%lld/%lld)\n", nm[i], + la_tot[i]?100.0*la_hit[i]/la_tot[i]:0.0, (long long)la_hit[i], (long long)la_tot[i]); + } if(stats) stats_dump(&m,stats); return 0; }