From 1453dab7aeebbc7282515e759501fac000bbf3b1 Mon Sep 17 00:00:00 2001 From: ZacharyZcR Date: Fri, 10 Jul 2026 18:48:53 +0800 Subject: [PATCH] =?UTF-8?q?REPIN=20follows=20live=20session=20heat=20(deca?= =?UTF-8?q?ying=20heat=20map;=20.coli=5Fusage=20stays=20the=20persistent?= =?UTF-8?q?=20signal)=20+=20disk=E2=86=92RAM=E2=86=92VRAM=20promotion=20(#?= =?UTF-8?q?26)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 8 ++++- c/Makefile | 5 +++- c/coli | 3 ++ c/glm.c | 72 ++++++++++++++++++++++++++++++--------------- c/tests/test_tier.c | 22 ++++++++++++++ c/tier.h | 31 +++++++++++++++++++ 6 files changed, 115 insertions(+), 26 deletions(-) create mode 100644 c/tests/test_tier.c create mode 100644 c/tier.h diff --git a/README.md b/README.md index 2bf9eb5..5d2237c 100644 --- a/README.md +++ b/README.md @@ -211,7 +211,7 @@ works against the colibrì OpenAI-compatible server (in review, #21) or any othe compatible endpoint. Nothing leaves the endpoint you configure. The terminal `coli chat` remains the first-class interface. -Useful knobs (env or flags): `--temp T` token sampling temperature (default 0.7 + nucleus 0.90 — tuned for int4; 0 = greedy), `--topp 0.7` adaptive expert top-p (30–40% less disk), `--ngen N` max tokens per answer (`:piu` in chat continues a truncated one), `AUTOPIN=0` disable the learning cache's auto-pin, `THINK=1` enable GLM-5.2's reasoning block, `DRAFT=n` MTP draft depth, `TF=1` teacher-forcing validation, `PILOT=1` router-lookahead disk prefetch (experimental — see below), `CAP_RAISE=0` don't auto-grow the expert cache. +Useful knobs (env or flags): `--temp T` token sampling temperature (default 0.7 + nucleus 0.90 — tuned for int4; 0 = greedy), `--topp 0.7` adaptive expert top-p (30–40% less disk), `--ngen N` max tokens per answer (`:piu` in chat continues a truncated one), `--repin N` adapt RAM/VRAM hot experts every N emitted tokens, `AUTOPIN=0` disable the learning cache's auto-pin, `THINK=1` enable GLM-5.2's reasoning block, `DRAFT=n` MTP draft depth, `TF=1` teacher-forcing validation, `PILOT=1` router-lookahead disk prefetch (experimental — see below), `CAP_RAISE=0` don't auto-grow the expert cache. **The expert cache auto-sizes to your RAM** (since 2026-07-10): the engine now *raises* the LRU cap to fill your `--ram` budget instead of only lowering it. Before this fix a 128 GB machine ran with the same 8-experts/layer cache as a 16 GB one (issue #12) — **if you benchmarked colibrì before this date, rerun: your numbers were capped.** @@ -219,6 +219,12 @@ Useful knobs (env or flags): `--temp T` token sampling temperature (default 0.7 **The learning cache**: the engine records which experts your usage actually routes to (`.coli_usage` next to the model, updated every turn) and at startup automatically pins the hottest ones in spare RAM. colibrì literally gets faster the more you use it. +**Live tier adaptation** (`--repin N`, opt-in): at safe turn boundaries, a decaying +session heat map replaces cold pinned experts with hotter streamed experts. Replacement +loads the expert from disk into the existing RAM slot; GPU-backed slots immediately +refresh the same VRAM tier budget. A 25% hysteresis and a four-swap limit prevent tier +thrashing. Persistent `.coli_usage` remains the long-term signal and is not decayed. + **Conversations reopen warm** (`.coli_kv`, since 2026-07-10): `coli chat` persists the compressed MLA KV-cache to disk after every turn (~182 KB/token, appended incrementally, crash-safe). Close the chat, reopen it tomorrow — the model still remembers the whole conversation and **zero re-prefill happens**: validated byte-identical to an uninterrupted session. `:reset` clears it, `KVSAVE=0` disables it. ## Got a better machine? Try it — here's what to expect diff --git a/c/Makefile b/c/Makefile index 5921c5c..6fe3ab4 100644 --- a/c/Makefile +++ b/c/Makefile @@ -37,7 +37,7 @@ CUDA_ARCH ?= native NVCCFLAGS ?= -O3 -std=c++17 -arch=$(CUDA_ARCH) -Xcompiler=-Wall,-Wextra PYTHON ?= python3 CUDA_OBJ = -TEST_BINS = tests/test_json tests/test_st +TEST_BINS = tests/test_json tests/test_st tests/test_tier ifeq ($(CUDA),1) ifeq ($(UNAME_S),Darwin) $(error CUDA=1 is supported only on Linux) @@ -74,6 +74,9 @@ tests/test_json: tests/test_json.c json.h tests/test_st: tests/test_st.c st.h json.h compat.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_tier: tests/test_tier.c tier.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + test-c: $(TEST_BINS) @for test in $(TEST_BINS); do ./$$test || exit 1; done diff --git a/c/coli b/c/coli index 052d857..b2f2a03 100755 --- a/c/coli +++ b/c/coli @@ -14,6 +14,7 @@ CLI per far girare GLM-5.2 (744B) in locale, su CPU, in ~15-26 GB di RAM. Config via env o flag (validi anche dopo il sottocomando): COLI_MODEL= modello (default /home/vincenzo/glm52_i4) --ram N budget RAM in GB (auto-cap cache expert) + --repin N adatta gli expert RAM/VRAM ogni N token --topp P top-p adattivo sugli expert --topk N top-k fisso --ngen N token massimi per risposta --cap N slot cache/layer """ @@ -104,6 +105,7 @@ def env_for(a): if a.topp: e["TOPP"]=str(a.topp) if a.topk: e["TOPK"]=str(a.topk) if a.temp is not None: e["TEMP"]=str(a.temp) # 0 = greedy; default motore: 1.0 + nucleus 0.95 + if a.repin: e["REPIN"]=str(a.repin) return e # ---------- rendering markdown in STREAMING per il terminale ---------- @@ -426,6 +428,7 @@ def cmd_convert(a): def main(): common=argparse.ArgumentParser(add_help=False) common.add_argument("--model", default=DEF_MODEL); common.add_argument("--ram", type=int, default=0) # 0 = auto (il motore usa l'88% della RAM disponibile) + common.add_argument("--repin", type=int, default=0, help="adatta gli expert RAM/VRAM ogni N token") common.add_argument("--cap", type=int, default=8); common.add_argument("--ngen", type=int, default=1024) # rete di sicurezza: la fine vera la decidono gli stop token common.add_argument("--topp", type=float, default=0); common.add_argument("--topk", type=int, default=0) common.add_argument("--temp", type=float, default=None) # temperatura token (0=greedy, default 1.0+nucleus .95) diff --git a/c/glm.c b/c/glm.c index 95f3531..b690861 100644 --- a/c/glm.c +++ b/c/glm.c @@ -31,6 +31,7 @@ #endif #include "st.h" #include "tok.h" +#include "tier.h" #ifdef COLI_CUDA #include #include "backend_cuda.h" @@ -112,7 +113,8 @@ typedef struct { ESlot **ecache; int *ecn; int ecap; /* LRU expert per-layer */ ESlot ws[64]; /* working set del layer corrente (load paralleli) */ ESlot **pin; int *npin; /* HOT-STORE: expert pinnati in RAM (mai evicted) */ - uint32_t **eusage; /* contatori uso expert per layer (per STATS/PIN) */ + uint32_t **eusage; /* contatori persistenti (per STATS/PIN) */ + uint32_t **eheat; /* calore recente per promotion/demotion live */ /* DSA lightning indexer (attivo solo se i pesi out-idx-* sono presenti) */ int has_dsa; QT *ix_wq, *ix_wk, *ix_wp; /* per layer FULL: wq_b, wk, weights_proj */ @@ -708,7 +710,7 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits m->ecap=cap; m->ecache=calloc(NR,sizeof(ESlot*)); m->ecn=calloc(NR,sizeof(int)); m->eroute=calloc(NR,sizeof(int*)); m->enr=calloc(NR,sizeof(int)); m->pin=calloc(NR,sizeof(ESlot*)); m->npin=calloc(NR,sizeof(int)); - m->eusage=calloc(NR,sizeof(uint32_t*)); + m->eusage=calloc(NR,sizeof(uint32_t*)); m->eheat=calloc(NR,sizeof(uint32_t*)); m->kv_start=calloc(NR,sizeof(int)); for(int i=0;in_layers;i++){ Layer *l=&m->L[i]; @@ -737,6 +739,7 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits m->ecache[i]=calloc(cap,sizeof(ESlot)); m->eroute[i]=calloc(c->topk,sizeof(int)); /* metodo C: ultimo routing del layer */ m->eusage[i]=calloc(c->n_experts,sizeof(uint32_t)); + m->eheat[i]=calloc(c->n_experts,sizeof(uint32_t)); } #undef P } @@ -781,6 +784,7 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits m->ecache[i]=calloc(cap,sizeof(ESlot)); m->eroute[i]=calloc(c->topk,sizeof(int)); m->eusage[i]=calloc(c->n_experts,sizeof(uint32_t)); + m->eheat[i]=calloc(c->n_experts,sizeof(uint32_t)); m->kv_start[i]=-1; /* KV MTP: parte dalla prima posizione di decode */ #undef PM } @@ -1157,7 +1161,10 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out){ float cum=0; for(int kk=0;kk=g_topp*tot){ Ke=kk+1; break; } } } keff[s]=Ke; m->ereq+=Ke; - for(int kk=0;kkeusage[layer][idx[kk]]++; + for(int kk=0;kkeusage[layer][idx[kk]]++; + if(m->eheat[layer][idx[kk]]eheat[layer][idx[kk]]++; + } if(c->norm_topk){ float sm=0; for(int kk=0;kkrouted_scale; for(int d=0;dc; int nb=0; for(int l=0;ln_layers;l++){ - if(!m->npin || m->npin[l]<1 || !m->eusage[l]) continue; - uint32_t *u=m->eusage[l]; ESlot *P=m->pin[l]; - int zp=0; for(int z=1;znpin[l];z++) if(u[P[z].eid]n_experts;e++){ - int pinned=0; for(int z=0;znpin[l];z++) if(P[z].eid==e){pinned=1;break;} - if(!pinned && u[e]>fu){ fu=u[e]; eu=e; } /* non-pin piu' caldo / hottest unpinned */ - } - if(eu<0) continue; - uint32_t fp=u[P[zp].eid]; - if(fu <= fp + (fp>>2) + 4) continue; /* isteresi 25% / hysteresis */ - long g=(long)fu-(long)fp; + if(!m->npin || m->npin[l]<1 || !m->eheat[l]) continue; + ESlot *P=m->pin[l]; int ids[4096], zp, eu; long g; + int np=m->npin[l]; if(np>4096) np=4096; + for(int z=0;zeheat[l],c->n_experts,ids,np,&zp,&eu,&g)) continue; if(nbout[w].gain){ out[w].gain=g; out[w].l=l; out[w].slot=zp; out[w].eid=eu; } } @@ -1805,13 +1805,37 @@ static void repin_pass(Model *m){ g_last_repin = m->n_emit; RepinCand cd[4]; int nb=repin_pick(m,cd,4); for(int b=0;bpin[cd[b].l][cd[b].slot].eid; + ESlot *s=&m->pin[cd[b].l][cd[b].slot]; + int old=s->eid; + uint32_t old_heat=m->eheat[cd[b].l][old], new_heat=m->eheat[cd[b].l][cd[b].eid]; +#ifdef COLI_CUDA + int gpu=s->g.cuda_eligible; + int64_t old_gpu=gpu ? (int64_t)coli_cuda_tensor_bytes(s->g.cuda) + +(int64_t)coli_cuda_tensor_bytes(s->u.cuda) + +(int64_t)coli_cuda_tensor_bytes(s->d.cuda) : 0; +#endif double t0=now_s(); - expert_load(m, cd[b].l, cd[b].eid, &m->pin[cd[b].l][cd[b].slot]); - fprintf(stderr,"[REPIN] layer %d: esce/out %d (f=%u) <- entra/in %d (f=%u) in %.0f ms\n", - cd[b].l, old, m->eusage[cd[b].l][old], cd[b].eid, m->eusage[cd[b].l][cd[b].eid], - (now_s()-t0)*1e3); + expert_load(m,cd[b].l,cd[b].eid,s); /* disk -> RAM, same resident slot */ + const char *tier="RAM"; +#ifdef COLI_CUDA + if(gpu){ /* refresh the same VRAM slot now, not lazily */ + if(qt_cuda_upload(&s->g) && qt_cuda_upload(&s->u) && qt_cuda_upload(&s->d)){ + int64_t now_gpu=(int64_t)coli_cuda_tensor_bytes(s->g.cuda) + +(int64_t)coli_cuda_tensor_bytes(s->u.cuda) + +(int64_t)coli_cuda_tensor_bytes(s->d.cuda); + m->gpu_expert_bytes+=now_gpu-old_gpu; tier="VRAM"; + } else { + qt_cuda_reset(&s->g); qt_cuda_reset(&s->u); qt_cuda_reset(&s->d); + s->g.cuda_eligible=s->u.cuda_eligible=s->d.cuda_eligible=0; + m->gpu_expert_count--; m->gpu_expert_bytes-=old_gpu; + fprintf(stderr,"[REPIN] upload VRAM fallito; slot degradato a RAM\n"); + } + } +#endif + fprintf(stderr,"[REPIN] %s layer %d: esce/out %d (heat=%u) <- entra/in %d (heat=%u) in %.0f ms\n", + tier,cd[b].l,old,old_heat,cd[b].eid,new_heat,(now_s()-t0)*1e3); } + for(int l=0;lc.n_layers;l++) if(m->eheat[l]) tier_decay(m->eheat[l],m->c.n_experts); } /* ---- KV SU DISCO: la conversazione si riapre CALDA (KVSAVE=0 disattiva) ---- * Il re-prefill di una chat riaperta costa ore su questo disco; la KV compressa MLA diff --git a/c/tests/test_tier.c b/c/tests/test_tier.c new file mode 100644 index 0000000..9b68981 --- /dev/null +++ b/c/tests/test_tier.c @@ -0,0 +1,22 @@ +#include +#include "../tier.h" + +static int fail(const char *message){ + fprintf(stderr,"tier test failed: %s\n",message); + return 1; +} + +int main(void){ + uint32_t heat[6]={20,2,8,3,30,1}; + int pinned[2]={0,1}, slot=-1, eid=-1; long gain=0; + if(!tier_pick_swap(heat,6,pinned,2,&slot,&eid,&gain)) return fail("hot expert not promoted"); + if(slot!=1 || eid!=4 || gain!=28) return fail("wrong promotion candidate"); + + uint32_t stable[4]={20,18,24,4}; int resident[2]={0,1}; + if(tier_pick_swap(stable,4,resident,2,&slot,&eid,&gain)) return fail("hysteresis did not block churn"); + + tier_decay(heat,6); + if(heat[0]!=10 || heat[1]!=1 || heat[4]!=15) return fail("heat decay"); + puts("tier tests: ok"); + return 0; +} diff --git a/c/tier.h b/c/tier.h new file mode 100644 index 0000000..8f66cdd --- /dev/null +++ b/c/tier.h @@ -0,0 +1,31 @@ +#ifndef COLIBRI_TIER_H +#define COLIBRI_TIER_H + +#include + +/* Pick one RAM/VRAM hot-store slot to replace from recent routing heat. + * The fixed margin handles tiny samples; the 25% margin prevents ping-pong. */ +static int tier_pick_swap(const uint32_t *heat, int nexpert, + const int *pinned, int npin, + int *slot, int *eid, long *gain){ + if(!heat || !pinned || npin<1 || nexpert<1) return 0; + int cold=0; + for(int z=1;zfh){ fh=heat[e]; hot=e; } + } + if(hot<0) return 0; + uint32_t fc=heat[pinned[cold]]; + if(fh<=fc+(fc>>2)+4) return 0; + *slot=cold; *eid=hot; *gain=(long)fh-(long)fc; + return 1; +} + +static void tier_decay(uint32_t *heat, int nexpert){ + for(int e=0;e>=1; +} + +#endif