danielkrause/colibri-strix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Grammar-forced speculative drafts: GBNF grammar as a third draft source, guaranteed-accepted forced spans, lossless + opt-in (#48, #70)

Browse Source 
New byte-level GBNF-subset engine (c/grammar.h: parser + set-of-stacks PDA
walker) wired into spec_decode as a third draft source ("metodo F"), tried
before MTP/n-gram. Wherever the grammar admits exactly one legal byte, the
forced span is tokenized and injected as drafts; the existing batch-union
verification confirms them, so a wrong or out-of-sync grammar can never
change the output. Lazy arming skips preambles; adaptive guard (same
pattern as MTP) disables the source below 50% acceptance; grammar-accepted
tokens no longer pollute the MTP acceptance counter.

GRAMMAR=file.gbnf enables it in run and serve modes (also with DRAFT=0 and
with the int4 MTP head from #8); GRAMMAR_DRAFT=n caps the span (default 24).

Measured on M3 Max / int8-MTP container, greedy, MTP=0 DRAFT=0, NDJSON
classification: 0.37 -> 0.50 tok/s (1.60 tok/forward, 81 fw per 130 tok),
100% acceptance (48/48), output byte-identical to baseline.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Fabio Rovai
2026-07-12 00:35:39 +01:00
committed by GitHub
parent 1bab3243f7
commit cec7d6b648
6 changed files with 619 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
c/glm.c
+98 -7
View File
@@ -32,6 +32,7 @@
#include "st.h"
#include "tok.h"
#include "tier.h"
#include "grammar.h" /* metodo F: draft grammaticali (#48) */
#ifdef COLI_CUDA
#include <omp.h>
#include "backend_cuda.h"
@@ -569,6 +570,19 @@ static int g_draft=0; /* metodo E: DRAFT=n token auto-speculati per forward v
* misurato sul run reale (2026-07-03) acceptance ~5% -> ogni draft
* rifiutato paga comunque i suoi expert dal disco = ~3x piu' lento.
* Opt-in (DRAFT=4) per testi ripetitivi dove l'acceptance e' alta. */
/* metodo F (#48): GRAMMAR=<file.gbnf> -> terza sorgente di draft, la grammatica stessa.
* Nei workload a output vincolato (JSON/NDJSON, function calling) i byte FORZATI dalla
* grammatica (chiavi, punteggiatura, valori enum) sono draft gratuiti ad acceptance ~1:
* nessuna testa, nessuna lookup table, e si aggancia anche dove la testa MTP int4 non
* parte (#8). MAI un vincolo sul sampling: solo proposte, la verifica batch-union
* decide — grammatica sbagliata = draft rifiutati, output identico.
* GRAMMAR_DRAFT=n (default 24) limita i token forzati per forward. */
static Grammar g_gram; static GrState g_gst;
static Tok *g_gr_T=NULL;
static int g_gr_on=0; /* grammatica caricata e walker vivo */
static int g_gr_armed=0; /* lazy: parte dal primo byte ammesso dalla radice (salta i preamboli) */
static int g_gr_max=24;
static uint64_t g_gr_prop=0, g_gr_acc=0;
static int g_looka=0; /* LOOKA=1: misura (solo contatori, zero effetti) quanto il routing MoE
* e' predicibile IN ANTICIPO — la domanda che decide se un prefetch
* pilotato dal router puo' riempire i tempi morti del disco.
@@ -1526,6 +1540,68 @@ static inline int argmax_v(const float *lo, int V){
int b=0; float bv=lo[0]; for(int i=1;i<V;i++) if(lo[i]>bv){bv=lo[i];b=i;} return b;
}
/* ---- METODO F: draft grammaticale (#48) ----
* gr_feed consuma i byte di ogni token EMESSO e tiene il walker in sync con l'output;
* grammar_draft propone lo span FORZATO successivo (un solo byte legale per posizione)
* gia' tokenizzato. Il confine di tokenizzazione non e' garantito coincidere con quello
* del modello: la verifica assorbe la differenza (al peggio l'ultimo draft e' rifiutato). */
static void grammar_setup(Tok *T){
const char *gf=getenv("GRAMMAR"); if(!gf||!*gf) return;
FILE *f=fopen(gf,"rb");
if(!f){ fprintf(stderr,"[GRAMMAR] impossibile aprire %s\n",gf); return; }
fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET);
char *txt=malloc((size_t)n+1);
if(!txt || fread(txt,1,(size_t)n,f)!=(size_t)n){
fprintf(stderr,"[GRAMMAR] lettura fallita: %s\n",gf); fclose(f); free(txt); return; }
fclose(f); txt[n]=0;
if(gr_parse(&g_gram,txt)){ fprintf(stderr,"[GRAMMAR] %s: %s\n",gf,g_gram.err); free(txt); return; }
free(txt);
gr_state_init(&g_gst,&g_gram);
if(!g_gst.alive){ fprintf(stderr,"[GRAMMAR] %s: grammatica non trattabile (ricorsione sinistra?)\n",gf); return; }
if(getenv("GRAMMAR_DRAFT")) g_gr_max=atoi(getenv("GRAMMAR_DRAFT"));
if(g_gr_max<1) g_gr_max=1;
if(g_gr_max>48) g_gr_max=48;
g_gr_T=T; g_gr_on=1;
fprintf(stderr,"[GRAMMAR] %s: %d regole, span forzato max %d token/forward\n",gf,g_gram.n,g_gr_max);
}
/* stato pulito all'inizio di ogni RISPOSTA (non tra i \x02MORE, che continuano) */
static void grammar_reset(void){
if(!g_gr_on) return;
gr_state_init(&g_gst,&g_gram); g_gr_armed=0;
if(!g_gst.alive) g_gr_on=0;
}
/* consuma i byte di un token emesso. Preambolo (prima dell'arming): ignorato.
* Desync dopo l'arming: si riarma in attesa del prossimo inizio valido — al peggio
* i draft vengono rifiutati dalla verifica, l'output non cambia MAI. */
static void gr_feed(int t){
if(!g_gr_on||!g_gr_T) return;
char b[64]; int n=tok_decode(g_gr_T,&t,1,b,63);
for(int i=0;i<n;i++){
int r=gr_accept(&g_gst,(unsigned char)b[i]);
if(r==1){ g_gr_armed=1; continue; }
if(r<0){ g_gr_on=0; return; } /* walker spento: fine dei draft */
if(!g_gr_armed) continue; /* preambolo: aspetta l'inizio */
gr_state_init(&g_gst,&g_gram); g_gr_armed=0; /* desync: riparti dalla radice */
if(!g_gst.alive){ g_gr_on=0; return; }
if(gr_accept(&g_gst,(unsigned char)b[i])==1) g_gr_armed=1;
}
}
/* propone lo span forzato come token (max cap); 0 se la grammatica dirama qui */
static int grammar_draft(int *draft, int cap){
if(!g_gr_on||!g_gr_armed||!g_gr_T||cap<1) return 0;
if(g_gr_prop>=32 && g_gr_acc*2<g_gr_prop){ /* guardia adattiva, come per MTP:
acceptance sotto il 50% = tokenizzazione fuori asse, meglio spegnersi */
g_gr_on=0;
fprintf(stderr,"[GRAMMAR] acceptance %.0f%% dopo %llu proposte: draft grammaticali disattivati\n",
100.0*g_gr_acc/g_gr_prop,(unsigned long long)g_gr_prop);
return 0;
}
char fb[512]; int nb=gr_forced(&g_gst,fb,(int)sizeof fb-1);
if(nb<=0) return 0;
int g=tok_encode(g_gr_T,fb,nb,draft,cap);
return g>0?g:0;
}
/* ---- SAMPLING (temperatura + nucleus) con verifica speculativa LOSSLESS ----
* Il draft (MTP/n-gram) e' DETERMINISTICO (argmax della testa): q = massa puntuale.
* Rejection sampling di Leviathan: accetta il draft x_d con prob p(x_d); al rifiuto
@@ -1599,9 +1675,15 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *lo
int next=pick_tok(logit,V,carry_ban); carry_ban=-1; free(logit); logit=NULL;
if((eos>=0 && next==eos) || is_stop(next)) break;
emit(next,ud); all[kv]=next; emitted++; m->n_emit++;
gr_feed(next); /* il walker segue l'output emesso */
if(emitted>=n_new) break; /* l'ultimo token non serve forwardarlo */
int g = 0;
if(g_draft>0){
int g = 0, gsrc = 0; /* sorgente: 1=grammatica 2=MTP/n-gram */
if(g_gr_on){ /* metodo F: prima la grammatica — dove
* forza, l'acceptance e' ~1 (#48) */
g=grammar_draft(draft,g_gr_max);
if(g>0) gsrc=1;
}
if(!g && g_draft>0){
/* auto-off adattivo: draft che non vengono mai accettati = solo tassa disco */
if(m->has_mtp && m->mtp_prop>=24 && m->mtp_acc*10 < m->mtp_prop){
g_draft=0;
@@ -1609,13 +1691,14 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *lo
100.0*m->mtp_acc/m->mtp_prop, (unsigned long long)m->mtp_prop);
}
}
if(g_draft>0){
if(m->has_mtp){ g=mtp_draft(m,next,kv,g_draft,draft); m->mtp_prop+=g; }
else g=ngram_draft(all,kv+1,g_draft,draft);
if(!g && g_draft>0){
if(m->has_mtp){ g=mtp_draft(m,next,kv,g_draft,draft); m->mtp_prop+=g; if(g)gsrc=2; }
else { g=ngram_draft(all,kv+1,g_draft,draft); if(g)gsrc=2; }
}
if(g>n_new-emitted) g=n_new-emitted;
if(kv+1+g+1>m->max_t) g=m->max_t-kv-2;
if(g<0) g=0;
if(gsrc==1) g_gr_prop+=(uint64_t)g;
int S=1+g; int batch[64]; batch[0]=next; memcpy(batch+1,draft,g*sizeof(int));
float *lo=step_all(m,batch,S,kv); m->n_fw++;
int k=0; /* verifica: accetta finche' coincide */
@@ -1628,9 +1711,11 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *lo
accept = (rndu() < g_pbuf[draft[k]]); }
if(!accept){ if(g_temp>0) carry_ban=draft[k]; break; }
if((eos>=0 && draft[k]==eos) || is_stop(draft[k])){ done=1; break; }
emit(draft[k],ud); all[kv+1+k]=draft[k]; emitted++; m->n_emit++; k++;
emit(draft[k],ud); all[kv+1+k]=draft[k]; emitted++; m->n_emit++;
gr_feed(draft[k]); k++;
}
if(m->has_mtp) m->mtp_acc+=k;
if(gsrc==1) g_gr_acc+=(uint64_t)k;
else if(gsrc==2 && m->has_mtp) m->mtp_acc+=k;
if(m->has_mtp && k>=1) mtp_absorb(m, all+kv+1, m->h_all, k, kv); /* KV MTP in sync coi verificati */
/* hlast deve corrispondere all'ultima posizione ACCETTATA (kv+k), non a fine batch */
if(m->h_all && k<S-1) memcpy(m->hlast, m->h_all+(int64_t)k*m->c.hidden, m->c.hidden*sizeof(float));
@@ -1759,6 +1844,7 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){
Tok T; tok_load(&T,tkp);
int eos=tok_id_of(&T,"<|endoftext|>");
stops_arm(&m->c, eos);
grammar_setup(&T); /* metodo F: GRAMMAR=file.gbnf (#48) */
if(g_temp<0) g_temp=0.7f; /* auto: 0.7, NON l'1.0 ufficiale — la coda della
* distribuzione int4 e' rumore di quantizzazione */
int cap=(int)strlen(prompt)+16; int *pids=malloc(cap*sizeof(int));
@@ -1771,6 +1857,7 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){
double t=now_s();
float *logit=step(m,pids,np,0);
EmitStream es={&T,m,t,0,0};
grammar_reset();
int produced=spec_decode(m,all,np,ngen,eos,logit,emit_stream,&es,NULL);
double dt=now_s()-t;
double tot=m->hits+m->miss;
@@ -1782,6 +1869,8 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){
printf("speculazione: %.2f token/forward (%llu fw per %llu tok) | MTP acceptance %.0f%% (%llu/%llu)\n",
m->n_fw?(double)m->n_emit/m->n_fw:1.0, (unsigned long long)m->n_fw, (unsigned long long)m->n_emit,
m->mtp_prop?100.0*m->mtp_acc/m->mtp_prop:0.0, (unsigned long long)m->mtp_acc, (unsigned long long)m->mtp_prop);
if(g_gr_prop) printf("grammatica: acceptance %.0f%% (%llu/%llu draft forzati)\n",
100.0*g_gr_acc/g_gr_prop, (unsigned long long)g_gr_acc, (unsigned long long)g_gr_prop);
#ifdef COLI_CUDA
if(m->gpu_expert_count) printf("CUDA expert tier: %d residenti (%.2f GB) | %llu chiamate servite da VRAM\n",
m->gpu_expert_count,m->gpu_expert_bytes/1e9,(unsigned long long)m->gpu_expert_calls);
@@ -1977,6 +2066,7 @@ static void run_serve(Model *m, const char *snap){
Tok T; tok_load(&T,tkp);
int eos=tok_id_of(&T,"<|endoftext|>");
stops_arm(&m->c, eos);
grammar_setup(&T); /* metodo F: GRAMMAR=file.gbnf (#48) */
if(g_temp<0) g_temp=0.7f; /* auto: 0.7, NON l'1.0 ufficiale — la coda della
* distribuzione int4 e' rumore di quantizzazione */
int ngen=getenv("NGEN")?atoi(getenv("NGEN")):256;
@@ -2084,6 +2174,7 @@ static void run_serve(Model *m, const char *snap){
else logit=step(m,hist+len-1,1,len-1); /* prompt identico/prefisso: rigenera i logits */
EmitStream es={&T,m,now_s(),0,1};
int prod=0;
grammar_reset(); /* nuova risposta = nuovo documento (MORE invece continua) */
if(cur>0) prod=spec_decode(m,hist,len,cur,eos,logit,emit_stream,&es,&len);
else free(logit);
double tdt=now_s()-tt0; if(tdt<1e-6) tdt=1e-6;