diff --git a/.gitignore b/.gitignore index 13870d7..ade2deb 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,8 @@ c/backend_cuda.o c/backend_cuda_test c/tests/test_json c/tests/test_st +c/tests/test_tier +c/tests/test_grammar # oracoli tiny generati (make_glm_oracle.py) e dati benchmark scaricati c/glm_tiny/ diff --git a/README.md b/README.md index 03cbce1..f4a9ace 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ The engine is a single C file (`c/glm.c`, ~2,400 lines) plus small headers. No B - **MLA attention** (q/kv-LoRA, interleaved partial RoPE) with **compressed KV-cache**: 576 floats/token instead of 32,768 (57× smaller — GLM-5.2 has 64 heads and no GQA). - **DeepSeek-V3-style sigmoid router** (noaux_tc, routed_scaling_factor), shared expert, first-3-dense layers. - **Native MTP speculative decoding** — GLM-5.2's own multi-token-prediction head (layer 78) drafts tokens that the main model verifies in one batched forward. **The head must be int8** (the converter does this by default): at int4 draft acceptance collapses to 0–4% and speculation never engages; at int8 it's 39–59% acceptance, **2.2–2.8 tokens/forward** (community-measured, [#8](https://github.com/JustVugg/colibri/issues/8)). Lossless — *and stays lossless under sampling* via rejection sampling. Honest caveat from the same measurement: on a **cold** cache each verified draft routes to extra experts (~660 → ~1100 expert-loads/token), so speculation can be a net *time* loss until the cache/pin warms up — the adaptive guard and `DRAFT=0` are there for that. +- **Grammar-forced speculative drafts** (`GRAMMAR=file.gbnf`, [#48](https://github.com/JustVugg/colibri/issues/48)) — on constrained-output workloads (JSON/NDJSON, function calling, structured extraction) the grammar itself is a third draft source: wherever it admits exactly **one** legal byte (braces, quotes, key names, enum bodies), that forced span is tokenized and injected as pre-accepted drafts with ~1.0 acceptance — no draft head, no lookup table, and it engages even with the int4 MTP head from [#8](https://github.com/JustVugg/colibri/issues/8). It never constrains sampling: forced spans are verified in the same batch-union forward as any draft, so a wrong or out-of-sync grammar cannot change the output — worst case is rejected drafts, and an adaptive guard turns the source off below 50% acceptance. Byte-level GBNF subset (literals, char classes, `| ( ) ? * +`, comments); `GRAMMAR_DRAFT=n` caps the forced span per forward (default 24). Composes with `DRAFT`/MTP, which fill the free-text gaps between forced spans. - **True sampling** — temperature + nucleus, defaults tuned for int4 reality (0.7 / 0.90; the official 1.0 / 0.95 samples quantization noise from the tail). - **Integer-dot kernels** (Q8_0-style int8 activations, AVX2 `maddubs`): int8 matmuls 1.4–2.5× faster (119 GFLOP/s measured), int4 1.8× in batch — routing decided per shape by measurement (int4 single-row stays f32: it measured slower). - **MLA weight absorption** (DeepSeek trick) for decode: no per-token k/v reconstruction — the query absorbs `kv_b`, context is projected after attention. Validated exact: TF 32/32 and generation 20/20 with absorption forced everywhere. @@ -312,7 +313,7 @@ works against the colibrì OpenAI-compatible server (in review, #21) or any othe compatible endpoint. Nothing leaves the endpoint you configure. The terminal `coli chat` remains the first-class interface. -Useful knobs (env or flags): `--temp T` token sampling temperature (default 0.7 + nucleus 0.90 — tuned for int4; 0 = greedy), `--topp 0.7` adaptive expert top-p (30–40% less disk), `--ngen N` max tokens per answer (`:piu` in chat continues a truncated one), `--repin N` adapt RAM/VRAM hot experts every N emitted tokens, `AUTOPIN=0` disable the learning cache's auto-pin, `THINK=1` enable GLM-5.2's reasoning block, `DRAFT=n` MTP draft depth, `TF=1` teacher-forcing validation, `PILOT=1` router-lookahead disk prefetch (experimental — see below), `CAP_RAISE=0` don't auto-grow the expert cache. +Useful knobs (env or flags): `--temp T` token sampling temperature (default 0.7 + nucleus 0.90 — tuned for int4; 0 = greedy), `--topp 0.7` adaptive expert top-p (30–40% less disk), `--ngen N` max tokens per answer (`:piu` in chat continues a truncated one), `--repin N` adapt RAM/VRAM hot experts every N emitted tokens, `AUTOPIN=0` disable the learning cache's auto-pin, `THINK=1` enable GLM-5.2's reasoning block, `DRAFT=n` MTP draft depth, `GRAMMAR=g.gbnf` grammar-forced drafts for constrained JSON/NDJSON output (`GRAMMAR_DRAFT=n` caps the forced span), `TF=1` teacher-forcing validation, `PILOT=1` router-lookahead disk prefetch (experimental — see below), `CAP_RAISE=0` don't auto-grow the expert cache. **The expert cache auto-sizes to your RAM** (since 2026-07-10): the engine now *raises* the LRU cap to fill your `--ram` budget instead of only lowering it. Before this fix a 128 GB machine ran with the same 8-experts/layer cache as a 16 GB one (issue #12) — **if you benchmarked colibrì before this date, rerun: your numbers were capped.** diff --git a/c/Makefile b/c/Makefile index 94120e8..77bdf63 100644 --- a/c/Makefile +++ b/c/Makefile @@ -53,7 +53,7 @@ CUDA_ARCH ?= native NVCCFLAGS ?= -O3 -std=c++17 -arch=$(CUDA_ARCH) -Xcompiler=-Wall,-Wextra PYTHON ?= python3 CUDA_OBJ = -TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) +TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) ifeq ($(CUDA),1) ifeq ($(UNAME_S),Darwin) $(error CUDA=1 is supported only on Linux) @@ -72,7 +72,7 @@ all: glm$(EXE) # phony 'glm' → 'glm.exe' on Windows (so 'make glm' and 'coli build' work on every platform) glm: glm$(EXE) -glm$(EXE): glm.c st.h json.h tok.h tok_unicode.h compat.h $(CUDA_OBJ) +glm$(EXE): glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h $(CUDA_OBJ) $(CC) $(CFLAGS) glm.c $(CUDA_OBJ) -o glm$(EXE) $(LDFLAGS) backend_cuda.o: backend_cuda.cu backend_cuda.h @@ -103,6 +103,9 @@ tests/test_st$(EXE): tests/test_st.c st.h json.h compat.h tests/test_tier$(EXE): tests/test_tier.c tier.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_grammar$(EXE): tests/test_grammar.c grammar.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + test-c: $(TEST_BINS) @for test in $(TEST_BINS); do ./$$test || exit 1; done diff --git a/c/glm.c b/c/glm.c index 3a29610..d615582 100644 --- a/c/glm.c +++ b/c/glm.c @@ -32,6 +32,7 @@ #include "st.h" #include "tok.h" #include "tier.h" +#include "grammar.h" /* metodo F: draft grammaticali (#48) */ #ifdef COLI_CUDA #include #include "backend_cuda.h" @@ -569,6 +570,19 @@ static int g_draft=0; /* metodo E: DRAFT=n token auto-speculati per forward v * misurato sul run reale (2026-07-03) acceptance ~5% -> ogni draft * rifiutato paga comunque i suoi expert dal disco = ~3x piu' lento. * Opt-in (DRAFT=4) per testi ripetitivi dove l'acceptance e' alta. */ +/* metodo F (#48): GRAMMAR= -> terza sorgente di draft, la grammatica stessa. + * Nei workload a output vincolato (JSON/NDJSON, function calling) i byte FORZATI dalla + * grammatica (chiavi, punteggiatura, valori enum) sono draft gratuiti ad acceptance ~1: + * nessuna testa, nessuna lookup table, e si aggancia anche dove la testa MTP int4 non + * parte (#8). MAI un vincolo sul sampling: solo proposte, la verifica batch-union + * decide — grammatica sbagliata = draft rifiutati, output identico. + * GRAMMAR_DRAFT=n (default 24) limita i token forzati per forward. */ +static Grammar g_gram; static GrState g_gst; +static Tok *g_gr_T=NULL; +static int g_gr_on=0; /* grammatica caricata e walker vivo */ +static int g_gr_armed=0; /* lazy: parte dal primo byte ammesso dalla radice (salta i preamboli) */ +static int g_gr_max=24; +static uint64_t g_gr_prop=0, g_gr_acc=0; static int g_looka=0; /* LOOKA=1: misura (solo contatori, zero effetti) quanto il routing MoE * e' predicibile IN ANTICIPO — la domanda che decide se un prefetch * pilotato dal router puo' riempire i tempi morti del disco. @@ -1526,6 +1540,68 @@ static inline int argmax_v(const float *lo, int V){ int b=0; float bv=lo[0]; for(int i=1;ibv){bv=lo[i];b=i;} return b; } +/* ---- METODO F: draft grammaticale (#48) ---- + * gr_feed consuma i byte di ogni token EMESSO e tiene il walker in sync con l'output; + * grammar_draft propone lo span FORZATO successivo (un solo byte legale per posizione) + * gia' tokenizzato. Il confine di tokenizzazione non e' garantito coincidere con quello + * del modello: la verifica assorbe la differenza (al peggio l'ultimo draft e' rifiutato). */ +static void grammar_setup(Tok *T){ + const char *gf=getenv("GRAMMAR"); if(!gf||!*gf) return; + FILE *f=fopen(gf,"rb"); + if(!f){ fprintf(stderr,"[GRAMMAR] impossibile aprire %s\n",gf); return; } + fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET); + char *txt=malloc((size_t)n+1); + if(!txt || fread(txt,1,(size_t)n,f)!=(size_t)n){ + fprintf(stderr,"[GRAMMAR] lettura fallita: %s\n",gf); fclose(f); free(txt); return; } + fclose(f); txt[n]=0; + if(gr_parse(&g_gram,txt)){ fprintf(stderr,"[GRAMMAR] %s: %s\n",gf,g_gram.err); free(txt); return; } + free(txt); + gr_state_init(&g_gst,&g_gram); + if(!g_gst.alive){ fprintf(stderr,"[GRAMMAR] %s: grammatica non trattabile (ricorsione sinistra?)\n",gf); return; } + if(getenv("GRAMMAR_DRAFT")) g_gr_max=atoi(getenv("GRAMMAR_DRAFT")); + if(g_gr_max<1) g_gr_max=1; + if(g_gr_max>48) g_gr_max=48; + g_gr_T=T; g_gr_on=1; + fprintf(stderr,"[GRAMMAR] %s: %d regole, span forzato max %d token/forward\n",gf,g_gram.n,g_gr_max); +} +/* stato pulito all'inizio di ogni RISPOSTA (non tra i \x02MORE, che continuano) */ +static void grammar_reset(void){ + if(!g_gr_on) return; + gr_state_init(&g_gst,&g_gram); g_gr_armed=0; + if(!g_gst.alive) g_gr_on=0; +} +/* consuma i byte di un token emesso. Preambolo (prima dell'arming): ignorato. + * Desync dopo l'arming: si riarma in attesa del prossimo inizio valido — al peggio + * i draft vengono rifiutati dalla verifica, l'output non cambia MAI. */ +static void gr_feed(int t){ + if(!g_gr_on||!g_gr_T) return; + char b[64]; int n=tok_decode(g_gr_T,&t,1,b,63); + for(int i=0;i=32 && g_gr_acc*20?g:0; +} + /* ---- SAMPLING (temperatura + nucleus) con verifica speculativa LOSSLESS ---- * Il draft (MTP/n-gram) e' DETERMINISTICO (argmax della testa): q = massa puntuale. * Rejection sampling di Leviathan: accetta il draft x_d con prob p(x_d); al rifiuto @@ -1599,9 +1675,15 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *lo int next=pick_tok(logit,V,carry_ban); carry_ban=-1; free(logit); logit=NULL; if((eos>=0 && next==eos) || is_stop(next)) break; emit(next,ud); all[kv]=next; emitted++; m->n_emit++; + gr_feed(next); /* il walker segue l'output emesso */ if(emitted>=n_new) break; /* l'ultimo token non serve forwardarlo */ - int g = 0; - if(g_draft>0){ + int g = 0, gsrc = 0; /* sorgente: 1=grammatica 2=MTP/n-gram */ + if(g_gr_on){ /* metodo F: prima la grammatica — dove + * forza, l'acceptance e' ~1 (#48) */ + g=grammar_draft(draft,g_gr_max); + if(g>0) gsrc=1; + } + if(!g && g_draft>0){ /* auto-off adattivo: draft che non vengono mai accettati = solo tassa disco */ if(m->has_mtp && m->mtp_prop>=24 && m->mtp_acc*10 < m->mtp_prop){ g_draft=0; @@ -1609,13 +1691,14 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *lo 100.0*m->mtp_acc/m->mtp_prop, (unsigned long long)m->mtp_prop); } } - if(g_draft>0){ - if(m->has_mtp){ g=mtp_draft(m,next,kv,g_draft,draft); m->mtp_prop+=g; } - else g=ngram_draft(all,kv+1,g_draft,draft); + if(!g && g_draft>0){ + if(m->has_mtp){ g=mtp_draft(m,next,kv,g_draft,draft); m->mtp_prop+=g; if(g)gsrc=2; } + else { g=ngram_draft(all,kv+1,g_draft,draft); if(g)gsrc=2; } } if(g>n_new-emitted) g=n_new-emitted; if(kv+1+g+1>m->max_t) g=m->max_t-kv-2; if(g<0) g=0; + if(gsrc==1) g_gr_prop+=(uint64_t)g; int S=1+g; int batch[64]; batch[0]=next; memcpy(batch+1,draft,g*sizeof(int)); float *lo=step_all(m,batch,S,kv); m->n_fw++; int k=0; /* verifica: accetta finche' coincide */ @@ -1628,9 +1711,11 @@ static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *lo accept = (rndu() < g_pbuf[draft[k]]); } if(!accept){ if(g_temp>0) carry_ban=draft[k]; break; } if((eos>=0 && draft[k]==eos) || is_stop(draft[k])){ done=1; break; } - emit(draft[k],ud); all[kv+1+k]=draft[k]; emitted++; m->n_emit++; k++; + emit(draft[k],ud); all[kv+1+k]=draft[k]; emitted++; m->n_emit++; + gr_feed(draft[k]); k++; } - if(m->has_mtp) m->mtp_acc+=k; + if(gsrc==1) g_gr_acc+=(uint64_t)k; + else if(gsrc==2 && m->has_mtp) m->mtp_acc+=k; if(m->has_mtp && k>=1) mtp_absorb(m, all+kv+1, m->h_all, k, kv); /* KV MTP in sync coi verificati */ /* hlast deve corrispondere all'ultima posizione ACCETTATA (kv+k), non a fine batch */ if(m->h_all && khlast, m->h_all+(int64_t)k*m->c.hidden, m->c.hidden*sizeof(float)); @@ -1759,6 +1844,7 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ Tok T; tok_load(&T,tkp); int eos=tok_id_of(&T,"<|endoftext|>"); stops_arm(&m->c, eos); + grammar_setup(&T); /* metodo F: GRAMMAR=file.gbnf (#48) */ if(g_temp<0) g_temp=0.7f; /* auto: 0.7, NON l'1.0 ufficiale — la coda della * distribuzione int4 e' rumore di quantizzazione */ int cap=(int)strlen(prompt)+16; int *pids=malloc(cap*sizeof(int)); @@ -1771,6 +1857,7 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ double t=now_s(); float *logit=step(m,pids,np,0); EmitStream es={&T,m,t,0,0}; + grammar_reset(); int produced=spec_decode(m,all,np,ngen,eos,logit,emit_stream,&es,NULL); double dt=now_s()-t; double tot=m->hits+m->miss; @@ -1782,6 +1869,8 @@ static void run_text(Model *m, const char *snap, const char *prompt, int ngen){ printf("speculazione: %.2f token/forward (%llu fw per %llu tok) | MTP acceptance %.0f%% (%llu/%llu)\n", m->n_fw?(double)m->n_emit/m->n_fw:1.0, (unsigned long long)m->n_fw, (unsigned long long)m->n_emit, m->mtp_prop?100.0*m->mtp_acc/m->mtp_prop:0.0, (unsigned long long)m->mtp_acc, (unsigned long long)m->mtp_prop); + if(g_gr_prop) printf("grammatica: acceptance %.0f%% (%llu/%llu draft forzati)\n", + 100.0*g_gr_acc/g_gr_prop, (unsigned long long)g_gr_acc, (unsigned long long)g_gr_prop); #ifdef COLI_CUDA if(m->gpu_expert_count) printf("CUDA expert tier: %d residenti (%.2f GB) | %llu chiamate servite da VRAM\n", m->gpu_expert_count,m->gpu_expert_bytes/1e9,(unsigned long long)m->gpu_expert_calls); @@ -1977,6 +2066,7 @@ static void run_serve(Model *m, const char *snap){ Tok T; tok_load(&T,tkp); int eos=tok_id_of(&T,"<|endoftext|>"); stops_arm(&m->c, eos); + grammar_setup(&T); /* metodo F: GRAMMAR=file.gbnf (#48) */ if(g_temp<0) g_temp=0.7f; /* auto: 0.7, NON l'1.0 ufficiale — la coda della * distribuzione int4 e' rumore di quantizzazione */ int ngen=getenv("NGEN")?atoi(getenv("NGEN")):256; @@ -2084,6 +2174,7 @@ static void run_serve(Model *m, const char *snap){ else logit=step(m,hist+len-1,1,len-1); /* prompt identico/prefisso: rigenera i logits */ EmitStream es={&T,m,now_s(),0,1}; int prod=0; + grammar_reset(); /* nuova risposta = nuovo documento (MORE invece continua) */ if(cur>0) prod=spec_decode(m,hist,len,cur,eos,logit,emit_stream,&es,&len); else free(logit); double tdt=now_s()-tt0; if(tdt<1e-6) tdt=1e-6; diff --git a/c/grammar.h b/c/grammar.h new file mode 100644 index 0000000..6c0631e --- /dev/null +++ b/c/grammar.h @@ -0,0 +1,364 @@ +/* grammar.h — draft grammaticale (#48): GBNF (sottoinsieme) valutata a livello di BYTE. + * + * Idea: nei workload a output vincolato (JSON/NDJSON, function calling, estrazione + * strutturata) una frazione dei token e' DETERMINISTICA data la grammatica: parentesi, + * virgolette, nomi delle chiavi, separatori, valori enum. Quegli span sono draft + * gratuiti ad acceptance ~1: nessuna testa, nessuna lookup table — la verifica + * batch-union li conferma e paga UN forward per piu' token. E si aggancia anche dove + * la testa MTP int4 non parte (#8). + * + * La grammatica non vincola MAI il campionamento: propone solo draft, che la verifica + * accetta o rifiuta come qualunque altro draft. Grammatica sbagliata o fuori sync => + * draft rifiutati, output IDENTICO. E' un acceleratore puro, mai un filtro. + * + * Sottoinsieme GBNF (stile llama.cpp), valutato sui BYTE: + * root ::= obj+ # la regola di partenza si chiama "root" + * obj ::= "{" pair ("," pair)* "}" "\n" + * str ::= "\"" [^"\\]* "\"" + * Supportato: letterali "..." (escape \" \\ \n \r \t \xHH), classi [a-z0-9-] anche + * negate [^...], riferimenti a regole, gruppi (...), postfissi ? * +, commenti #, + * alternate con |, epsilon come "". Le regole possono estendersi su piu' righe: una + * nuova regola inizia dove un identificatore e' seguito da "::=". + * NON supportato: ripetizioni {m,n}, range unicode nelle classi (le classi lavorano + * sui byte; per l'UTF-8 multibyte usare i letterali, che passano i byte grezzi). + * Ricorsione sinistra: intercettata dal tetto di profondita' -> il walker si spegne + * (alive=0) e la generazione prosegue senza draft. Mai un blocco, mai un crash. + * + * Il walker e' un PDA con INSIEME di stack (come llama.cpp): ogni stack in forma + * normale ha in cima un simbolo terminale (classe di byte) oppure e' vuoto (parse + * completabile qui). gr_forced() estende il prefisso finche' esiste UN SOLO byte + * legale e il parse non e' terminabile: quel prefisso e' il draft forzato. + */ +#ifndef COLI_GRAMMAR_H +#define COLI_GRAMMAR_H + +#include +#include +#include +#include + +#define GR_MAX_RULES 1024 +#define GR_MAX_STACKS 64 /* ambiguita' massima seguita in parallelo */ +#define GR_MAX_DEPTH 64 /* profondita' massima di uno stack del PDA */ + +typedef struct { uint8_t bits[32]; } GrCls; /* insieme di byte ammessi */ +enum { GR_CLS = 0, GR_REF = 1 }; +typedef struct { uint8_t t; int16_t ref; GrCls c; } GrSym; +typedef struct { GrSym *s; int n, cap; } GrAlt; /* una sequenza di simboli */ +typedef struct { GrAlt *a; int n, cap; char name[64]; } GrRule; +typedef struct { GrRule r[GR_MAX_RULES]; int n; int root; char err[160]; } Grammar; + +/* frame = posizione dentro un alternate: (regola, alternate, simbolo) */ +typedef struct { int16_t r, a, s; } GrFrame; +typedef struct { GrFrame f[GR_MAX_DEPTH]; int16_t n; } GrStack; +typedef struct { Grammar *G; GrStack st[GR_MAX_STACKS]; int n; int alive; } GrState; + +/* ---------- costruzione ---------- */ + +static int gr__alt_new(Grammar *G, int ri){ + GrRule *R=&G->r[ri]; + if(R->n==R->cap){ int nc=R->cap?R->cap*2:4; + GrAlt *na=(GrAlt*)realloc(R->a,(size_t)nc*sizeof(GrAlt)); if(!na) return -1; + R->a=na; R->cap=nc; } + memset(&R->a[R->n],0,sizeof(GrAlt)); + return R->n++; +} +static int gr__push(Grammar *G, int ri, int ai, const GrSym *sy){ + GrAlt *A=&G->r[ri].a[ai]; + if(A->n==A->cap){ int nc=A->cap?A->cap*2:8; + GrSym *ns=(GrSym*)realloc(A->s,(size_t)nc*sizeof(GrSym)); if(!ns) return -1; + A->s=ns; A->cap=nc; } + A->s[A->n++]=*sy; return 0; +} +static int gr__rule(Grammar *G, const char *name, int len){ + if(len>63) len=63; + for(int i=0;in;i++) + if((int)strlen(G->r[i].name)==len && !memcmp(G->r[i].name,name,(size_t)len)) return i; + if(G->n>=GR_MAX_RULES) return -1; + GrRule *R=&G->r[G->n]; memset(R,0,sizeof *R); + memcpy(R->name,name,(size_t)len); + return G->n++; +} +static int gr__anon(Grammar *G){ /* regola sintetica ($n non collide: '$' non e' un identificatore */ + if(G->n>=GR_MAX_RULES) return -1; + GrRule *R=&G->r[G->n]; memset(R,0,sizeof *R); + snprintf(R->name,sizeof R->name,"$%d",G->n); + return G->n++; +} + +/* ---------- parser GBNF ---------- */ + +static const char* gr__ws(const char *p){ + for(;;){ + while(*p==' '||*p=='\t'||*p=='\r'||*p=='\n') p++; + if(*p=='#'){ while(*p && *p!='\n') p++; continue; } + return p; + } +} +static int gr__idch(char c){ + return (c>='a'&&c<='z')||(c>='A'&&c<='Z')||(c>='0'&&c<='9')||c=='_'||c=='-'; +} +static int gr__idlen(const char *p){ int n=0; while(gr__idch(p[n])) n++; return n; } +static int gr__hex(char c){ + if(c>='0'&&c<='9') return c-'0'; + if(c>='a'&&c<='f') return c-'a'+10; + if(c>='A'&&c<='F') return c-'A'+10; + return -1; +} +static int gr__esc(const char **pp){ /* dopo la barra: byte 0-255 o -1 */ + const char *p=*pp; int c=-1; + switch(*p){ + case 'n': c='\n'; break; case 'r': c='\r'; break; case 't': c='\t'; break; + case '"': c='"'; break; case '\\':c='\\'; break; + case '[': c='['; break; case ']': c=']'; break; + case '-': c='-'; break; case '^': c='^'; break; + case 'x': { int h=gr__hex(p[1]), l=gr__hex(p[2]); + if(h>=0&&l>=0){ c=h*16+l; p+=2; } break; } + default: return -1; + } + if(c<0) return -1; + *pp=p+1; return c; +} +static int gr__lit(Grammar *G, int ri, int ai, const char **pp){ + const char *p=*pp+1; + while(*p && *p!='"'){ + int b; + if(*p=='\\'){ p++; b=gr__esc(&p); + if(b<0){ snprintf(G->err,sizeof G->err,"escape non valido nel letterale"); return -1; } } + else b=(unsigned char)*p++; + GrSym s; memset(&s,0,sizeof s); s.t=GR_CLS; s.c.bits[b>>3]|=(uint8_t)(1u<<(b&7)); + if(gr__push(G,ri,ai,&s)){ snprintf(G->err,sizeof G->err,"memoria esaurita"); return -1; } + } + if(*p!='"'){ snprintf(G->err,sizeof G->err,"letterale non chiuso"); return -1; } + *pp=p+1; return 0; +} +static int gr__cls(Grammar *G, int ri, int ai, const char **pp){ + const char *p=*pp+1; int neg=0; + GrSym s; memset(&s,0,sizeof s); s.t=GR_CLS; + if(*p=='^'){ neg=1; p++; } + while(*p && *p!=']'){ + int lo, hi; + if(*p=='\\'){ p++; lo=gr__esc(&p); + if(lo<0){ snprintf(G->err,sizeof G->err,"escape non valido nella classe"); return -1; } } + else lo=(unsigned char)*p++; + hi=lo; + if(*p=='-' && p[1] && p[1]!=']'){ + p++; + if(*p=='\\'){ p++; hi=gr__esc(&p); + if(hi<0){ snprintf(G->err,sizeof G->err,"escape non valido nella classe"); return -1; } } + else hi=(unsigned char)*p++; + } + if(hi>3]|=(uint8_t)(1u<<(b&7)); + } + if(*p!=']'){ snprintf(G->err,sizeof G->err,"classe non chiusa"); return -1; } + if(neg) for(int i=0;i<32;i++) s.c.bits[i]=(uint8_t)~s.c.bits[i]; + *pp=p+1; + if(gr__push(G,ri,ai,&s)){ snprintf(G->err,sizeof G->err,"memoria esaurita"); return -1; } + return 0; +} +/* postfisso ? * + sull'ITEM appena letto (simboli [n0, n) dell'alternate corrente). + * L'item diventa una regola anonima I; poi: ? -> R ::= I | "" + * * -> R ::= I R | "" + * + -> R ::= I R | I */ +static int gr__postfix(Grammar *G, int ri, int ai, int n0, char op){ + int k=G->r[ri].a[ai].n-n0; + if(k<=0) return 0; /* postfisso su "" : no-op */ + int ii=gr__anon(G); if(ii<0) goto full; + int ia=gr__alt_new(G,ii); if(ia<0) goto full; + for(int j=0;jr[ri].a[ai].s[n0+j])) goto full; + G->r[ri].a[ai].n=n0; + int rr=gr__anon(G); if(rr<0) goto full; + GrSym I; memset(&I,0,sizeof I); I.t=GR_REF; I.ref=(int16_t)ii; + GrSym R; memset(&R,0,sizeof R); R.t=GR_REF; R.ref=(int16_t)rr; + int a0=gr__alt_new(G,rr); if(a0<0) goto full; + if(gr__push(G,rr,a0,&I)) goto full; + if(op=='*'||op=='+') if(gr__push(G,rr,a0,&R)) goto full; + int a1=gr__alt_new(G,rr); if(a1<0) goto full; /* "" per ? e *, I per + */ + if(op=='+') if(gr__push(G,rr,a1,&I)) goto full; + if(gr__push(G,ri,ai,&R)) goto full; /* l'item nell'alternate diventa R */ + return 0; +full: + snprintf(G->err,sizeof G->err,"grammatica troppo grande"); + return -1; +} +static int gr__alts(Grammar *G, int ri, const char **pp, int depth, int in_group){ + if(depth>32){ snprintf(G->err,sizeof G->err,"gruppi troppo annidati"); return -1; } + const char *p=*pp; + int ai=gr__alt_new(G,ri); + if(ai<0){ snprintf(G->err,sizeof G->err,"memoria esaurita"); return -1; } + for(;;){ + p=gr__ws(p); + if(!*p){ + if(in_group){ snprintf(G->err,sizeof G->err,"manca ')'"); return -1; } + break; + } + if(*p==')'){ + if(!in_group){ snprintf(G->err,sizeof G->err,"')' inatteso"); return -1; } + break; + } + if(*p=='|'){ + p++; + ai=gr__alt_new(G,ri); + if(ai<0){ snprintf(G->err,sizeof G->err,"memoria esaurita"); return -1; } + continue; + } + int n0=G->r[ri].a[ai].n; + if(*p=='"'){ + if(gr__lit(G,ri,ai,&p)) return -1; + } else if(*p=='['){ + if(gr__cls(G,ri,ai,&p)) return -1; + } else if(*p=='('){ + p++; + int gi=gr__anon(G); + if(gi<0){ snprintf(G->err,sizeof G->err,"grammatica troppo grande"); return -1; } + if(gr__alts(G,gi,&p,depth+1,1)) return -1; + p=gr__ws(p); + if(*p!=')'){ snprintf(G->err,sizeof G->err,"manca ')'"); return -1; } + p++; + GrSym s; memset(&s,0,sizeof s); s.t=GR_REF; s.ref=(int16_t)gi; + if(gr__push(G,ri,ai,&s)){ snprintf(G->err,sizeof G->err,"memoria esaurita"); return -1; } + } else if(gr__idch(*p)){ + int nl=gr__idlen(p); + const char *after=gr__ws(p+nl); + if(!in_group && !strncmp(after,"::=",3)) break; /* inizia la prossima regola */ + int ref=gr__rule(G,p,nl); + if(ref<0){ snprintf(G->err,sizeof G->err,"troppe regole"); return -1; } + p+=nl; + GrSym s; memset(&s,0,sizeof s); s.t=GR_REF; s.ref=(int16_t)ref; + if(gr__push(G,ri,ai,&s)){ snprintf(G->err,sizeof G->err,"memoria esaurita"); return -1; } + } else { + snprintf(G->err,sizeof G->err,"carattere inatteso '%c'",*p); return -1; + } + p=gr__ws(p); + if(*p=='?'||*p=='*'||*p=='+'){ if(gr__postfix(G,ri,ai,n0,*p)) return -1; p++; } + } + *pp=p; + return 0; +} +/* parse del testo GBNF. 0 = ok; -1 = errore (messaggio in G->err). */ +static int gr_parse(Grammar *G, const char *src){ + memset(G,0,sizeof *G); G->root=-1; + const char *p=src; + for(;;){ + p=gr__ws(p); + if(!*p) break; + int nl=gr__idlen(p); + if(nl<=0){ snprintf(G->err,sizeof G->err,"attesa una regola, trovato '%c'",*p); return -1; } + const char *name=p; + const char *q=gr__ws(p+nl); + if(strncmp(q,"::=",3)){ snprintf(G->err,sizeof G->err,"atteso '::=' dopo '%.*s'",nl,name); return -1; } + p=q+3; + int ri=gr__rule(G,name,nl); + if(ri<0){ snprintf(G->err,sizeof G->err,"troppe regole"); return -1; } + if(G->r[ri].n>0){ snprintf(G->err,sizeof G->err,"regola '%.*s' duplicata",nl,name); return -1; } + if(gr__alts(G,ri,&p,0,0)) return -1; + } + for(int i=0;in;i++){ + if(!strcmp(G->r[i].name,"root")) G->root=i; + if(G->r[i].n==0){ snprintf(G->err,sizeof G->err,"regola '%s' usata ma mai definita",G->r[i].name); return -1; } + } + if(G->root<0){ snprintf(G->err,sizeof G->err,"manca la regola 'root'"); return -1; } + return 0; +} +static void gr_free(Grammar *G){ + for(int i=0;in;i++){ + for(int a=0;ar[i].n;a++) free(G->r[i].a[a].s); + free(G->r[i].a); + } + G->n=0; +} + +/* ---------- walker (PDA a insieme di stack) ---------- */ + +static int gr__set_add(GrState *S, const GrStack *k){ + for(int i=0;in;i++) + if(S->st[i].n==k->n && !memcmp(S->st[i].f,k->f,(size_t)k->n*sizeof(GrFrame))) return 1; + if(S->n>=GR_MAX_STACKS) return 0; /* troppa ambiguita': fail-safe */ + S->st[S->n++]=*k; return 1; +} +/* porta lo stack in forma normale (cima = terminale, o stack vuoto = parse completo), + * diramando sugli alternate delle regole referenziate. 0 = overflow (fail-safe). */ +static int gr__normalize(Grammar *G, GrStack *k, GrState *out, int depth){ + for(;;){ + if(k->n==0) return gr__set_add(out,k); + GrFrame *t=&k->f[k->n-1]; + GrAlt *A=&G->r[t->r].a[t->a]; + if(t->s>=A->n){ k->n--; continue; } /* alternate esaurito: pop */ + GrSym *sy=&A->s[t->s]; + if(sy->t==GR_CLS) return gr__set_add(out,k); + if(depth>=GR_MAX_DEPTH) return 0; /* ricorsione sinistra / epsilon-ciclo */ + t->s++; /* il chiamante riprende OLTRE il ref */ + GrRule *C=&G->r[sy->ref]; + for(int a=0;an;a++){ + if(k->n>=GR_MAX_DEPTH) return 0; + GrStack cp=*k; + cp.f[cp.n].r=sy->ref; cp.f[cp.n].a=(int16_t)a; cp.f[cp.n].s=0; cp.n++; + if(!gr__normalize(G,&cp,out,depth+1)) return 0; + } + return 1; + } +} +static void gr_state_init(GrState *S, Grammar *G){ + S->G=G; S->n=0; S->alive=1; + GrRule *R=&G->r[G->root]; + for(int a=0;an;a++){ + GrStack k; k.n=1; + k.f[0].r=(int16_t)G->root; k.f[0].a=(int16_t)a; k.f[0].s=0; + if(!gr__normalize(G,&k,S,0)){ S->alive=0; return; } + } + if(S->n==0) S->alive=0; +} +/* avanza di un byte. 1 = consumato; 0 = byte non ammesso (stato INVARIATO); + * -1 = walker spento (overflow: da qui in poi niente piu' draft). */ +static int gr_accept(GrState *S, unsigned char b){ + if(!S->alive) return -1; + GrState out; out.G=S->G; out.n=0; out.alive=1; + for(int i=0;in;i++){ + GrStack *k=&S->st[i]; + if(k->n==0) continue; /* parse gia' completo: non consuma */ + GrFrame *t=&k->f[k->n-1]; + GrSym *sy=&S->G->r[t->r].a[t->a].s[t->s]; + if(!(sy->c.bits[b>>3]&(1u<<(b&7)))) continue; + GrStack cp=*k; cp.f[cp.n-1].s++; + if(!gr__normalize(S->G,&cp,&out,0)){ S->alive=0; return -1; } + } + if(out.n==0) return 0; + S->n=out.n; + memcpy(S->st,out.st,(size_t)out.n*sizeof(GrStack)); + return 1; +} +/* insieme dei byte ammessi adesso (bitmap 256). Ritorna il conteggio; + * *can_end = 1 se il parse puo' terminare qui (quindi il modello puo' emettere EOS). */ +static int gr_admissible(const GrState *S, unsigned char mask[32], int *can_end){ + memset(mask,0,32); int end=0; + for(int i=0;in;i++){ + const GrStack *k=&S->st[i]; + if(k->n==0){ end=1; continue; } + const GrFrame *t=&k->f[k->n-1]; + const GrSym *sy=&S->G->r[t->r].a[t->a].s[t->s]; + for(int j=0;j<32;j++) mask[j]|=sy->c.bits[j]; + } + int cnt=0; + for(int j=0;j<32;j++){ unsigned v=mask[j]; while(v){ cnt+=v&1; v>>=1; } } + if(can_end)*can_end=end; + return cnt; +} +/* prefisso FORZATO: si estende finche' c'e' UN SOLO byte legale e il parse non e' + * terminabile (li' il modello potrebbe fermarsi). Non muta S. Ritorna i byte scritti. */ +static int gr_forced(const GrState *S, char *out, int max){ + if(!S->alive||S->n==0) return 0; + GrState cp=*S; + int n=0; + while(n>3]&(1u<<(b&7)))) b++; + if(b>=256 || gr_accept(&cp,(unsigned char)b)!=1) break; + out[n++]=(char)b; + } + return n; +} + +#endif /* COLI_GRAMMAR_H */ diff --git a/c/tests/test_grammar.c b/c/tests/test_grammar.c new file mode 100644 index 0000000..6603f74 --- /dev/null +++ b/c/tests/test_grammar.c @@ -0,0 +1,148 @@ +#include +#include + +#include "../grammar.h" + +#define CHECK(condition) do { \ + if (!(condition)) { \ + fprintf(stderr, "%s:%d: check failed: %s\n", __FILE__, __LINE__, #condition); \ + return 1; \ + } \ +} while (0) + +/* alimenta il walker con una stringa; ritorna quanti byte sono stati consumati */ +static int feed(GrState *S, const char *s){ + int n=0; + while(s[n]){ if(gr_accept(S,(unsigned char)s[n])!=1) break; n++; } + return n; +} + +int main(void){ + static Grammar G; + GrState S; + char buf[512]; + + /* letterale: tutto il prefisso e' forzato, poi il parse termina */ + CHECK(gr_parse(&G,"root ::= \"{\\\"id\\\":\"")==0); + gr_state_init(&S,&G); + CHECK(S.alive); + CHECK(gr_forced(&S,buf,sizeof buf)==6); + CHECK(!memcmp(buf,"{\"id\":",6)); + CHECK(feed(&S,"{\"id\":")==6); + unsigned char m[32]; int end; + CHECK(gr_admissible(&S,m,&end)==0 && end==1); /* parse completo: solo EOS */ + gr_free(&G); + + /* alternate: il forcing si ferma alla diramazione e riparte dopo */ + CHECK(gr_parse(&G,"root ::= \"a\" (\"b\" | \"c\") \"d\"")==0); + gr_state_init(&S,&G); + CHECK(gr_forced(&S,buf,sizeof buf)==1 && buf[0]=='a'); + CHECK(feed(&S,"ab")==2); + CHECK(gr_forced(&S,buf,sizeof buf)==1 && buf[0]=='d'); + gr_free(&G); + + /* enum stile #48: forzata la virgoletta, poi dal primo byte l'intero valore */ + CHECK(gr_parse(&G, + "root ::= \"\\\"\" val \"\\\"\"\n" + "val ::= \"no_fit\" | \"partial_fit\" | \"good_fit\"")==0); + gr_state_init(&S,&G); + CHECK(gr_forced(&S,buf,sizeof buf)==1 && buf[0]=='\"'); + CHECK(feed(&S,"\"n")==2); + CHECK(gr_forced(&S,buf,sizeof buf)==6 && !memcmp(buf,"o_fit\"",6)); + gr_free(&G); + + /* classi con range, star: niente forcing dove la grammatica dirama */ + CHECK(gr_parse(&G,"root ::= \"a\" [0-9]* \"b\"")==0); + gr_state_init(&S,&G); + CHECK(feed(&S,"a")==1); + CHECK(gr_admissible(&S,m,&end)==11 && end==0); /* 0-9 oppure b */ + CHECK(gr_forced(&S,buf,sizeof buf)==0); + CHECK(feed(&S,"42b")==3); + CHECK(gr_admissible(&S,m,&end)==0 && end==1); + gr_free(&G); + + /* plus su gruppo: il forcing si ferma dove il parse e' terminabile */ + CHECK(gr_parse(&G,"root ::= (\"x\" \"\\n\")+")==0); + gr_state_init(&S,&G); + CHECK(gr_forced(&S,buf,sizeof buf)==2 && buf[0]=='x' && buf[1]=='\n'); + CHECK(feed(&S,"x\n")==2); + CHECK(gr_admissible(&S,m,&end)==1 && end==1); /* puo' chiudere o aprire una riga */ + CHECK(gr_forced(&S,buf,sizeof buf)==0); + gr_free(&G); + + /* postfisso su letterale multi-byte: ripete l'INTERO letterale */ + CHECK(gr_parse(&G,"root ::= \"ab\"+ \"c\"")==0); + gr_state_init(&S,&G); + CHECK(gr_forced(&S,buf,sizeof buf)==2 && !memcmp(buf,"ab",2)); + CHECK(feed(&S,"ab")==2); + CHECK(gr_admissible(&S,m,&end)==2 && end==0); /* 'a' (ripete) o 'c' (chiude) */ + CHECK(feed(&S,"abc")==3); + CHECK(gr_admissible(&S,m,&end)==0 && end==1); + gr_free(&G); + + /* classe negata: l'unione con la chiusura copre tutti i byte -> nessun forcing */ + CHECK(gr_parse(&G,"root ::= \"\\\"\" [^\"]* \"\\\"\"")==0); + gr_state_init(&S,&G); + CHECK(feed(&S,"\"")==1); + CHECK(gr_admissible(&S,m,&end)==256 && end==0); + CHECK(feed(&S,"ciao \\ mondo\"")==13); + CHECK(gr_admissible(&S,m,&end)==0 && end==1); + gr_free(&G); + + /* desync: byte non ammesso NON muta lo stato */ + CHECK(gr_parse(&G,"root ::= \"ab\"")==0); + gr_state_init(&S,&G); + CHECK(gr_accept(&S,'x')==0); + CHECK(gr_accept(&S,'a')==1 && gr_accept(&S,'b')==1); + gr_free(&G); + + /* escape esadecimale e commenti; regole su piu' righe */ + CHECK(gr_parse(&G, + "# grammatica di prova\n" + "root ::= \"\\x41\" # una A\n" + " [\\x30-\\x32]\n")==0); + gr_state_init(&S,&G); + CHECK(gr_forced(&S,buf,sizeof buf)==1 && buf[0]=='A'); + CHECK(feed(&S,"A1")==2); + CHECK(gr_admissible(&S,m,&end)==0 && end==1); + gr_free(&G); + + /* errori di parse: regola indefinita, root mancante, ')' fuori posto */ + CHECK(gr_parse(&G,"root ::= foo")!=0); + CHECK(gr_parse(&G,"a ::= \"x\"")!=0); + CHECK(gr_parse(&G,"root ::= \"x\" )")!=0); + + /* ricorsione sinistra: parse ok, walker si spegne senza bloccare (fail-safe) */ + CHECK(gr_parse(&G,"root ::= root \"a\" | \"b\"")==0); + gr_state_init(&S,&G); + CHECK(!S.alive); + CHECK(gr_forced(&S,buf,sizeof buf)==0); + gr_free(&G); + + /* grammatica NDJSON realistica (il workload di #48): span forzati lunghi */ + CHECK(gr_parse(&G, + "root ::= riga+\n" + "riga ::= \"{\\\"id\\\":\\\"\" chiave \"\\\",\\\"fit_category\\\":\\\"\" cat \"\\\"}\" \"\\n\"\n" + "chiave ::= [a-z0-9-]+\n" + "cat ::= \"no_fit\" | \"partial_fit\" | \"good_fit\"\n")==0); + gr_state_init(&S,&G); + CHECK(gr_forced(&S,buf,sizeof buf)==7 && !memcmp(buf,"{\"id\":\"",7)); + CHECK(feed(&S,"{\"id\":\"ocds-123\"")==16); + int nf=gr_forced(&S,buf,sizeof buf); + buf[nf]=0; + CHECK(nf==17 && !strcmp(buf,",\"fit_category\":\"")); + CHECK(feed(&S,",\"fit_category\":\"p")==18); + nf=gr_forced(&S,buf,sizeof buf); buf[nf]=0; + CHECK(nf==13 && !strcmp(buf,"artial_fit\"}\n")); + CHECK(feed(&S,"artial_fit\"}\n")==13); + /* a fine riga il parse e' terminabile (riga+): niente forcing — il modello puo' + * fermarsi qui. Il forcing riparte appena il modello apre la riga successiva. */ + CHECK(gr_admissible(&S,m,&end)==1 && end==1); + CHECK(gr_forced(&S,buf,sizeof buf)==0); + CHECK(feed(&S,"{")==1); + CHECK(gr_forced(&S,buf,sizeof buf)==6 && !memcmp(buf,"\"id\":\"",6)); + gr_free(&G); + + puts("test_grammar: ok"); + return 0; +}