danielkrause/colibri-strix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
97c756a06472b2f632453f35a541b2f0553bec00
colibri-strix/c/glm.c
T
ZacharyZcR cbd599024e Unify continuous batching + heterogeneous runtime: decode batching, physical-core planning, disjoint VRAM/RAM placement, topp-policy warning (CPU-validated, CUDA on 6x5090) (#68) 
* Fuse CUDA expert MLP execution

* Group CUDA expert transfers by device

* Instrument grouped CUDA expert execution

* Bound grouped CUDA decode scratch

* Execute expert groups across GPUs in parallel

* Release host backing for multi-GPU experts

* Define quality-preserving memory policies

* Overlap cold expert loading with resident compute

* Adapt expert placement with session LFRU

* Fuse q4 expert gate and up dispatch

* Plan CPU work on physical cores

* Batch grouped expert CUDA kernels

* Separate VRAM and RAM expert placement

* Add ragged multi-sequence decode forward

* feat(runtime): add continuous decode scheduler

* Route concurrent API requests through batch scheduler

* Harden multiplex request lifecycle and framing

* Cancel disconnected multiplex requests

* Bind API port before starting the engine

* fix automatic KV slot allocation

* add native int4 Tensor Core grouped GEMM

* add Tensor Core throughput benchmark

* optimize packed int4 low-row kernels

* add asynchronous CUDA staging streams

* document validated six-GPU dense acceleration

* tune six-GPU expert hot set

* raise validated expert hot-set target

* add CUDA MLA absorption core

* fuse grouped expert gate and up projections

* Warn for explicit lossy routing flags
2026-07-13 14:30:36 +02:00

3776 lines
212 KiB
C
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
/* Motore GLM-5.2 (architettura glm_moe_dsa) in C puro.
* Stadio B: replica fedele del forward di transformers (modeling_glm_moe_dsa.py):
* - attenzione MLA (q/kv-LoRA, RoPE interleaved parziale)
* - router sigmoid + noaux_tc (n_group=1) con routed_scaling_factor
* - shared expert + expert routed in streaming dal disco (per-expert)
* - primi first_k_dense_replace layer densi
* Il DSA indexer e' un NO-OP per seq <= index_topk (seleziona tutte le key): qui si usa
* attenzione causale densa -> output identico all'oracolo su prompt corti.
*
* QUANTIZZAZIONE: gli expert (streaming) e la parte DENSA residente (attenzione, lm_head,
* embed, mlp densa, shared expert) sono tenuti in int8 per-riga + scala (dequant-on-use).
* E' cio' che fa entrare GLM-5.2 nei 15 GB: ~17B param residenti a int4 ~= 8.7 GB.
* Norme/router/bias restano f32 (piccoli e sensibili).
*
* Validazione: stessi token id di ref_glm.json (oracolo transformers, c/tools/make_glm_oracle.py).
* build: make glm run: SNAP=./glm_tiny ./glm <cap> <expert_bits> <dense_bits>
* TF=1 -> teacher-forcing (valida il prefill su tutta la sequenza)
*/
#define _GNU_SOURCE
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <time.h>
#include <limits.h>
#include <pthread.h> /* thread I/O del PILOTA */
#include <stdatomic.h> /* PIPE ready-flags/job queue + PILOT_REAL cross-layer handshake */
#include <sched.h> /* sched_yield: PIPE spin / PILOT barrier */
#include <unistd.h>
#include <sys/select.h>
#if defined(__APPLE__) || defined(__linux__)
#include <sys/resource.h>
#include <sys/mman.h> /* mlock: inchioda le pagine in RAM / wire pages into RAM */
#include <sys/stat.h> /* fstat per mmap degli shard (COLI_MMAP) */
#endif
#include "st.h"
#include "tok.h"
#include "tier.h"
#include "grammar.h" /* metodo F: draft grammaticali (#48) */
#include "decode_batch.h"
#ifdef _OPENMP
#include <omp.h> /* scratch per-thread nell'attention */
#else
static inline int omp_get_max_threads(void){ return 1; }
static inline int omp_get_thread_num(void){ return 0; }
#endif
#ifdef COLI_CUDA
#include "backend_cuda.h"
#endif
#ifdef COLI_METAL
#include "backend_metal.h"
#include <omp.h>
static int g_metal_enabled;
static int g_metal_gemm_min=16; /* COLI_METAL_GEMM_MIN: min rows to send a matmul_qt GEMM to GPU */
/* routing precalcolata dalla GPU (layer CB): moe() la usa e salta la FASE A */
static const int *g_pre_idx; static const float *g_pre_w; static const int *g_pre_keff;
static const float *g_pre_sh; /* output dello shared expert gia' calcolato su GPU */
#endif
#ifdef __AVX2__
#include <immintrin.h>
static inline float hsum256(__m256 v){ /* somma orizzontale di 8 float */
__m128 lo=_mm256_castps256_ps128(v), hi=_mm256_extractf128_ps(v,1);
lo=_mm_add_ps(lo,hi); __m128 sh=_mm_movehl_ps(lo,lo); lo=_mm_add_ps(lo,sh);
sh=_mm_shuffle_ps(lo,lo,1); lo=_mm_add_ss(lo,sh); return _mm_cvtss_f32(lo);
}
#elif defined(__ARM_NEON)
#include <arm_neon.h> /* Apple Silicon / aarch64: kernel NEON */
#elif defined(__VSX__)
#include <altivec.h> /* POWER8+ (ppc64le): kernel VSX */
#undef vector /* igiene: si usano __vector/__bool espliciti */
#undef pixel
#undef bool
#endif
#ifdef __APPLE__
#include <mach/mach.h> /* host_statistics64: MemAvailable di macOS */
#endif
typedef struct {
int hidden, n_layers, n_heads, n_experts, topk, moe_inter, dense_inter;
int first_dense, q_lora, kv_lora, qk_nope, qk_rope, qk_head, v_head, n_shared, vocab;
int n_group, topk_group, norm_topk;
int stop_ids[8], n_stop; /* eos_token_id dal config (GLM-5.2 ne ha 3!) */
int index_topk, index_nh, index_hd; /* DSA lightning indexer */
int8_t idx_type[128]; /* per layer: 1=full (calcola), 0=shared (riusa) */
float eps, theta, attn_scale, routed_scale;
} Cfg;
/* tensore [O,I] in uno di tre formati:
* fmt=0 F32 -> qf
* fmt=1 INT8 -> q8 (1 byte/param) + scala per riga
* fmt=2 INT4 -> q4 (2 valori per byte, impacchettati) + scala per riga
* INT4 e' cio' che fa stare la densa residente nei 15 GB (0.5 byte/param). */
/* fmt: 0 F32, 1 INT8, 2 INT4 (2/byte), 3 INT2 (4/byte). q4 ospita sia int4 che int2 packed. */
typedef struct {
int fmt; float *qf; int8_t *q8; uint8_t *q4; float *s; int O, I;
#ifdef COLI_CUDA
ColiCudaTensor *cuda;
#endif
int cuda_eligible, cuda_failed, cuda_device; /* resident tensor, never a reused expert slot */
} QT;
static int64_t qt_bytes(const QT *t){ /* byte residenti del tensore */
int64_t n=(int64_t)t->O*t->I;
if(t->fmt==0) return n*4;
if(t->fmt==1) return n + (int64_t)t->O*4;
if(t->fmt==3) return (int64_t)t->O*((t->I+3)/4) + (int64_t)t->O*4;
return (int64_t)t->O*((t->I+1)/2) + (int64_t)t->O*4;
}
typedef struct {
float *in_ln, *post_ln;
/* MLA (densa, quantizzata) */
QT q_a, q_b, kv_a, kv_b, o; float *q_a_ln, *kv_a_ln;
int sparse;
/* dense mlp (sparse==0) */
QT gate_proj, up_proj, down_proj;
/* moe (sparse==1) */
float *router, *router_bias; /* router f32 (sensibile) */
QT sh_gate, sh_up, sh_down; /* shared expert */
} Layer;
/* slot di un expert: pesi quantizzati + scale. Nel container pre-quantizzato g/u/d sono
* VISTE dentro `slab` (una sola pread coalescente); nel fallback hanno buffer propri.
* slab_cap/fslab_cap: capienza allocata — gli slot ws[] sono riusati TRA layer e gli
* expert non hanno tutti la stessa taglia (layer MTP int8 = 2x i layer int4). */
typedef struct { int eid; QT g,u,d; uint8_t *slab; float *fslab;
int64_t slab_cap, fslab_cap; uint64_t used; } ESlot;
typedef struct {
float **Lc, **Rc, **Ic;
int *kv_start, max_t;
int disk_nrec;
char disk_path[2048];
} KVState;
typedef struct {
KVState *kv;
int token, pos;
} DecodeRow;
typedef struct {
Cfg c; shards S;
int ebits, dbits; /* bit expert / bit densa */
QT embed, lm_head; float *final_norm;
Layer *L;
/* KV-cache MLA COMPRESSA: per token si tiene solo il latente normato [kv_lora] e
* k_rot [qk_rope] (576 vs 32768 valori/token). k_nope e value si ricostruiscono al
* volo con kv_b. E' cio' che rende gestibile il contesto su 15 GB (64 teste, no GQA). */
float **Lc, **Rc; int max_t; /* alias della KVState attiva */
int *kv_start; /* prima pos valida nella KV del layer (MTP: parziale) */
KVState *kv;
ESlot **ecache; int *ecn; int ecap; /* LRU expert per-layer */
ESlot ws[64]; /* working set del layer corrente (load paralleli) */
ESlot **pin; int *npin; /* HOT-STORE: expert pinnati in RAM (mai evicted) */
uint32_t **eusage; /* contatori persistenti (per STATS/PIN) */
uint32_t **eheat; /* calore recente per promotion/demotion live */
uint32_t **elast, eaccess_clock; /* recency per LFRU session-local */
/* DSA lightning indexer (attivo solo se i pesi out-idx-* sono presenti) */
int has_dsa;
QT *ix_wq, *ix_wk, *ix_wp; /* per layer FULL: wq_b, wk, weights_proj */
float **ix_knw, **ix_knb; /* k_norm (LayerNorm, eps 1e-6) */
float **Ic; /* alias KVState: cache indexer [max_t*hd] */
int *dsa_sel, *dsa_nsel; int dsa_scap; /* selezione per posizione del batch corrente */
/* testa MTP (layer n_layers, stile DeepSeek-V3): draft nativi ad alta acceptance */
int has_mtp; Layer mtpL; QT eh_proj;
float *enorm, *hnorm, *mtp_norm;
float *hlast, *h_all; /* hidden pre-norm: ultima pos / tutte le pos batch */
uint64_t mtp_prop, mtp_acc; /* statistica acceptance */
int **eroute; int *enr; /* metodo C: routing dell'ULTIMO token per layer */
uint64_t eclock, hits, miss, ereq;
uint64_t gpu_expert_calls; int gpu_expert_count; int64_t gpu_expert_bytes;
uint64_t n_fw, n_emit; /* metodo E: forward di decode / token emessi */
double t_edisk, t_ewait, t_emm, t_attn, t_kvb, t_head;/* profiling: dove va il tempo */
double t_aproj,t_acore,t_aout; /* attention breakdown */
int64_t resident_bytes;
} Model;
static void usage_save(Model *m); /* cache che impara: definita accanto a stats_dump */
#ifdef COLI_CUDA
static int g_cuda_enabled;
static double g_cuda_expert_gb;
static int g_cuda_dense;
static int g_cuda_release_host;
static int g_cuda_devices[COLI_CUDA_MAX_DEVICES], g_cuda_ndev, g_cuda_rr;
static int64_t g_cuda_dense_projected[COLI_CUDA_MAX_DEVICES];
static void qt_cuda_reset(QT *t){
if(t->cuda){ coli_cuda_tensor_free(t->cuda); t->cuda=NULL; }
t->cuda_failed=0;
}
static int qt_cuda_upload(QT *t){
const void *weights = t->fmt==0 ? (const void*)t->qf
: t->fmt==1 ? (const void*)t->q8 : (const void*)t->q4;
return coli_cuda_tensor_upload(&t->cuda,weights,t->s,t->fmt,t->I,t->O,t->cuda_device);
}
static void cuda_stats_print(void){
size_t n=0,b=0; coli_cuda_stats(-1,&n,&b);
fprintf(stderr,"[CUDA] resident set: %zu tensors, %.2f GB VRAM\n",n,b/1e9);
if(g_cuda_ndev>1) for(int i=0;i<g_cuda_ndev;i++){
coli_cuda_stats(g_cuda_devices[i],&n,&b);
fprintf(stderr,"[CUDA] device %d: %zu tensors, %.2f GB\n",g_cuda_devices[i],n,b/1e9);
}
uint64_t calls=0,experts=0,rows=0; double h2d=0,kernel=0,d2h=0;
coli_cuda_group_stats(&calls,&experts,&rows,&h2d,&kernel,&d2h);
if(calls) fprintf(stderr,"[CUDA] expert groups: %llu call, %llu expert, %llu righe "
"(%.2f expert/call)%s\n",(unsigned long long)calls,(unsigned long long)experts,
(unsigned long long)rows,(double)experts/calls,
getenv("COLI_CUDA_PROFILE")?"; timing sotto":"");
if(calls&&getenv("COLI_CUDA_PROFILE")) fprintf(stderr,
"[CUDA] expert groups timing: H2D %.1f ms | kernel %.1f ms | D2H %.1f ms\n",h2d,kernel,d2h);
}
static int parse_cuda_devices(const char *list, int *out){
if(!list||!*list) return 0;
int n=0; const char *p=list;
while(*p){
char *end=NULL; long v=strtol(p,&end,10);
if(end==p||v<0||v>INT_MAX||n>=COLI_CUDA_MAX_DEVICES) return 0;
for(int i=0;i<n;i++) if(out[i]==(int)v) return 0;
out[n++]=(int)v; p=end;
while(*p==' '||*p=='\t') p++;
if(!*p) break;
if(*p++!=',') return 0;
while(*p==' '||*p=='\t') p++;
if(!*p) return 0;
}
return n;
}
#endif
static double now_s(void){ struct timespec t; clock_gettime(CLOCK_MONOTONIC,&t); return t.tv_sec+t.tv_nsec*1e-9; }
static double rss_gb(void){ struct rusage r; getrusage(RUSAGE_SELF,&r);
#ifdef __APPLE__
return r.ru_maxrss/(1024.0*1024.0*1024.0); /* macOS: ru_maxrss in BYTE */
#else
return r.ru_maxrss/(1024.0*1024.0); /* Linux: in KB */
#endif
}
static float *falloc(int64_t n){
/* guardia anti-wrap (report PR #25): n assurdo da file modello ostili non deve
* diventare una malloc piccola. Niente calloc: il memset nel percorso caldo costa. */
if(n<0 || (uint64_t)n > SIZE_MAX/sizeof(float)){ fprintf(stderr,"falloc: n=%lld is out of range\n",(long long)n); exit(1); }
float *p=malloc((size_t)n*sizeof(float)); if(!p){fprintf(stderr,"OOM\n");exit(1);} return p; }
/* y[S,O] = x[S,I] @ W^T, W[O,I] f32 */
static void matmul(float *y, const float *x, const float *W, int S, int I, int O){
#pragma omp parallel for schedule(static)
for (int o=0;o<O;o++){ const float *w=W+(int64_t)o*I;
for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; for(int i=0;i<I;i++) a+=xs[i]*w[i]; y[(int64_t)s*O+o]=a; } }
}
/* y[S,O] = x[S,I] @ W^T con W quantizzato int8 per-riga + scala[O] (dequant-on-use) */
static void matmul_q(float *y, const float *x, const int8_t *q, const float *scale, int S, int I, int O){
#pragma omp parallel for schedule(static)
for (int o=0;o<O;o++){ const int8_t *w=q+(int64_t)o*I; float sc=scale[o];
for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; int i=0;
#ifdef __AVX2__
__m256 acc=_mm256_setzero_ps();
for(;i+8<=I;i+=8){ __m256i wi=_mm256_cvtepi8_epi32(_mm_loadl_epi64((const __m128i*)(w+i)));
acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), _mm256_cvtepi32_ps(wi), acc); }
a=hsum256(acc);
#elif defined(__ARM_NEON)
float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0);
for(;i+8<=I;i+=8){ int16x8_t w16=vmovl_s8(vld1_s8(w+i));
ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w16))));
ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w16)))); }
a=vaddvq_f32(vaddq_f32(ac0,ac1));
#endif
for(;i<I;i++) a+=xs[i]*(float)w[i]; y[(int64_t)s*O+o]=a*sc; } }
}
/* y[S,O] = x[S,I] @ W^T con W int4 impacchettato (2 valori/byte) + scala[O]. */
static void matmul_i4(float *y, const float *x, const uint8_t *q4, const float *scale, int S, int I, int O){
int rb=(I+1)/2;
#pragma omp parallel for schedule(static)
for (int o=0;o<O;o++){ const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o];
for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; int i=0;
#ifdef __AVX2__
const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8);
__m256 acc=_mm256_setzero_ps();
for(;i+16<=I;i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1))); /* 8 byte=16 nibble */
__m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
__m128i nib=_mm_unpacklo_epi8(lo,hi); /* nibble in ordine */
__m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8));
__m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8));
acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc);
acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); }
a=hsum256(acc);
#elif defined(__ARM_NEON)
const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8);
float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0);
for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1)); /* 8 byte=16 nibble */
uint8x8x2_t z=vzip_u8(vand_u8(by,m4), vshr_n_u8(by,4)); /* nibble in ordine */
int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[0]),b8));
int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(z.val[1]),b8));
ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0))));
ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0))));
ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1))));
ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); }
a=vaddvq_f32(vaddq_f32(ac0,ac1));
#endif
for(;i+1<I;i+=2){ uint8_t byte=w[i>>1]; int lo=(int)(byte&0xF)-8, hi=(int)(byte>>4)-8;
a += xs[i]*(float)lo + xs[i+1]*(float)hi; }
if(i<I){ uint8_t byte=w[i>>1]; int lo=(int)(byte&0xF)-8; a += xs[i]*(float)lo; }
y[(int64_t)s*O+o]=a*sc; } }
}
/* Decode hot path for gate+up: same exact q4 dot products as matmul_i4, but one
* OpenMP dispatch covers both matrices. KTransformers uses persistent pools;
* this keeps colibri dependency-free while removing one team launch/expert. */
static void matmul_i4_pair(float *yg, float *yu, const float *x,
const uint8_t *qg, const float *sg,
const uint8_t *qu, const float *su, int I, int O){
int rb=(I+1)/2;
#pragma omp parallel for schedule(static)
for(int z=0;z<2*O;z++){
int o=z<O?z:z-O; const uint8_t *w=(z<O?qg:qu)+(int64_t)o*rb;
float a=0; int i=0;
#ifdef __AVX2__
const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8);
__m256 acc=_mm256_setzero_ps();
for(;i+16<=I;i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1)));
__m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
__m128i nib=_mm_unpacklo_epi8(lo,hi);
__m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8));
__m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8));
acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i),w0,acc);
acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i+8),w1,acc); }
a=hsum256(acc);
#elif defined(__ARM_NEON)
const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8);
float32x4_t ac0=vdupq_n_f32(0),ac1=vdupq_n_f32(0);
for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1));
uint8x8x2_t n=vzip_u8(vand_u8(by,m4),vshr_n_u8(by,4));
int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[0]),b8));
int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[1]),b8));
ac0=vfmaq_f32(ac0,vld1q_f32(x+i),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0))));
ac1=vfmaq_f32(ac1,vld1q_f32(x+i+4),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0))));
ac0=vfmaq_f32(ac0,vld1q_f32(x+i+8),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1))));
ac1=vfmaq_f32(ac1,vld1q_f32(x+i+12),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); }
a=vaddvq_f32(vaddq_f32(ac0,ac1));
#endif
for(;i+1<I;i+=2){ uint8_t b=w[i>>1]; a+=x[i]*(float)((b&15)-8)+x[i+1]*(float)((b>>4)-8); }
if(i<I) a+=x[i]*(float)((w[i>>1]&15)-8);
(z<O?yg:yu)[o]=a*(z<O?sg:su)[o];
}
}
static void matmul_qt(float *y,const float *x,QT *w,int S);
static void expert_gate_up(float *g,float *u,const float *x,QT *wg,QT *wu,int S){
if(S==1&&wg->fmt==2&&wu->fmt==2&&wg->I==wu->I&&wg->O==wu->O)
matmul_i4_pair(g,u,x,wg->q4,wg->s,wu->q4,wu->s,wg->I,wg->O);
else { matmul_qt(g,x,wg,S); matmul_qt(u,x,wu,S); }
}
/* y[S,O] = x[S,I] @ W^T con W int2 impacchettato (4 valori/byte) + scala[O]. nibble 2-bit -> [-2,1]. */
static void matmul_i2(float *y, const float *x, const uint8_t *q2, const float *scale, int S, int I, int O){
int rb=(I+3)/4;
#pragma omp parallel for schedule(static)
for (int o=0;o<O;o++){ const uint8_t *w=q2+(int64_t)o*rb; float sc=scale[o];
for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; int i=0;
#ifdef __AVX2__
const __m128i m2=_mm_set1_epi8(0x03); const __m256i b2=_mm256_set1_epi32(2);
__m256 acc=_mm256_setzero_ps();
for(;i+16<=I;i+=16){ __m128i by=_mm_cvtsi32_si128(*(const int*)(w+(i>>2))); /* 4 byte=16 valori */
__m128i p0=_mm_and_si128(by,m2), p1=_mm_and_si128(_mm_srli_epi16(by,2),m2);
__m128i p2=_mm_and_si128(_mm_srli_epi16(by,4),m2), p3=_mm_and_si128(_mm_srli_epi16(by,6),m2);
__m128i lo=_mm_unpacklo_epi8(p0,p1), hi=_mm_unpacklo_epi8(p2,p3);
__m128i nib=_mm_unpacklo_epi16(lo,hi); /* 16 valori in ordine */
__m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b2));
__m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b2));
acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i), w0, acc);
acc=_mm256_fmadd_ps(_mm256_loadu_ps(xs+i+8), w1, acc); }
a=hsum256(acc);
#elif defined(__ARM_NEON)
const uint8x8_t m2v=vdup_n_u8(3); const int8x8_t b2v=vdup_n_s8(2);
float32x4_t ac0=vdupq_n_f32(0), ac1=vdupq_n_f32(0);
for(;i+16<=I;i+=16){ uint32_t wd; memcpy(&wd, w+(i>>2), 4); /* 4 byte=16 valori */
uint8x8_t by=vreinterpret_u8_u32(vdup_n_u32(wd));
uint8x8x2_t z01=vzip_u8(vand_u8(by,m2v), vand_u8(vshr_n_u8(by,2),m2v));
uint8x8x2_t z23=vzip_u8(vand_u8(vshr_n_u8(by,4),m2v), vshr_n_u8(by,6));
uint16x4x2_t zz=vzip_u16(vreinterpret_u16_u8(z01.val[0]), vreinterpret_u16_u8(z23.val[0]));
int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[0]),b2v)); /* 16 valori in ordine */
int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u16(zz.val[1]),b2v));
ac0=vfmaq_f32(ac0, vld1q_f32(xs+i), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0))));
ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+4), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0))));
ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1))));
ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); }
a=vaddvq_f32(vaddq_f32(ac0,ac1));
#endif
for(;i<I;i++){ uint8_t byte=w[i>>2]; int sh=(i&3)*2; a += xs[i]*(float)((int)((byte>>sh)&3)-2); }
y[(int64_t)s*O+o]=a*sc; } }
}
/* ---- KERNEL INTERI (IDOT): attivazioni quantizzate a int8 per riga (absmax/127,
* stile Q8_0), prodotto scalare INTERO via maddubs/madd AVX2 — niente conversione
* f32 dei pesi nel ciclo caldo. ~2-3x sui matmul quantizzati; errore aggiunto ~0.3%
* RMS per matmul (attivazione int8), IDOT=0 torna al percorso f32 esatto. */
#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
#define IDOT_KERNEL "avx512-vnni"
#elif defined(__AVX2__)
#define IDOT_KERNEL "avx2"
#elif defined(__ARM_NEON)
#define IDOT_KERNEL "neon"
#elif defined(__VSX__)
#define IDOT_KERNEL "vsx"
#else
#define IDOT_KERNEL "scalar"
#endif
static int g_idot=1;
#if defined(__ARM_NEON) && defined(__ARM_FEATURE_DOTPROD)
static int g_i4s=1; /* SDOT presente: int4 IDOT conviene anche a S=1 (decode). Misurato
* su Apple M-series: +14%%, expert-matmul -16%%. EN: with SDOT, int4
* IDOT pays even at S=1 (decode); measured on Apple M-series. */
#elif defined(__VSX__)
static int g_i4s=1; /* POWER8 vec_msum: qui il fallback f32 e' SCALARE, quindi l'IDOT
* int4 conviene anche a S=1. Misurato su POWER8 S824 (vedi PR).
* EN: on VSX the f32 fallback is plain scalar C, so int4 IDOT
* pays even at S=1. Measured on a POWER8 S824 (see PR). */
#else
static int g_i4s=2; /* senza SDOT / altrove: soglia originale (misura AVX2 dell'autore).
* EN: without SDOT / elsewhere: original threshold (author's AVX2). */
#endif
static inline float qrow_i8(const float *x, int8_t *q, int I){
float amax=0; for(int i=0;i<I;i++){ float a=fabsf(x[i]); if(a>amax)amax=a; }
float s=amax/127.f; if(s<1e-12f) s=1e-12f; float inv=1.f/s;
for(int i=0;i<I;i++) q[i]=(int8_t)lrintf(x[i]*inv);
return s;
}
#ifdef __AVX2__
static inline int hsum256_i32(__m256i v){
__m128i lo=_mm256_castsi256_si128(v), hi=_mm256_extracti128_si256(v,1);
lo=_mm_add_epi32(lo,hi); lo=_mm_hadd_epi32(lo,lo); lo=_mm_hadd_epi32(lo,lo);
return _mm_cvtsi128_si32(lo);
}
#endif
/* dot int8·int8: trucco del segno (|w| unsigned × x·sign(w) signed). Sicuro:
* coppie <= 128*127*2 = 32512 < 32767, accumulo s32 fino a I=16384. */
static inline int32_t dot_i8i8(const int8_t *w, const int8_t *x, int I){
int32_t sum=0; int i=0;
#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
/* VNNI: vpdpbusd u8*s8 -> s32 directly, 64 bytes/iter, no 16-bit intermediate.
* AVX-512 has no vpsignb: |w| via abs, sign folded into x with a mask-negate
* (w==0 -> product 0 either way). |x|<=127 (qrow_i8), |w|<=128 as u8: each
* s32 lane adds <= 4*128*127, safe up to I=16384 like the AVX2 bound. */
__m512i acc=_mm512_setzero_si512();
for(;i+64<=I;i+=64){
__m512i wv=_mm512_loadu_si512((const void*)(w+i));
__m512i xv=_mm512_loadu_si512((const void*)(x+i));
__mmask64 neg=_mm512_movepi8_mask(wv);
__m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv);
acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs);
}
sum=_mm512_reduce_add_epi32(acc);
#elif defined(__AVX2__)
__m256i acc=_mm256_setzero_si256(); const __m256i ones=_mm256_set1_epi16(1);
for(;i+32<=I;i+=32){
__m256i wv=_mm256_loadu_si256((const __m256i*)(w+i));
__m256i xv=_mm256_loadu_si256((const __m256i*)(x+i));
__m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv));
acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones));
}
sum=hsum256_i32(acc);
#elif defined(__ARM_NEON)
/* ARM: SDOT nativo se disponibile (Apple Silicon: sempre); altrimenti vmull/vpadal.
* Stesso bound anti-overflow del trucco AVX2: coppie <= 128*127*2 = 32512 < 32767. */
int32x4_t acc=vdupq_n_s32(0);
for(;i+16<=I;i+=16){
int8x16_t wv=vld1q_s8(w+i), xv=vld1q_s8(x+i);
#if defined(__ARM_FEATURE_DOTPROD)
acc=vdotq_s32(acc,wv,xv);
#else
int16x8_t p=vmull_s8(vget_low_s8(wv),vget_low_s8(xv));
p=vmlal_s8(p,vget_high_s8(wv),vget_high_s8(xv));
acc=vpadalq_s16(acc,p);
#endif
}
sum=vaddvq_s32(acc);
#elif defined(__VSX__)
/* POWER8: vec_msum (s8 x u8 -> s32) somma i prodotti byte DIRETTAMENTE in lane
* s32, 16 byte/iter: il bound anti-saturazione a 16 bit di maddubs qui non serve.
* Stesso trucco del segno (|w| u8 per x*sign(w) s8), ma |w| via select+sub MODULO
* e non vec_abs: -128 deve diventare 128 u8, non saturare a 127.
* EN: vec_msum accumulates byte products straight into s32 lanes; |w| is built
* with a modulo subtract select instead of vec_abs so w=-128 wraps to 128 (u8)
* rather than saturating to 127. |x|<=127 from qrow_i8, so x negation is safe. */
__vector signed int acc=vec_splats(0);
const __vector signed char vz=vec_splats((signed char)0);
for(;i+16<=I;i+=16){
__vector signed char wv=vec_xl(0,(const signed char*)(w+i));
__vector signed char xv=vec_xl(0,(const signed char*)(x+i));
__vector __bool char neg=vec_cmplt(wv,vz);
__vector signed char xs=vec_sel(xv,vec_sub(vz,xv),neg);
__vector unsigned char wa=(__vector unsigned char)vec_sel(wv,vec_sub(vz,wv),neg);
acc=vec_msum(xs,wa,acc);
}
sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3);
#endif
for(;i<I;i++) sum+=(int32_t)w[i]*x[i];
return sum;
}
/* dot int4(packed)·int8: nibble -> int8 [-8,7] al volo, poi stesso trucco */
static inline int32_t dot_i4i8(const uint8_t *w4, const int8_t *x, int I){
int32_t sum=0; int i=0;
#if defined(__AVX512VNNI__) && defined(__AVX512BW__)
/* 32 bytes = 64 nibbles -> int8 in [-8,7], one vpdpbusd per 64 values.
* 256-bit unpack leaves values in per-128-lane order [0-15][32-47]/[16-31][48-63];
* dot pairing is order-invariant, so permute x's 128-bit blocks to match
* instead of re-ordering w (one vpermq per iter, off the critical unpack path). */
const __m256i m4v=_mm256_set1_epi8(0x0F);
const __m512i b8v=_mm512_set1_epi8(8);
const __m512i xidx=_mm512_setr_epi64(0,1,4,5,2,3,6,7);
__m512i acc=_mm512_setzero_si512();
for(;i+64<=I;i+=64){
__m256i by=_mm256_loadu_si256((const __m256i*)(w4+(i>>1)));
__m256i lo=_mm256_and_si256(by,m4v), hi=_mm256_and_si256(_mm256_srli_epi16(by,4),m4v);
__m256i z0=_mm256_unpacklo_epi8(lo,hi), z1=_mm256_unpackhi_epi8(lo,hi);
__m512i wv=_mm512_sub_epi8(_mm512_inserti64x4(_mm512_castsi256_si512(z0),z1,1),b8v);
__m512i xv=_mm512_permutexvar_epi64(xidx,_mm512_loadu_si512((const void*)(x+i)));
__mmask64 neg=_mm512_movepi8_mask(wv);
__m512i xs=_mm512_mask_sub_epi8(xv,neg,_mm512_setzero_si512(),xv);
acc=_mm512_dpbusd_epi32(acc,_mm512_abs_epi8(wv),xs);
}
sum=_mm512_reduce_add_epi32(acc);
#elif defined(__AVX2__)
const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi8(8);
const __m256i ones=_mm256_set1_epi16(1);
__m256i acc=_mm256_setzero_si256();
for(;i+32<=I;i+=32){
__m128i by=_mm_loadu_si128((const __m128i*)(w4+(i>>1))); /* 16 byte = 32 nibble */
__m128i lo=_mm_and_si128(by,m4), hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
__m128i n0=_mm_unpacklo_epi8(lo,hi), n1=_mm_unpackhi_epi8(lo,hi); /* in ordine */
__m256i wv=_mm256_sub_epi8(_mm256_set_m128i(n1,n0),b8);
__m256i xv=_mm256_loadu_si256((const __m256i*)(x+i));
__m256i p=_mm256_maddubs_epi16(_mm256_sign_epi8(wv,wv),_mm256_sign_epi8(xv,wv));
acc=_mm256_add_epi32(acc,_mm256_madd_epi16(p,ones));
}
sum=hsum256_i32(acc);
#elif defined(__ARM_NEON)
const uint8x16_t m4q=vdupq_n_u8(0x0F); const int8x16_t b8q=vdupq_n_s8(8);
int32x4_t acc=vdupq_n_s32(0);
for(;i+32<=I;i+=32){
uint8x16_t by=vld1q_u8(w4+(i>>1)); /* 16 byte = 32 nibble */
uint8x16x2_t z=vzipq_u8(vandq_u8(by,m4q), vshrq_n_u8(by,4)); /* nibble in ordine */
int8x16_t w0=vsubq_s8(vreinterpretq_s8_u8(z.val[0]),b8q);
int8x16_t w1=vsubq_s8(vreinterpretq_s8_u8(z.val[1]),b8q);
int8x16_t x0=vld1q_s8(x+i), x1=vld1q_s8(x+i+16);
#if defined(__ARM_FEATURE_DOTPROD)
acc=vdotq_s32(acc,w0,x0); acc=vdotq_s32(acc,w1,x1);
#else
int16x8_t p=vmull_s8(vget_low_s8(w0),vget_low_s8(x0)); /* |w|<=8: nessun overflow */
p=vmlal_s8(p,vget_high_s8(w0),vget_high_s8(x0));
acc=vpadalq_s16(acc,p);
p=vmull_s8(vget_low_s8(w1),vget_low_s8(x1));
p=vmlal_s8(p,vget_high_s8(w1),vget_high_s8(x1));
acc=vpadalq_s16(acc,p);
#endif
}
sum=vaddvq_s32(acc);
#elif defined(__VSX__)
/* 16 byte = 32 nibble. vec_mergeh/vec_mergel su ppc64le (GCC) interallacciano come
* unpacklo/unpackhi x86 (verificato empiricamente su POWER8): i nibble escono in
* ordine di memoria. |w|<=8 dopo il -8, quindi stesso trucco del segno di dot_i8i8.
* EN: vec_mergeh/l on ppc64le interleave like x86 unpacklo/hi (verified on POWER8),
* so nibbles come out in memory order; then the same sign trick as dot_i8i8. */
const __vector unsigned char m4v=vec_splats((unsigned char)0x0F);
const __vector unsigned char sh4=vec_splats((unsigned char)4);
const __vector signed char b8v=vec_splats((signed char)8);
const __vector signed char vz=vec_splats((signed char)0);
__vector signed int acc=vec_splats(0);
for(;i+32<=I;i+=32){
__vector unsigned char by=vec_xl(0,w4+(i>>1)); /* 16 byte = 32 nibble */
__vector unsigned char lo=vec_and(by,m4v), hi=vec_sr(by,sh4);
__vector signed char w0=vec_sub((__vector signed char)vec_mergeh(lo,hi),b8v);
__vector signed char w1=vec_sub((__vector signed char)vec_mergel(lo,hi),b8v);
__vector signed char x0=vec_xl(0,(const signed char*)(x+i));
__vector signed char x1=vec_xl(0,(const signed char*)(x+i+16));
__vector __bool char n0=vec_cmplt(w0,vz), n1=vec_cmplt(w1,vz);
acc=vec_msum(vec_sel(x0,vec_sub(vz,x0),n0),
(__vector unsigned char)vec_sel(w0,vec_sub(vz,w0),n0),acc);
acc=vec_msum(vec_sel(x1,vec_sub(vz,x1),n1),
(__vector unsigned char)vec_sel(w1,vec_sub(vz,w1),n1),acc);
}
sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3);
#endif
for(;i+1<I;i+=2){ uint8_t b=w4[i>>1]; sum+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; }
if(i<I){ uint8_t b=w4[i>>1]; sum+=((int)(b&0xF)-8)*x[i]; }
return sum;
}
static void matmul_q_idot(float *y, const int8_t *xq, const float *sx, const int8_t *q,
const float *scale, int S, int I, int O){
#pragma omp parallel for schedule(static)
for(int o=0;o<O;o++){ const int8_t *w=q+(int64_t)o*I; float sc=scale[o];
for(int s=0;s<S;s++) y[(int64_t)s*O+o]=(float)dot_i8i8(w,xq+(int64_t)s*I,I)*sc*sx[s]; }
}
static void matmul_i4_idot(float *y, const int8_t *xq, const float *sx, const uint8_t *q4,
const float *scale, int S, int I, int O){
int rb=(I+1)/2;
#pragma omp parallel for schedule(static)
for(int o=0;o<O;o++){ const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o];
for(int s=0;s<S;s++) y[(int64_t)s*O+o]=(float)dot_i4i8(w,xq+(int64_t)s*I,I)*sc*sx[s]; }
}
typedef struct { int8_t *xq; size_t xq_cap; float *sx; size_t sx_cap; } QScratch;
static _Thread_local QScratch g_qscratch;
static void quant_scratch(size_t xn, size_t sn, int8_t **xq, float **sx){
if(xn>g_qscratch.xq_cap){
int8_t *p=realloc(g_qscratch.xq,xn);
if(!p){ fprintf(stderr,"OOM quant scratch\n"); exit(1); }
g_qscratch.xq=p; g_qscratch.xq_cap=xn;
}
if(sn>g_qscratch.sx_cap){
float *p=realloc(g_qscratch.sx,sn*sizeof(float));
if(!p){ fprintf(stderr,"OOM quant scales\n"); exit(1); }
g_qscratch.sx=p; g_qscratch.sx_cap=sn;
}
*xq=g_qscratch.xq; *sx=g_qscratch.sx;
}
static void matmul_qt(float *y, const float *x, QT *w, int S){
#ifdef COLI_METAL
/* Large row-batches (prefill: kv_b reconstruction, o_proj, dense MLP, step_all logits)
* amortize Metal's ~5ms submit latency; small-S decode matmuls stay on CPU (NEON wins).
* Weights must be registered (all dense QT allocs are, via qalloc). */
if(g_metal_enabled && S>=g_metal_gemm_min && (w->fmt==1||w->fmt==2) && !omp_in_parallel()){
const void *wp = w->fmt==1 ? (const void*)w->q8 : (const void*)w->q4;
if(coli_metal_gemm(y,x,wp,w->s,w->fmt,S,w->I,w->O)) return;
}
#endif
#ifdef COLI_CUDA
/* The CUDA backend owns persistent copies only for model-resident tensors.
* Streaming expert slots are reused for different IDs and must never enter
* this cache. Nested OpenMP calls stay on CPU because each device context
* intentionally owns one synchronous scratch stream in this stage. */
if(g_cuda_enabled && w->cuda_eligible && !w->cuda_failed && !omp_in_parallel()){
const void *weights = w->fmt==0 ? (const void*)w->qf
: w->fmt==1 ? (const void*)w->q8 : (const void*)w->q4;
if(coli_cuda_matmul(&w->cuda,y,x,weights,w->s,w->fmt,S,w->I,w->O,w->cuda_device)) return;
w->cuda_failed=1;
fprintf(stderr,"[CUDA] tensor [%d,%d] on device %d disabled after an error; falling back to CPU\n",
w->O,w->I,w->cuda_device);
}
#endif
if(w->fmt==0){ matmul(y,x,w->qf,S,w->I,w->O); return; }
/* int8 IDOT vince sempre (1.4-2.5x). int4 IDOT: l'autore su AVX2 trovo' che a S=1
* non ripaga (soglia S>=2); ma su ARM/SDOT il singolo token CONVIENE (vedi g_i4s /
* PR #9 per il gemello VNNI). Soglia configurabile con I4S.
* EN: int8 IDOT always wins (1.4-2.5x). int4 IDOT: on AVX2 the author found S=1 didn't
* pay (S>=2 gate); on ARM/SDOT single-token DOES pay (see g_i4s / PR #9 for the VNNI
* twin). Threshold configurable via I4S. */
if(g_idot && (w->fmt==1 || (w->fmt==2 && S>=g_i4s))){
int I=w->I; int8_t *xq; float *sx;
if(S<0 || I<0 || (size_t)S>SIZE_MAX/(size_t)(I?I:1)){ fprintf(stderr,"matmul_qt: shape overflow\n"); exit(1); }
quant_scratch((size_t)S*I,(size_t)S,&xq,&sx);
for(int s=0;s<S;s++) sx[s]=qrow_i8(x+(int64_t)s*I, xq+(int64_t)s*I, I);
if(w->fmt==1) matmul_q_idot(y,xq,sx,w->q8,w->s,S,I,w->O);
else matmul_i4_idot(y,xq,sx,w->q4,w->s,S,I,w->O);
return;
}
if(w->fmt==1) matmul_q(y,x,w->q8,w->s,S,w->I,w->O);
else if(w->fmt==3) matmul_i2(y,x,w->q4,w->s,S,w->I,w->O);
else matmul_i4(y,x,w->q4,w->s,S,w->I,w->O);
}
/* quantizza w[O,I] f32 -> int8 q[O,I] + scala[O] simmetrica per riga */
static void quantize_rows(const float *w, int8_t *q, float *scale, int O, int I, int bits){
int qmax=(1<<(bits-1))-1;
#pragma omp parallel for schedule(static)
for(int o=0;o<O;o++){ const float *wr=w+(int64_t)o*I; float amax=0;
for(int i=0;i<I;i++){ float a=fabsf(wr[i]); if(a>amax)amax=a; }
float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s;
int8_t *qr=q+(int64_t)o*I;
for(int i=0;i<I;i++){ int v=(int)lrintf(wr[i]/s); if(v>qmax)v=qmax; if(v<-qmax-1)v=-qmax-1; qr[i]=(int8_t)v; }
}
}
/* quantizza w[O,I] f32 -> int4 impacchettato (2/byte) + scala[O].
* bits<=4: valori in [-qmax-1,qmax] stanno in un nibble [-8,7]; memorizzati come v+8 (0..15). */
static void pack_int4(const float *w, uint8_t *q4, float *scale, int O, int I, int bits){
int qmax=(1<<(bits-1))-1, rb=(I+1)/2;
#pragma omp parallel for schedule(static)
for(int o=0;o<O;o++){ const float *wr=w+(int64_t)o*I; float amax=0;
for(int i=0;i<I;i++){ float a=fabsf(wr[i]); if(a>amax)amax=a; }
float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s;
uint8_t *qr=q4+(int64_t)o*rb;
for(int i=0;i<I;i+=2){
int v0=(int)lrintf(wr[i]/s); if(v0>qmax)v0=qmax; if(v0<-8)v0=-8;
int v1=0; if(i+1<I){ v1=(int)lrintf(wr[i+1]/s); if(v1>qmax)v1=qmax; if(v1<-8)v1=-8; }
qr[i>>1] = (uint8_t)((v0+8) | ((v1+8)<<4));
}
}
}
/* quantizza w[O,I] f32 -> int2 impacchettato (4/byte) + scala[O]. valori nibble 2-bit in [-2,1]. */
static void pack_int2(const float *w, uint8_t *q2, float *scale, int O, int I, int bits){
int qmax=(1<<(bits-1))-1, rb=(I+3)/4;
#pragma omp parallel for schedule(static)
for(int o=0;o<O;o++){ const float *wr=w+(int64_t)o*I; float amax=0;
for(int i=0;i<I;i++){ float a=fabsf(wr[i]); if(a>amax)amax=a; }
float s=amax/qmax; if(s<1e-8f)s=1e-8f; scale[o]=s;
uint8_t *qr=q2+(int64_t)o*rb;
for(int i=0;i<I;i+=4){ uint8_t byte=0;
for(int k=0;k<4 && i+k<I;k++){ int v=(int)lrintf(wr[i+k]/s); if(v>qmax)v=qmax; if(v<-2)v=-2; byte|=(uint8_t)((v+2)<<(k*2)); }
qr[i>>2]=byte;
}
}
}
static int g_nopack=0; /* NOPACK=1 -> tiene i valori <=4bit in contenitore int8 (per validare il packing) */
static int g_drop=0; /* DROP=1 -> scarta le pagine expart dopo l'uso. Default 0: le lascia in
* page-cache (buff/cache, NON RSS) come L2 gratuito -> sfrutta lo
* sbilanciamento del routing MoE (pochi expert "caldi" riusati). */
static int g_prefetch=0; /* PREFETCH=1 -> riabilita il WILLNEED cross-layer (metodo C). Default
* OFF: i load VERI in parallelo lo hanno reso superfluo, e sotto
* pressione di memoria il readahead speculativo veniva rievictato. */
static int g_direct=0; /* DIRECT=1 -> O_DIRECT sugli slab expert. Default OFF: su questo host
* (VHDX su NVMe DRAM-less, latenza serializzata ~60ms/req) il buffered
* liscio e' risultato il migliore; su NVMe veri DIRECT=1 rende di piu'. */
static float g_temp=-1; /* TEMP: temperatura di sampling sui TOKEN. <0 = auto (1.0 in chat/testo,
* 0=greedy in validazione). 0 = greedy puro. */
static float g_nuc=0.95f;/* NUCLEUS: top-p sul vocabolario (default dal generation_config GLM-5.2) */
static int g_topk=0; /* TOPK=n -> usa n expert/token invece di config (ricerca: meno disco) */
static float g_topp=0; /* TOPP=p (0..1) -> top-p adattivo: tieni gli expert fino a peso cumulato p */
static int g_spec=1; /* metodo C: SPEC=0 disabilita il prefetch speculativo cross-layer */
static int g_draft=0; /* metodo E: DRAFT=n token auto-speculati per forward via n-gram lookup
* (0=off). LOSSLESS: verifica = output identico al greedy. Default OFF:
* misurato sul run reale (2026-07-03) acceptance ~5% -> ogni draft
* rifiutato paga comunque i suoi expert dal disco = ~3x piu' lento.
* Opt-in (DRAFT=4) per testi ripetitivi dove l'acceptance e' alta. */
/* metodo F (#48): GRAMMAR=<file.gbnf> -> terza sorgente di draft, la grammatica stessa.
* Nei workload a output vincolato (JSON/NDJSON, function calling) i byte FORZATI dalla
* grammatica (chiavi, punteggiatura, valori enum) sono draft gratuiti ad acceptance ~1:
* nessuna testa, nessuna lookup table, e si aggancia anche dove la testa MTP int4 non
* parte (#8). MAI un vincolo sul sampling: solo proposte, la verifica batch-union
* decide — grammatica sbagliata = draft rifiutati, output identico.
* GRAMMAR_DRAFT=n (default 24) limita i token forzati per forward. */
static Grammar g_gram; static GrState g_gst;
static Tok *g_gr_T=NULL;
static int g_gr_on=0; /* grammatica caricata e walker vivo */
static int g_gr_armed=0; /* lazy: parte dal primo byte ammesso dalla radice (salta i preamboli) */
static int g_gr_max=24;
static uint64_t g_gr_prop=0, g_gr_acc=0;
static int g_looka=0; /* LOOKA=1: misura (solo contatori, zero effetti) quanto il routing MoE
* e' predicibile IN ANTICIPO — la domanda che decide se un prefetch
* pilotato dal router puo' riempire i tempi morti del disco.
* [0] token precedente, stesso layer (cio' che usa gia' SPEC/PREFETCH)
* [1] ingresso del layer -> routing dello STESSO layer (salta l'attention)
* [2] post-attention del layer L -> routing di L+1 (un residuo MoE e
* un'attention di anticipo: il punto dove il prefetch avrebbe
* un intero giro di disco per lavorare in ombra). */
static int64_t la_hit[3], la_tot[3];
static int la_pred[2][130][16]; static signed char la_val[2][130];
static int g_pilot=0; /* PILOT=1: prefetch pilotato dal router (vedi pilot_prefetch) */
static int g_pilot_k=8; /* PILOT_K=k: prefetcha solo le prime k predizioni per posizione */
/* Aligned allocator for dense QT weights/scales: under METAL, page-align + register so the
* GPU reads them zero-copy (no upload duplicate). Plain malloc otherwise. */
static void *qalloc(size_t n){
#ifdef COLI_METAL
if(g_metal_enabled){ void *p; size_t r=(n+16383)&~(size_t)16383;
if(posix_memalign(&p,16384,r)){fprintf(stderr,"OOM qalloc\n");exit(1);}
coli_metal_register(p,r); return p; }
#endif
return malloc(n);
}
static float *qsalloc(int O){ return (float*)qalloc((size_t)O*sizeof(float)); }
static int g_pilot_real=0;/* PILOT_REAL=1: il pilota fa LOAD VERI cross-layer dentro ecache[L+1]
* (non il semplice WILLNEED). Implica PILOT=1. Default OFF: hint-only. */
/* Handshake main<->pilota per il load-vero cross-layer. Invariante di sicurezza in DUE parti:
* 1) Percorso MATMUL (moe): il pilota scrive SOLO ecache[layer] con layer > g_cur_moe_layer;
* il matmul in moe() legge SOLO ecache[layer]==g_cur_moe_layer, e la barriera a inizio moe()
* aspetta l'eventuale load in volo su QUEL layer. Quindi NESSUNO slot mezzo-caricato viene
* mai matmul-ato: il matmul e il pilota non toccano mai lo stesso layer contemporaneamente.
* 2) Percorso SCAN (pilot_prefetch, anch'esso sul MAIN): la scansione di residenza gira sul
* layer FUTURO (lnext = layer corrente + 1), esattamente il layer che il pilota sta scrivendo
* -> QUI i due thread toccano davvero la stessa ecache. Percio' quella scansione prende
* g_pilot_mx (lo stesso lock del worker): letture e pubblicazione degli slot sono serializzate,
* niente torn read di ecn[]/eid. Il pilota non altera MAI il valore di un expert, solo QUALE
* expert e' residente: con un load andato a buon fine l'output resta byte-identico all'OFF. */
static pthread_mutex_t g_pilot_mx=PTHREAD_MUTEX_INITIALIZER;
static _Atomic int g_cur_moe_layer=-1; /* massimo layer moe in cui il MAIN e' entrato (per forward) */
static _Atomic int g_pilot_inflight=-1; /* layer che il worker sta REAL-caricando adesso (-1 = idle) */
static _Atomic long g_pilot_loads=0; /* load cross-layer VERI completati (banda spesa) */
static _Atomic long g_pilot_drops=0; /* predizioni scartate perche' il main possiede gia' il layer */
/* sceglie il formato da `bits`: >=16 f32, 5..8 int8, <=4 int4-packed */
static void qt_alloc(QT *t, int O, int I, int bits){
t->O=O; t->I=I; t->qf=NULL; t->q8=NULL; t->q4=NULL; t->s=NULL;
if(bits>=16){ t->fmt=0; t->qf=falloc((int64_t)O*I); }
else if(bits>=5 || g_nopack){ t->fmt=1; t->q8=qalloc((int64_t)O*I); t->s=qsalloc(O); }
else if(bits>=3){ t->fmt=2; t->q4=qalloc((int64_t)O*((I+1)/2)); t->s=qsalloc(O); }
else { t->fmt=3; t->q4=qalloc((int64_t)O*((I+3)/4)); t->s=qsalloc(O); }
}
static void qt_fill(QT *t, const float *w, int bits){
if(t->fmt==0) memcpy(t->qf, w, (int64_t)t->O*t->I*sizeof(float));
else if(t->fmt==1) quantize_rows(w, t->q8, t->s, t->O, t->I, bits);
else if(t->fmt==3) pack_int2(w, t->q4, t->s, t->O, t->I, bits);
else pack_int4(w, t->q4, t->s, t->O, t->I, bits);
}
static void rmsnorm(float *out, const float *x, const float *w, int D, float eps){
double ms=0; for(int i=0;i<D;i++) ms+=(double)x[i]*x[i];
float r=1.f/sqrtf((float)(ms/D)+eps); for(int i=0;i<D;i++) out[i]=x[i]*r*w[i];
}
/* LayerNorm classica (media+varianza, weight+bias) — usata dal k_norm dell'indexer DSA */
static void layernorm(float *v, const float *w, const float *b, int n, float eps){
double mu=0; for(int i=0;i<n;i++) mu+=v[i]; mu/=n;
double var=0; for(int i=0;i<n;i++){ double d=v[i]-mu; var+=d*d; } var/=n;
float r=1.f/sqrtf((float)var+eps);
for(int i=0;i<n;i++) v[i]=((float)(v[i]-mu))*r*w[i]+b[i];
}
static void softmax(float *x,int n){ float m=-1e30f; for(int i=0;i<n;i++) if(x[i]>m)m=x[i];
float s=0; for(int i=0;i<n;i++){x[i]=expf(x[i]-m);s+=x[i];} for(int i=0;i<n;i++) x[i]/=s; }
static inline float sigmoidf(float x){ return 1.f/(1.f+expf(-x)); }
static inline float siluf(float x){ return x/(1.f+expf(-x)); }
/* RoPE interleaved su un vettore di dimensione qk_rope a posizione pos */
static void rope_interleave(float *v, int pos, const Cfg *c){
int half = c->qk_rope/2; float in[256]; memcpy(in,v,c->qk_rope*sizeof(float));
for(int j=0;j<half;j++){
float inv = powf(c->theta, -2.0f*j/c->qk_rope);
float ang = pos*inv, cs=cosf(ang), sn=sinf(ang);
float a=in[2*j], b=in[2*j+1];
v[j] = a*cs - b*sn;
v[half+j] = b*cs + a*sn;
}
}
/* ---------- config ---------- */
static jval* cfg_root(const char *snap, char **arena){
char p[2048]; snprintf(p,sizeof(p),"%s/config.json",snap);
FILE *f=fopen(p,"rb"); if(!f){perror(p);exit(1);}
fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET);
char *b=malloc(n+1); if(fread(b,1,n,f)!=(size_t)n){} b[n]=0; fclose(f);
return json_parse(b,arena);
}
static int gi(jval*r,const char*k){ jval*v=json_get(r,k); return v?(int)v->num:0; }
static void load_cfg(Cfg *c, const char *snap){
char *ar=NULL; jval *r=cfg_root(snap,&ar);
c->hidden=gi(r,"hidden_size"); c->n_layers=gi(r,"num_hidden_layers");
c->n_heads=gi(r,"num_attention_heads"); c->n_experts=gi(r,"n_routed_experts");
c->topk=gi(r,"num_experts_per_tok"); c->moe_inter=gi(r,"moe_intermediate_size");
c->dense_inter=gi(r,"intermediate_size"); c->first_dense=gi(r,"first_k_dense_replace");
c->q_lora=gi(r,"q_lora_rank"); c->kv_lora=gi(r,"kv_lora_rank");
c->qk_nope=gi(r,"qk_nope_head_dim"); c->qk_rope=gi(r,"qk_rope_head_dim");
c->v_head=gi(r,"v_head_dim"); c->n_shared=gi(r,"n_shared_experts"); c->vocab=gi(r,"vocab_size");
c->n_group=gi(r,"n_group"); c->topk_group=gi(r,"topk_group");
jval *nt=json_get(r,"norm_topk_prob"); c->norm_topk=(nt&&nt->t==J_BOOL)?nt->boolean:0;
jval *ep=json_get(r,"rms_norm_eps"); c->eps=ep?(float)ep->num:1e-5f;
jval *rs=json_get(r,"routed_scaling_factor"); c->routed_scale=rs?(float)rs->num:1.f;
jval *rp=json_get(r,"rope_parameters"); jval *th=rp?json_get(rp,"rope_theta"):NULL;
c->theta = th?(float)th->num:10000.f;
/* token di stop: GLM-5.2 ne ha TRE (endoftext, user, observation). Fermarsi solo sul
* primo = generare spazzatura invisibile dopo la fine del turno (5-10x token sprecati). */
c->n_stop=0;
jval *eo=json_get(r,"eos_token_id");
if(eo){ if(eo->t==J_NUM) c->stop_ids[c->n_stop++]=(int)eo->num;
else if(eo->t==J_ARR) for(int i=0;i<eo->len && c->n_stop<8;i++)
c->stop_ids[c->n_stop++]=(int)eo->kids[i]->num; }
/* DSA lightning indexer: parametri + tipo per-layer (lista esplicita o formula freq/offset) */
c->index_topk=gi(r,"index_topk"); c->index_nh=gi(r,"index_n_heads"); c->index_hd=gi(r,"index_head_dim");
{ jval *it=json_get(r,"indexer_types");
int freq=gi(r,"index_topk_freq"); if(freq<1) freq=1;
jval *of=json_get(r,"index_skip_topk_offset"); int off=of?(int)of->num:2;
for(int i=0;i<c->n_layers && i<128;i++){
if(it && it->t==J_ARR && i<it->len && it->kids[i]->str)
c->idx_type[i] = !strcmp(it->kids[i]->str,"full");
else { int v=i-off+1; if(v<0) v=0; c->idx_type[i] = (v%freq)==0; }
} }
c->qk_head=c->qk_nope+c->qk_rope;
c->attn_scale = 1.f / sqrtf((float)c->qk_head);
if(c->n_group!=1){ fprintf(stderr,"this engine requires n_group=1 (GLM-5.2)\n"); exit(1); }
/* VALIDAZIONE (report PR #25): il config.json arriva da mirror non fidati — dimensioni
* ostili non devono superare questo punto. Un solo choke point protegge ogni alloc a valle. */
#define CKR(name,v,lo,hi) if((v)<(lo)||(v)>(hi)){ \
fprintf(stderr,"config: %s=%d is outside [%d,%d]\n",name,(int)(v),(int)(lo),(int)(hi)); exit(1); }
CKR("hidden_size",c->hidden,1,1<<20) CKR("num_hidden_layers",c->n_layers,1,128)
CKR("num_attention_heads",c->n_heads,1,1024) CKR("n_routed_experts",c->n_experts,1,4096)
CKR("num_experts_per_tok",c->topk,1,64) CKR("moe_intermediate_size",c->moe_inter,1,1<<20)
CKR("intermediate_size",c->dense_inter,1,1<<24) CKR("first_k_dense_replace",c->first_dense,0,c->n_layers)
CKR("q_lora_rank",c->q_lora,0,1<<20) CKR("kv_lora_rank",c->kv_lora,1,1<<20)
CKR("qk_nope_head_dim",c->qk_nope,1,1<<16) CKR("qk_rope_head_dim",c->qk_rope,1,1<<16)
CKR("v_head_dim",c->v_head,1,1<<16) CKR("n_shared_experts",c->n_shared,0,64)
CKR("vocab_size",c->vocab,1,1<<24) CKR("index_topk",c->index_topk,0,1<<20)
CKR("index_n_heads",c->index_nh,0,1024) CKR("index_head_dim",c->index_hd,0,1<<16)
#undef CKR
free(ar);
}
/* costruisce un QT [O,I] dal disco in `t` (buffer riusabili tra chiamate).
* - se esiste `name.qs`: pesi GIA' quantizzati nel container (U8 qdata + F32 scala) -> letti diretti
* - altrimenti: tensore pieno (f32/bf16) -> quantizzato a runtime a `bits` (oracolo tiny / pesi pieni)
* drop=1 -> fadvise DONTNEED (streaming expert). */
static void qt_from_disk(Model *m, const char *name, int O, int I, int bits, int drop, QT *t){
char sn[300]; snprintf(sn,sizeof(sn),"%s.qs",name);
if(st_has(&m->S,sn)){
int64_t nb=st_nbytes(&m->S,name);
int fmt = (nb==(int64_t)O*I)?1 : (nb==(int64_t)O*((I+1)/2))?2 : 3; /* int8 / int4 / int2 dai byte */
if(fmt==1){ if(t->fmt!=1||!t->q8){ t->fmt=1; t->O=O; t->I=I; t->q8=qalloc(nb); t->s=qsalloc(O); } st_read_raw(&m->S,name,t->q8,drop); }
else { if(t->fmt!=fmt||!t->q4){ t->fmt=fmt; t->O=O; t->I=I; t->q4=qalloc(nb); t->s=qsalloc(O); } st_read_raw(&m->S,name,t->q4,drop); }
st_read_f32(&m->S,sn,t->s,drop);
} else {
if(!t->qf && !t->q8 && !t->q4) qt_alloc(t,O,I,bits);
if(t->fmt==0) st_read_f32(&m->S,name,t->qf,drop);
else { float *tmp=falloc((int64_t)O*I); st_read_f32(&m->S,name,tmp,drop); qt_fill(t,tmp,bits); free(tmp); }
}
}
static QT qt_load(Model *m, const char *name, int O, int I, int bits){
QT t; memset(&t,0,sizeof(t)); qt_from_disk(m,name,O,I,bits,0,&t);
#ifdef COLI_CUDA
if(g_cuda_enabled&&g_cuda_dense){
t.cuda_eligible=1;
int slot=g_cuda_rr++%g_cuda_ndev; t.cuda_device=g_cuda_devices[slot];
g_cuda_dense_projected[slot]+=qt_bytes(&t);
}
#endif
return t;
}
static float *ld(Model *m, const char *name){ /* tensore 1D f32 residente (norme/bias) */
int64_t n=st_numel(&m->S,name); if(n<0){fprintf(stderr,"missing %s\n",name);exit(1);}
float *p=(float*)qalloc((size_t)n*sizeof(float)); /* registrato per la GPU sotto METAL */
st_read_f32(&m->S,name,p,0); return p;
}
static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits){
memset(m,0,sizeof(*m)); m->ebits=ebits; m->dbits=dbits;
load_cfg(&m->c,snap); st_init(&m->S,snap);
Cfg *c=&m->c; char nm[256]; int H=c->n_heads, D=c->hidden;
/* embed e lm_head sono il confine I/O: tenerli ad alta precisione (come i quant dynamic
* reali). A bf16 ~1.9GB su GLM reale: trascurabile. dbits>=8 -> qui f32; piu' basso -> dbits. */
int io_bits = dbits>=8 ? 16 : dbits;
m->embed = qt_load(m,"model.embed_tokens.weight", c->vocab, D, io_bits);
m->lm_head = qt_load(m,"lm_head.weight", c->vocab, D, io_bits);
m->final_norm = ld(m,"model.norm.weight");
m->L=calloc(c->n_layers,sizeof(Layer));
int NR=c->n_layers+1; /* +1: riga del layer MTP */
m->ecap=cap; m->ecache=calloc(NR,sizeof(ESlot*)); m->ecn=calloc(NR,sizeof(int));
m->eroute=calloc(NR,sizeof(int*)); m->enr=calloc(NR,sizeof(int));
m->pin=calloc(NR,sizeof(ESlot*)); m->npin=calloc(NR,sizeof(int));
m->eusage=calloc(NR,sizeof(uint32_t*)); m->eheat=calloc(NR,sizeof(uint32_t*));
m->elast=calloc(NR,sizeof(uint32_t*));
m->kv=calloc(1,sizeof(KVState));
m->kv_start=m->kv->kv_start=calloc(NR,sizeof(int));
for(int i=0;i<c->n_layers;i++){
Layer *l=&m->L[i];
#define P(s) (snprintf(nm,sizeof(nm),"model.layers.%d." s,i),nm)
l->in_ln=ld(m,P("input_layernorm.weight"));
l->post_ln=ld(m,P("post_attention_layernorm.weight"));
l->q_a = qt_load(m,P("self_attn.q_a_proj.weight"), c->q_lora, D, dbits);
l->q_a_ln= ld(m,P("self_attn.q_a_layernorm.weight"));
l->q_b = qt_load(m,P("self_attn.q_b_proj.weight"), H*c->qk_head, c->q_lora, dbits);
l->kv_a = qt_load(m,P("self_attn.kv_a_proj_with_mqa.weight"), c->kv_lora+c->qk_rope, D, dbits);
l->kv_a_ln= ld(m,P("self_attn.kv_a_layernorm.weight"));
l->kv_b = qt_load(m,P("self_attn.kv_b_proj.weight"), H*(c->qk_nope+c->v_head), c->kv_lora, dbits);
l->o = qt_load(m,P("self_attn.o_proj.weight"), D, H*c->v_head, dbits);
l->sparse = (i >= c->first_dense);
if(!l->sparse){
l->gate_proj = qt_load(m,P("mlp.gate_proj.weight"), c->dense_inter, D, dbits);
l->up_proj = qt_load(m,P("mlp.up_proj.weight"), c->dense_inter, D, dbits);
l->down_proj = qt_load(m,P("mlp.down_proj.weight"), D, c->dense_inter, dbits);
} else {
l->router=ld(m,P("mlp.gate.weight"));
l->router_bias=ld(m,P("mlp.gate.e_score_correction_bias"));
int sI=c->moe_inter*c->n_shared;
l->sh_gate = qt_load(m,P("mlp.shared_experts.gate_proj.weight"), sI, D, dbits);
l->sh_up = qt_load(m,P("mlp.shared_experts.up_proj.weight"), sI, D, dbits);
l->sh_down = qt_load(m,P("mlp.shared_experts.down_proj.weight"), D, sI, dbits);
m->ecache[i]=calloc(cap,sizeof(ESlot));
m->eroute[i]=calloc(c->topk,sizeof(int)); /* metodo C: ultimo routing del layer */
m->eusage[i]=calloc(c->n_experts,sizeof(uint32_t));
m->eheat[i]=calloc(c->n_experts,sizeof(uint32_t));
m->elast[i]=calloc(c->n_experts,sizeof(uint32_t));
}
#undef P
}
/* testa MTP (layer n_layers): presente solo se convertita con --mtp */
{
/* MTP attiva SOLO se il set e' COMPLETO (i tensori vivono su 3 shard: durante la
* conversione parziale ne esiste solo una parte). MTP=0 la disabilita comunque. */
const char *req[]={"eh_proj.weight","enorm.weight","hnorm.weight","shared_head.norm.weight",
"input_layernorm.weight","post_attention_layernorm.weight",
"self_attn.q_a_proj.weight","self_attn.q_b_proj.weight","self_attn.kv_a_proj_with_mqa.weight",
"self_attn.kv_b_proj.weight","self_attn.o_proj.weight","mlp.gate.weight",
"mlp.shared_experts.gate_proj.weight","mlp.shared_experts.down_proj.weight",
"mlp.experts.0.gate_proj.weight","mlp.experts.255.down_proj.weight"};
char mn[256]; m->has_mtp=1;
for(unsigned q=0;q<sizeof(req)/sizeof(req[0]);q++){
snprintf(mn,sizeof(mn),"model.layers.%d.%s",c->n_layers,req[q]);
if(!st_has(&m->S,mn)){ m->has_mtp=0; break; }
}
if(getenv("MTP") && atoi(getenv("MTP"))==0) m->has_mtp=0;
if(m->has_mtp){
int i=c->n_layers; Layer *l=&m->mtpL;
#define PM(s) (snprintf(nm,sizeof(nm),"model.layers.%d." s,i),nm)
l->in_ln=ld(m,PM("input_layernorm.weight"));
l->post_ln=ld(m,PM("post_attention_layernorm.weight"));
l->q_a = qt_load(m,PM("self_attn.q_a_proj.weight"), c->q_lora, D, dbits);
l->q_a_ln= ld(m,PM("self_attn.q_a_layernorm.weight"));
l->q_b = qt_load(m,PM("self_attn.q_b_proj.weight"), H*c->qk_head, c->q_lora, dbits);
l->kv_a = qt_load(m,PM("self_attn.kv_a_proj_with_mqa.weight"), c->kv_lora+c->qk_rope, D, dbits);
l->kv_a_ln= ld(m,PM("self_attn.kv_a_layernorm.weight"));
l->kv_b = qt_load(m,PM("self_attn.kv_b_proj.weight"), H*(c->qk_nope+c->v_head), c->kv_lora, dbits);
l->o = qt_load(m,PM("self_attn.o_proj.weight"), D, H*c->v_head, dbits);
l->sparse=1;
l->router=ld(m,PM("mlp.gate.weight"));
l->router_bias=ld(m,PM("mlp.gate.e_score_correction_bias"));
int sI=c->moe_inter*c->n_shared;
l->sh_gate = qt_load(m,PM("mlp.shared_experts.gate_proj.weight"), sI, D, dbits);
l->sh_up = qt_load(m,PM("mlp.shared_experts.up_proj.weight"), sI, D, dbits);
l->sh_down = qt_load(m,PM("mlp.shared_experts.down_proj.weight"), D, sI, dbits);
m->eh_proj = qt_load(m,PM("eh_proj.weight"), D, 2*D, dbits);
m->enorm=ld(m,PM("enorm.weight")); m->hnorm=ld(m,PM("hnorm.weight"));
m->mtp_norm=ld(m,PM("shared_head.norm.weight"));
m->ecache[i]=calloc(cap,sizeof(ESlot));
m->eroute[i]=calloc(c->topk,sizeof(int));
m->eusage[i]=calloc(c->n_experts,sizeof(uint32_t));
m->eheat[i]=calloc(c->n_experts,sizeof(uint32_t));
m->elast[i]=calloc(c->n_experts,sizeof(uint32_t));
m->kv_start[i]=-1; /* KV MTP: parte dalla prima posizione di decode */
#undef PM
}
}
/* DSA lightning indexer: attivo SOLO se i pesi (conversione --indexer) ci sono per
* TUTTI i layer full. Auto-rilevamento come per MTP: niente flag, niente passi extra. */
{
m->has_dsa = (c->index_topk>0 && c->index_nh>0 && c->index_hd>0 && c->index_hd<=256);
char inm[300];
for(int i=0;i<c->n_layers && m->has_dsa;i++){
if(!c->idx_type[i]) continue;
snprintf(inm,sizeof(inm),"model.layers.%d.self_attn.indexer.wq_b.weight",i);
if(!st_has(&m->S,inm)) m->has_dsa=0;
}
if(getenv("DSA") && atoi(getenv("DSA"))==0) m->has_dsa=0;
if(m->has_dsa){
m->ix_wq=calloc(c->n_layers,sizeof(QT)); m->ix_wk=calloc(c->n_layers,sizeof(QT));
m->ix_wp=calloc(c->n_layers,sizeof(QT));
m->ix_knw=calloc(c->n_layers,sizeof(float*)); m->ix_knb=calloc(c->n_layers,sizeof(float*));
for(int i=0;i<c->n_layers;i++){
if(!c->idx_type[i]) continue;
#define PI(s) (snprintf(nm,sizeof(nm),"model.layers.%d.self_attn.indexer." s,i),nm)
m->ix_wq[i]=qt_load(m,PI("wq_b.weight"), c->index_nh*c->index_hd, c->q_lora, dbits);
m->ix_wk[i]=qt_load(m,PI("wk.weight"), c->index_hd, D, dbits);
m->ix_wp[i]=qt_load(m,PI("weights_proj.weight"), c->index_nh, D, dbits);
m->ix_knw[i]=ld(m,PI("k_norm.weight")); m->ix_knb[i]=ld(m,PI("k_norm.bias"));
#undef PI
}
fprintf(stderr,"[DSA] indexer active: top-%d sparse attention beyond %d context tokens\n",
c->index_topk, c->index_topk);
}
}
m->hlast=falloc(D); m->h_all=falloc((int64_t)64*D);
/* byte della parte DENSA residente (embed+lm_head+attn+mlp densa+shared+norme) */
int64_t rb=qt_bytes(&m->embed)+qt_bytes(&m->lm_head);
for(int i=0;i<c->n_layers;i++){ Layer *l=&m->L[i];
rb+=qt_bytes(&l->q_a)+qt_bytes(&l->q_b)+qt_bytes(&l->kv_a)+qt_bytes(&l->kv_b)+qt_bytes(&l->o);
if(!l->sparse) rb+=qt_bytes(&l->gate_proj)+qt_bytes(&l->up_proj)+qt_bytes(&l->down_proj);
else rb+=qt_bytes(&l->sh_gate)+qt_bytes(&l->sh_up)+qt_bytes(&l->sh_down);
}
if(m->has_mtp){ Layer *l=&m->mtpL;
rb+=qt_bytes(&l->q_a)+qt_bytes(&l->q_b)+qt_bytes(&l->kv_a)+qt_bytes(&l->kv_b)+qt_bytes(&l->o);
rb+=qt_bytes(&l->sh_gate)+qt_bytes(&l->sh_up)+qt_bytes(&l->sh_down)+qt_bytes(&m->eh_proj);
}
if(m->has_dsa) for(int i=0;i<c->n_layers;i++) if(c->idx_type[i])
rb+=qt_bytes(&m->ix_wq[i])+qt_bytes(&m->ix_wk[i])+qt_bytes(&m->ix_wp[i]);
m->resident_bytes=rb;
}
/* embed: dequantizza la riga del token (scala per-riga) in x[hidden] */
static void embed_row(Model *m, int tok, float *x){
int D=m->c.hidden; QT *e=&m->embed;
if(e->fmt==0){ memcpy(x, e->qf+(int64_t)tok*D, D*sizeof(float)); return; }
if(e->fmt==1){ const int8_t *q=e->q8+(int64_t)tok*D; float s=e->s[tok];
for(int i=0;i<D;i++) x[i]=(float)q[i]*s; return; }
if(e->fmt==2){ const uint8_t *q=e->q4+(int64_t)tok*((D+1)/2); float s=e->s[tok]; /* int4 */
for(int i=0;i<D;i+=2){ uint8_t byte=q[i>>1]; x[i]=(float)((int)(byte&0xF)-8)*s;
if(i+1<D) x[i+1]=(float)((int)(byte>>4)-8)*s; }
return; }
const uint8_t *q=e->q4+(int64_t)tok*((D+3)/4); float s=e->s[tok]; /* int2 */
for(int i=0;i<D;i++){ uint8_t byte=q[i>>2]; int sh=(i&3)*2; x[i]=(float)((int)((byte>>sh)&3)-2)*s; }
}
/* COLI_MMAP=1: gli expert diventano VISTE dentro mmap dei file safetensors (niente pread,
* niente slab, niente copia: la page cache del kernel E' la cache). Le mappe sono
* registrate con Metal (newBufferWithBytesNoCopy su pagine file-backed, come llama.cpp),
* quindi la GPU legge gli stessi byte. Fallback allo slab path su disallineamento. */
static int g_mmap=0;
static struct { int fd; void *base; size_t len; } g_maps[512]; static int g_nmaps;
static pthread_mutex_t g_map_mtx = PTHREAD_MUTEX_INITIALIZER; /* expert_load e' OMP-parallel */
static void *map_of_fd(int fd){
pthread_mutex_lock(&g_map_mtx);
for(int i=0;i<g_nmaps;i++) if(g_maps[i].fd==fd){ void *b=g_maps[i].base; pthread_mutex_unlock(&g_map_mtx); return b; }
void *base=NULL; struct stat st;
if(g_nmaps<512 && fstat(fd,&st)==0){
size_t len=((size_t)st.st_size+16383)&~(size_t)16383;
void *p=mmap(NULL,len,PROT_READ,MAP_SHARED,fd,0);
if(p!=MAP_FAILED){
base=p; g_maps[g_nmaps].fd=fd; g_maps[g_nmaps].base=p; g_maps[g_nmaps].len=len; g_nmaps++;
#ifdef COLI_METAL
if(g_metal_enabled) coli_metal_register(p,len);
#endif
}
}
pthread_mutex_unlock(&g_map_mtx);
return base;
}
/* carica un expert nello slot. Container pre-quantizzato: le 3 matrici sono contigue nel
* file -> UNA pread coalescente da ~19 MB dentro `slab` (+ le scale in fslab); i QT sono
* viste dentro lo slab (zero copie). Fallback per modelli non quantizzati (oracolo tiny).
* THREAD-SAFE su slot distinti (pread posizionale, st_find read-only). */
/* Load one expert's weights into slot `s`. Returns 0 on success, -1 on failure.
* fatal=1 (all main / on-demand / REPIN / pin callers): preserve the original
* exit-on-error contract byte-for-byte — any missing tensor, OOM, short read or
* pread error aborts the process. fatal=0 (speculative pilot only): the same
* errors instead abandon the load and return -1 without touching s->eid, so a
* mispredicted cross-layer prefetch can never kill the server. */
static int expert_load(Model *m, int layer, int eid, ESlot *s, int fatal){
#ifdef COLI_CUDA
/* A live REPIN may reuse a GPU-enabled pinned slot for a different expert.
* Keep its tier assignment, but invalidate the old device weights. */
if(s->eid!=eid){ qt_cuda_reset(&s->g); qt_cuda_reset(&s->u); qt_cuda_reset(&s->d); }
#endif
Cfg *c=&m->c; int I=c->moe_inter, D=c->hidden, b=m->ebits;
char nm[3][288]; const char *suf[3]={"gate_proj","up_proj","down_proj"};
for(int k=0;k<3;k++) snprintf(nm[k],sizeof(nm[k]),"model.layers.%d.mlp.experts.%d.%s.weight",layer,eid,suf[k]);
char qn[300]; snprintf(qn,sizeof(qn),"%s.qs",nm[0]);
if(!st_has(&m->S,qn)){ /* fallback: tensori pieni, quantizza a runtime.
* Reachable ONLY for unquantized models (no .qs);
* GLM always has .qs, so the pilot never hits it. */
qt_from_disk(m,nm[0],I,D,b,g_drop,&s->g);
qt_from_disk(m,nm[1],I,D,b,g_drop,&s->u);
qt_from_disk(m,nm[2],D,I,b,g_drop,&s->d);
s->eid=eid; return 0;
}
st_tensor *tw[3], *tq[3];
for(int k=0;k<3;k++){
tw[k]=st_find(&m->S,nm[k]);
snprintf(qn,sizeof(qn),"%s.qs",nm[k]); tq[k]=st_find(&m->S,qn);
if(!tw[k]||!tq[k]){ fprintf(stderr,"missing %s\n",nm[k]); if(fatal) exit(1); return -1; }
}
if(g_mmap){
void *bw[3],*bq[3]; int okm=1;
for(int k=0;k<3;k++){
bw[k]=map_of_fd(tw[k]->fd); bq[k]=map_of_fd(tq[k]->fd);
if(!bw[k]||!bq[k]||((tw[k]->off)&3)||((tq[k]->off)&3)) okm=0;
}
if(okm){
QT *qt[3]={&s->g,&s->u,&s->d}; int OO[3]={I,I,D}, II[3]={D,D,I};
for(int k=0;k<3;k++){
int64_t nb=tw[k]->nbytes;
int fmt=(nb==(int64_t)OO[k]*II[k])?1:(nb==(int64_t)OO[k]*((II[k]+1)/2))?2:3;
qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->qf=NULL;
qt[k]->q8=(int8_t*)((char*)bw[k]+tw[k]->off); qt[k]->q4=(uint8_t*)((char*)bw[k]+tw[k]->off);
qt[k]->s=(float*)((char*)bq[k]+tq[k]->off);
}
/* CPU pre-touch: fault the pages in HERE (cheap, parallel, overlapped with the
* resident-experts GPU submit) so the GPU never demand-faults file-backed pages
* (measured catastrophic). madvise starts async readahead, the touch guarantees
* residency. This is pread's I/O without the copy and without the slab. */
for(int k=0;k<3;k++){
char *p=(char*)bw[k]+tw[k]->off; size_t n=(size_t)tw[k]->nbytes;
madvise((void*)((uintptr_t)p & ~16383UL), n+16384, MADV_WILLNEED);
volatile char acc=0;
for(size_t i=0;i<n;i+=4096) acc+=p[i];
acc+=p[n-1]; (void)acc;
char *q=(char*)bq[k]+tq[k]->off; size_t nq=(size_t)tq[k]->nbytes;
for(size_t i=0;i<nq;i+=4096) acc+=q[i];
}
s->eid=eid; return 0;
}
}
int64_t wtot=tw[0]->nbytes+tw[1]->nbytes+tw[2]->nbytes;
int64_t ftot=(tq[0]->nbytes+tq[1]->nbytes+tq[2]->nbytes)/4;
/* rialloca se lo slot (riusato tra layer) e' troppo piccolo per QUESTO expert:
* pread oltre la mappatura = short-read o CORRUZIONE silenziosa dei vicini */
if(!s->slab || wtot+8192 > s->slab_cap){
#ifdef COLI_METAL
/* page-align + zero-copy wrap: the GPU reads this slab in place (unified memory) */
if(s->slab && g_metal_enabled) coli_metal_unregister(s->slab);
compat_aligned_free(s->slab);
size_t need=((size_t)wtot+8192+16383)&~(size_t)16383;
if(posix_memalign((void**)&s->slab,16384,need)){fprintf(stderr,"OOM slab\n"); if(fatal) exit(1); s->slab=NULL; s->slab_cap=0; return -1;}
s->slab_cap=need;
if(g_metal_enabled) coli_metal_register(s->slab,need);
#else
compat_aligned_free(s->slab);
if(posix_memalign((void**)&s->slab,4096,wtot+8192)){fprintf(stderr,"OOM slab\n"); if(fatal) exit(1); s->slab=NULL; s->slab_cap=0; return -1;}
s->slab_cap=wtot+8192;
#endif
}
if(!s->fslab || ftot > s->fslab_cap){
#ifdef COLI_METAL
/* page-align + register: the GPU reads the scales in place (unified memory).
* Honours `fatal` exactly like the CPU arm below — a speculative pilot load
* that hits OOM must unwind into a clean hidden slot, never exit(). */
if(s->fslab && g_metal_enabled) coli_metal_unregister(s->fslab);
free(s->fslab);
size_t fb=(((size_t)ftot*sizeof(float))+16383)&~(size_t)16383;
if(ftot<0 || (uint64_t)ftot > SIZE_MAX/sizeof(float) ||
posix_memalign((void**)&s->fslab,16384,fb)){
fprintf(stderr,"OOM fslab\n"); if(fatal) exit(1);
compat_aligned_free(s->slab); s->slab=NULL; s->slab_cap=0; /* clean, hidden slot (eid stays -1) */
s->fslab=NULL; s->fslab_cap=0; return -1;
}
s->fslab_cap=ftot;
if(g_metal_enabled) coli_metal_register(s->fslab,fb);
#else
free(s->fslab);
if(fatal){ s->fslab=falloc(ftot); } /* main path: byte-identical exit-on-OOM */
else { /* speculative pilot: checked alloc, never exit() */
/* replicate falloc's anti-wrap guard + malloc (no zeroing/alignment) */
if(ftot<0 || (uint64_t)ftot > SIZE_MAX/sizeof(float) ||
!(s->fslab=malloc((size_t)ftot*sizeof(float)))){
fprintf(stderr,"OOM fslab\n");
compat_aligned_free(s->slab); s->slab=NULL; s->slab_cap=0; /* leave a clean, hidden slot (eid stays -1) */
s->fslab=NULL; s->fslab_cap=0; return -1;
}
}
s->fslab_cap=ftot;
#endif
}
int ord[3]={0,1,2}; /* ordina per offset nel file */
for(int a=0;a<3;a++) for(int bb=a+1;bb<3;bb++) if(tw[ord[bb]]->off<tw[ord[a]]->off){ int t=ord[a]; ord[a]=ord[bb]; ord[bb]=t; }
int contig = tw[ord[0]]->fd==tw[ord[1]]->fd && tw[ord[1]]->fd==tw[ord[2]]->fd
&& tw[ord[0]]->off+tw[ord[0]]->nbytes==tw[ord[1]]->off
&& tw[ord[1]]->off+tw[ord[1]]->nbytes==tw[ord[2]]->off;
int64_t pos[3]; int done=0;
if(contig){
int64_t off0=tw[ord[0]]->off;
int dfd = g_direct ? st_direct_fd(&m->S, tw[ord[0]]->fd) : -1;
if(dfd>=0){ /* O_DIRECT: offset/len allineati a 4K */
int64_t base=off0 & ~4095LL, need=(off0-base)+wtot;
int64_t len=(need+4095)&~4095LL;
ssize_t r=pread(dfd, s->slab, len, base);
if(r>=need){
pos[ord[0]]=off0-base; pos[ord[1]]=pos[ord[0]]+tw[ord[0]]->nbytes;
pos[ord[2]]=pos[ord[1]]+tw[ord[1]]->nbytes; done=1;
}
}
if(!done){ /* fallback bufferizzato */
if(pread(tw[ord[0]]->fd, s->slab, wtot, off0)!=wtot){ perror("pread expert"); if(fatal) exit(1); return -1; }
pos[ord[0]]=0; pos[ord[1]]=tw[ord[0]]->nbytes; pos[ord[2]]=tw[ord[0]]->nbytes+tw[ord[1]]->nbytes; done=1;
}
}
if(!done){ /* non contigui: 3 pread bufferizzate */
int64_t o=0;
for(int a=0;a<3;a++){ int k=ord[a];
if(pread(tw[k]->fd, s->slab+o, tw[k]->nbytes, tw[k]->off)!=tw[k]->nbytes){ perror("pread expert"); if(fatal) exit(1); return -1; }
pos[k]=o; o+=tw[k]->nbytes; }
}
float *fp[3]; int64_t fo=0; /* scale (piccole) */
for(int k=0;k<3;k++){
if(pread(tq[k]->fd, (char*)(s->fslab+fo), tq[k]->nbytes, tq[k]->off)!=tq[k]->nbytes){ perror("pread qs"); if(fatal) exit(1); return -1; }
fp[k]=s->fslab+fo; fo+=tq[k]->nbytes/4; }
if(g_drop){ /* scarta subito le pagine: evita che la page
* cache in pressione strangoli il throughput */
posix_fadvise(tw[ord[0]]->fd, tw[ord[0]]->off, wtot, POSIX_FADV_DONTNEED);
for(int k=0;k<3;k++) posix_fadvise(tq[k]->fd, tq[k]->off, tq[k]->nbytes, POSIX_FADV_DONTNEED);
}
QT *qt[3]={&s->g,&s->u,&s->d}; int OO[3]={I,I,D}, II[3]={D,D,I};
for(int k=0;k<3;k++){
int64_t nb=tw[k]->nbytes;
int fmt = (nb==(int64_t)OO[k]*II[k])?1 : (nb==(int64_t)OO[k]*((II[k]+1)/2))?2 : 3;
qt[k]->fmt=fmt; qt[k]->O=OO[k]; qt[k]->I=II[k]; qt[k]->qf=NULL;
qt[k]->q8=(int8_t*)(s->slab+pos[k]); qt[k]->q4=s->slab+pos[k]; qt[k]->s=fp[k];
}
s->eid=eid; return 0;
}
/* ============================ PIPE: load ‖ matmul ============================
* Overlap NVMe expert-weight loads with expert matmul. A small persistent pool
* of I/O worker pthreads runs the misses' pread (expert_load) into distinct
* ws[] slabs and sets a per-slot `ready` flag; the MAIN thread walks the block's
* experts in order, waiting on ready[q] only for the expert it needs right now,
* and does all matmul_qt on itself (matmul_qt parallelises internally via OpenMP
* and checks !omp_in_parallel() for GPU dispatch — so it must stay off the omp
* team and off these I/O threads).
*
* Cross-generation safety is provided by a single generation-tagged, lock-free
* cursor `cur = (gen<<8) | index`. The main thread is the sole writer of `gen`
* (monotonic bump, so no ABA); workers grab jobs by CAS-advancing the low 8-bit
* index. THE INVARIANT: a worker reads eids[i]/layer only AFTER its winning CAS,
* and that CAS's comparand carries the generation — so if `cur`'s gen advanced
* (a new batch was published), the CAS fails and the worker re-reads, seeing the
* new generation. A straggler preempted anywhere (wake gap, post-cursor) can
* therefore NEVER grab a wrong-generation job or read torn batch state: its
* first act is a gen-checked CAS. dispatch publishes all batch state with
* relaxed stores and then RELEASE-stores `cur`; each worker ACQUIRE-loads `cur`,
* so the ready[] reset + eids[]/njobs/layer are visible before any worker acts.
* The per-expert pipe_wait(ready[q]) in the matmul loop makes every grabbed job
* complete before the block ends, so no grab outlives its generation — which is
* why the old `active` counter AND the end-of-block drain barrier are gone (both
* were redundant with those per-slot waits + the gen-tagged cursor). The mutex/
* condvar exist ONLY to park/wake idle workers, never for correctness. Gated
* behind PIPE=1; OFF => the original blocking-load + serial-matmul path runs
* byte-identically. */
static int g_pipe=0; /* PIPE=1: async expert-load pipeline (default OFF) */
static int g_pipe_nw=8; /* PIPE_WORKERS=n: I/O worker threads (disk-parallel reads) */
typedef struct {
_Atomic uint64_t cur; /* (gen<<8)|index; gen main-only, index 0..njobs (≤64) */
_Atomic int njobs; /* current batch job count */
_Atomic int eids[64]; /* current batch expert ids */
_Atomic int layer; /* current batch layer */
_Atomic int ready[64]; /* per-slot load-done flag */
pthread_mutex_t mx; pthread_cond_t cv; /* ONLY for parking/waking idle workers */
Model *m;
pthread_t th[16]; int nw; int started;
} PipePool;
static PipePool g_pp;
static void *pipe_worker(void *arg){
(void)arg; PipePool *p=&g_pp; uint64_t seen=0;
for(;;){
pthread_mutex_lock(&p->mx);
while((atomic_load_explicit(&p->cur,memory_order_relaxed)>>8)==seen)
pthread_cond_wait(&p->cv,&p->mx);
pthread_mutex_unlock(&p->mx);
for(;;){
uint64_t c=atomic_load_explicit(&p->cur,memory_order_acquire);
seen=c>>8;
uint32_t i=(uint32_t)(c & 0xFF);
if(i >= (uint32_t)atomic_load_explicit(&p->njobs,memory_order_relaxed))
break; /* batch drained → re-park */
if(atomic_compare_exchange_weak_explicit(&p->cur,&c,c+1,
memory_order_acq_rel,memory_order_relaxed)){
int L =atomic_load_explicit(&p->layer,memory_order_relaxed);
int eid=atomic_load_explicit(&p->eids[i],memory_order_relaxed); /* AFTER winning CAS */
expert_load(p->m,L,eid,&p->m->ws[i],1); /* needed-now load: fatal on I/O error (matches serial path) */
atomic_store_explicit(&p->ready[i],1,memory_order_release);
}
/* CAS failed → another worker advanced index (or gen advanced): re-loop */
}
}
return NULL;
}
static void pipe_init(Model *m){
if(g_pp.started) return;
g_pp.m=m; g_pp.nw=g_pipe_nw; if(g_pp.nw>16) g_pp.nw=16; if(g_pp.nw<1) g_pp.nw=1;
atomic_store(&g_pp.cur,0); atomic_store(&g_pp.njobs,0);
pthread_mutex_init(&g_pp.mx,NULL); pthread_cond_init(&g_pp.cv,NULL);
for(int i=0;i<g_pp.nw;i++) pthread_create(&g_pp.th[i],NULL,pipe_worker,NULL);
g_pp.started=1;
}
/* enqueue `njobs` loads (slots ws[0..njobs)); returns immediately, workers run ahead.
* Order is load-bearing: write all batch state RELAXED, then RELEASE-store cur to
* publish it, then wake parked workers. */
static void pipe_dispatch(Model *m,int layer,const int *eids,int njobs){
g_pp.m=m;
atomic_store_explicit(&g_pp.njobs,njobs,memory_order_relaxed);
atomic_store_explicit(&g_pp.layer,layer,memory_order_relaxed);
for(int q=0;q<njobs;q++) atomic_store_explicit(&g_pp.eids[q],eids[q],memory_order_relaxed);
for(int q=0;q<njobs;q++) atomic_store_explicit(&g_pp.ready[q],0,memory_order_relaxed); /* reset BEFORE publish */
uint64_t g=(atomic_load_explicit(&g_pp.cur,memory_order_relaxed)>>8)+1;
atomic_store_explicit(&g_pp.cur,(g<<8),memory_order_release); /* PUBLISH */
pthread_mutex_lock(&g_pp.mx); pthread_cond_broadcast(&g_pp.cv); pthread_mutex_unlock(&g_pp.mx);
}
static inline void pipe_wait(int q){
while(!atomic_load_explicit(&g_pp.ready[q],memory_order_acquire)) sched_yield();
}
#ifdef COLI_CUDA
static void expert_host_release(Model *m, ESlot *s){
if(!s->slab&&!s->fslab) return;
#if defined(__APPLE__) || defined(__linux__)
if(s->slab) munlock(s->slab,(size_t)s->slab_cap);
if(s->fslab) munlock(s->fslab,(size_t)s->fslab_cap*sizeof(float));
#endif
int64_t bytes=qt_bytes(&s->g)+qt_bytes(&s->u)+qt_bytes(&s->d);
free(s->slab); free(s->fslab); s->slab=NULL; s->fslab=NULL; s->slab_cap=s->fslab_cap=0;
QT *q[3]={&s->g,&s->u,&s->d};
for(int k=0;k<3;k++){ q[k]->qf=NULL; q[k]->q8=NULL; q[k]->q4=NULL; q[k]->s=NULL; }
m->resident_bytes-=bytes; if(m->resident_bytes<0) m->resident_bytes=0;
}
static void expert_host_ensure(Model *m, int layer, ESlot *s){
if(!s->slab) expert_load(m,layer,s->eid,s,1);
}
#endif
/* prefetch asincrono dei pesi di un expert (e delle sue scale .qs): avvia il readahead
* cosi' le letture sincrone successive trovano la page-cache calda. */
static void expert_prefetch(Model *m, int layer, int eid){
char nm[300];
const char *suf[3]={"gate_proj.weight","up_proj.weight","down_proj.weight"};
for(int k=0;k<3;k++){
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.%d.%s",layer,eid,suf[k]); st_prefetch(&m->S,nm);
char qs[320]; snprintf(qs,sizeof(qs),"%s.qs",nm); st_prefetch(&m->S,qs);
}
}
/* ---- helper per l'ABSORPTION: accesso per-riga ai QT quantizzati ---- */
/* acc[0..I) += coef * W[row,:] (dequant al volo) */
static void qt_addrow(const QT *t, int row, float coef, float *acc){
int I=t->I;
if(t->fmt==0){ const float *w=t->qf+(int64_t)row*I; for(int i=0;i<I;i++) acc[i]+=coef*w[i]; return; }
float c=coef*t->s[row];
if(t->fmt==1){ const int8_t *w=t->q8+(int64_t)row*I; for(int i=0;i<I;i++) acc[i]+=c*(float)w[i]; return; }
if(t->fmt==2){ const uint8_t *w=t->q4+(int64_t)row*((I+1)/2);
for(int i=0;i+1<I;i+=2){ uint8_t b=w[i>>1]; acc[i]+=c*((int)(b&0xF)-8); acc[i+1]+=c*((int)(b>>4)-8); }
if(I&1){ uint8_t b=w[I>>1]; acc[I-1]+=c*((int)(b&0xF)-8); } return; }
const uint8_t *w=t->q4+(int64_t)row*((I+3)/4);
for(int i=0;i<I;i++){ uint8_t b=w[i>>2]; acc[i]+=c*((int)((b>>((i&3)*2))&3)-2); }
}
/* y[0..n) = W[r0+j,:]·x (matvec su una FETTA di righe del QT) */
static void qt_matvec_rows(const QT *t, int r0, int n, const float *x, float *y){
int I=t->I;
for(int j=0;j<n;j++){ int row=r0+j; double a=0;
if(t->fmt==0){ const float *w=t->qf+(int64_t)row*I; for(int i=0;i<I;i++) a+=(double)w[i]*x[i]; }
else if(t->fmt==1){ const int8_t *w=t->q8+(int64_t)row*I; float s=t->s[row];
float acc=0; for(int i=0;i<I;i++) acc+=(float)w[i]*x[i]; a=acc*s; }
else if(t->fmt==2){ const uint8_t *w=t->q4+(int64_t)row*((I+1)/2); float s=t->s[row]; float acc=0;
for(int i=0;i+1<I;i+=2){ uint8_t b=w[i>>1]; acc+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; }
if(I&1){ uint8_t b=w[I>>1]; acc+=((int)(b&0xF)-8)*x[I-1]; } a=acc*s; }
else { const uint8_t *w=t->q4+(int64_t)row*((I+3)/4); float s=t->s[row]; float acc=0;
for(int i=0;i<I;i++){ uint8_t b=w[i>>2]; acc+=((int)((b>>((i&3)*2))&3)-2)*x[i]; } a=acc*s; }
y[j]=(float)a;
}
}
static int g_absorb=-1; /* ABSORB: -1 auto (decode S<=4), 0 mai, 1 sempre (test) */
static int g_dsa_force=0; /* DSA_FORCE=1: selezione sempre attiva (test: top-min(k,T)=denso) */
static int cmp_fdesc(const void *a,const void *b){
float x=*(const float*)a, y=*(const float*)b; return x<y?1:x>y?-1:0; }
/* attenzione MLA con KV-cache compressa, su token nuovi x[S,hidden], pos_base = pos del primo */
/* kvs/pos describe a ragged decode batch: each row may belong to a different
* sequence. NULL keeps the original contiguous, currently-bound KV path. */
static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int pos_base,
KVState *const *kvs, const int *positions, float *out){
Cfg *c=&m->c; int H=c->n_heads, D=c->hidden, qh=c->qk_head, vh=c->v_head;
int kvb_dim=H*(c->qk_nope+vh), Tk=pos_base+S;
double ta0=now_s();
#ifdef COLI_METAL
/* Fused decode attention on GPU: whole layer in one command buffer (keeps the GPU hot).
* S<=4 absorption path with st0==0, DSA selection inactive, and GLM-5.2 int4 dims. */
if(g_metal_enabled && S<=4 && (g_absorb==1||(g_absorb<0&&S<=4)) && m->kv_start[layer]==0
&& D==6144 && H==64 && c->q_lora==2048 && c->kv_lora==512 && c->qk_nope==192
&& c->qk_rope==64 && vh==256 && l->kv_b.fmt==2){
int sel_active = m->has_dsa && layer<c->n_layers && c->idx_type[layer] && (pos_base+S) > c->index_topk;
if(!sel_active){
if(m->has_dsa && layer<c->n_layers && c->idx_type[layer]){ /* index keys for future selection */
for(int s=0;s<S;s++){ int pos=pos_base+s; float *kd=m->Ic[layer]+(int64_t)pos*c->index_hd;
matmul_qt(kd, x+(int64_t)s*D, &m->ix_wk[layer], 1);
layernorm(kd, m->ix_knw[layer], m->ix_knb[layer], c->index_hd, 1e-6f);
rope_interleave(kd, pos, c); }
}
#define WP_(q) ((q).fmt==1?(const void*)(q).q8:(const void*)(q).q4)
int ok = coli_metal_attn_decode(x,
WP_(l->q_a), l->q_a.s, l->q_a.fmt, l->q_a_ln,
WP_(l->q_b), l->q_b.s, l->q_b.fmt,
WP_(l->kv_a), l->kv_a.s, l->kv_a.fmt, l->kv_a_ln,
WP_(l->kv_b), l->kv_b.s, l->kv_b.fmt,
WP_(l->o), l->o.s, l->o.fmt,
m->Lc[layer], m->Rc[layer], S, pos_base, m->kv_start[layer], c->eps, c->theta, c->attn_scale, out);
#undef WP_
if(ok){ m->t_attn += now_s()-ta0; return; }
}
}
#endif
float *ctx=falloc((int64_t)S*H*vh);
float *Q=falloc((int64_t)S*H*qh); /* query (roped) dei token nuovi */
float *QR=falloc((int64_t)S*c->q_lora), *comp=falloc(c->kv_lora+c->qk_rope);
/* 1) per ogni token nuovo: query roped + latente normato e k_rot roped -> in cache.
* QR tiene il residuo q_a per TUTTE le posizioni: serve anche all'indexer DSA. */
for(int s=0;s<S;s++){
KVState *ks=kvs?kvs[s]:m->kv;
const float *xs=x+(int64_t)s*D; int pos=positions?positions[s]:pos_base+s;
float *qresid=QR+(int64_t)s*c->q_lora;
matmul_qt(qresid, xs, &l->q_a, 1);
rmsnorm(qresid, qresid, l->q_a_ln, c->q_lora, c->eps);
float *qfull=Q+(int64_t)s*H*qh; matmul_qt(qfull, qresid, &l->q_b, 1);
for(int h=0;h<H;h++) rope_interleave(qfull+(int64_t)h*qh+c->qk_nope, pos, c);
matmul_qt(comp, xs, &l->kv_a, 1);
float *Ldst=coli_kv_row(ks->Lc[layer],pos,c->kv_lora);
float *Rdst=coli_kv_row(ks->Rc[layer],pos,c->qk_rope);
memcpy(Ldst, comp, c->kv_lora*sizeof(float));
rmsnorm(Ldst, Ldst, l->kv_a_ln, c->kv_lora, c->eps); /* latente normato */
memcpy(Rdst, comp+c->kv_lora, c->qk_rope*sizeof(float));
rope_interleave(Rdst, pos, c); /* k_rot roped, condiviso fra teste */
}
/* ---- DSA lightning indexer ----
* Layer FULL: k_idx dei token nuovi in cache + selezione top-k per query (riusata
* dai layer SHARED successivi). Selezione attiva solo con contesto > index_topk
* (o DSA_FORCE=1 per il test: selezionare TUTTO deve dare l'output denso esatto). */
const int *dsel=NULL, *dnsel=NULL; int dtopk=0;
if(m->has_dsa && layer<c->n_layers && ((!kvs && m->kv_start[layer]==0) || kvs)){
int nh=c->index_nh, hd=c->index_hd; dtopk=c->index_topk;
if(c->idx_type[layer]){
for(int s=0;s<S;s++){
KVState *ks=kvs?kvs[s]:m->kv;
const float *xs=x+(int64_t)s*D; int pos=positions?positions[s]:pos_base+s;
float *kd=coli_kv_row(ks->Ic[layer],pos,hd);
matmul_qt(kd, xs, &m->ix_wk[layer], 1);
layernorm(kd, m->ix_knw[layer], m->ix_knb[layer], hd, 1e-6f);
rope_interleave(kd, pos, c); /* primi qk_rope dim, interleaved */
}
if((int64_t)S*dtopk > m->dsa_scap){
free(m->dsa_sel); free(m->dsa_nsel);
m->dsa_scap=(int64_t)S*dtopk;
m->dsa_sel=malloc((size_t)m->dsa_scap*sizeof(int));
m->dsa_nsel=malloc((size_t)S*sizeof(int));
}
#pragma omp parallel for schedule(dynamic,1)
for(int s=0;s<S;s++){
KVState *ks=kvs?kvs[s]:m->kv;
int pos=positions?positions[s]:pos_base+s, nk=pos+1;
if(ks->kv_start[layer]!=0){ m->dsa_nsel[s]=0; continue; }
if(nk<=dtopk && !g_dsa_force){ m->dsa_nsel[s]=0; continue; }
int keep = nk<dtopk ? nk : dtopk;
float *qi=falloc((int64_t)nh*hd);
matmul_qt(qi, QR+(int64_t)s*c->q_lora, &m->ix_wq[layer], 1);
for(int h=0;h<nh;h++) rope_interleave(qi+(int64_t)h*hd, pos, c);
float *w32=falloc(nh);
matmul_qt(w32, x+(int64_t)s*D, &m->ix_wp[layer], 1);
float wsc=1.f/sqrtf((float)nh), rs=1.f/sqrtf((float)hd);
float *isc=falloc(nk);
for(int t=0;t<nk;t++){
const float *kt=coli_kv_row(ks->Ic[layer],t,hd);
float a=0;
for(int h=0;h<nh;h++){ const float *qhp=qi+(int64_t)h*hd;
float d0=0; for(int i=0;i<hd;i++) d0+=qhp[i]*kt[i];
d0*=rs; if(d0>0) a+=w32[h]*d0; /* ReLU sullo score, poi peso */
}
isc[t]=a*wsc;
}
/* top-keep: soglia via qsort desc, poi scan in ordine di posizione */
float *tmp=falloc(nk); memcpy(tmp,isc,nk*sizeof(float));
qsort(tmp,nk,sizeof(float),cmp_fdesc);
float thr=tmp[keep-1];
int *dst=m->dsa_sel+(int64_t)s*dtopk, nd=0;
for(int t=0;t<nk && nd<keep;t++) if(isc[t]>thr) dst[nd++]=t;
for(int t=0;t<nk && nd<keep;t++) if(isc[t]==thr) dst[nd++]=t;
m->dsa_nsel[s]=nd;
free(qi); free(w32); free(isc); free(tmp);
}
}
if(m->dsa_nsel){ dsel=m->dsa_sel; dnsel=m->dsa_nsel; }
}
/* WEIGHT ABSORPTION (DeepSeek): per S piccoli (decode/verifica MTP) NON si ricostruisce
* k/v per ogni token del contesto. Per linearita':
* q·k_nope_t = (W_K^hT q_nope)·L_t ctx^h = W_V^h (Σ_t a_t L_t)
* costo per step ~O(T·kv_lora) invece di O(T·H·(nope+vh)) del matmul kvb_all. */
int absorb = kvs || g_absorb==1 || (g_absorb<0 && S<=4);
if(absorb && c->kv_lora<=512){
m->t_aproj+=now_s()-ta0; double tac=now_s();
int kvl=c->kv_lora, r0v=c->qk_nope; /* offset righe V dentro il blocco di testa */
/* punteggi per-thread sul HEAP (vedi dev): cap Tk+1 copre anche il kv_start
* per-slot del percorso kvs (MTP: kv_start=-1 -> nt=Tk+1). */
int64_t sc_cap = (int64_t)Tk+1;
float *sc_all = falloc((int64_t)omp_get_max_threads()*sc_cap);
int cuda_core=0;
#ifdef COLI_CUDA
if(S<=4&&g_cuda_enabled&&getenv("COLI_CUDA_ATTN")&&atoi(getenv("COLI_CUDA_ATTN"))&&
l->kv_b.cuda_eligible&&qt_cuda_upload(&l->kv_b)){
cuda_core=1;
for(int s=0;s<S&&cuda_core;s++){
KVState *ks=kvs?kvs[s]:m->kv;int pos=positions?positions[s]:pos_base+s;
int st0=ks->kv_start[layer],nt=pos+1-st0;
if(dnsel&&dnsel[s]>0){cuda_core=0;break;}
cuda_core=coli_cuda_attention_absorb(l->kv_b.cuda,ctx+(int64_t)s*H*vh,
Q+(int64_t)s*H*qh,coli_kv_row(ks->Lc[layer],st0,kvl),
coli_kv_row(ks->Rc[layer],st0,c->qk_rope),H,c->qk_nope,c->qk_rope,
vh,kvl,nt,c->attn_scale);
}
}
#endif
if(!cuda_core){
#pragma omp parallel for collapse(2) schedule(static)
for(int s=0;s<S;s++) for(int h=0;h<H;h++){
KVState *ks=kvs?kvs[s]:m->kv;
int pos=positions?positions[s]:pos_base+s;
const float *qp=Q+(int64_t)s*H*qh+(int64_t)h*qh;
const float *qr=qp+c->qk_nope;
int rbase=h*(c->qk_nope+vh);
float qabs[512]; memset(qabs,0,kvl*sizeof(float));
for(int d=0;d<c->qk_nope;d++) qt_addrow(&l->kv_b, rbase+d, qp[d], qabs);
float *sc = sc_all + (int64_t)omp_get_thread_num()*sc_cap;
int st0=ks->kv_start[layer];
int ns = (dnsel && dnsel[s]>0) ? dnsel[s] : 0; /* DSA: lista top-k o range pieno */
const int *tlist = ns ? dsel+(int64_t)s*dtopk : NULL;
int nt = ns ? ns : pos+1-st0;
for(int jj=0;jj<nt;jj++){ int t = tlist ? tlist[jj] : st0+jj;
const float *Lt=coli_kv_row(ks->Lc[layer],t,kvl);
const float *kr=coli_kv_row(ks->Rc[layer],t,c->qk_rope);
float a=0; for(int i=0;i<kvl;i++) a+=qabs[i]*Lt[i];
for(int d=0;d<c->qk_rope;d++) a+=qr[d]*kr[d];
sc[jj]=a*c->attn_scale;
}
softmax(sc,nt);
float clat[512]; memset(clat,0,kvl*sizeof(float));
for(int jj=0;jj<nt;jj++){ int t = tlist ? tlist[jj] : st0+jj;
const float *Lt=coli_kv_row(ks->Lc[layer],t,kvl);
float a=sc[jj]; for(int i=0;i<kvl;i++) clat[i]+=a*Lt[i]; }
qt_matvec_rows(&l->kv_b, rbase+r0v, vh, clat, ctx+((int64_t)s*H+h)*vh);
}
}
m->t_acore+=now_s()-tac; double tao=now_s();
matmul_qt(out, ctx, &l->o, S); m->t_aout+=now_s()-tao;
free(ctx); free(Q); free(QR); free(comp); free(sc_all);
m->t_attn += now_s()-ta0;
return;
}
/* 2) ricostruzione di k_nope+value per TUTTI i token 0..Tk-1 (un solo matmul su kv_b) */
m->t_aproj+=now_s()-ta0; double tk0=now_s();
int stL=m->kv_start[layer];
float *kvb_all=falloc((int64_t)Tk*kvb_dim);
matmul_qt(kvb_all+(int64_t)stL*kvb_dim, m->Lc[layer]+(int64_t)stL*c->kv_lora, &l->kv_b, Tk-stL);
m->t_kvb += now_s()-tk0;
/* 3) attenzione causale: score = q_pass·k_nope + q_rot·k_rot
* (punteggi sul heap, per-thread: vedi il commento nel ramo absorb) */
int64_t sc_cap = Tk - stL;
float *sc_all = falloc((int64_t)omp_get_max_threads()*sc_cap);
double tac=now_s();
#pragma omp parallel for collapse(2) schedule(static)
for(int s=0;s<S;s++) for(int h=0;h<H;h++){
int pos=pos_base+s;
const float *qp=Q+(int64_t)s*H*qh+(int64_t)h*qh; /* [qk_nope | qk_rope] */
const float *qr=qp+c->qk_nope;
float *sc = sc_all + (int64_t)omp_get_thread_num()*sc_cap;
int st0=m->kv_start[layer];
int ns = (dnsel && dnsel[s]>0) ? dnsel[s] : 0; /* DSA: lista top-k o range pieno */
const int *tlist = ns ? dsel+(int64_t)s*dtopk : NULL;
int nt = ns ? ns : pos+1-st0;
for(int jj=0;jj<nt;jj++){ int t = tlist ? tlist[jj] : st0+jj;
const float *kn=kvb_all+(int64_t)t*kvb_dim+(int64_t)h*(c->qk_nope+vh);
const float *kr=m->Rc[layer]+(int64_t)t*c->qk_rope;
float a=0; for(int d=0;d<c->qk_nope;d++) a+=qp[d]*kn[d];
for(int d=0;d<c->qk_rope;d++) a+=qr[d]*kr[d];
sc[jj]=a*c->attn_scale;
}
softmax(sc,nt);
float *cx=ctx+((int64_t)s*H+h)*vh; for(int d=0;d<vh;d++) cx[d]=0;
for(int jj=0;jj<nt;jj++){ int t = tlist ? tlist[jj] : st0+jj;
const float *vv=kvb_all+(int64_t)t*kvb_dim+(int64_t)h*(c->qk_nope+vh)+c->qk_nope;
float a=sc[jj]; for(int d=0;d<vh;d++) cx[d]+=a*vv[d]; }
}
m->t_acore+=now_s()-tac; double tao=now_s();
matmul_qt(out, ctx, &l->o, S); m->t_aout+=now_s()-tao;
free(ctx); free(Q); free(QR); free(comp); free(kvb_all); free(sc_all);
m->t_attn += now_s()-ta0;
}
static void attention(Model *m, Layer *l, int layer, float *x, int S, int pos_base, float *out){
attention_rows(m,l,layer,x,S,pos_base,NULL,NULL,out);
}
/* MoE GLM su x[S,hidden] -> out (router sigmoid/noaux_tc, n_group=1, + shared expert).
* BATCH-UNION: per S>1 (prefill, verifica MTP) ogni expert UNICO del batch viene caricato
* una volta sola e moltiplicato per tutte le posizioni che lo usano (pesi letti 1 volta);
* lo shared expert e' un unico matmul a S righe. Per posizione l'accumulo resta
* nell'ordine (routed nel loro ordine di union, poi shared). */
static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out){
if(g_pilot_real){ /* barriera cross-layer: prendi possesso di QUESTO layer e aspetta
* l'eventuale load-pilota in volo sullo stesso layer (dopodiche' il
* worker droppa ogni nuovo load <= layer -> ecache[layer] e' stabile
* per tutto il resolve/matmul/promozione qui sotto). */
for(;;){
pthread_mutex_lock(&g_pilot_mx);
atomic_store_explicit(&g_cur_moe_layer,layer,memory_order_release);
int inf=atomic_load_explicit(&g_pilot_inflight,memory_order_acquire);
pthread_mutex_unlock(&g_pilot_mx);
if(inf!=layer) break;
sched_yield();
}
}
Cfg *c=&m->c; int D=c->hidden, E=c->n_experts, K=c->topk, I=c->moe_inter;
float *logit=falloc(E), *choice=falloc(E);
int sI=c->moe_inter*c->n_shared;
/* ---- FASE A: routing di tutte le S posizioni ---- */
int *idxs=malloc((size_t)S*K*sizeof(int)); float *ws=malloc((size_t)S*K*sizeof(float));
int *keff=malloc(S*sizeof(int));
#ifdef COLI_METAL
if(g_pre_idx){ /* routing gia' calcolata dal layer CB (GPU) */
memcpy(idxs,g_pre_idx,(size_t)S*K*sizeof(int));
memcpy(ws,g_pre_w,(size_t)S*K*sizeof(float));
memcpy(keff,g_pre_keff,(size_t)S*sizeof(int));
for(int s=0;s<S;s++){
m->ereq+=keff[s];
for(int kk=0;kk<keff[s];kk++){
m->eusage[layer][idxs[(int64_t)s*K+kk]]++;
if(m->eheat[layer][idxs[(int64_t)s*K+kk]]<UINT32_MAX) m->eheat[layer][idxs[(int64_t)s*K+kk]]++;
}
for(int d=0;d<D;d++) out[(int64_t)s*D+d]=0;
}
} else
#endif
for(int s=0;s<S;s++){
const float *xs=x+(int64_t)s*D;
matmul(logit, xs, l->router, 1, D, E);
for(int e=0;e<E;e++){ logit[e]=sigmoidf(logit[e]); choice[e]=logit[e]+l->router_bias[e]; }
int *idx=idxs+(int64_t)s*K; float *w=ws+(int64_t)s*K;
int Ksel = g_topk>0 ? (g_topk<K?g_topk:K) : K;
for(int kk=0;kk<Ksel;kk++){ int best=-1; float bv=-1e30f;
for(int e=0;e<E;e++){ int tk=0; for(int j=0;j<kk;j++) if(idx[j]==e){tk=1;break;}
if(!tk && choice[e]>bv){bv=choice[e];best=e;} }
idx[kk]=best; w[kk]=logit[best];
}
int Ke=Ksel;
if(g_topp>0 && g_topp<1.f){
for(int a=1;a<Ksel;a++){ int ii=idx[a]; float ww=w[a]; int b=a-1;
while(b>=0 && w[b]<ww){ w[b+1]=w[b]; idx[b+1]=idx[b]; b--; } w[b+1]=ww; idx[b+1]=ii; }
float tot=1e-20f; for(int kk=0;kk<Ksel;kk++) tot+=w[kk];
float cum=0; for(int kk=0;kk<Ksel;kk++){ cum+=w[kk]; if(cum>=g_topp*tot){ Ke=kk+1; break; } }
}
keff[s]=Ke; m->ereq+=Ke;
for(int kk=0;kk<Ke;kk++){
m->eusage[layer][idx[kk]]++;
if(m->eheat[layer][idx[kk]]<UINT32_MAX) m->eheat[layer][idx[kk]]++;
m->elast[layer][idx[kk]]=++m->eaccess_clock;
}
if(c->norm_topk){ float sm=0; for(int kk=0;kk<Ke;kk++) sm+=w[kk]; sm+=1e-20f; for(int kk=0;kk<Ke;kk++) w[kk]/=sm; }
for(int kk=0;kk<Ke;kk++) w[kk]*=c->routed_scale;
for(int d=0;d<D;d++) out[(int64_t)s*D+d]=0;
}
if(g_looka && S==1 && layer<c->n_layers){
int Ke=keff[0];
if(m->enr[layer]>0){ /* [0] vs routing del token precedente */
for(int kk=0;kk<Ke;kk++) for(int z=0;z<m->enr[layer];z++)
if(m->eroute[layer][z]==idxs[kk]){ la_hit[0]++; break; }
la_tot[0]+=Ke;
}
for(int kind=0;kind<2;kind++) if(la_val[kind][layer]){ /* [1]/[2] vs predizioni */
for(int kk=0;kk<Ke;kk++) for(int z=0;z<K;z++)
if(la_pred[kind][layer][z]==idxs[kk]){ la_hit[1+kind]++; break; }
la_tot[1+kind]+=Ke; la_val[kind][layer]=0;
}
}
m->enr[layer]=keff[S-1]; for(int kk=0;kk<keff[S-1];kk++) m->eroute[layer][kk]=idxs[(int64_t)(S-1)*K+kk];
/* ---- FASE B: union degli expert del batch ---- */
int *uniq=malloc((size_t)E*sizeof(int)); int nu=0;
unsigned char seen[E]; memset(seen,0,(size_t)E);
for(int s=0;s<S;s++) for(int kk=0;kk<keff[s];kk++){
int e=idxs[(int64_t)s*K+kk];
if(!seen[e]){ seen[e]=1; uniq[nu++]=e; }
}
/* ---- FASE C/D: risolvi (pin/cache/disco) e calcola, a blocchi di 64 unici ---- */
float *xg=falloc((int64_t)S*D), *gg=falloc((int64_t)S*I), *uu=falloc((int64_t)S*I), *hh=falloc((int64_t)S*D);
int *rows=malloc(S*sizeof(int)); float *rw=malloc(S*sizeof(float));
#ifdef COLI_CUDA
int group_enabled=S<=64;
float *group_x=group_enabled?falloc((int64_t)S*K*D):NULL;
float *group_y=group_enabled?falloc((int64_t)S*K*D):NULL;
int *group_row=group_enabled?malloc((size_t)64*S*sizeof(int)):NULL;
float *group_weight=group_enabled?malloc((size_t)64*S*sizeof(float)):NULL;
#endif
int shared_on_gpu=0; (void)shared_on_gpu; /* set by the Metal path when Phase E was fused */
for(int base=0;base<nu;base+=64){
int nb = nu-base<64 ? nu-base : 64;
ESlot *use[64]; int missk[64]; int qof[64]; int nmiss=0;
for(int j=0;j<nb;j++){ int eid=uniq[base+j]; use[j]=NULL; qof[j]=-1;
ESlot *P=m->pin[layer];
for(int z=0;z<m->npin[layer];z++) if(P[z].eid==eid){ m->hits++; use[j]=&P[z]; break; }
if(!use[j]){ ESlot *Sl=m->ecache[layer]; int nn=m->ecn[layer];
for(int z=0;z<nn;z++) if(Sl[z].eid==eid){ m->hits++; Sl[z].used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); use[j]=&Sl[z]; break; } }
if(!use[j]){ qof[j]=nmiss; use[j]=&m->ws[nmiss]; missk[nmiss++]=j; m->miss++; }
}
int metal_done=0;
#ifdef COLI_METAL
/* GPU/disk OVERLAP: submit the RESIDENT experts (pin/LRU hits, + shared expert on
* the first block) to the GPU BEFORE loading the missed experts from disk, so the
* preads run while the GPU computes; the missed subset follows in a second submit.
* Per-subset CPU fallback on unresolved slab / bad fmt / GPU fault. */
int is_miss[64]={0}; ColiMetalMoeHandle *mh=NULL;
int cpu_res=1, cpu_miss=1, mh_shared=0, nbb=0, Rtot=0, mfmt=-1, sh_in=0;
const void *MG[65],*MU[65],*MD[65]; const float *MGS[65],*MUS[65],*MDS[65];
int xoffb[65],nrb[65];
float *mxg=NULL; int *mrows=NULL; float *mrw=NULL;
/* subset builder: experts with is_miss==WANTMISS (+ shared expert when TRY_SH) */
#define MB_BUILD(WANTMISS, TRY_SH) do{ \
nbb=0; Rtot=0; mfmt=-1; sh_in=0; \
for(int j=0;j<nb;j++){ if(is_miss[j]!=(WANTMISS)) continue; \
int eid=uniq[base+j]; ESlot *e=use[j]; int cnt=0; \
for(int s=0;s<S;s++) for(int kk=0;kk<keff[s];kk++) \
if(idxs[(int64_t)s*K+kk]==eid){ cnt++; break; } \
if(!cnt) continue; \
if(mfmt<0) mfmt=e->g.fmt; \
MG[nbb]=e->g.fmt==1?(const void*)e->g.q8:(const void*)e->g.q4; \
MU[nbb]=e->u.fmt==1?(const void*)e->u.q8:(const void*)e->u.q4; \
MD[nbb]=e->d.fmt==1?(const void*)e->d.q8:(const void*)e->d.q4; \
MGS[nbb]=e->g.s; MUS[nbb]=e->u.s; MDS[nbb]=e->d.s; \
xoffb[nbb]=Rtot; nrb[nbb]=cnt; Rtot+=cnt; nbb++; \
} \
if(TRY_SH){ int shf = mfmt<0 ? l->sh_gate.fmt : mfmt; \
if(c->n_shared==1 && sI==I && l->sh_gate.fmt==shf && l->sh_up.fmt==shf && l->sh_down.fmt==shf){ \
if(mfmt<0) mfmt=shf; \
MG[nbb]=shf==1?(const void*)l->sh_gate.q8:(const void*)l->sh_gate.q4; \
MU[nbb]=shf==1?(const void*)l->sh_up.q8 :(const void*)l->sh_up.q4; \
MD[nbb]=shf==1?(const void*)l->sh_down.q8:(const void*)l->sh_down.q4; \
MGS[nbb]=l->sh_gate.s; MUS[nbb]=l->sh_up.s; MDS[nbb]=l->sh_down.s; \
xoffb[nbb]=Rtot; nrb[nbb]=S; Rtot+=S; nbb++; sh_in=1; } } \
int p=0; \
for(int j=0;j<nb;j++){ if(is_miss[j]!=(WANTMISS)) continue; int eid=uniq[base+j]; \
for(int s=0;s<S;s++) for(int kk=0;kk<keff[s];kk++) \
if(idxs[(int64_t)s*K+kk]==eid){ \
memcpy(mxg+(int64_t)p*D, x+(int64_t)s*D, D*sizeof(float)); \
mrows[p]=s; mrw[p]=ws[(int64_t)s*K+kk]; p++; break; } } \
if(sh_in) for(int s=0;s<S;s++){ \
memcpy(mxg+(int64_t)p*D, x+(int64_t)s*D, D*sizeof(float)); \
mrows[p]=s; mrw[p]=1.0f; p++; } \
}while(0)
if(g_metal_enabled){
for(int q=0;q<nmiss;q++) is_miss[missk[q]]=1;
mxg=falloc((int64_t)(nb+1)*S*D);
mrows=malloc((size_t)(nb+1)*S*sizeof(int)); mrw=malloc((size_t)(nb+1)*S*sizeof(float));
MB_BUILD(0, base==0 && !g_pre_sh);
if(nbb>0){
double t0=now_s();
mh=coli_metal_moe_block_begin(nbb,D,I,mfmt,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw);
m->t_emm += now_s()-t0;
if(mh){ cpu_res=0; mh_shared=sh_in; }
} else cpu_res=0;
}
#endif
/* Expert loads run HERE, after the resident-experts GPU submit above: under METAL the
* preads overlap the GPU compute (that submit is async). With METAL off the submit block
* is a no-op / compiled out, so this sits exactly where dev put it and CPU behaviour is
* unchanged. */
if(nmiss){
if(g_pipe){ /* PIPE: launch loads async, matmul overlaps them */
if(!g_pp.started) pipe_init(m);
double t0=now_s();
int eids[64]; for(int q=0;q<nmiss;q++) eids[q]=uniq[base+missk[q]];
pipe_dispatch(m,layer,eids,nmiss);
m->t_edisk += now_s()-t0; /* dispatch only; real reads hide behind matmul */
} else { double t0=now_s(); /* ORIGINALE: blocking parallel load */
#pragma omp parallel for schedule(dynamic,1)
for(int q=0;q<nmiss;q++) expert_load(m,layer,uniq[base+missk[q]],&m->ws[q],1);
m->t_edisk += now_s()-t0; }
}
/* I/O ASINCRONO: readahead (WILLNEED) del blocco SUCCESSIVO mentre calcoliamo
* questo — il kernel legge in background, le pread dopo trovano cache calda */
if(base+64<nu){
int nb2 = nu-(base+64)<64 ? nu-(base+64) : 64;
for(int j=0;j<nb2;j++){ int eid=uniq[base+64+j]; int found=0;
ESlot *P=m->pin[layer];
for(int z=0;z<m->npin[layer] && !found;z++) if(P[z].eid==eid) found=1;
ESlot *Sl=m->ecache[layer];
for(int z=0;z<m->ecn[layer] && !found;z++) if(Sl[z].eid==eid) found=1;
if(!found) expert_prefetch(m,layer,eid);
}
}
#ifdef COLI_CUDA
ESlot *group_e[64]; int group_n[64]; int ngroup=0;
#endif
#ifdef COLI_METAL
if(g_metal_enabled){
/* PIPE drain. Two reasons this barrier is mandatory here, and not optional:
* 1) MB_BUILD(1) hands the missed experts' slabs straight to the GPU — a slot still
* being pread by an I/O worker would be matmul-ed half-loaded.
* 2) PIPE's only drain barrier is the per-expert pipe_wait() in the CPU matmul loop
* below, which metal_done SKIPS ENTIRELY. Without this, a still-writing worker
* would race the end-of-block LRU swap that recycles ws[].
* pipe_wait() is an idempotent spin on ready[q], so the per-expert waits below stay
* correct (and free) when a subset falls back to the CPU. */
if(g_pipe && nmiss){ double tw=now_s();
for(int q=0;q<nmiss;q++) pipe_wait(q);
m->t_edisk += now_s()-tw; }
MB_BUILD(1, 0); /* missed experts, now loaded */
if(nbb>0){
double t0=now_s();
if(coli_metal_moe_block(nbb,D,I,mfmt,MG,MU,MD,MGS,MUS,MDS,mxg,xoffb,nrb,mrows,mrw,out,S)) cpu_miss=0;
m->t_emm += now_s()-t0;
} else cpu_miss=0;
if(mh){ double t0=now_s();
if(coli_metal_moe_block_end(mh,out)){ if(mh_shared) shared_on_gpu=1; }
else cpu_res=1;
m->t_emm += now_s()-t0; mh=NULL; }
metal_done = (!cpu_res && !cpu_miss);
free(mxg); free(mrows); free(mrw);
}
#undef MB_BUILD
#endif
if(!metal_done)
for(int j=0;j<nb;j++){ int eid=uniq[base+j]; ESlot *e=use[j];
/* Drain this miss's async load BEFORE the nr==0 early-exit below: every
* dispatched slot must be waited before the end-of-block LRU swap can reuse
* its ws[] slab, so correctness does not depend on the nr>=1 routing invariant.
* Stays ABOVE the METAL skip: a subset that fell back to the CPU still needs its
* slot drained here, and under METAL the block-level drain above already ran (this
* spin is then a no-op). */
if(g_pipe && qof[j]>=0){ double tw=now_s(); pipe_wait(qof[j]); m->t_edisk += now_s()-tw; }
#ifdef COLI_METAL
/* skip the subsets already computed on GPU */
if(g_metal_enabled && ((is_miss[j] && !cpu_miss) || (!is_miss[j] && !cpu_res))) continue;
#endif
int nr=0; /* righe (posizioni) che usano questo expert */
for(int s=0;s<S;s++) for(int kk=0;kk<keff[s];kk++)
if(idxs[(int64_t)s*K+kk]==eid){ rows[nr]=s; rw[nr]=ws[(int64_t)s*K+kk]; nr++; break; }
if(!nr) continue;
#ifdef COLI_CUDA
if(g_cuda_enabled && e->g.cuda_eligible) m->gpu_expert_calls++;
if(group_enabled && g_cuda_enabled && e->g.cuda_eligible && e->u.cuda_eligible && e->d.cuda_eligible &&
!omp_in_parallel()){
group_e[ngroup]=e; group_n[ngroup]=nr;
for(int r=0;r<nr;r++){ group_row[(int64_t)ngroup*S+r]=rows[r]; group_weight[(int64_t)ngroup*S+r]=rw[r]; }
ngroup++; continue;
}
#endif
for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D, x+(int64_t)rows[r]*D, D*sizeof(float));
double t0=now_s();
#ifdef COLI_CUDA
if(!group_enabled && g_cuda_enabled && e->g.cuda_eligible && e->u.cuda_eligible &&
e->d.cuda_eligible && !omp_in_parallel() &&
coli_cuda_expert_mlp(e->g.cuda,e->u.cuda,e->d.cuda,hh,xg,nr)){
for(int r=0;r<nr;r++){ float *os=out+(int64_t)rows[r]*D,wgt=rw[r],*hr=hh+(int64_t)r*D;
for(int d=0;d<D;d++) os[d]+=wgt*hr[d]; }
m->t_emm+=now_s()-t0; continue;
}
if(!e->slab) expert_host_ensure(m,layer,e);
#endif
expert_gate_up(gg,uu,xg,&e->g,&e->u,nr);
for(int64_t z=0;z<(int64_t)nr*I;z++) gg[z]=siluf(gg[z])*uu[z];
matmul_qt(hh, gg, &e->d, nr);
for(int r=0;r<nr;r++){ float *os=out+(int64_t)rows[r]*D, wgt=rw[r], *hr=hh+(int64_t)r*D;
for(int d=0;d<D;d++) os[d]+=wgt*hr[d]; }
m->t_emm += now_s()-t0;
}
#ifdef COLI_CUDA
ColiCudaTensor *dev_g[COLI_CUDA_MAX_DEVICES][64],*dev_u[COLI_CUDA_MAX_DEVICES][64];
ColiCudaTensor *dev_d[COLI_CUDA_MAX_DEVICES][64];
int dev_rows[COLI_CUDA_MAX_DEVICES][64],dev_which[COLI_CUDA_MAX_DEVICES][64];
int dev_nc[COLI_CUDA_MAX_DEVICES]={0},dev_total[COLI_CUDA_MAX_DEVICES]={0};
int dev_off[COLI_CUDA_MAX_DEVICES]={0},dev_ok[COLI_CUDA_MAX_DEVICES]={0};
for(int di=0;di<g_cuda_ndev;di++) for(int q=0;q<ngroup;q++)
if(group_e[q]->g.cuda_device==g_cuda_devices[di]) dev_total[di]+=group_n[q];
for(int di=1;di<g_cuda_ndev;di++) dev_off[di]=dev_off[di-1]+dev_total[di-1];
for(int di=0;di<g_cuda_ndev;di++){
int cursor=0,device=g_cuda_devices[di];
for(int q=0;q<ngroup;q++) if(group_e[q]->g.cuda_device==device){
int nc=dev_nc[di]++; ESlot *e=group_e[q];
dev_g[di][nc]=e->g.cuda; dev_u[di][nc]=e->u.cuda; dev_d[di][nc]=e->d.cuda;
dev_rows[di][nc]=group_n[q]; dev_which[di][nc]=q;
for(int r=0;r<group_n[q];r++) memcpy(group_x+(int64_t)(dev_off[di]+cursor+r)*D,
x+(int64_t)group_row[(int64_t)q*S+r]*D,D*sizeof(float));
cursor+=group_n[q];
}
}
double tg=now_s();
#pragma omp parallel for if(g_cuda_ndev>1) schedule(static)
for(int di=0;di<g_cuda_ndev;di++) if(dev_nc[di])
dev_ok[di]=coli_cuda_expert_group(dev_g[di],dev_u[di],dev_d[di],dev_rows[di],dev_nc[di],
group_y+(int64_t)dev_off[di]*D,group_x+(int64_t)dev_off[di]*D);
for(int di=0;di<g_cuda_ndev;di++){
int off=dev_off[di];
for(int q=0;q<dev_nc[di];q++){
int gi=dev_which[di][q],nr=group_n[gi]; ESlot *e=group_e[gi];
if(!dev_ok[di]){
for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D,x+(int64_t)group_row[(int64_t)gi*S+r]*D,D*sizeof(float));
if(!coli_cuda_expert_mlp(e->g.cuda,e->u.cuda,e->d.cuda,hh,xg,nr)){
expert_host_ensure(m,layer,e);
expert_gate_up(gg,uu,xg,&e->g,&e->u,nr);
for(int64_t z=0;z<(int64_t)nr*I;z++) gg[z]=siluf(gg[z])*uu[z];
matmul_qt(hh,gg,&e->d,nr);
}
}
float *src=dev_ok[di]?group_y+(int64_t)off*D:hh;
for(int r=0;r<nr;r++){ float *os=out+(int64_t)group_row[(int64_t)gi*S+r]*D,wgt=group_weight[(int64_t)gi*S+r];
for(int d=0;d<D;d++) os[d]+=wgt*src[(int64_t)r*D+d]; }
off+=nr;
}
}
m->t_emm+=now_s()-tg;
#endif
/* No drain barrier: the per-expert pipe_wait(qof[j]) above (issued for every
* dispatched miss slot, before the nr==0 skip) already waited on all ws[] loads
* for this block, so they are complete before the LRU swap — and the gen-tagged
* cursor keeps any still-spinning worker off a wrong-generation slot. */
{ ESlot *Sl=m->ecache[layer]; int *nn=&m->ecn[layer]; /* promozione LRU (swap buffer) */
int promo = nmiss<m->ecap ? nmiss : m->ecap;
for(int a=0;a<promo;a++){ int q=nmiss-1-a; ESlot *dst;
if(*nn<m->ecap) dst=&Sl[(*nn)++];
else { int lru=0; for(int z=1;z<*nn;z++) if(Sl[z].used<Sl[lru].used) lru=z; dst=&Sl[lru]; }
ESlot tmp=*dst; *dst=m->ws[q]; m->ws[q]=tmp; dst->used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED); }
}
}
/* ---- FASE E: shared expert, un matmul a S righe (skipped se fuso nel blocco GPU) ---- */
float *sg=falloc((int64_t)S*sI), *su=falloc((int64_t)S*sI);
#ifdef COLI_METAL
if(g_pre_sh){ for(int64_t z=0;z<(int64_t)S*D;z++) out[z]+=g_pre_sh[z]; shared_on_gpu=1; }
#endif
if(!shared_on_gpu){
matmul_qt(sg, x, &l->sh_gate, S);
matmul_qt(su, x, &l->sh_up, S);
for(int64_t z=0;z<(int64_t)S*sI;z++) sg[z]=siluf(sg[z])*su[z];
matmul_qt(hh, sg, &l->sh_down, S);
for(int64_t z=0;z<(int64_t)S*D;z++) out[z]+=hh[z];
}
free(logit); free(choice); free(idxs); free(ws); free(keff); free(uniq);
free(xg); free(gg); free(uu); free(hh); free(rows); free(rw); free(sg); free(su);
#ifdef COLI_CUDA
free(group_x); free(group_y); free(group_row); free(group_weight);
#endif
}
static void dense_mlp(Layer *l, float *x, int S, int D, int I, float *out){
float *g=falloc((int64_t)S*I), *u=falloc((int64_t)S*I);
matmul_qt(g, x, &l->gate_proj, S);
matmul_qt(u, x, &l->up_proj, S);
for(int64_t i=0;i<(int64_t)S*I;i++) g[i]=siluf(g[i])*u[i];
matmul_qt(out, g, &l->down_proj, S);
free(g); free(u);
}
/* LOOKA: predice il top-K del router del layer `target` dallo stato h (residual stream),
* usando la STESSA pipeline del routing vero (post_ln -> router -> sigmoid+bias, top-K).
* kind 0 = stesso layer saltando l'attention, kind 1 = layer successivo. */
static void la_predict(Model *m, int target, const float *h, int kind){
Cfg *c=&m->c; Layer *l=&m->L[target]; int D=c->hidden, E=c->n_experts, K=c->topk;
float *nrm=falloc(D), *ch=falloc(E);
rmsnorm(nrm,h,l->post_ln,D,c->eps);
matmul(ch,nrm,l->router,1,D,E);
for(int e=0;e<E;e++) ch[e]=sigmoidf(ch[e])+l->router_bias[e];
int *pred=la_pred[kind][target];
for(int kk=0;kk<K;kk++){ int best=-1; float bv=-1e30f;
for(int e=0;e<E;e++){ int tk=0; for(int j=0;j<kk;j++) if(pred[j]==e){tk=1;break;}
if(!tk && ch[e]>bv){bv=ch[e];best=e;} }
pred[kk]=best; }
la_val[kind][target]=1;
free(nrm); free(ch);
}
/* PILOTA: prefetch guidato dal router. Predice il top-K del layer L+1 dallo stato
* post-attention di L (recall misurato 71.6% su GLM-5.2, vs 41.3% del token precedente)
* e lancia il WILLNEED degli expert mancanti MENTRE il MoE di L legge i suoi: il disco
* lavora nei tempi morti del calcolo invece di aspettare il routing vero. Con MTP attiva
* predice per TUTTE le posizioni del draft: la speculazione pilota anche l'I/O.
* PILOT_K limita alle prime k predizioni (la testa del ranking e' piu' affidabile
* della coda: meno banda sprecata sulle predizioni sbagliate).
*
* I WILLNEED partono da un THREAD I/O dedicato: con la coda disco satura la submit
* del fadvise BLOCCA (~0.5ms x 169k chiamate = +92s/48 token, misurato) — inline
* il pilota costava piu' di quanto rendesse. Ring lock-free 1P/1C; pieno = scarta
* (un hint perso non e' un errore). */
static struct { int l,e; } pilot_q[4096];
static volatile unsigned pilot_w=0, pilot_r=0;
static Model *pilot_m=NULL;
/* PILOT_REAL: load VERO dell'expert predetto dentro la LRU del layer FUTURO. Vedi
* l'invariante di sicurezza accanto a g_pilot_real. Il pread (lento) gira FUORI dal lock;
* il lock protegge solo la scelta/pubblicazione dello slot e l'handshake col main. */
static void pilot_realload(Model *m, int layer, int eid){
pthread_mutex_lock(&g_pilot_mx);
if(layer <= atomic_load_explicit(&g_cur_moe_layer,memory_order_acquire)){
atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed);
pthread_mutex_unlock(&g_pilot_mx); return; /* il main possiede gia' questo layer */
}
ESlot *P=m->pin[layer]; /* gia' residente (pin o ecache)? skip */
for(int z=0;z<m->npin[layer];z++) if(P[z].eid==eid){ pthread_mutex_unlock(&g_pilot_mx); return; }
ESlot *Sl=m->ecache[layer]; int nn=m->ecn[layer];
for(int z=0;z<nn;z++) if(Sl[z].eid==eid){ pthread_mutex_unlock(&g_pilot_mx); return; }
int slot,isnew; /* cresci se c'e' posto, altrimenti LRU */
if(nn<m->ecap){ slot=nn; isnew=1; }
else { int lru=0; for(int z=1;z<nn;z++) if(Sl[z].used<Sl[lru].used) lru=z; slot=lru; isnew=0; }
ESlot *dst=&Sl[slot];
dst->eid=-1; /* nascondi dagli scan-hint mentre carica */
atomic_store_explicit(&g_pilot_inflight,layer,memory_order_release);
pthread_mutex_unlock(&g_pilot_mx);
int rc=expert_load(m,layer,eid,dst,0); /* pread VERO — fuori dal lock, sovrapposto al compute; fatal=0: un errore su una speculazione NON deve uccidere il server */
pthread_mutex_lock(&g_pilot_mx);
if(rc==0){
dst->used=(uint64_t)__atomic_add_fetch(&m->eclock,1,__ATOMIC_RELAXED);
if(isnew) m->ecn[layer]=slot+1; /* pubblica lo slot SOLO ora che eid e' valido */
atomic_fetch_add_explicit(&g_pilot_loads,1,memory_order_relaxed);
} else {
atomic_fetch_add_explicit(&g_pilot_drops,1,memory_order_relaxed); /* load fallito: slot resta nascosto (eid=-1), mai pubblicato */
}
atomic_store_explicit(&g_pilot_inflight,-1,memory_order_release);
pthread_mutex_unlock(&g_pilot_mx);
if(rc!=0) /* mai swallow silenzioso: logga (una riga) e prosegui */
fprintf(stderr,"[PILOT] load speculativo abbandonato: layer %d expert %d (I/O error/short read) — nessun impatto sull'output\n",layer,eid);
}
static void *pilot_worker(void *arg){
(void)arg;
for(;;){
unsigned r=__atomic_load_n(&pilot_r,__ATOMIC_ACQUIRE);
unsigned w=__atomic_load_n(&pilot_w,__ATOMIC_ACQUIRE);
if(r==w){ usleep(200); continue; }
if(g_pilot_real) pilot_realload(pilot_m, pilot_q[r&4095].l, pilot_q[r&4095].e);
else expert_prefetch(pilot_m, pilot_q[r&4095].l, pilot_q[r&4095].e);
__atomic_store_n(&pilot_r,r+1,__ATOMIC_RELEASE);
}
return NULL;
}
static void pilot_prefetch(Model *m, int lnext, const float *x, int S){
Cfg *c=&m->c; Layer *l=&m->L[lnext]; int D=c->hidden, E=c->n_experts;
int K = g_pilot_k<c->topk ? g_pilot_k : c->topk;
if(!pilot_m){ pilot_m=m; pthread_t t; pthread_create(&t,NULL,pilot_worker,NULL); }
float *nrm=falloc(D), *ch=falloc(E);
for(int s=0;s<S;s++){
rmsnorm(nrm, x+(int64_t)s*D, l->post_ln, D, c->eps);
matmul(ch, nrm, l->router, 1, D, E);
for(int e=0;e<E;e++) ch[e]=sigmoidf(ch[e])+l->router_bias[e];
for(int kk=0;kk<K;kk++){
int best=0; for(int e=1;e<E;e++) if(ch[e]>ch[best]) best=e;
ch[best]=-2e30f;
/* Residency scan of the FUTURE layer lnext under g_pilot_mx: with
* PILOT_REAL=1 the pilot worker mutates ecache[lnext]/ecn[lnext]
* concurrently, so read them under the same lock (Option A). Decide
* under the lock, then enqueue AFTER unlocking — the pilot_q ring is
* lock-free (pilot_w/pilot_r atomics, not g_pilot_mx) so there is no
* re-entrant double-lock, and the worker re-checks residency under the
* lock anyway, making a racing redundant enqueue harmless. */
int found=0;
pthread_mutex_lock(&g_pilot_mx);
ESlot *P=m->pin[lnext];
for(int z=0;z<m->npin[lnext] && !found;z++) if(P[z].eid==best) found=1;
ESlot *Sl=m->ecache[lnext];
for(int z=0;z<m->ecn[lnext] && !found;z++) if(Sl[z].eid==best) found=1;
pthread_mutex_unlock(&g_pilot_mx);
if(!found){
unsigned w=__atomic_load_n(&pilot_w,__ATOMIC_RELAXED);
if(w-__atomic_load_n(&pilot_r,__ATOMIC_ACQUIRE)<4096){
pilot_q[w&4095].l=lnext; pilot_q[w&4095].e=best;
__atomic_store_n(&pilot_w,w+1,__ATOMIC_RELEASE);
}
}
}
}
free(nrm); free(ch);
}
/* forward di UN layer (usato dai 78 principali e dal layer MTP) */
static void layer_forward_rows(Model *m, Layer *l, int li, float *x, int S, int pos_base,
KVState *const *kvs, const int *positions, float *nrm, float *tmp){
Cfg *c=&m->c; int D=c->hidden;
if(g_spec && g_prefetch && l->sparse && m->enr[li]>0)
for(int z=0;z<m->enr[li];z++) expert_prefetch(m,li,m->eroute[li][z]);
if(g_looka && S==1 && li<c->n_layers && l->sparse) la_predict(m,li,x,0);
#ifdef COLI_METAL
/* FULL-LAYER CB: in_ln + attention + residuo + post_ln + shared expert + router/top-K
* in un solo submit GPU; la CPU legge il routing e fa solo resolve/disk/expert-CB.
* Fallback: qualsiasi condizione mancante -> percorso CPU intero qui sotto. */
if(g_metal_enabled && S<=4 && li<c->n_layers && l->sparse
&& (g_absorb==1||(g_absorb<0&&S<=4)) && m->kv_start[li]==0
&& D==6144 && c->n_heads==64 && c->q_lora==2048 && c->kv_lora==512
&& c->qk_nope==192 && c->qk_rope==64 && c->v_head==256 && l->kv_b.fmt==2
&& c->n_experts==256 && c->topk==8 && c->n_shared==1 && c->moe_inter==2048){
int sel_active = m->has_dsa && c->idx_type[li] && (pos_base+S) > c->index_topk;
if(!sel_active){
static float *linrm,*lnrm,*lsh,*lw; static int *lidx,*lkeff;
if(!linrm){ linrm=falloc(4*(int64_t)D); lnrm=falloc(4*(int64_t)D); lsh=falloc(4*(int64_t)D);
lidx=malloc(4*8*sizeof(int)); lw=malloc(4*8*sizeof(float)); lkeff=malloc(4*sizeof(int)); }
int Ksel = g_topk>0 ? (g_topk<8?g_topk:8) : 8;
float tp = (g_topp>0 && g_topp<1.f) ? g_topp : 0.f;
double ta0=now_s();
#define WP_(q) ((q).fmt==1?(const void*)(q).q8:(const void*)(q).q4)
int ok = coli_metal_layer_decode(x, l->in_ln, l->post_ln,
WP_(l->q_a), l->q_a.s, l->q_a.fmt, l->q_a_ln,
WP_(l->q_b), l->q_b.s, l->q_b.fmt,
WP_(l->kv_a), l->kv_a.s, l->kv_a.fmt, l->kv_a_ln,
WP_(l->kv_b), l->kv_b.s, l->kv_b.fmt,
WP_(l->o), l->o.s, l->o.fmt,
WP_(l->sh_gate), l->sh_gate.s, l->sh_gate.fmt,
WP_(l->sh_up), l->sh_up.s, l->sh_up.fmt,
WP_(l->sh_down), l->sh_down.s, l->sh_down.fmt,
l->router, l->router_bias,
c->n_experts, c->topk, Ksel, tp, c->norm_topk, c->routed_scale,
m->Lc[li], m->Rc[li], S, pos_base, m->kv_start[li],
c->eps, c->theta, c->attn_scale,
linrm, lnrm, lsh, lidx, lw, lkeff);
#undef WP_
if(ok){
m->t_attn += now_s()-ta0;
if(m->has_dsa && c->idx_type[li]){ /* index key per selezioni future */
for(int s=0;s<S;s++){ int pos=pos_base+s;
float *kd=m->Ic[li]+(int64_t)pos*c->index_hd;
matmul_qt(kd, linrm+(int64_t)s*D, &m->ix_wk[li], 1);
layernorm(kd, m->ix_knw[li], m->ix_knb[li], c->index_hd, 1e-6f);
rope_interleave(kd, pos, c);
}
}
if(g_pilot && S<=8 && li+1<c->n_layers && m->L[li+1].sparse) pilot_prefetch(m,li+1,x,S);
if(g_looka && S==1 && li+1<c->n_layers && m->L[li+1].sparse) la_predict(m,li+1,x,1);
g_pre_idx=lidx; g_pre_w=lw; g_pre_keff=lkeff; g_pre_sh=lsh;
moe(m,l,li,lnrm,S,tmp);
g_pre_idx=NULL; g_pre_w=NULL; g_pre_keff=NULL; g_pre_sh=NULL;
for(int64_t j=0;j<(int64_t)S*D;j++) x[j]+=tmp[j];
return;
}
}
}
#endif
for(int s=0;s<S;s++) rmsnorm(nrm+(int64_t)s*D, x+(int64_t)s*D, l->in_ln, D, c->eps);
attention_rows(m,l,li,nrm,S,pos_base,kvs,positions,tmp);
for(int64_t j=0;j<(int64_t)S*D;j++) x[j]+=tmp[j];
if(g_pilot && S<=8 && li+1<c->n_layers && m->L[li+1].sparse) pilot_prefetch(m,li+1,x,S);
if(g_looka && S==1 && li+1<c->n_layers && m->L[li+1].sparse) la_predict(m,li+1,x,1);
for(int s=0;s<S;s++) rmsnorm(nrm+(int64_t)s*D, x+(int64_t)s*D, l->post_ln, D, c->eps);
if(l->sparse) moe(m,l,li,nrm,S,tmp); else dense_mlp(l,nrm,S,D,c->dense_inter,tmp);
for(int64_t j=0;j<(int64_t)S*D;j++) x[j]+=tmp[j];
}
static void layer_forward(Model *m, Layer *l, int li, float *x, int S, int pos_base, float *nrm, float *tmp){
layer_forward_rows(m,l,li,x,S,pos_base,NULL,NULL,nrm,tmp);
}
static void layers_forward_rows(Model *m, float *x, int S, int pos_base,
KVState *const *kvs, const int *positions){
Cfg *c=&m->c; int D=c->hidden;
if(g_pilot_real){ /* nuovo forward: il possesso-layer riparte da -1 (i layer si rifanno da 0) */
pthread_mutex_lock(&g_pilot_mx);
atomic_store_explicit(&g_cur_moe_layer,-1,memory_order_release);
pthread_mutex_unlock(&g_pilot_mx);
}
float *nrm=falloc((int64_t)S*D), *tmp=falloc((int64_t)S*D);
for(int i=0;i<c->n_layers;i++){
/* progresso su stderr per i batch grossi (prefill): il primo byte di risposta
* puo' arrivare dopo MINUTI di streaming — al buio sembra un blocco. */
if(S>=8 && (i%4==0 || i==c->n_layers-1))
fprintf(stderr,"[prefill] layer %d/%d · %d token\n", i+1, c->n_layers, S);
layer_forward_rows(m,&m->L[i],i,x,S,pos_base,kvs,positions,nrm,tmp);
}
free(nrm); free(tmp);
}
static void layers_forward(Model *m, float *x, int S, int pos_base){
layers_forward_rows(m,x,S,pos_base,NULL,NULL);
}
static void kv_alloc(Model *m, int max_t){
Cfg *c=&m->c;
KVState *k=m->kv;
if(k->Lc){ for(int i=0;i<c->n_layers+1;i++){
#ifdef COLI_METAL
if(g_metal_enabled){ coli_metal_unregister(k->Lc[i]); coli_metal_unregister(k->Rc[i]); }
#endif
free(k->Lc[i]); free(k->Rc[i]); } free(k->Lc); free(k->Rc); }
if(k->Ic){ for(int i=0;i<c->n_layers;i++) free(k->Ic[i]); free(k->Ic); k->Ic=NULL; }
if(m->has_dsa){
k->Ic=calloc(c->n_layers,sizeof(float*));
for(int i=0;i<c->n_layers;i++) if(c->idx_type[i]) k->Ic[i]=falloc((int64_t)max_t*c->index_hd);
}
k->max_t=max_t;
int NR=c->n_layers+1; /* riga extra: KV del layer MTP */
k->Lc=calloc(NR,sizeof(float*)); k->Rc=calloc(NR,sizeof(float*));
for(int i=0;i<NR;i++){ k->Lc[i]=falloc((int64_t)max_t*c->kv_lora);
k->Rc[i]=falloc((int64_t)max_t*c->qk_rope);
#ifdef COLI_METAL
/* page-align + register Lc/Rc for zero-copy GPU attention. falloc isn't 16K-aligned,
* so re-allocate aligned and register the exact byte length. */
if(g_metal_enabled){
size_t lb=(((size_t)max_t*c->kv_lora*sizeof(float))+16383)&~(size_t)16383;
size_t rb=(((size_t)max_t*c->qk_rope*sizeof(float))+16383)&~(size_t)16383;
free(k->Lc[i]); free(k->Rc[i]); void *lp,*rp;
if(posix_memalign(&lp,16384,lb)||posix_memalign(&rp,16384,rb)){fprintf(stderr,"OOM kv\n");exit(1);}
k->Lc[i]=lp; k->Rc[i]=rp;
coli_metal_register(k->Lc[i],lb); coli_metal_register(k->Rc[i],rb);
}
#endif
}
m->Lc=k->Lc; m->Rc=k->Rc; m->Ic=k->Ic; m->max_t=k->max_t; m->kv_start=k->kv_start;
}
static void kv_bind(Model *m, KVState *k){
m->kv=k; m->Lc=k->Lc; m->Rc=k->Rc; m->Ic=k->Ic;
m->max_t=k->max_t; m->kv_start=k->kv_start;
}
static void mtp_absorb(Model *m, const int *next_ids, const float *x, int S, int pos_base);
static float *step(Model *m, const int *ids, int S, int pos_base){
Cfg *c=&m->c; int D=c->hidden;
float *x=falloc((int64_t)S*D);
for(int s=0;s<S;s++) embed_row(m, ids[s], x+(int64_t)s*D);
layers_forward(m,x,S,pos_base);
if(m->hlast) memcpy(m->hlast, x+(int64_t)(S-1)*D, D*sizeof(float));
if(m->has_mtp && S>=2 && g_draft>0) mtp_absorb(m, ids+1, x, S-1, pos_base);
float *last=falloc(D); rmsnorm(last, x+(int64_t)(S-1)*D, m->final_norm, D, c->eps);
double th0=now_s();
float *logit=falloc(c->vocab); matmul_qt(logit,last,&m->lm_head,1);
m->t_head += now_s()-th0;
free(x); free(last); return logit;
}
/* come step(), ma ritorna i logits di TUTTE le S posizioni [S,vocab] (per la verifica spec) */
static float *step_all(Model *m, const int *ids, int S, int pos_base){
Cfg *c=&m->c; int D=c->hidden;
float *x=falloc((int64_t)S*D);
for(int s=0;s<S;s++) embed_row(m, ids[s], x+(int64_t)s*D);
layers_forward(m,x,S,pos_base);
if(m->h_all) memcpy(m->h_all, x, (int64_t)S*D*sizeof(float)); /* hidden di TUTTE le pos (S<=64) */
if(m->hlast) memcpy(m->hlast, x+(int64_t)(S-1)*D, D*sizeof(float));
float *lo=falloc((int64_t)S*c->vocab), *row=falloc(D);
for(int s=0;s<S;s++){ rmsnorm(row, x+(int64_t)s*D, m->final_norm, D, c->eps);
matmul_qt(lo+(int64_t)s*c->vocab, row, &m->lm_head, 1); }
free(x); free(row); return lo;
}
/* One decode token from each independent sequence, evaluated as a single MoE
* batch. Prefill and speculative batches retain their contiguous-KV path. */
static float *step_decode_batch(Model *m, const DecodeRow *rows, int S){
Cfg *c=&m->c; int D=c->hidden;
/* Ragged KV currently uses MLA absorption; the stack kernel is sized to 512. */
if(!rows || S<1 || S>64 || c->kv_lora>512) return NULL;
KVState *kvs[64]; int positions[64];
float *x=falloc((int64_t)S*D);
for(int s=0;s<S;s++){
if(!rows[s].kv || !rows[s].kv->Lc || !rows[s].kv->Rc || !rows[s].kv->kv_start ||
rows[s].token<0 || rows[s].token>=c->vocab ||
rows[s].pos<0 || rows[s].pos>=rows[s].kv->max_t){
free(x); return NULL;
}
for(int l=0;l<c->n_layers;l++){
if(!rows[s].kv->Lc[l] || !rows[s].kv->Rc[l] ||
rows[s].kv->kv_start[l]<0 || rows[s].kv->kv_start[l]>rows[s].pos ||
(m->has_dsa && c->idx_type[l] &&
(!rows[s].kv->Ic || !rows[s].kv->Ic[l]))){ free(x); return NULL; }
}
for(int p=0;p<s;p++) if(rows[p].kv==rows[s].kv){ free(x); return NULL; }
kvs[s]=rows[s].kv; positions[s]=rows[s].pos;
embed_row(m,rows[s].token,x+(int64_t)s*D);
}
layers_forward_rows(m,x,S,0,kvs,positions);
float *norm=falloc((int64_t)S*D);
for(int s=0;s<S;s++)
rmsnorm(norm+(int64_t)s*D,x+(int64_t)s*D,m->final_norm,D,c->eps);
double th0=now_s();
float *logit=falloc((int64_t)S*c->vocab);
matmul_qt(logit,norm,&m->lm_head,S);
m->t_head+=now_s()-th0;
free(x); free(norm);
return logit;
}
/* METODO E — prompt-lookup: cerca l'occorrenza piu' recente dell'ultimo bigramma nel
* contesto e propone i token che la seguirono. Zero pesi extra, zero costo: e' solo
* un'ipotesi che il modello verifichera'. */
static int ngram_draft(const int *ids, int len, int G, int *draft){
if(len<4 || G<1) return 0;
int a=ids[len-2], b=ids[len-1];
for(int i=len-3;i>=1;i--)
if(ids[i-1]==a && ids[i]==b){
int n=0; for(int j=i+1;j<len && n<G;j++) draft[n++]=ids[j];
return n;
}
return 0;
}
/* METODO MTP: propone fino a G draft con la testa multi-token nativa di GLM-5.2.
* Input: next_tok (appena emesso, posizione kv) e hlast (hidden pre-norm della pos kv-1).
* Catena DeepSeek-V3: h' = Layer78( eh_proj[ enorm(emb(tok)) ; hnorm(h) ] ),
* draft = argmax(lm_head(shared_head.norm(h'))). La KV del layer MTP vive alla riga n_layers
* ed e' valida da kv_start (niente prefill: finestra di solo-decode, basta per il draft). */
static int mtp_argmax(const float *lo, int V){
int b=0; float bv=lo[0]; for(int i=1;i<V;i++) if(lo[i]>bv){bv=lo[i];b=i;} return b;
}
static int mtp_draft(Model *m, int next_tok, int kv, int G, int *draft){
Cfg *c=&m->c; int D=c->hidden, li=c->n_layers;
int p=kv-1; if(p<0||G<1) return 0;
if(m->kv_start[li]<0 || m->kv_start[li]>p) m->kv_start[li]=p;
float *x=falloc(D), *cat=falloc(2*D), *hx=falloc(D), *nrm=falloc(D), *tmp=falloc(D);
float *row=falloc(D), *logit=falloc(c->vocab), *h=falloc(D);
memcpy(h, m->hlast, D*sizeof(float));
int tok=next_tok, n=0;
int prenorm = getenv("MTP_PRENORM")!=NULL;
for(int g=0; g<G; g++){
int pos=p+g; if(pos+2>=m->max_t) break;
embed_row(m, tok, x);
rmsnorm(x, x, m->enorm, D, c->eps);
if(g==0 && !prenorm) rmsnorm(h, h, m->final_norm, D, c->eps); /* h vero: post model.norm */
rmsnorm(h, h, m->hnorm, D, c->eps);
if(getenv("MTP_SWAP")){ memcpy(cat, h, D*sizeof(float)); memcpy(cat+D, x, D*sizeof(float)); }
else { memcpy(cat, x, D*sizeof(float)); memcpy(cat+D, h, D*sizeof(float)); }
matmul_qt(hx, cat, &m->eh_proj, 1);
double n_eh=0; for(int d=0;d<D;d++) n_eh+=hx[d]*hx[d];
int dbg = getenv("MTP_DEBUG") && atoi(getenv("MTP_DEBUG"))>=2;
int t_pre=-1;
if(dbg){ rmsnorm(row, hx, m->mtp_norm, D, c->eps); matmul_qt(logit, row, &m->lm_head, 1);
t_pre=mtp_argmax(logit, c->vocab); }
layer_forward(m, &m->mtpL, li, hx, 1, pos, nrm, tmp);
double n_post=0; for(int d=0;d<D;d++) n_post+=hx[d]*hx[d];
rmsnorm(row, hx, m->mtp_norm, D, c->eps);
matmul_qt(logit, row, &m->lm_head, 1);
int t2=mtp_argmax(logit, c->vocab);
if(dbg) fprintf(stderr,"[mtp2] pos=%d in_tok=%d ||eh||=%.1f ||post||=%.1f pre_blk=%d post_blk=%d\n",
pos, tok, sqrt(n_eh), sqrt(n_post), t_pre, t2);
draft[n++]=t2; tok=t2; memcpy(h, hx, D*sizeof(float));
}
free(x); free(cat); free(hx); free(nrm); free(tmp); free(row); free(logit); free(h);
return n;
}
/* assorbe nella KV della testa MTP le coppie VERIFICATE (emb(token@pos+1), h_vero@pos):
* next_ids[i] = token alla posizione pos_base+i+1; x[i] = hidden VERO a pos_base+i.
* Un solo passaggio batch del layer MTP (il batch-union rende economici gli expert). */
static void mtp_absorb(Model *m, const int *next_ids, const float *x, int S, int pos_base){
if(!m->has_mtp || S<1) return;
Cfg *c=&m->c; int D=c->hidden, li=c->n_layers;
if(m->kv_start[li]<0 || m->kv_start[li]>pos_base) m->kv_start[li]=pos_base;
float *hx=falloc((int64_t)S*D), *cat=falloc(2*D), *e=falloc(D), *hn=falloc(D), *hf=falloc(D);
int prenorm = getenv("MTP_PRENORM")!=NULL;
for(int i=0;i<S;i++){
embed_row(m,next_ids[i],e);
rmsnorm(e,e,m->enorm,D,c->eps);
if(prenorm) rmsnorm(hn,x+(int64_t)i*D,m->hnorm,D,c->eps);
else { rmsnorm(hf,x+(int64_t)i*D,m->final_norm,D,c->eps); /* vLLM: h POST model.norm */
rmsnorm(hn,hf,m->hnorm,D,c->eps); }
if(getenv("MTP_SWAP")){ memcpy(cat,hn,D*sizeof(float)); memcpy(cat+D,e,D*sizeof(float)); }
else { memcpy(cat,e,D*sizeof(float)); memcpy(cat+D,hn,D*sizeof(float)); }
matmul_qt(hx+(int64_t)i*D, cat, &m->eh_proj, 1);
}
float *nrm=falloc((int64_t)S*D), *tmp=falloc((int64_t)S*D);
layer_forward(m,&m->mtpL,li,hx,S,pos_base,nrm,tmp);
free(hx); free(cat); free(e); free(hn); free(hf); free(nrm); free(tmp);
}
static inline int argmax_v(const float *lo, int V){
int b=0; float bv=lo[0]; for(int i=1;i<V;i++) if(lo[i]>bv){bv=lo[i];b=i;} return b;
}
/* ---- METODO F: draft grammaticale (#48) ----
* gr_feed consuma i byte di ogni token EMESSO e tiene il walker in sync con l'output;
* grammar_draft propone lo span FORZATO successivo (un solo byte legale per posizione)
* gia' tokenizzato. Il confine di tokenizzazione non e' garantito coincidere con quello
* del modello: la verifica assorbe la differenza (al peggio l'ultimo draft e' rifiutato). */
static void grammar_setup(Tok *T){
const char *gf=getenv("GRAMMAR"); if(!gf||!*gf) return;
FILE *f=fopen(gf,"rb");
if(!f){ fprintf(stderr,"[GRAMMAR] cannot open %s\n",gf); return; }
fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET);
char *txt=malloc((size_t)n+1);
if(!txt || fread(txt,1,(size_t)n,f)!=(size_t)n){
fprintf(stderr,"[GRAMMAR] failed to read %s\n",gf); fclose(f); free(txt); return; }
fclose(f); txt[n]=0;
if(gr_parse(&g_gram,txt)){ fprintf(stderr,"[GRAMMAR] %s: %s\n",gf,g_gram.err); free(txt); return; }
free(txt);
gr_state_init(&g_gst,&g_gram);
if(!g_gst.alive){ fprintf(stderr,"[GRAMMAR] %s: grammar cannot be evaluated (left recursion?)\n",gf); return; }
if(getenv("GRAMMAR_DRAFT")) g_gr_max=atoi(getenv("GRAMMAR_DRAFT"));
if(g_gr_max<1) g_gr_max=1;
if(g_gr_max>48) g_gr_max=48;
g_gr_T=T; g_gr_on=1;
fprintf(stderr,"[GRAMMAR] %s: %d rules, forced span capped at %d tokens/forward\n",gf,g_gram.n,g_gr_max);
}
/* stato pulito all'inizio di ogni RISPOSTA (non tra i \x02MORE, che continuano) */
static void grammar_reset(void){
if(!g_gr_on) return;
gr_state_init(&g_gst,&g_gram); g_gr_armed=0;
if(!g_gst.alive) g_gr_on=0;
}
/* consuma i byte di un token emesso. Preambolo (prima dell'arming): ignorato.
* Desync dopo l'arming: si riarma in attesa del prossimo inizio valido — al peggio
* i draft vengono rifiutati dalla verifica, l'output non cambia MAI. */
static void gr_feed(int t){
if(!g_gr_on||!g_gr_T) return;
char b[64]; int n=tok_decode(g_gr_T,&t,1,b,63);
for(int i=0;i<n;i++){
int r=gr_accept(&g_gst,(unsigned char)b[i]);
if(r==1){ g_gr_armed=1; continue; }
if(r<0){ g_gr_on=0; return; } /* walker spento: fine dei draft */
if(!g_gr_armed) continue; /* preambolo: aspetta l'inizio */
gr_state_init(&g_gst,&g_gram); g_gr_armed=0; /* desync: riparti dalla radice */
if(!g_gst.alive){ g_gr_on=0; return; }
if(gr_accept(&g_gst,(unsigned char)b[i])==1) g_gr_armed=1;
}
}
/* propone lo span forzato come token (max cap); 0 se la grammatica dirama qui */
static int grammar_draft(int *draft, int cap){
if(!g_gr_on||!g_gr_armed||!g_gr_T||cap<1) return 0;
if(g_gr_prop>=32 && g_gr_acc*2<g_gr_prop){ /* guardia adattiva, come per MTP:
acceptance sotto il 50% = tokenizzazione fuori asse, meglio spegnersi */
g_gr_on=0;
fprintf(stderr,"[GRAMMAR] %.0f%% acceptance after %llu proposals: grammar drafts disabled\n",
100.0*g_gr_acc/g_gr_prop,(unsigned long long)g_gr_prop);
return 0;
}
char fb[512]; int nb=gr_forced(&g_gst,fb,(int)sizeof fb-1);
if(nb<=0) return 0;
int g=tok_encode(g_gr_T,fb,nb,draft,cap);
return g>0?g:0;
}
/* ---- SAMPLING (temperatura + nucleus) con verifica speculativa LOSSLESS ----
* Il draft (MTP/n-gram) e' DETERMINISTICO (argmax della testa): q = massa puntuale.
* Rejection sampling di Leviathan: accetta il draft x_d con prob p(x_d); al rifiuto
* ricampiona da p con x_d azzerato e rinormalizzato. La distribuzione risultante e'
* ESATTAMENTE p: la speculazione resta invisibile all'output anche col sampling. */
static uint64_t g_rng=0x9E3779B97F4A7C15ULL;
static inline double rndu(void){ g_rng^=g_rng<<13; g_rng^=g_rng>>7; g_rng^=g_rng<<17;
return (double)(g_rng>>11)*(1.0/9007199254740992.0); }
static float *g_pbuf=NULL; static int *g_pidx=NULL; /* buffer riusati (decode single-thread) */
static int cmp_pdesc(const void *a,const void *b){
float pa=g_pbuf[*(const int*)a], pb=g_pbuf[*(const int*)b];
return pa<pb ? 1 : pa>pb ? -1 : 0; }
/* costruisce in g_pbuf la distribuzione target: softmax(lo/temp) troncata a top-p g_nuc */
static void dist_build(const float *lo, int V){
if(!g_pbuf){ g_pbuf=falloc(V); g_pidx=malloc(V*sizeof(int)); }
float mx=lo[0]; for(int i=1;i<V;i++) if(lo[i]>mx) mx=lo[i];
double s=0; float invt=1.f/(g_temp>1e-4f?g_temp:1e-4f);
for(int i=0;i<V;i++){ g_pbuf[i]=expf((lo[i]-mx)*invt); s+=g_pbuf[i]; }
for(int i=0;i<V;i++) g_pbuf[i]/=(float)s;
if(g_nuc>0 && g_nuc<1.f){
for(int i=0;i<V;i++) g_pidx[i]=i;
qsort(g_pidx,V,sizeof(int),cmp_pdesc);
double cum=0; int keep=V;
for(int i=0;i<V;i++){ cum+=g_pbuf[g_pidx[i]]; if(cum>=g_nuc){ keep=i+1; break; } }
double s2=0; for(int i=keep;i<V;i++) g_pbuf[g_pidx[i]]=0;
for(int i=0;i<keep;i++) s2+=g_pbuf[g_pidx[i]];
for(int i=0;i<keep;i++) g_pbuf[g_pidx[i]]/=(float)s2;
}
}
/* campiona da g_pbuf; ban>=0 -> quel token e' escluso (rinormalizzando al volo) */
static int dist_sample(int V, int ban){
double z = 1.0 - (ban>=0 ? g_pbuf[ban] : 0.0); if(z<=1e-12) z=1e-12;
double u = rndu()*z, cum=0;
for(int i=0;i<V;i++){ if(i==ban) continue; cum+=g_pbuf[i]; if(cum>=u) return i; }
for(int i=V-1;i>=0;i--) if(i!=ban && g_pbuf[i]>0) return i;
return 0;
}
/* prossimo token dai logits: greedy se g_temp<=0, altrimenti sampling.
* ban = token escluso perche' rifiutato dalla verifica speculativa precedente. */
static int pick_tok(const float *lo, int V, int ban){
if(g_temp<=0) return argmax_v(lo,V);
dist_build(lo,V);
return dist_sample(V,ban);
}
/* stop-set attivo (popolato da run_text/run_serve dal config; vuoto in validazione,
* dove si genera un numero fisso di token da confrontare con l'oracolo) */
static int g_stop[9], g_nstop=0;
static inline int is_stop(int t){ for(int i=0;i<g_nstop;i++) if(t==g_stop[i]) return 1; return 0; }
static void stops_arm(const Cfg *c, int tok_eos){
g_nstop=0;
for(int i=0;i<c->n_stop;i++) g_stop[g_nstop++]=c->stop_ids[i];
if(tok_eos>=0 && !is_stop(tok_eos)) g_stop[g_nstop++]=tok_eos;
fprintf(stderr,"[stop] %d stop tokens:",g_nstop);
for(int i=0;i<g_nstop;i++) fprintf(stderr," %d",g_stop[i]);
fprintf(stderr,"\n");
}
/* decode greedy con SELF-SPECULATION n-gram: LOSSLESS (output identico al greedy puro).
* Ogni forward verifica fino a g_draft token proposti dal contesto: i token accettati
* costano UNA sola passata sui pesi -> disco e banda RAM ammortizzati su piu' token.
* all: storia token (capacita' >= kv+n_new+g_draft+2), kv = token gia' in KV.
* logit = logits della posizione kv-1 (dal prefill); viene liberato qui.
* emit(tok,ud) per ogni token emesso. Ritorna i token emessi; *kv_out = nuova kv. */
static int spec_decode(Model *m, int *all, int kv, int n_new, int eos, float *logit,
void (*emit)(int,void*), void *ud, int *kv_out){
Cfg *c=&m->c; int V=c->vocab; int emitted=0, done=0;
int draft[64]; if(g_draft>63) g_draft=63;
int carry_ban=-1; /* token rifiutato dalla verifica: escluso dal resample */
while(emitted<n_new && !done){
int next=pick_tok(logit,V,carry_ban); carry_ban=-1; free(logit); logit=NULL;
if((eos>=0 && next==eos) || is_stop(next)) break;
emit(next,ud); all[kv]=next; emitted++; m->n_emit++;
gr_feed(next); /* il walker segue l'output emesso */
if(emitted>=n_new) break; /* l'ultimo token non serve forwardarlo */
int g = 0, gsrc = 0; /* sorgente: 1=grammatica 2=MTP/n-gram */
if(g_gr_on){ /* metodo F: prima la grammatica — dove
* forza, l'acceptance e' ~1 (#48) */
g=grammar_draft(draft,g_gr_max);
if(g>0) gsrc=1;
}
if(!g && g_draft>0){
/* auto-off adattivo: draft che non vengono mai accettati = solo tassa disco */
if(m->has_mtp && m->mtp_prop>=24 && m->mtp_acc*10 < m->mtp_prop){
g_draft=0;
fprintf(stderr,"[MTP] %.0f%% acceptance after %llu proposals: drafts disabled\n",
100.0*m->mtp_acc/m->mtp_prop, (unsigned long long)m->mtp_prop);
}
}
if(!g && g_draft>0){
if(m->has_mtp){ g=mtp_draft(m,next,kv,g_draft,draft); m->mtp_prop+=g; if(g)gsrc=2; }
else { g=ngram_draft(all,kv+1,g_draft,draft); if(g)gsrc=2; }
}
if(g>n_new-emitted) g=n_new-emitted;
if(kv+1+g+1>m->max_t) g=m->max_t-kv-2;
if(g<0) g=0;
if(gsrc==1) g_gr_prop+=(uint64_t)g;
int S=1+g; int batch[64]; batch[0]=next; memcpy(batch+1,draft,g*sizeof(int));
float *lo=step_all(m,batch,S,kv); m->n_fw++;
int k=0; /* verifica: accetta finche' coincide */
if(g>0 && getenv("MTP_DEBUG")){ int veri=argmax_v(lo,V);
fprintf(stderr,"[mtpdbg] draft0=%d verified=%d %s\n", draft[0], veri, draft[0]==veri?"HIT":"miss"); }
while(k<g && emitted<n_new){
int accept;
if(g_temp<=0) accept = (argmax_v(lo+(int64_t)k*V,V)==draft[k]);
else { dist_build(lo+(int64_t)k*V,V); /* rejection sampling: p(draft) */
accept = (rndu() < g_pbuf[draft[k]]); }
if(!accept){ if(g_temp>0) carry_ban=draft[k]; break; }
if((eos>=0 && draft[k]==eos) || is_stop(draft[k])){ done=1; break; }
emit(draft[k],ud); all[kv+1+k]=draft[k]; emitted++; m->n_emit++;
gr_feed(draft[k]); k++;
}
if(gsrc==1) g_gr_acc+=(uint64_t)k;
else if(gsrc==2 && m->has_mtp) m->mtp_acc+=k;
if(m->has_mtp && k>=1) mtp_absorb(m, all+kv+1, m->h_all, k, kv); /* KV MTP in sync coi verificati */
/* hlast deve corrispondere all'ultima posizione ACCETTATA (kv+k), non a fine batch */
if(m->h_all && k<S-1) memcpy(m->hlast, m->h_all+(int64_t)k*m->c.hidden, m->c.hidden*sizeof(float));
kv += 1+k; /* KV oltre kv e' stantia: verra' sovrascritta */
logit=falloc(V); memcpy(logit, lo+(int64_t)k*V, V*sizeof(float)); free(lo);
}
if(logit) free(logit);
if(kv_out) *kv_out=kv;
return emitted;
}
/* emit callback: accumula in un array (validazione) */
typedef struct { int *dst; int n; } EmitStore;
static void emit_store(int t, void *ud){ EmitStore *e=(EmitStore*)ud; e->dst[e->n++]=t; }
/* emit callback: detokenizza e stampa in streaming (chat/run), con heartbeat */
typedef struct { Tok *T; Model *m; double t0; int count; int quiet; } EmitStream;
static void emit_stream(int t, void *ud){
EmitStream *e=(EmitStream*)ud; char dec[64];
int dn=tok_decode(e->T,&t,1,dec,63); dec[dn]=0; fputs(dec,stdout); fflush(stdout);
if(!e->quiet && ++e->count%16==0){ double tt=e->m->hits+e->m->miss;
fprintf(stderr,"\n[t=%d RSS %.2f GB hit %.0f%% %.2f tok/s %.2f tok/fw]\n", e->count,
rss_gb(), tt?100.0*e->m->hits/tt:0.0, e->count/(now_s()-e->t0),
e->m->n_fw?(double)e->m->n_emit/e->m->n_fw:1.0); }
}
/* teacher-forcing: un solo forward su ids[S], argmax per posizione in pred[S] */
static void forward_all(Model *m, const int *ids, int S, int *pred){
Cfg *c=&m->c; int D=c->hidden;
kv_alloc(m,S);
float *x=falloc((int64_t)S*D);
for(int s=0;s<S;s++) embed_row(m, ids[s], x+(int64_t)s*D);
layers_forward(m,x,S,0);
float *lo=falloc(c->vocab);
for(int s=0;s<S;s++){
float row[8192]; rmsnorm(row, x+(int64_t)s*D, m->final_norm, D, c->eps);
matmul_qt(lo, row, &m->lm_head, 1);
int best=0; float bv=lo[0]; for(int i=1;i<c->vocab;i++) if(lo[i]>bv){bv=lo[i];best=i;}
pred[s]=best;
}
free(x); free(lo);
}
/* log-prob (log-softmax) del token target dato il vettore di logit; *am=1 se e' l'argmax */
static double logprob_target(const float *lo, int V, int target, int *am){
float mx=lo[0]; int best=0; for(int i=1;i<V;i++){ if(lo[i]>mx){mx=lo[i];best=i;} }
double se=0; for(int i=0;i<V;i++) se+=exp((double)lo[i]-mx);
if(am)*am=(best==target);
return (double)(lo[target]-mx) - log(se);
}
/* modalita' SCORING per i benchmark (stile lm-eval, log-likelihood):
* input: file con righe "<ctxlen> <contlen> <id0> .. <id_{T-1}>" (T=ctxlen+contlen)
* output: riga "<logprob_continuazione> <contlen> <greedy 0/1>" per richiesta.
* Un solo forward per richiesta (teacher-forcing): niente generazione -> fattibile a bassa velocita'. */
static void run_score(Model *m, const char *path){
Cfg *c=&m->c; int D=c->hidden;
FILE *f=fopen(path,"rb"); if(!f){perror(path);exit(1);}
int maxT=1; { char *ln=NULL; size_t cp=0;
while(getline(&ln,&cp,f)>0){ int a,b; if(sscanf(ln,"%d %d",&a,&b)==2 && a+b>maxT) maxT=a+b; }
free(ln); }
kv_alloc(m,maxT);
float *x=falloc((int64_t)maxT*D), *lo=falloc(c->vocab), *row=falloc(D);
int *ids=malloc(maxT*sizeof(int));
rewind(f); char *ln=NULL; size_t cp=0; int nreq=0; double t0=now_s();
while(getline(&ln,&cp,f)>0){
char *p=ln; int ctxlen=strtol(p,&p,10), contlen=strtol(p,&p,10), T=ctxlen+contlen;
if(T<=0||ctxlen<1){ printf("0 0 0\n"); fflush(stdout); continue; }
for(int i=0;i<T;i++) ids[i]=strtol(p,&p,10);
for(int s=0;s<T;s++) embed_row(m, ids[s], x+(int64_t)s*D);
layers_forward(m,x,T,0);
double lp=0; int greedy=1;
for(int pos=ctxlen-1; pos<T-1; pos++){
rmsnorm(row, x+(int64_t)pos*D, m->final_norm, D, c->eps);
matmul_qt(lo,row,&m->lm_head,1);
int am; lp += logprob_target(lo,c->vocab,ids[pos+1],&am); if(!am) greedy=0;
}
printf("%.6f %d %d\n", lp, contlen, greedy); fflush(stdout);
if(++nreq%5==0) fprintf(stderr,"[score %d req | %.1fs | RSS %.2f GB | hit %.0f%%]\n",
nreq, now_s()-t0, rss_gb(), (m->hits+m->miss)?100.0*m->hits/(m->hits+m->miss):0.0);
}
free(ln); free(ids); free(x); free(lo); free(row); fclose(f);
}
static void generate(Model *m, const int *prompt, int np, int n_new, int *out){
kv_alloc(m,np+n_new+g_draft+2);
for(int i=0;i<np;i++) out[i]=prompt[i];
float *logit=step(m,prompt,np,0);
EmitStore es={out+np,0};
spec_decode(m,out,np,n_new,-1,logit,emit_store,&es,NULL);
}
static void profile_print(Model *m, double elapsed){
double accounted=m->t_ewait+m->t_emm+m->t_attn+m->t_head;
printf("PROFILE: expert-disk %.3fs service / %.3fs wait | expert-matmul %.3fs | attention %.3fs "
"(including kvb %.3fs) | lm_head %.3fs | other %.3fs\n",
m->t_edisk,m->t_ewait,m->t_emm,m->t_attn,m->t_kvb,m->t_head,elapsed-accounted);
printf("ATTENTION: projection/RoPE %.3fs | score-softmax-value %.3fs | output projection %.3fs\n",
m->t_aproj,m->t_acore,m->t_aout);
#ifdef COLI_METAL
if(g_metal_enabled){ uint64_t ok=0,fb=0,ex=0; double su=0,gp=0,sc=0;
coli_metal_moe_counts(&ok,&fb,&ex); coli_metal_moe_times(&su,&gp,&sc);
{ uint64_t aok=0; double aw=0,ak=0; coli_metal_attn_counts(&aok,&aw,&ak);
if(aok){ double ks=0,gs=0; coli_metal_attn_lat(&ks,&gs);
printf("METAL-ATTN: layer GPU %llu | gpu-wall %.2fs (kernel %.2fs | cpu-sched %.2fs gpu-sched %.2fs)\n",(unsigned long long)aok,aw,ak,ks,gs); } }
printf("METAL: blocchi GPU %llu | fallback CPU %llu | expert su GPU %llu | setup %.2fs gpu-wall %.2fs (kernel %.2fs) scatter %.2fs\n",
(unsigned long long)ok,(unsigned long long)fb,(unsigned long long)ex,su,gp,coli_metal_moe_kernel_time(),sc); }
#endif
}
/* Fixed-token decode benchmark: prefill all but the prompt's last token, then
* replay the oracle sequence one token at a time. CPU and CUDA therefore see
* identical hidden-state inputs even if their argmax predictions differ. */
static void run_replay(Model *m, const int *full, int nfull, int np){
if(np<2||nfull<=np){ fprintf(stderr,"REPLAY requires a non-empty prompt and continuation\n"); return; }
kv_alloc(m,nfull+2);
float *logit=step(m,full,np-1,0); free(logit);
m->hits=m->miss=m->ereq=m->gpu_expert_calls=0;
m->t_edisk=m->t_ewait=m->t_emm=m->t_attn=m->t_kvb=m->t_head=0;
m->t_aproj=m->t_acore=m->t_aout=0;
double t0=now_s(); int steps=0;
for(int i=np-1;i<nfull-1;i++){
logit=step(m,full+i,1,i); free(logit); steps++;
}
double dt=now_s()-t0, tot=m->hits+m->miss;
printf("REPLAY decode: %d tokens in %.3fs | %.2f tok/s | expert hit %.1f%%\n",
steps,dt,steps/dt,tot?100.0*m->hits/tot:0.0);
profile_print(m,dt);
#ifdef COLI_CUDA
if(m->gpu_expert_count) printf("CUDA expert tier: %d resident experts (%.2f GB) | %llu calls served from VRAM\n",
m->gpu_expert_count,m->gpu_expert_bytes/1e9,(unsigned long long)m->gpu_expert_calls);
if(g_cuda_enabled) cuda_stats_print();
#endif
}
/* generazione reale: tokenizza PROMPT, prefill + decode greedy con stop su EOS,
* detokenizza e stampa il testo in streaming. */
static void run_text(Model *m, const char *snap, const char *prompt, int ngen){
Cfg *c=&m->c; char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap);
Tok T; tok_load(&T,tkp);
int eos=tok_id_of(&T,"<|endoftext|>");
stops_arm(&m->c, eos);
grammar_setup(&T); /* metodo F: GRAMMAR=file.gbnf (#48) */
if(g_temp<0) g_temp=0.7f; /* auto: 0.7, NON l'1.0 ufficiale — la coda della
* distribuzione int4 e' rumore di quantizzazione */
int cap=(int)strlen(prompt)+16; int *pids=malloc(cap*sizeof(int));
int np=tok_encode(&T,prompt,(int)strlen(prompt),pids,cap);
if(np<1){ fprintf(stderr,"prompt is empty after tokenization\n"); return; }
printf("prompt: %d tokens | generating up to %d (EOS stop=%d) | n-gram draft=%d\n", np, ngen, eos, g_draft);
fputs(prompt,stdout); fflush(stdout);
kv_alloc(m, np+ngen+g_draft+2);
int *all=malloc((np+ngen+g_draft+2)*sizeof(int)); memcpy(all,pids,np*sizeof(int));
double t=now_s();
float *logit=step(m,pids,np,0);
EmitStream es={&T,m,t,0,0};
grammar_reset();
int produced=spec_decode(m,all,np,ngen,eos,logit,emit_stream,&es,NULL);
double dt=now_s()-t;
double tot=m->hits+m->miss;
int nsp=0; for(int i=0;i<c->n_layers;i++) if(m->L[i].sparse) nsp++;
printf("\n---\n%d tokens in %.2fs (%.2f tok/s) | expert hit rate %.1f%% | RSS %.2f GB\n",
produced, dt, produced/dt, tot?100.0*m->hits/tot:0.0, rss_gb());
printf("experts loaded/token: %.1f (per-layer %.2f across %d; baseline topk=%d) | TOPK=%d TOPP=%.2f\n",
produced?(double)m->ereq/produced:0.0, (produced&&nsp)?(double)m->ereq/produced/nsp:0.0, nsp, c->topk, g_topk, g_topp);
printf("speculation: %.2f tokens/forward (%llu forwards per %llu tokens) | MTP acceptance %.0f%% (%llu/%llu)\n",
m->n_fw?(double)m->n_emit/m->n_fw:1.0, (unsigned long long)m->n_fw, (unsigned long long)m->n_emit,
m->mtp_prop?100.0*m->mtp_acc/m->mtp_prop:0.0, (unsigned long long)m->mtp_acc, (unsigned long long)m->mtp_prop);
if(g_gr_prop) printf("grammar: %.0f%% acceptance (%llu/%llu forced drafts)\n",
100.0*g_gr_acc/g_gr_prop, (unsigned long long)g_gr_acc, (unsigned long long)g_gr_prop);
#ifdef COLI_CUDA
if(m->gpu_expert_count) printf("CUDA expert tier: %d resident experts (%.2f GB) | %llu calls served from VRAM\n",
m->gpu_expert_count,m->gpu_expert_bytes/1e9,(unsigned long long)m->gpu_expert_calls);
if(g_cuda_enabled) cuda_stats_print();
#endif
profile_print(m,dt);
if(g_pilot_real) printf("PILOT_REAL: %ld load cross-layer completati, %ld scartati (main gia' sul layer) | PILOT_K=%d\n",
(long)atomic_load_explicit(&g_pilot_loads,memory_order_relaxed),
(long)atomic_load_explicit(&g_pilot_drops,memory_order_relaxed), g_pilot_k);
if(g_looka){
const char *nm[3]={"previous token (=SPEC prefetch)","layer input, skip attention","next layer (one step ahead)"};
printf("LOOKAHEAD routing — recall of true experts in predicted top-8:\n");
for(int i=0;i<3;i++) printf(" %-38s %5.1f%% (%lld/%lld)\n", nm[i],
la_tot[i]?100.0*la_hit[i]/la_tot[i]:0.0, (long long)la_hit[i], (long long)la_tot[i]);
}
free(pids); free(all);
usage_save(m);
}
/* modalita' SERVE (per la CLI 'coli'): carica il modello UNA volta, poi CHAT conversazionale.
* KV-cache PERSISTENTE tra i turni: la storia resta in cache, si fa il prefill solo dei
* token NUOVI -> il modello RICORDA la conversazione e non ri-processa il passato (lossless,
* piu' umano, piu' veloce). Template chat GLM con token speciali (CHAT_TEMPLATE=0 -> grezzo).
* Protocollo: "\x01\x01" "READY" "\x01\x01\n" dopo il load; risposta in streaming; "\x01\x01" "END" "\x01\x01\n" a fine turno.
* ":reset" (riga "\x02RESET") azzera la memoria. EOF -> esce. */
/* ---- RFC: RE-PIN A CALDO / LIVE RE-PIN (opt-in, REPIN=n, default OFF) ----
* Upstream fa AUTOPIN allo START (dalla storia .coli_usage). Questo aggiunge un re-pin
* TRA I TURNI: nel punto sicuro dopo la risposta scambia i pin peggiori con i non-pinnati
* piu' caldi, cosi' l'hot-store insegue il carico VIVO senza un profilo a parte. Isteresi
* 25% (+4) contro il ping-pong; max 4 scambi/passata (~20 MB di disco l'uno). Una heat
* map separata decade a ogni passata: la storia persistente .coli_usage resta intatta.
* EN: upstream AUTOPINs at START (from .coli_usage). This adds a between-turns re-pin: at
* the safe point after the reply, swap the worst pins for the hottest unpinned, so the
* hot-store tracks the LIVE workload without a separate profile. 25% (+4) hysteresis vs
* ping-pong; max 4 swaps/pass (~20 MB disk each). A separate decaying heat map keeps
* persistent .coli_usage intact while adapting to the current workload. */
static int g_repin=0;
static uint64_t g_last_repin=0;
typedef struct { long gain; int l, slot, eid; } RepinCand;
static int repin_pick(Model *m, RepinCand *out, int maxc){
Cfg *c=&m->c; int nb=0;
for(int l=0;l<c->n_layers;l++){
if(!m->npin || m->npin[l]<1 || !m->eheat[l]) continue;
ESlot *P=m->pin[l]; int ids[4096], zp, eu; long g;
int np=m->npin[l]; if(np>4096) np=4096;
for(int z=0;z<np;z++) ids[z]=P[z].eid;
if(!tier_pick_lfru(m->eheat[l],m->elast[l],m->eaccess_clock,
c->n_experts,ids,np,&zp,&eu,&g)) continue;
if(nb<maxc){ out[nb].gain=g; out[nb].l=l; out[nb].slot=zp; out[nb].eid=eu; nb++; }
else { int w=0; for(int b=1;b<maxc;b++) if(out[b].gain<out[w].gain) w=b;
if(g>out[w].gain){ out[w].gain=g; out[w].l=l; out[w].slot=zp; out[w].eid=eu; } }
}
return nb;
}
static void repin_pass(Model *m){
if(g_repin<=0) return;
if(m->n_emit - g_last_repin < (uint64_t)g_repin) return;
g_last_repin = m->n_emit;
RepinCand cd[4]; int nb=repin_pick(m,cd,4);
for(int b=0;b<nb;b++){
ESlot *s=&m->pin[cd[b].l][cd[b].slot];
int old=s->eid;
uint32_t old_heat=m->eheat[cd[b].l][old], new_heat=m->eheat[cd[b].l][cd[b].eid];
#ifdef COLI_CUDA
int gpu=s->g.cuda_eligible;
int64_t old_gpu=gpu ? (int64_t)coli_cuda_tensor_bytes(s->g.cuda)
+(int64_t)coli_cuda_tensor_bytes(s->u.cuda)
+(int64_t)coli_cuda_tensor_bytes(s->d.cuda) : 0;
#endif
double t0=now_s();
expert_load(m,cd[b].l,cd[b].eid,s,1); /* disk -> RAM, same resident slot */
const char *tier="RAM";
#ifdef COLI_CUDA
if(gpu){ /* refresh the same VRAM slot now, not lazily */
if(qt_cuda_upload(&s->g) && qt_cuda_upload(&s->u) && qt_cuda_upload(&s->d)){
int64_t now_gpu=(int64_t)coli_cuda_tensor_bytes(s->g.cuda)
+(int64_t)coli_cuda_tensor_bytes(s->u.cuda)
+(int64_t)coli_cuda_tensor_bytes(s->d.cuda);
m->gpu_expert_bytes+=now_gpu-old_gpu; tier="VRAM";
if(g_cuda_release_host) expert_host_release(m,s);
} else {
qt_cuda_reset(&s->g); qt_cuda_reset(&s->u); qt_cuda_reset(&s->d);
s->g.cuda_eligible=s->u.cuda_eligible=s->d.cuda_eligible=0;
m->gpu_expert_count--; m->gpu_expert_bytes-=old_gpu;
fprintf(stderr,"[REPIN] VRAM upload failed; slot downgraded to RAM\n");
}
}
#endif
fprintf(stderr,"[REPIN] %s layer %d: evict %d (heat=%u) <- admit %d (heat=%u) in %.0f ms\n",
tier,cd[b].l,old,old_heat,cd[b].eid,new_heat,(now_s()-t0)*1e3);
}
for(int l=0;l<m->c.n_layers;l++) if(m->eheat[l]) tier_decay(m->eheat[l],m->c.n_experts);
}
/* ---- KV SU DISCO: la conversazione si riapre CALDA (KVSAVE=0 disattiva) ----
* Il re-prefill di una chat riaperta costa ore su questo disco; la KV compressa MLA
* costa ~182 KB/token. File <SNAP>/.coli_kv append-only: header (magic + dimensioni +
* nrec) e un record per posizione [tok i32][Lc+Rc dei 78 layer][Ic DSA]. A fine turno
* si appendono SOLO le posizioni nuove e si riscrive nrec per ultimo: un crash a meta'
* append lascia nrec vecchio = file coerente. La riga KV del layer MTP non si salva:
* al resume kv_start=-1 e la finestra di draft riparte da sola. */
static int g_kvsave=1;
#define KV_MAGIC "COLIKV1\0"
static void kv_hdr(Model *m, int32_t *h, int nrec){
Cfg *c=&m->c; int nic=0;
for(int i=0;i<c->n_layers;i++) if(m->Ic && m->Ic[i]) nic++;
h[0]=c->n_layers; h[1]=c->kv_lora; h[2]=c->qk_rope;
h[3]=m->has_dsa?c->index_hd:0; h[4]=nic; h[5]=c->vocab; h[6]=nrec; h[7]=0;
}
static void kv_disk_truncate(Model *m, int nrec){
if(!g_kvsave) return;
KVState *k=m->kv;
FILE *f=fopen(k->disk_path,"r+b");
if(!f){ k->disk_nrec=0; return; }
k->disk_nrec=nrec;
int32_t nr=nrec; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); fclose(f);
}
static void kv_disk_reset(Model *m){ kv_disk_truncate(m,0); }
static void kv_disk_append(Model *m, const int *hist, int len){
KVState *k=m->kv;
if(!g_kvsave || len<=k->disk_nrec) return;
Cfg *c=&m->c;
FILE *f=fopen(k->disk_path,"r+b");
if(!f){ f=fopen(k->disk_path,"wb"); if(!f) return;
int32_t h[8]; kv_hdr(m,h,0); fwrite(KV_MAGIC,1,8,f); fwrite(h,4,8,f); }
int64_t rec = 4 + (int64_t)c->n_layers*(c->kv_lora+c->qk_rope)*4;
if(m->has_dsa) for(int i=0;i<c->n_layers;i++) if(m->Ic[i]) rec+=(int64_t)c->index_hd*4;
fseek(f, 8+8*4 + (int64_t)k->disk_nrec*rec, SEEK_SET);
for(int p=k->disk_nrec;p<len;p++){
int32_t tk=hist[p]; fwrite(&tk,4,1,f);
for(int i=0;i<c->n_layers;i++){
fwrite(m->Lc[i]+(int64_t)p*c->kv_lora, 4, c->kv_lora, f);
fwrite(m->Rc[i]+(int64_t)p*c->qk_rope, 4, c->qk_rope, f);
}
if(m->has_dsa) for(int i=0;i<c->n_layers;i++) if(m->Ic[i])
fwrite(m->Ic[i]+(int64_t)p*c->index_hd, 4, c->index_hd, f);
}
fflush(f); /* dati prima, contatore poi */
int32_t nr=len; fseek(f,8+6*4,SEEK_SET); fwrite(&nr,4,1,f); fclose(f);
k->disk_nrec=len;
}
static int kv_disk_load(Model *m, int *hist, int maxctx){
if(!g_kvsave) return 0;
KVState *k=m->kv;
Cfg *c=&m->c;
FILE *f=fopen(k->disk_path,"rb"); if(!f) return 0;
char mg[8]; int32_t h[8], w[8]; kv_hdr(m,w,0);
if(fread(mg,1,8,f)!=8 || memcmp(mg,KV_MAGIC,8) || fread(h,4,8,f)!=8 ||
h[0]!=w[0]||h[1]!=w[1]||h[2]!=w[2]||h[3]!=w[3]||h[4]!=w[4]||h[5]!=w[5]){
fprintf(stderr,"[KV] ignoring .coli_kv from a different model or version\n"); fclose(f); return 0; }
int nrec=h[6];
if(nrec<1){ fclose(f); return 0; }
if(nrec>=maxctx-8-g_draft){
fprintf(stderr,"[KV] saved conversation (%d tokens) exceeds the context: starting over\n",nrec);
fclose(f); return 0; }
double t0=now_s();
for(int p=0;p<nrec;p++){
int32_t tk; if(fread(&tk,4,1,f)!=1){ nrec=p; break; } hist[p]=tk;
for(int i=0;i<c->n_layers;i++){
if(fread(m->Lc[i]+(int64_t)p*c->kv_lora, 4, c->kv_lora, f)!=(size_t)c->kv_lora ||
fread(m->Rc[i]+(int64_t)p*c->qk_rope, 4, c->qk_rope, f)!=(size_t)c->qk_rope){ nrec=p; goto out; }
}
if(m->has_dsa) for(int i=0;i<c->n_layers;i++) if(m->Ic[i])
if(fread(m->Ic[i]+(int64_t)p*c->index_hd, 4, c->index_hd, f)!=(size_t)c->index_hd){ nrec=p; goto out; }
}
out:
fclose(f);
if(nrec>0){
if(m->has_mtp) m->kv_start[c->n_layers]=-1; /* la finestra MTP riparte da sola */
fprintf(stderr,"[KV] resumed conversation from disk: %d tokens in %.1fs (no re-prefill)\n",
nrec, now_s()-t0);
}
k->disk_nrec=nrec;
return nrec;
}
typedef struct { KVState kv; int *hist, len, first; } ServeCtx;
static double kv_pool_bytes(Model *m, int max_ctx);
static void serve_ctx_init(Model *m, ServeCtx *s, const char *snap, int slot, int maxctx){
s->kv.kv_start=calloc(m->c.n_layers+1,sizeof(int));
if(m->has_mtp) s->kv.kv_start[m->c.n_layers]=-1;
kv_bind(m,&s->kv); kv_alloc(m,maxctx);
s->hist=malloc(maxctx*sizeof(int)); s->first=1;
if(slot==0) snprintf(s->kv.disk_path,sizeof(s->kv.disk_path),"%s/.coli_kv",snap);
else snprintf(s->kv.disk_path,sizeof(s->kv.disk_path),"%s/.coli_kv.%d",snap,slot);
s->len=kv_disk_load(m,s->hist,maxctx); if(s->len>0) s->first=0;
}
static void serve_ctx_free(Model *m, ServeCtx *s){
KVState *k=&s->kv; int NR=m->c.n_layers+1;
if(k->Lc) for(int i=0;i<NR;i++){ free(k->Lc[i]); free(k->Rc[i]); }
if(k->Ic) for(int i=0;i<m->c.n_layers;i++) free(k->Ic[i]);
free(k->Lc); free(k->Rc); free(k->Ic); free(k->kv_start); free(s->hist);
}
typedef struct {
int active, pending, emitted, maximum, prompt_tokens, length_limited;
unsigned long long id;
float temp, top_p;
double started;
uint64_t hits0, miss0;
} ServeReq;
static void mux_data(Tok *T, unsigned long long id, int token){
char out[256]; int n=tok_decode(T,&token,1,out,sizeof(out));
printf("DATA %llu %d\n",id,n); if(n>0) fwrite(out,1,(size_t)n,stdout); putchar('\n');
fflush(stdout);
}
static void mux_done(Model *m, ServeCtx *sc, ServeReq *r){
double dt=now_s()-r->started; if(dt<1e-6) dt=1e-6;
double dh=(double)(m->hits-r->hits0), dm=(double)(m->miss-r->miss0);
printf("DONE %llu STAT %d %.2f %.1f %.2f %d %d\n",r->id,r->emitted,
r->emitted/dt,(dh+dm)>0?100.0*dh/(dh+dm):0.0,rss_gb(),
r->prompt_tokens,r->length_limited);
fflush(stdout); kv_bind(m,&sc->kv); kv_disk_append(m,sc->hist,sc->len);
r->active=0;
}
/* Read and prefill one request. Returns -1 on EOF, 0 for a rejected frame and
* 1 for an accepted request. Prefill deliberately remains serial: continuous
* batching starts at decode, where every active slot contributes one row. */
static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, int nctx,
int maxctx, int eos){
char *line=NULL; size_t cap=0; ssize_t nr=getline(&line,&cap,stdin);
if(nr<0){ free(line); return -1; }
if(nr && line[nr-1]=='\n') line[--nr]=0;
if(!strncmp(line,"CANCEL ",7)){
unsigned long long id=0; char tail;
if(sscanf(line+7,"%llu %c",&id,&tail)!=1 || id==0){
printf("ERROR 0 BAD_REQUEST\n"); fflush(stdout); free(line); return 0;
}
for(int i=0;i<nctx;i++) if(req[i].active && req[i].id==id){
req[i].active=0; kv_bind(m,&ctx[i].kv);
kv_disk_append(m,ctx[i].hist,ctx[i].len);
printf("ERROR %llu CANCELLED\n",id); fflush(stdout); free(line); return 0;
}
printf("ERROR %llu NOT_FOUND\n",id); fflush(stdout); free(line); return 0;
}
ColiSubmit sub; int valid=coli_submit_parse(line,&sub);
if(!valid){ printf("ERROR 0 BAD_REQUEST\n"); fflush(stdout); free(line); return 0; }
char *raw=malloc((size_t)sub.bytes+1);
if(!raw){ fprintf(stderr,"OOM multiplex payload\n"); exit(1); }
if(fread(raw,1,(size_t)sub.bytes,stdin)!=(size_t)sub.bytes){ free(raw); free(line); return -1; }
int delim=fgetc(stdin);
if(delim!='\n'){
printf("ERROR %llu BAD_FRAME\n",sub.id); fflush(stdout);
free(raw); free(line); return -1;
}
raw[sub.bytes]=0;
if(sub.slot>=nctx || memchr(raw,0,(size_t)sub.bytes)){
printf("ERROR %llu BAD_REQUEST\n",sub.id); fflush(stdout); free(raw); free(line); return 0;
}
if(req[sub.slot].active){
printf("ERROR %llu SLOT_BUSY\n",sub.id); fflush(stdout); free(raw); free(line); return 0;
}
for(int i=0;i<nctx;i++) if(req[i].active && req[i].id==sub.id){
printf("ERROR %llu DUPLICATE_ID\n",sub.id); fflush(stdout); free(raw); free(line); return 0;
}
ServeCtx *sc=&ctx[sub.slot]; kv_bind(m,&sc->kv);
int *tmp=malloc(maxctx*sizeof(int));
int nt=tok_encode(T,raw,(int)sub.bytes,tmp,maxctx-2);
free(raw); free(line);
if(nt<1){ free(tmp); printf("ERROR %llu EMPTY_PROMPT\n",sub.id); fflush(stdout); return 0; }
int prefix=0; while(prefix<sc->len && prefix<nt && sc->hist[prefix]==tmp[prefix]) prefix++;
if(prefix<sc->len){ sc->len=prefix; if(m->has_mtp) m->kv_start[m->c.n_layers]=-1;
kv_disk_truncate(m,sc->len); }
int add=nt-sc->len;
if(add>0) memcpy(sc->hist+sc->len,tmp+sc->len,(size_t)add*sizeof(int));
free(tmp);
float *logit = add>0 ? step(m,sc->hist+sc->len,add,sc->len)
: step(m,sc->hist+sc->len-1,1,sc->len-1);
sc->len+=add; sc->first=0;
ServeReq *r=&req[sub.slot]; memset(r,0,sizeof(*r));
r->id=sub.id; r->maximum=sub.max_tokens; r->temp=sub.temperature; r->top_p=sub.top_p;
r->prompt_tokens=nt; r->started=now_s(); r->hits0=m->hits; r->miss0=m->miss;
int room=maxctx-sc->len-1; if(r->maximum>room){r->maximum=room; r->length_limited=1;}
g_temp=r->temp; g_nuc=r->top_p;
int next=pick_tok(logit,m->c.vocab,-1); free(logit);
if(r->maximum<=0 || next==eos || is_stop(next)){ mux_done(m,sc,r); return 1; }
r->pending=next; r->emitted=1; r->active=1; sc->hist[sc->len]=next; m->n_emit++;
mux_data(T,r->id,next);
if(r->emitted>=r->maximum) mux_done(m,sc,r);
return 1;
}
static void run_serve_mux(Model *m, const char *snap){
char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap);
Tok T; tok_load(&T,tkp); int eos=tok_id_of(&T,"<|endoftext|>"); stops_arm(&m->c,eos);
g_draft=0; /* one scheduler owns every forward; MTP/speculation is not ragged-safe */
int maxctx=getenv("CTX")?atoi(getenv("CTX")):4096;
int nctx=getenv("KV_SLOTS")?atoi(getenv("KV_SLOTS")):1;
if(nctx<1||nctx>16){fprintf(stderr,"KV_SLOTS deve essere tra 1 e 16\n");exit(2);}
g_kvsave=getenv("KVSAVE")?atoi(getenv("KVSAVE")):1;
KVState *initial=m->kv; free(initial->kv_start); free(initial);
ServeCtx *ctx=calloc(nctx,sizeof(*ctx)); ServeReq *req=calloc(nctx,sizeof(*req));
for(int i=0;i<nctx;i++) serve_ctx_init(m,&ctx[i],snap,i,maxctx);
setvbuf(stdin,NULL,_IONBF,0);
printf("\x01\x01READY\x01\x01\nSTAT 0 0.00 0.0 %.2f\n",rss_gb()); fflush(stdout);
int eof=0;
for(;;){
int active=0; for(int i=0;i<nctx;i++) active+=req[i].active;
fd_set rfds; FD_ZERO(&rfds); FD_SET(STDIN_FILENO,&rfds);
struct timeval tv={0,0}, *ptv=active?&tv:NULL;
int ready=eof?0:select(STDIN_FILENO+1,&rfds,NULL,NULL,ptv);
if(ready>0 && FD_ISSET(STDIN_FILENO,&rfds)) if(mux_submit(m,&T,ctx,req,nctx,maxctx,eos)<0) eof=1;
active=0; for(int i=0;i<nctx;i++) active+=req[i].active;
if(!active){ if(eof) break; continue; }
DecodeRow rows[16]; int slots[16], S=0;
for(int i=0;i<nctx;i++) if(req[i].active){
rows[S]=(DecodeRow){&ctx[i].kv,req[i].pending,ctx[i].len}; slots[S++]=i;
}
float *lo=step_decode_batch(m,rows,S); if(!lo){fprintf(stderr,"decode batch failed\n");break;}
m->n_fw++;
for(int s=0;s<S;s++){
int i=slots[s]; ServeCtx *sc=&ctx[i]; ServeReq *r=&req[i];
sc->len++; g_temp=r->temp; g_nuc=r->top_p;
int next=pick_tok(lo+(int64_t)s*m->c.vocab,m->c.vocab,-1);
if(next==eos || is_stop(next)){mux_done(m,sc,r);continue;}
r->pending=next; sc->hist[sc->len]=next; r->emitted++; m->n_emit++;
mux_data(&T,r->id,next);
if(r->emitted>=r->maximum) mux_done(m,sc,r);
}
free(lo);
}
usage_save(m);
for(int i=0;i<nctx;i++) serve_ctx_free(m,&ctx[i]); free(ctx); free(req);
m->kv=NULL; m->Lc=m->Rc=m->Ic=NULL; m->kv_start=NULL; m->max_t=0;
}
static void run_serve(Model *m, const char *snap){
char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap);
Tok T; tok_load(&T,tkp);
int eos=tok_id_of(&T,"<|endoftext|>");
stops_arm(&m->c, eos);
grammar_setup(&T); /* metodo F: GRAMMAR=file.gbnf (#48) */
if(g_temp<0) g_temp=0.7f; /* auto: 0.7, NON l'1.0 ufficiale — la coda della
* distribuzione int4 e' rumore di quantizzazione */
int ngen=getenv("NGEN")?atoi(getenv("NGEN")):256;
int maxctx=getenv("CTX")?atoi(getenv("CTX")):4096;
int templ=getenv("CHAT_TEMPLATE")?atoi(getenv("CHAT_TEMPLATE")):1;
g_kvsave = getenv("KVSAVE")?atoi(getenv("KVSAVE")):1;
int nctx=getenv("KV_SLOTS")?atoi(getenv("KV_SLOTS")):1;
if(nctx<1||nctx>16){ fprintf(stderr,"KV_SLOTS must be between 1 and 16\n"); exit(2); }
KVState *initial=m->kv; free(initial->kv_start); free(initial);
ServeCtx *ctx=calloc(nctx,sizeof(ServeCtx));
for(int i=0;i<nctx;i++) serve_ctx_init(m,&ctx[i],snap,i,maxctx);
int active=0; ServeCtx *sc=&ctx[0]; kv_bind(m,&sc->kv);
fprintf(stderr,"[KV] context slots: %d x %d tokens, projected pool %.2f GB\n",
nctx,maxctx,kv_pool_bytes(m,maxctx)/1e9);
#define hist (sc->hist)
#define len (sc->len)
#define first (sc->first)
char *line=NULL; size_t cap=0; ssize_t nr; char *buf=malloc(1<<16);
printf("\x01\x01" "READY" "\x01\x01\n"); printf("STAT 0 0.00 0.0 %.2f\n", rss_gb()); fflush(stdout);
while((nr=getline(&line,&cap,stdin))>0){
if(nr>0 && line[nr-1]=='\n') line[--nr]=0;
if(!strcmp(line,"\x02RESET")){ len=0; first=1; if(m->has_mtp) m->kv_start[m->c.n_layers]=-1;
kv_disk_reset(m);
printf("\x01\x01" "END" "\x01\x01\n"); printf("STAT 0 0.00 0.0 %.2f\n", rss_gb()); fflush(stdout); continue; }
if(!strcmp(line,"\x02MORE")){ /* continua la risposta troncata da NGEN:
la storia e' gia' in KV, basta ri-forwardare l'ULTIMO token per riavere i logits */
if(len<1){ printf("\x01\x01" "END" "\x01\x01\n"); printf("STAT 0 0.00 0.0 %.2f\n", rss_gb()); fflush(stdout); continue; }
int cur=ngen; if(len+cur+g_draft+2>=maxctx) cur=maxctx-len-g_draft-2;
uint64_t h0=m->hits, ms0=m->miss; double tt0=now_s();
float *logit=step(m,hist+len-1,1,len-1);
EmitStream es={&T,m,now_s(),0,1};
int prod=0;
if(cur>0) prod=spec_decode(m,hist,len,cur,eos,logit,emit_stream,&es,&len);
else free(logit);
double tdt=now_s()-tt0; if(tdt<1e-6) tdt=1e-6;
double dh=(double)(m->hits-h0), dm=(double)(m->miss-ms0);
printf("\n\x01\x01" "END" "\x01\x01\n");
printf("STAT %d %.2f %.1f %.2f\n", prod, prod/tdt, (dh+dm)>0?100.0*dh/(dh+dm):0.0, rss_gb());
fflush(stdout); kv_disk_append(m,hist,len); repin_pass(m); continue; } /* RFC: re-pin a caldo tra i turni / live re-pin between turns */
if(nr<1){ printf("\x01\x01" "END" "\x01\x01\n"); printf("STAT 0 0.00 0.0 %.2f\n", rss_gb()); fflush(stdout); continue; }
/* API mode: an exact, length-prefixed prompt. Unlike the interactive
* line protocol this accepts newlines. The tokenized prompt is matched
* against hist so the common KV prefix survives stateless HTTP turns.
* Per-request generation controls follow the byte count:
* \x02PROMPT <bytes> <max_tokens> <temperature> <top_p> [kv_slot]\n<prompt>\n */
char *raw=NULL, *input=line;
int input_n=(int)nr, raw_mode=0, req_ngen=ngen, prompt_tokens=0;
float base_temp=g_temp, base_nuc=g_nuc;
if(!strncmp(line,"\x02PROMPT ",8)){
unsigned long long nb=0; double rt=0, rp=0; int slot=0;
int nf=sscanf(line+8,"%llu %d %lf %lf %d",&nb,&req_ngen,&rt,&rp,&slot);
if(nf<4 || nb>(16u<<20) || req_ngen<1 || rt<0 || rt>2 || rp<=0 || rp>1 ||
slot<0 || slot>=nctx){
printf("\x01\x01" "END" "\x01\x01\n"); printf("STAT 0 0.00 0.0 %.2f 0 0\n",rss_gb()); fflush(stdout); continue;
}
active=slot; sc=&ctx[active]; kv_bind(m,&sc->kv);
raw=malloc((size_t)nb+1); if(!raw){fprintf(stderr,"OOM raw prompt\n");exit(1);}
if(fread(raw,1,(size_t)nb,stdin)!=(size_t)nb){free(raw);break;}
int delim=fgetc(stdin); if(delim!='\n' && delim!=EOF) ungetc(delim,stdin);
if(memchr(raw,0,(size_t)nb)){free(raw); printf("\x01\x01" "END" "\x01\x01\n");
printf("STAT 0 0.00 0.0 %.2f 0 0\n",rss_gb()); fflush(stdout); continue;}
raw[nb]=0; input=raw; input_n=(int)nb; raw_mode=1;
if(req_ngen>ngen) req_ngen=ngen;
g_temp=(float)rt; g_nuc=(float)rp;
} else { active=0; sc=&ctx[0]; kv_bind(m,&sc->kv); }
int bl=0, k=0; /* costruisce/tokenizza il turno */
/* template UFFICIALE GLM-5.2 (chat_template.jinja): niente \n dopo i ruoli, e dopo
* <|assistant|> serve SEMPRE il blocco think — <think></think> lo DISATTIVA (nothink):
* col template sbagliato il modello farfuglia e non emette mai lo stop. THINK=1 lo abilita. */
const char *tk = getenv("THINK")&&atoi(getenv("THINK"))? "<think>" : "<think></think>";
if(raw_mode){
int *tmp=malloc(maxctx*sizeof(int)); if(!tmp){fprintf(stderr,"OOM raw tokens\n");exit(1);}
prompt_tokens=tok_encode(&T,input,input_n,tmp,maxctx-8-g_draft);
int old_len=len, prefix=0;
while(prefix<old_len && prefix<prompt_tokens && hist[prefix]==tmp[prefix]) prefix++;
if(prefix<old_len){
len=prefix;
if(m->has_mtp) m->kv_start[m->c.n_layers]=-1;
kv_disk_truncate(m,len); /* il prossimo append sovrascrive solo la coda */
}
k=prompt_tokens-len;
if(k>0) memcpy(hist+len,tmp+len,k*sizeof(int));
fprintf(stderr,"[API] KV slot %d prefix %d/%d token, prefill %d\n",
active,len,prompt_tokens,k);
free(tmp);
} else {
if(templ){ if(first) bl+=snprintf(buf+bl,(1<<16)-bl,"[gMASK]<sop>");
bl+=snprintf(buf+bl,(1<<16)-bl,"<|user|>%s<|assistant|>%s",input,tk); }
else bl+=snprintf(buf+bl,(1<<16)-bl,"%s",input);
k=tok_encode(&T,buf,bl,hist+len,maxctx-len); prompt_tokens=k;
if(len+k+8+g_draft>=maxctx){ len=0; first=1; kv_disk_reset(m);
bl=0; if(templ){ bl+=snprintf(buf+bl,(1<<16)-bl,"[gMASK]<sop><|user|>%s<|assistant|>%s",input,tk); }
else bl+=snprintf(buf+bl,(1<<16)-bl,"%s",input);
k=tok_encode(&T,buf,bl,hist,maxctx); if(k>maxctx-8-g_draft) k=maxctx-8-g_draft;
prompt_tokens=k;
}
}
if(prompt_tokens<1){ free(raw); g_temp=base_temp; g_nuc=base_nuc;
printf("\x01\x01" "END" "\x01\x01\n"); printf("STAT 0 0.00 0.0 %.2f 0 0\n", rss_gb()); fflush(stdout); continue; }
first=0;
int cur=req_ngen; if(len+k+cur+g_draft+2>=maxctx) cur=maxctx-len-k-g_draft-2;
uint64_t h0=m->hits, ms0=m->miss; double tt0=now_s();
float *logit;
if(k>0){ logit=step(m,hist+len,k,len); len+=k; }
else logit=step(m,hist+len-1,1,len-1); /* prompt identico/prefisso: rigenera i logits */
EmitStream es={&T,m,now_s(),0,1};
int prod=0;
grammar_reset(); /* nuova risposta = nuovo documento (MORE invece continua) */
if(cur>0) prod=spec_decode(m,hist,len,cur,eos,logit,emit_stream,&es,&len);
else free(logit);
double tdt=now_s()-tt0; if(tdt<1e-6) tdt=1e-6;
double dh=(double)(m->hits-h0), dm=(double)(m->miss-ms0);
printf("%s\x01\x01" "END" "\x01\x01\n",raw_mode?"":"\n");
printf("STAT %d %.2f %.1f %.2f %d %d\n", prod, prod/tdt,
(dh+dm)>0?100.0*dh/(dh+dm):0.0, rss_gb(), prompt_tokens, prod>=cur);
fflush(stdout);
free(raw); g_temp=base_temp; g_nuc=base_nuc;
usage_save(m); /* la cache che impara: storia aggiornata a ogni turno */
kv_disk_append(m,hist,len); /* KV su disco: il prossimo avvio riparte da qui */
repin_pass(m); /* safe request boundary: adapt session-local hot tier */
}
free(line); free(buf);
usage_save(m);
#undef hist
#undef len
#undef first
for(int i=0;i<nctx;i++) serve_ctx_free(m,&ctx[i]);
free(ctx); m->kv=NULL; m->Lc=m->Rc=m->Ic=NULL; m->kv_start=NULL; m->max_t=0;
}
static int *read_arr(jval*o,const char*k,int*n){ jval*a=json_get(o,k); int*r=malloc(a->len*sizeof(int));
for(int i=0;i<a->len;i++) r[i]=(int)a->kids[i]->num; *n=a->len; return r; }
/* byte residenti di un tensore [O,I] al numero di bit dato (specchio di qt_bytes) */
static int64_t tbytes(int O,int I,int bits){
if(bits>=16) return (int64_t)O*I*4;
if(bits>=5) return (int64_t)O*I + (int64_t)O*4;
return (int64_t)O*((I+1)/2) + (int64_t)O*4;
}
/* byte VERI di un expert: dal container se pre-quantizzato, altrimenti stima da ebits */
static int64_t expert_bytes_probe(Model *m, int ebits){
Cfg *c=&m->c; int64_t eb=0; char nm[256];
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.gate_proj.weight",c->first_dense);
if(st_nbytes(&m->S,nm)>0){
const char *suf[3]={"gate_proj","up_proj","down_proj"};
for(int k=0;k<3;k++){
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight",c->first_dense,suf[k]);
eb+=st_nbytes(&m->S,nm);
snprintf(nm,sizeof(nm),"model.layers.%d.mlp.experts.0.%s.weight.qs",c->first_dense,suf[k]);
int64_t q=st_nbytes(&m->S,nm); if(q>0) eb+=q;
}
}
if(eb<=0) eb = tbytes(c->moe_inter,c->hidden,ebits)*2 + tbytes(c->hidden,c->moe_inter,ebits);
return eb;
}
/* scarica su file l'istogramma d'uso degli expert: righe "layer eid count" (per PIN).
* Include la riga MTP (layer n_layers). Scrittura atomica (tmp+rename): viene chiamata
* anche a ogni turno di serve e il processo puo' morire in qualsiasi momento. */
static void stats_dump_q(Model *m, const char *path, int quiet){
char tmp[2100]; snprintf(tmp,sizeof(tmp),"%s.tmp",path);
FILE *f=fopen(tmp,"w"); if(!f){ if(!quiet) perror(tmp); return; }
Cfg *c=&m->c; int64_t tot=0, nz=0;
for(int i=0;i<=c->n_layers;i++){ if(!m->eusage[i]) continue;
for(int e=0;e<c->n_experts;e++) if(m->eusage[i][e]){ fprintf(f,"%d %d %u\n",i,e,m->eusage[i][e]); tot+=m->eusage[i][e]; nz++; } }
fclose(f); rename(tmp,path);
if(!quiet) fprintf(stderr,"[STATS] %lld selections across %lld distinct experts -> %s\n",(long long)tot,(long long)nz,path);
}
static void stats_dump(Model *m, const char *path){ stats_dump_q(m,path,0); }
/* CACHE CHE IMPARA: istogramma d'uso PERSISTENTE in <SNAP>/.coli_usage.
* Caricato all'avvio (i contatori ripartono dalla storia), salvato a ogni turno:
* piu' usi colibri', meglio l'auto-pin conosce i TUOI expert caldi. */
static char g_usage_path[2100]="";
static int64_t usage_load(Model *m, const char *path){
FILE *f=fopen(path,"r"); if(!f) return 0;
Cfg *c=&m->c; int l,e; uint32_t cnt; int64_t tot=0;
while(fscanf(f,"%d %d %u",&l,&e,&cnt)==3)
if(l>=0&&l<=c->n_layers&&e>=0&&e<c->n_experts&&m->eusage[l]){ m->eusage[l][e]+=cnt; tot+=cnt; }
fclose(f); return tot;
}
static void usage_save(Model *m){ if(g_usage_path[0]) stats_dump_q(m,g_usage_path,1); }
/* HOT-STORE ("il redis del colibri'"): carica in RAM, UNA VOLTA e per sempre, i top expert
* per frequenza d'uso misurata (file STATS di un run precedente), entro un budget in GB.
* Ogni hit evita una lettura dal disco lento. */
/* MLOCK: inchioda in RAM fisica gli expert pinnati cosi' il compressore di memoria di
* macOS non li comprime/evacua (visto: RSS reale < residente previsto -> "hit" lenti).
* -1 = auto (ON su macOS dove serve e RLIMIT_MEMLOCK e' permissivo; OFF altrove, dove
* il limite e' spesso minuscolo e va alzato a mano), 0 = off, 1 = force.
* EN: MLOCK: wire pinned experts into physical RAM so macOS's memory compressor can't
* compress/evict them (we saw actual RSS < intended resident -> slow "hits"). -1 = auto
* (ON on macOS where it matters and RLIMIT_MEMLOCK is permissive; OFF elsewhere, where the
* limit is often tiny and must be raised by hand), 0 = off, 1 = force. */
static int g_mlock=-1;
static int mem_should_wire(void){
if(g_mlock>=0) return g_mlock;
#if defined(__APPLE__)
return 1; /* macOS: default ON */
#else
return 0; /* Linux/altri: opt-in via MLOCK=1 / opt-in */
#endif
}
/* Inchioda [addr,addr+len) in RAM fisica. No-op fuori da POSIX (Windows ecc.).
* EN: wire [addr,addr+len) into physical RAM. No-op off POSIX (Windows, etc.). */
static int mem_wire(void *addr, size_t len){
#if defined(__APPLE__) || defined(__linux__)
return mlock(addr, len);
#else
(void)addr; (void)len; return 0;
#endif
}
/* Inchioda tutti gli slab degli expert pinnati (pesi + scale). Non fatale se fallisce.
* EN: wire all pinned-expert slabs (weights + scales). Non-fatal on failure. */
static void pin_wire(Model *m){
if(!mem_should_wire()) return;
Cfg *c=&m->c; double t0=now_s(); int64_t wired=0; long failed=0;
for(int i=0;i<c->n_layers;i++) for(int z=0;z<m->npin[i];z++){
ESlot *s=&m->pin[i][z];
if(s->slab){ if(mem_wire(s->slab, s->slab_cap)==0) wired+=s->slab_cap; else failed++; }
if(s->fslab){ size_t fl=(size_t)s->fslab_cap*sizeof(float);
if(mem_wire(s->fslab, fl)==0) wired+=fl; else failed++; }
}
if(failed)
fprintf(stderr,"[PIN] mlock: %.1f GB wired, %ld allocations failed "
"(raise the limit: ulimit -l unlimited) in %.0fs\n", wired/1e9, failed, now_s()-t0);
else
fprintf(stderr,"[PIN] mlock: %.1f GB wired in physical RAM "
"(no compression) in %.0fs\n", wired/1e9, now_s()-t0);
}
typedef struct { int l,e; uint32_t c; } PinRec;
static int pin_rec_cmp(const void *a,const void *b){
const PinRec *x=a,*y=b; return x->c<y->c?1:x->c>y->c?-1:0;
}
static void pin_load(Model *m, const char *statspath, double gb){
FILE *f=fopen(statspath,"r"); if(!f){ perror(statspath); return; }
Cfg *c=&m->c; int cap=(c->n_layers+1)*c->n_experts;
PinRec *r=malloc((size_t)cap*sizeof(PinRec)); int n=0;
unsigned char *seen=calloc((size_t)(c->n_layers+1)*c->n_experts,1);
int l,e; uint32_t cnt;
while(n<cap && fscanf(f,"%d %d %u",&l,&e,&cnt)==3){
int ok = l>=0 && e>=0 && e<c->n_experts &&
((l<c->n_layers && m->L[l].sparse) || (l==c->n_layers && m->has_mtp));
int64_t key=(int64_t)l*c->n_experts+e;
if(ok&&!seen[key]){ r[n++]=(PinRec){l,e,cnt}; seen[key]=1; }
}
fclose(f);
int fill=getenv("PIN_FILL")?atoi(getenv("PIN_FILL")):0;
#ifdef COLI_CUDA
if(!getenv("PIN_FILL")&&g_cuda_release_host) fill=1;
#endif
if(fill) for(int li=0;li<=c->n_layers;li++){
int sparse=(li<c->n_layers&&m->L[li].sparse)||(li==c->n_layers&&m->has_mtp);
if(sparse) for(int ei=0;ei<c->n_experts;ei++) if(!seen[(int64_t)li*c->n_experts+ei])
r[n++]=(PinRec){li,ei,0};
}
free(seen);
qsort(r,(size_t)n,sizeof(*r),pin_rec_cmp);
int64_t eb=expert_bytes_probe(m,m->ebits);
int npin=(int)(gb*1e9/eb); if(npin>n) npin=n;
if(npin<1){ free(r); return; }
int *cnt_l=calloc(c->n_layers+1,sizeof(int)); /* +1: riga MTP */
for(int a=0;a<npin;a++) cnt_l[r[a].l]++;
for(int i=0;i<=c->n_layers;i++) if(cnt_l[i]) m->pin[i]=calloc(cnt_l[i],sizeof(ESlot));
int *slot_of=malloc((size_t)npin*sizeof(int)), *next=calloc(c->n_layers+1,sizeof(int));
for(int a=0;a<npin;a++) slot_of[a]=next[r[a].l]++;
for(int i=0;i<=c->n_layers;i++) m->npin[i]=cnt_l[i];
double t0=now_s();
#ifdef COLI_CUDA
double remaining[COLI_CUDA_MAX_DEVICES]={0}, placed_b[COLI_CUDA_MAX_DEVICES]={0};
int placed_n[COLI_CUDA_MAX_DEVICES]={0}, gpu_prefix=0;
double budget=g_cuda_expert_gb*1e9, safe_total=0;
if(g_cuda_enabled&&g_cuda_expert_gb>0) for(int i=0;i<g_cuda_ndev;i++){
size_t free_b=0,total_b=0;
if(coli_cuda_mem_info(g_cuda_devices[i],&free_b,&total_b)){
remaining[i]=(double)free_b-(double)g_cuda_dense_projected[i]-2e9;
if(remaining[i]<0) remaining[i]=0; safe_total+=remaining[i];
}
}
if(budget>safe_total) budget=safe_total;
if(g_cuda_enabled&&g_cuda_release_host&&budget>0){ gpu_prefix=(int)(budget/eb)+g_cuda_ndev; if(gpu_prefix>npin)gpu_prefix=npin; }
#else
int gpu_prefix=0;
#endif
/* Load the VRAM-ranked prefix first. Once uploaded its host backing is
* released before the disjoint RAM-ranked suffix is allocated. */
#pragma omp parallel for schedule(dynamic,1)
for(int a=0;a<(gpu_prefix?gpu_prefix:npin);a++)
expert_load(m,r[a].l,r[a].e,&m->pin[r[a].l][slot_of[a]],1);
m->resident_bytes+=(int64_t)(gpu_prefix?gpu_prefix:npin)*eb;
#ifdef COLI_CUDA
if(g_cuda_enabled && g_cuda_expert_gb>0){
int gpu_limit=gpu_prefix?gpu_prefix:npin;
for(int a=0;a<gpu_limit && m->gpu_expert_bytes<budget;a++){
int li=r[a].l;
{ ESlot *s=&m->pin[li][slot_of[a]];
int64_t need=qt_bytes(&s->g)+qt_bytes(&s->u)+qt_bytes(&s->d);
if(m->gpu_expert_bytes+need>budget) break;
int tried[COLI_CUDA_MAX_DEVICES]={0}, placed=0;
for(int attempt=0;attempt<g_cuda_ndev && !placed;attempt++){
int best=-1;
for(int i=0;i<g_cuda_ndev;i++) if(!tried[i] && remaining[i]>=need &&
(best<0||placed_b[i]<placed_b[best])) best=i;
if(best<0) break;
tried[best]=1;
s->g.cuda_device=s->u.cuda_device=s->d.cuda_device=g_cuda_devices[best];
s->g.cuda_eligible=s->u.cuda_eligible=s->d.cuda_eligible=1;
if(qt_cuda_upload(&s->g) && qt_cuda_upload(&s->u) && qt_cuda_upload(&s->d)){
int64_t actual=(int64_t)coli_cuda_tensor_bytes(s->g.cuda)
+(int64_t)coli_cuda_tensor_bytes(s->u.cuda)
+(int64_t)coli_cuda_tensor_bytes(s->d.cuda);
m->gpu_expert_count++; m->gpu_expert_bytes+=actual;
remaining[best]-=actual; placed_b[best]+=actual; placed_n[best]++;
if(g_cuda_release_host) expert_host_release(m,s);
placed=1;
} else {
qt_cuda_reset(&s->g); qt_cuda_reset(&s->u); qt_cuda_reset(&s->d);
s->g.cuda_eligible=s->u.cuda_eligible=s->d.cuda_eligible=0;
remaining[best]=0; /* device rejected its projected capacity */
}
}
}
}
fprintf(stderr,"[CUDA] hot expert tier: %d/%d experts, VRAM %.2f GB (total budget %.1f GB)\n",
m->gpu_expert_count,npin,m->gpu_expert_bytes/1e9,g_cuda_expert_gb);
for(int i=0;i<g_cuda_ndev;i++) fprintf(stderr,"[CUDA] device %d: %d experts, %.2f GB\n",
g_cuda_devices[i],placed_n[i],placed_b[i]/1e9);
}
#endif
if(gpu_prefix>0&&gpu_prefix<npin){
#pragma omp parallel for schedule(dynamic,1)
for(int a=gpu_prefix;a<npin;a++)
expert_load(m,r[a].l,r[a].e,&m->pin[r[a].l][slot_of[a]],1);
m->resident_bytes+=(int64_t)(npin-gpu_prefix)*eb;
}
fprintf(stderr,"[PIN] placement: %d VRAM + %d RAM expert (%.1f GB warm) in %.0fs da %s\n",
m->gpu_expert_count,npin-m->gpu_expert_count,(npin-m->gpu_expert_count)*eb/1e9,now_s()-t0,statspath);
pin_wire(m); /* inchioda in RAM (no compressione) / wire in RAM (no compression) */
free(r); free(cnt_l); free(slot_of); free(next);
}
static double g_mem_avail_boot=0; /* MemAvailable all'avvio, prima di caricare il modello */
/* RAM disponibile ADESSO (GB): e' il tetto vero, non il totale. Linux: MemAvailable
* da /proc/meminfo. macOS: pagine free+inactive+purgeable da host_statistics64
* (stessa semantica: recuperabili senza swap). Senza questo ramo il fallback
* "assumo 8 GB" castrava la cache expert proprio sulle macchine con piu' RAM. */
static double mem_available_gb(void){
#ifdef __APPLE__
mach_msg_type_number_t cnt=HOST_VM_INFO64_COUNT;
vm_statistics64_data_t vm;
if(host_statistics64(mach_host_self(),HOST_VM_INFO64,(host_info64_t)&vm,&cnt)!=KERN_SUCCESS) return 0;
return ((double)vm.free_count+(double)vm.inactive_count+(double)vm.purgeable_count)
* (double)sysconf(_SC_PAGESIZE) / 1e9;
#elif defined(_WIN32)
double total, avail;
compat_meminfo(&total, &avail);
return avail;
#else
FILE *f=fopen("/proc/meminfo","r"); if(!f) return 0;
char ln[256]; double kb=0;
while(fgets(ln,sizeof(ln),f)) if(sscanf(ln,"MemAvailable: %lf",&kb)==1) break;
fclose(f); return kb/1e6;
#endif
}
static int kv_slot_count(void){
if(!getenv("SERVE")) return 1;
return getenv("KV_SLOTS")?atoi(getenv("KV_SLOTS")):1;
}
static double kv_pool_bytes(Model *m, int max_ctx){
Cfg *c=&m->c; double one=(double)(c->n_layers+1)*max_ctx*(c->kv_lora+c->qk_rope)*4.0;
if(m->has_dsa) for(int i=0;i<c->n_layers;i++) if(c->idx_type[i])
one+=(double)max_ctx*c->index_hd*4.0;
int slots=kv_slot_count(); if(slots<1||slots>16) slots=1;
return one*slots;
}
/* byte disponibili per gli expert (pin + LRU) nel budget — specchio del conto di cap_for_ram */
static double expert_avail(Model *m, double ram_gb, int ebits, int max_ctx){
Cfg *c=&m->c; int64_t eb=expert_bytes_probe(m,ebits);
if(ram_gb<=0){ ram_gb=g_mem_avail_boot*0.88; if(ram_gb<4) ram_gb=8; }
double slack = 1.2e9 + 2.5e9 + 64.0*(double)eb
+ kv_pool_bytes(m,max_ctx)
+ (double)max_ctx*c->n_heads*(c->qk_nope+c->v_head)*4.0;
return ram_gb*1e9 - (double)m->resident_bytes - slack;
}
/* clampa la cache expert a un budget RAM (GB): cap t.c. residente + cache + slack <= budget.
* ram_gb<=0 -> budget AUTO = 88% della RAM disponibile adesso (lascia respiro a OS+wrapper:
* sforare = OOM-kill del kernel a meta' generazione, molto peggio di una cache piu' piccola). */
static void cap_for_ram(Model *m, double ram_gb, int ebits, int max_ctx){
Cfg *c=&m->c; int nsp=0; for(int i=0;i<c->n_layers;i++) if(m->L[i].sparse) nsp++;
if(m->has_mtp) nsp+=2; /* riga cache MTP: conta ~doppia (expert int8 = 2x int4) */
int64_t eb=expert_bytes_probe(m,ebits);
int auto_b = ram_gb<=0;
if(auto_b){ ram_gb = g_mem_avail_boot*0.88; /* misurata PRIMA del load: il residente gia'
* allocato viene sottratto sotto, non due volte */
if(ram_gb<4){ fprintf(stderr,"[RAM] MemAvailable is unreadable or too low; assuming 8 GB\n"); ram_gb=8; } }
/* slack ONESTO, non forfettario (l'OOM del 2026-07-04 veniva da qui):
* ws[64] slab del working-set (si materializzano TUTTI nel prefill batch-union),
* KV cache a max_ctx, kvb_all della ricostruzione k/v in attention,
* attivazioni+logits+overhead ~1.2 GB */
double ws_b = 64.0*(double)eb;
double kv_b = kv_pool_bytes(m,max_ctx);
double kvb_b = (double)max_ctx*c->n_heads*(c->qk_nope+c->v_head)*4.0;
/* RISERVA PAGE-CACHE (misurato 2026-07-06): strangolarla fa crollare le pread
* buffered da ~800 a ~180 MB/s — gli ultimi GB di LRU rendono MENO di quanto
* costino in banda disco persa. 2.5 GB restano SEMPRE al kernel. */
double pc_b = 2.5e9;
double slack = 1.2e9 + pc_b + ws_b + kv_b + kvb_b;
double avail = ram_gb*1e9 - (double)m->resident_bytes - slack;
int capmax = (avail>0 && nsp>0) ? (int)(avail/((double)nsp*eb)) : 0;
if(capmax<1) capmax=1;
if(capmax < m->ecap){
fprintf(stderr,"[RAM_GB=%.1f%s] resident %.1f GB + reserve %.1f GB (ws %.1f, KV %dx%d %.1f, kvb %.1f), "
"experts %.1f MB x %d layers -> cap lowered %d->%d (projected peak %.1f GB)\n",
ram_gb,auto_b?" auto":"",m->resident_bytes/1e9,slack/1e9,ws_b/1e9,
kv_slot_count(),max_ctx,kv_b/1e9,kvb_b/1e9,
eb/1e6, nsp, m->ecap, capmax,
(m->resident_bytes + (double)capmax*nsp*eb + slack)/1e9);
m->ecap=capmax;
} else {
/* AUTO-RAISE (issue #12): il budget consente PIU' cache di quella chiesta.
* Senza questo, una macchina da 128 GB girava con la LRU di una da 16
* (cap=8 di default in coli): hit 23-28% con decine di GB inutilizzati.
* Tetto a n_experts: oltre, ogni layer avrebbe slot che non puo' riempire.
* CAP_RAISE=0 ripristina il comportamento fisso. */
int raise_on = getenv("CAP_RAISE")?atoi(getenv("CAP_RAISE")):1;
int newcap = capmax>c->n_experts ? c->n_experts : capmax;
if(raise_on && newcap>m->ecap){
for(int i=0;i<=c->n_layers;i++) if(m->ecache[i]){
m->ecache[i]=realloc(m->ecache[i],(size_t)newcap*sizeof(ESlot));
memset(m->ecache[i]+m->ecap,0,(size_t)(newcap-m->ecap)*sizeof(ESlot));
}
fprintf(stderr,"[RAM_GB=%.1f%s] cap raised %d->%d: budget allows it "
"(projected peak %.1f GB; set CAP_RAISE=0 to disable)\n",
ram_gb, auto_b?" auto":"", m->ecap, newcap,
(m->resident_bytes + (double)newcap*nsp*eb + slack)/1e9);
m->ecap=newcap;
} else
fprintf(stderr,"[RAM_GB=%.1f%s] cap=%d ok (projected peak %.1f GB)\n", ram_gb, auto_b?" auto":"", m->ecap,
(m->resident_bytes + (double)m->ecap*nsp*eb + slack)/1e9);
}
}
int main(int argc, char **argv){
/* ---- Permanent OpenMP hot-thread tuning. The per-expert matmul regions are
* tiny and back-to-back; with the default passive wait policy libgomp parks
* the worker team between regions and the re-wake latency dominates. Keeping
* the threads hot (active spin) collapses that overhead — measured matmul
* time 66.9s -> 20.9s on the Zen5 build, with no change to numerical output.
*
* libgomp reads the OMP_ / GOMP_ vars in a CONSTRUCTOR that runs before
* main(), so setenv() here and continuing would be too late (verified:
* setenv-in-main is ignored by the already-initialised runtime). Instead, on
* first entry seed the winning defaults — respecting anything the user
* already set (overwrite=0) — then re-exec self once so a fresh libgomp
* constructor picks them up. The COLI_OMP_TUNED sentinel guards the exec so
* we re-exec at most once. Fully overridable: any explicit OMP_/GOMP_ env the
* user sets wins (overwrite=0), pre-setting COLI_OMP_TUNED=1 skips the
* re-exec entirely (runs with whatever policy the environment already has),
* and COLI_NO_OMP_TUNE=1 is a documented kill-switch that disables the whole
* re-exec + tuning path (distinct from the internal COLI_OMP_TUNED sentinel).
*
* Must remain the FIRST statement in main(): argv is passed verbatim to execv(). */
if(!getenv("COLI_OMP_TUNED") && !getenv("COLI_NO_OMP_TUNE")){
setenv("OMP_WAIT_POLICY","active",0); /* keep the team hot across the tiny per-expert matmul regions */
setenv("GOMP_SPINCOUNT","200000",0); /* spin briefly, then yield so long disk waits don't burn a core */
setenv("OMP_PROC_BIND","close",0); /* pack the team onto adjacent cores for cache locality */
setenv("OMP_DYNAMIC","FALSE",0); /* fixed team size: no per-region thread-count churn */
setenv("COLI_OMP_TUNED","1",1);
#ifdef __linux__
fprintf(stderr,"[OMP] hot-thread tuning: re-exec once (COLI_NO_OMP_TUNE=1 to skip)\n");
execv("/proc/self/exe", argv); /* returns only on failure -> fall through and run untuned */
perror("[OMP] execv self-reexec failed, running untuned");
#endif
}
const char *snap=getenv("SNAP"); if(!snap){fprintf(stderr,"SNAP=<dir>\n");return 1;}
g_nopack = getenv("NOPACK")?1:0;
g_drop = getenv("DROP")?1:0;
g_prefetch = getenv("PREFETCH")?atoi(getenv("PREFETCH")):0;
g_mmap = getenv("COLI_MMAP")?atoi(getenv("COLI_MMAP")):0;
if(g_mmap) fprintf(stderr,"[MMAP] expert = viste zero-copy nei file (page cache = cache)\n");
g_topk = getenv("TOPK")?atoi(getenv("TOPK")):0;
g_topp = getenv("TOPP")?atof(getenv("TOPP")):0;
const char *policy=getenv("COLI_POLICY"); if(!policy) policy="quality";
int experimental=!strcmp(policy,"experimental-fast");
if(strcmp(policy,"quality")&&strcmp(policy,"balanced")&&!experimental){
fprintf(stderr,"COLI_POLICY non valida: quality, balanced o experimental-fast\n"); return 2;
}
if(!experimental&&(g_topk>0||g_topp>0)){
fprintf(stderr,"[policy] --topp/--topk drop low-weight experts (~1.6x fewer reads, small quality cost)\n");
}
g_mlock = getenv("MLOCK")?atoi(getenv("MLOCK")):-1; /* -1 auto (ON macOS), 0 off, 1 force / auto (ON macOS), 0 off, 1 force */
g_spec = getenv("SPEC")?atoi(getenv("SPEC")):1;
g_draft = getenv("DRAFT")?atoi(getenv("DRAFT")):-1; /* -1 = auto: 3 se MTP, 0 senza */
g_looka = getenv("LOOKA")?atoi(getenv("LOOKA")):0; /* 1 = misura predicibilita' routing */
g_pilot = getenv("PILOT")?atoi(getenv("PILOT")):0; /* 1 = prefetch pilotato dal router */
g_pilot_real = getenv("PILOT_REAL")?atoi(getenv("PILOT_REAL")):0; /* default OFF: load VERI cross-layer (value-preserving prefetch); PILOT_REAL=1 opta in */
if(g_pilot_real) g_pilot=1; /* PILOT_REAL implica il pilota attivo */
/* Default K: hint-only PILOT keeps 8 (WILLNEED hints are free, no eviction).
* Under PILOT_REAL the speculative loads are REAL and create LRU eviction
* pressure, so at ~28% mispredict a large K thrashes the cache — default to 6
* (best-measured this session) unless the user set PILOT_K explicitly. */
g_pilot_k = getenv("PILOT_K")?atoi(getenv("PILOT_K")):(g_pilot_real?6:8);
if(g_pilot_k<1) g_pilot_k=1;
g_pipe = getenv("PIPE")?atoi(getenv("PIPE")):0; /* default OFF: overlap expert load ‖ matmul (byte-identical; reorders I/O). PIPE=1 opts in */
g_pipe_nw = getenv("PIPE_WORKERS")?atoi(getenv("PIPE_WORKERS")):8; /* I/O worker threads */
if(g_pipe_nw<1) g_pipe_nw=1;
g_direct = getenv("DIRECT")?atoi(getenv("DIRECT")):0;
g_idot = getenv("IDOT")?atoi(getenv("IDOT")):1; /* 0 = kernel f32 esatti (A/B) */
g_repin = getenv("REPIN")?atoi(getenv("REPIN")):0; /* RFC: re-pin ogni n token emessi (0=off) / live re-pin every n emitted tokens (0=off) */
g_absorb = getenv("ABSORB")?atoi(getenv("ABSORB")):-1; /* -1 auto: assorbita per S<=4 */
g_dsa_force = getenv("DSA_FORCE")?atoi(getenv("DSA_FORCE")):0;
g_temp = getenv("TEMP")?atof(getenv("TEMP")):-1; /* -1 = auto (1.0 chat/testo, greedy altrove) */
g_nuc = getenv("NUCLEUS")?atof(getenv("NUCLEUS")):0.90f; /* piu' stretto dell'ufficiale 0.95: la coda int4 e' rumore */
if(getenv("SEED")) g_rng = (uint64_t)atoll(getenv("SEED"))*0x9E3779B97F4A7C15ULL+1;
else { struct timespec ts; clock_gettime(CLOCK_MONOTONIC,&ts); g_rng ^= (uint64_t)ts.tv_nsec<<20 ^ (uint64_t)getpid(); }
if(g_draft>63) g_draft=63; /* -1 = auto, risolto dopo model_init */
int cap = argc>1?atoi(argv[1]):64;
int ebits= argc>2?atoi(argv[2]):8;
int dbits= argc>3?atoi(argv[3]):ebits;
if(getenv("SERVE") && (kv_slot_count()<1 || kv_slot_count()>16)){
fprintf(stderr,"KV_SLOTS must be between 1 and 16\n"); return 2;
}
#ifdef COLI_CUDA
if(getenv("COLI_CUDA") && atoi(getenv("COLI_CUDA"))){
const char *one=getenv("COLI_GPU"), *many=getenv("COLI_GPUS");
if(one&&many){ fprintf(stderr,"use COLI_GPU or COLI_GPUS, not both\n"); return 2; }
if(many) g_cuda_ndev=parse_cuda_devices(many,g_cuda_devices);
else if(one) g_cuda_ndev=parse_cuda_devices(one,g_cuda_devices);
else { g_cuda_ndev=1; g_cuda_devices[0]=0; }
if(g_cuda_ndev<1){ fprintf(stderr,"invalid COLI_GPUS: use a list such as 0,1,2\n"); return 2; }
g_cuda_enabled=coli_cuda_init(g_cuda_devices,g_cuda_ndev);
if(!g_cuda_enabled){ fprintf(stderr,"[CUDA] requested backend is unavailable\n"); return 2; }
}
g_cuda_dense=getenv("CUDA_DENSE")?atoi(getenv("CUDA_DENSE")):0;
g_cuda_expert_gb=getenv("CUDA_EXPERT_GB")?atof(getenv("CUDA_EXPERT_GB")):0;
g_cuda_release_host=getenv("CUDA_RELEASE_HOST")?atoi(getenv("CUDA_RELEASE_HOST")):(g_cuda_ndev>1);
if((getenv("COLI_GPU")||getenv("COLI_GPUS"))&&!g_cuda_enabled){ fprintf(stderr,"COLI_GPU(S) requires COLI_CUDA=1\n"); return 2; }
if(g_cuda_dense&&!g_cuda_enabled){ fprintf(stderr,"CUDA_DENSE requires COLI_CUDA=1\n"); return 2; }
if(g_cuda_expert_gb>0 && !g_cuda_enabled){ fprintf(stderr,"CUDA_EXPERT_GB requires COLI_CUDA=1\n"); return 2; }
if(g_cuda_enabled) fprintf(stderr,"[CUDA] mode: routed experts%s%s\n",
g_cuda_dense?" + resident dense tensors":" only (resident dense on CPU)",
g_cuda_release_host?"; VRAM experts without host backing":"");
#else
if((getenv("COLI_CUDA") && atoi(getenv("COLI_CUDA"))) ||
getenv("COLI_GPU") || getenv("COLI_GPUS") ||
(getenv("CUDA_DENSE") && atoi(getenv("CUDA_DENSE"))) ||
(getenv("CUDA_EXPERT_GB") && atof(getenv("CUDA_EXPERT_GB"))>0)){
fprintf(stderr,"CUDA was requested, but this binary is CPU-only; rebuild with: make CUDA=1\n");
return 2;
}
#endif
#ifdef COLI_METAL
if(getenv("COLI_METAL") && atoi(getenv("COLI_METAL"))){
g_metal_enabled = coli_metal_init();
if(!g_metal_enabled){ fprintf(stderr,"[METAL] backend requested but not available\n"); return 2; }
fprintf(stderr,"[METAL] mode: batched routed experts on GPU (unified-memory zero-copy)\n");
if(getenv("COLI_METAL_SPIN") && atoi(getenv("COLI_METAL_SPIN"))){ coli_metal_spin_start(); fprintf(stderr,"[METAL] keep-alive spinner ON\n"); }
if(getenv("COLI_METAL_GEMM_MIN")) g_metal_gemm_min=atoi(getenv("COLI_METAL_GEMM_MIN"));
}
#else
if(getenv("COLI_METAL") && atoi(getenv("COLI_METAL"))){
fprintf(stderr,"METAL was requested, but this binary has no Metal backend; rebuild with: make METAL=1\n");
return 2;
}
#endif
printf("== GLM C engine (glm_moe_dsa), cache=%d experts/layer | experts@%d-bit dense@%d-bit | idot: " IDOT_KERNEL " ==\n", cap, ebits, dbits);
g_mem_avail_boot = mem_available_gb();
Model m; double t0=now_s(); model_init(&m,snap,cap,ebits,dbits);
if(g_draft<0){
#ifdef COLI_CUDA
g_draft = (m.has_mtp&&!g_cuda_enabled) ? 3 : 0;
#else
g_draft = m.has_mtp ? 3 : 0;
#endif
}
if(getenv("DSA_TOPK")) m.c.index_topk=atoi(getenv("DSA_TOPK")); /* override per test */
printf("loaded in %.2fs | resident dense: %.2f MB | layers=%d experts=%d | MTP %s (draft=%d)\n",
now_s()-t0, m.resident_bytes/(1024.0*1024.0), m.c.n_layers, m.c.n_experts,
m.has_mtp?"ACTIVE":"absent", g_draft);
/* anche su stderr: e' il canale che le UI (coli) mostrano all'utente */
fprintf(stderr,"[MTP] %s (draft=%d)\n", m.has_mtp?"active: native speculative decoding":"absent", g_draft);
if(!strncmp(snap,"/mnt/",5))
fprintf(stderr,"WARNING: the model is on %s (slow 9p/Windows filesystem; fadvise is ineffective).\n"
" Keep it on ext4 (for example, /home/...) for memory efficiency and speed.\n", snap);
/* HOT-STORE: PIN=<statsfile> [PIN_GB=g] -> top expert per frequenza fissi in RAM.
* Va PRIMA di cap_for_ram: i pinnati contano nel residente. */
if(getenv("PIN")) pin_load(&m, getenv("PIN"), getenv("PIN_GB")?atof(getenv("PIN_GB")):10.0);
/* CACHE CHE IMPARA: l'uso degli expert si accumula in <SNAP>/.coli_usage tra le sessioni;
* all'avvio i piu' usati vengono auto-pinnati in RAM (meta' del budget expert: il pin
* conosce la TUA storia, la LRU si adatta alla sessione). AUTOPIN=0 disattiva. */
{ double ram_env = getenv("RAM_GB")?atof(getenv("RAM_GB")):0.0;
int est_ctx = getenv("CTX")?atoi(getenv("CTX")):4096; /* stesso default di run_serve */
snprintf(g_usage_path,sizeof(g_usage_path),"%s/.coli_usage",snap);
int64_t hist = usage_load(&m,g_usage_path);
if(hist>0) fprintf(stderr,"[USAGE] expert history: %lld selections (%s)\n",(long long)hist,g_usage_path);
int autopin = getenv("AUTOPIN")?atoi(getenv("AUTOPIN")):1;
if(!getenv("PIN") && autopin && hist>=5000){
/* quota pin proporzionale alla FIDUCIA nella storia: con pochi dati il pin
* sbaglia expert e ruba slot alla LRU adattiva; a regime (>=200k selezioni,
* qualche ora di chat) arriva a meta' del budget expert. */
double conf = (double)hist/200000.0; if(conf>1) conf=1;
double pin_gb = expert_avail(&m,ram_env,ebits,est_ctx)*0.5*conf/1e9;
if(pin_gb>=0.5) pin_load(&m, g_usage_path, pin_gb);
}
/* SEMPRE: senza clamp la LRU cresce fino a cap*76 layer = decine di GB -> OOM-kill.
* RAM_GB assente o <=0 = budget automatico da MemAvailable. */
cap_for_ram(&m, ram_env, ebits, est_ctx); }
const char *stats=getenv("STATS"); /* STATS=<file> -> istogramma uso expert a fine run */
/* modo scoring per benchmark: SCORE=<requests.txt> -> log-likelihood per riga */
if(getenv("SCORE")){ run_score(&m, getenv("SCORE")); if(stats) stats_dump(&m,stats); return 0; }
/* modo serve persistente per la CLI 'coli': SERVE=1 */
if(getenv("SERVE")){
if(getenv("SERVE_BATCH") && atoi(getenv("SERVE_BATCH"))) run_serve_mux(&m,snap);
else run_serve(&m,snap);
if(stats) stats_dump(&m,stats); return 0;
}
/* modo testo reale: PROMPT="..." [NGEN=n] -> tokenizza, genera, detokenizza */
if(getenv("PROMPT")){
int ngen=getenv("NGEN")?atoi(getenv("NGEN")):64;
run_text(&m, snap, getenv("PROMPT"), ngen);
if(stats) stats_dump(&m,stats);
return 0;
}
/* altrimenti: validazione contro l'oracolo (ref_glm.json) */
const char *refpath=getenv("REF")?getenv("REF"):"ref_glm.json";
FILE *f=fopen(refpath,"rb"); if(!f){perror(refpath);return 1;}
fseek(f,0,SEEK_END); long n=ftell(f); fseek(f,0,SEEK_SET);
char *b=malloc(n+1); if(fread(b,1,n,f)!=(size_t)n){} b[n]=0; fclose(f);
char *ar=NULL; jval *ref=json_parse(b,&ar);
int np,nfull; int *prompt=read_arr(ref,"prompt_ids",&np); int *full=read_arr(ref,"full_ids",&nfull);
int n_new=nfull-np;
/* L'oracolo (ref_glm.json in repo) e' del modello TINY: contro il 744B da' 0/20
* garantito su OGNI piattaforma (prompt-token tiny = spazzatura per il modello vero).
* Non e' un bug del motore — vedi #76. */
{ int maxid=0; for(int i=0;i<nfull;i++) if(full[i]>maxid) maxid=full[i];
if(m.c.vocab>1000 && maxid<1000 && !getenv("REF_FORCE")){
fprintf(stderr,"ERRORE: ref_glm.json e' l'oracolo del modello TINY (token max %d, ma il tuo vocab e' %d).\n"
" Self-test motore: SNAP=./glm_tiny TF=1 ./glm 64 16 16 (atteso 32/32)\n"
" Prova reale: PROMPT=\"Ciao\" NGEN=32 SNAP=<modello> ./glm 64\n"
" REF_FORCE=1 per eseguire comunque il confronto (senza senso).\n", maxid, m.c.vocab);
return 1;
} }
if(getenv("REPLAY")){
run_replay(&m,full,nfull,np);
if(stats) stats_dump(&m,stats);
return 0;
}
if(getenv("TF")){
int *tf=read_arr(ref,"tf_pred",&(int){0});
int *pred=malloc(nfull*sizeof(int)); double tt=now_s();
forward_all(&m, full, nfull, pred); double tdt=now_s()-tt;
int ok=0; for(int i=0;i<nfull;i++){
if(pred[i]==tf[i]) ok++;
else fprintf(stderr,"[ORACLE] mismatch pos=%d expected=%d got=%d\n",i,tf[i],pred[i]);
}
printf("PREFILL (teacher-forcing) C vs oracle: %d/%d positions | %.1f pos/s\n",
ok,nfull,nfull/tdt);
if(ok<nfull) fprintf(stderr,
"[ORACLE] %d/%d mismatches — run: TF=1 DEBUG_LOGITS=1 for top-5 logit dump\n",
nfull-ok,nfull);
profile_print(&m,tdt);
#ifdef COLI_CUDA
if(g_cuda_enabled) cuda_stats_print();
#endif
return 0;
}
int *out=malloc((np+n_new)*sizeof(int));
double t=now_s(); generate(&m,prompt,np,n_new,out); double dt=now_s()-t;
int match=0;
printf("\nReference (oracle): "); for(int i=np;i<nfull;i++) printf("%d ", full[i]);
printf("\nGLM C engine : "); for(int i=np;i<nfull;i++){ printf("%d ", out[i]); if(out[i]==full[i])match++; }
printf("\nMatching tokens: %d/%d\n", match, n_new);
double tot=m.hits+m.miss;
printf("N-gram speculation (DRAFT=%d): %.2f tokens/forward (%llu forwards per %llu tokens)\n",
g_draft, m.n_fw?(double)m.n_emit/m.n_fw:1.0, (unsigned long long)m.n_fw, (unsigned long long)m.n_emit);
printf("Expert cache hit rate: %.1f%% (hit=%llu miss=%llu) | RSS: %.2f GB | %.1f tok/s\n",
tot?100.0*m.hits/tot:0.0, (unsigned long long)m.hits, (unsigned long long)m.miss, rss_gb(), n_new/dt);
profile_print(&m,dt);
#ifdef COLI_CUDA
if(m.gpu_expert_count) printf("CUDA expert tier: %d resident experts (%.2f GB) | %llu calls served from VRAM\n",
m.gpu_expert_count,m.gpu_expert_bytes/1e9,(unsigned long long)m.gpu_expert_calls);
if(g_cuda_enabled) cuda_stats_print();
#endif
if(g_looka){
const char *nm[3]={"previous token (=SPEC prefetch)","layer input, skip attention","next layer (one step ahead)"};
printf("LOOKAHEAD routing — recall of true experts in predicted top-8:\n");
for(int i=0;i<3;i++) printf(" %-38s %5.1f%% (%lld/%lld)\n", nm[i],
la_tot[i]?100.0*la_hit[i]/la_tot[i]:0.0, (long long)la_hit[i], (long long)la_tot[i]);
}
if(stats) stats_dump(&m,stats);
return 0;
}
Reference in New Issue View Git Blame Copy Permalink