Unify continuous batching + heterogeneous runtime: decode batching, physical-core planning, disjoint VRAM/RAM placement, topp-policy warning (CPU-validated, CUDA on 6x5090) (#68)
* Fuse CUDA expert MLP execution * Group CUDA expert transfers by device * Instrument grouped CUDA expert execution * Bound grouped CUDA decode scratch * Execute expert groups across GPUs in parallel * Release host backing for multi-GPU experts * Define quality-preserving memory policies * Overlap cold expert loading with resident compute * Adapt expert placement with session LFRU * Fuse q4 expert gate and up dispatch * Plan CPU work on physical cores * Batch grouped expert CUDA kernels * Separate VRAM and RAM expert placement * Add ragged multi-sequence decode forward * feat(runtime): add continuous decode scheduler * Route concurrent API requests through batch scheduler * Harden multiplex request lifecycle and framing * Cancel disconnected multiplex requests * Bind API port before starting the engine * fix automatic KV slot allocation * add native int4 Tensor Core grouped GEMM * add Tensor Core throughput benchmark * optimize packed int4 low-row kernels * add asynchronous CUDA staging streams * document validated six-GPU dense acceleration * tune six-GPU expert hot set * raise validated expert hot-set target * add CUDA MLA absorption core * fuse grouped expert gate and up projections * Warn for explicit lossy routing flags
This commit is contained in:
ZacharyZcR
@@ -27,6 +27,7 @@
|
||||
#include <stdatomic.h> /* PIPE ready-flags/job queue + PILOT_REAL cross-layer handshake */
|
||||
#include <sched.h> /* sched_yield: PIPE spin / PILOT barrier */
|
||||
#include <unistd.h>
|
||||
#include <sys/select.h>
|
||||
#if defined(__APPLE__) || defined(__linux__)
|
||||
#include <sys/resource.h>
|
||||
#include <sys/mman.h> /* mlock: inchioda le pagine in RAM / wire pages into RAM */
|
||||
@@ -36,6 +37,7 @@
|
||||
#include "tok.h"
|
||||
#include "tier.h"
|
||||
#include "grammar.h" /* metodo F: draft grammaticali (#48) */
|
||||
#include "decode_batch.h"
|
||||
#ifdef _OPENMP
|
||||
#include <omp.h> /* scratch per-thread nell'attention */
|
||||
#else
|
||||
@@ -130,6 +132,11 @@ typedef struct {
|
||||
char disk_path[2048];
|
||||
} KVState;
|
||||
|
||||
typedef struct {
|
||||
KVState *kv;
|
||||
int token, pos;
|
||||
} DecodeRow;
|
||||
|
||||
typedef struct {
|
||||
Cfg c; shards S;
|
||||
int ebits, dbits; /* bit expert / bit densa */
|
||||
@@ -146,6 +153,7 @@ typedef struct {
|
||||
ESlot **pin; int *npin; /* HOT-STORE: expert pinnati in RAM (mai evicted) */
|
||||
uint32_t **eusage; /* contatori persistenti (per STATS/PIN) */
|
||||
uint32_t **eheat; /* calore recente per promotion/demotion live */
|
||||
uint32_t **elast, eaccess_clock; /* recency per LFRU session-local */
|
||||
/* DSA lightning indexer (attivo solo se i pesi out-idx-* sono presenti) */
|
||||
int has_dsa;
|
||||
QT *ix_wq, *ix_wk, *ix_wp; /* per layer FULL: wq_b, wk, weights_proj */
|
||||
@@ -161,7 +169,8 @@ typedef struct {
|
||||
uint64_t eclock, hits, miss, ereq;
|
||||
uint64_t gpu_expert_calls; int gpu_expert_count; int64_t gpu_expert_bytes;
|
||||
uint64_t n_fw, n_emit; /* metodo E: forward di decode / token emessi */
|
||||
double t_edisk, t_emm, t_attn, t_kvb, t_head;/* profiling: dove va il tempo (sempre attivo) */
|
||||
double t_edisk, t_ewait, t_emm, t_attn, t_kvb, t_head;/* profiling: dove va il tempo */
|
||||
double t_aproj,t_acore,t_aout; /* attention breakdown */
|
||||
int64_t resident_bytes;
|
||||
} Model;
|
||||
|
||||
@@ -170,6 +179,7 @@ static void usage_save(Model *m); /* cache che impara: definita accanto a
|
||||
static int g_cuda_enabled;
|
||||
static double g_cuda_expert_gb;
|
||||
static int g_cuda_dense;
|
||||
static int g_cuda_release_host;
|
||||
static int g_cuda_devices[COLI_CUDA_MAX_DEVICES], g_cuda_ndev, g_cuda_rr;
|
||||
static int64_t g_cuda_dense_projected[COLI_CUDA_MAX_DEVICES];
|
||||
static void qt_cuda_reset(QT *t){
|
||||
@@ -188,6 +198,14 @@ static void cuda_stats_print(void){
|
||||
coli_cuda_stats(g_cuda_devices[i],&n,&b);
|
||||
fprintf(stderr,"[CUDA] device %d: %zu tensors, %.2f GB\n",g_cuda_devices[i],n,b/1e9);
|
||||
}
|
||||
uint64_t calls=0,experts=0,rows=0; double h2d=0,kernel=0,d2h=0;
|
||||
coli_cuda_group_stats(&calls,&experts,&rows,&h2d,&kernel,&d2h);
|
||||
if(calls) fprintf(stderr,"[CUDA] expert groups: %llu call, %llu expert, %llu righe "
|
||||
"(%.2f expert/call)%s\n",(unsigned long long)calls,(unsigned long long)experts,
|
||||
(unsigned long long)rows,(double)experts/calls,
|
||||
getenv("COLI_CUDA_PROFILE")?"; timing sotto":"");
|
||||
if(calls&&getenv("COLI_CUDA_PROFILE")) fprintf(stderr,
|
||||
"[CUDA] expert groups timing: H2D %.1f ms | kernel %.1f ms | D2H %.1f ms\n",h2d,kernel,d2h);
|
||||
}
|
||||
static int parse_cuda_devices(const char *list, int *out){
|
||||
if(!list||!*list) return 0;
|
||||
@@ -280,6 +298,53 @@ static void matmul_i4(float *y, const float *x, const uint8_t *q4, const float *
|
||||
if(i<I){ uint8_t byte=w[i>>1]; int lo=(int)(byte&0xF)-8; a += xs[i]*(float)lo; }
|
||||
y[(int64_t)s*O+o]=a*sc; } }
|
||||
}
|
||||
/* Decode hot path for gate+up: same exact q4 dot products as matmul_i4, but one
|
||||
* OpenMP dispatch covers both matrices. KTransformers uses persistent pools;
|
||||
* this keeps colibri dependency-free while removing one team launch/expert. */
|
||||
static void matmul_i4_pair(float *yg, float *yu, const float *x,
|
||||
const uint8_t *qg, const float *sg,
|
||||
const uint8_t *qu, const float *su, int I, int O){
|
||||
int rb=(I+1)/2;
|
||||
#pragma omp parallel for schedule(static)
|
||||
for(int z=0;z<2*O;z++){
|
||||
int o=z<O?z:z-O; const uint8_t *w=(z<O?qg:qu)+(int64_t)o*rb;
|
||||
float a=0; int i=0;
|
||||
#ifdef __AVX2__
|
||||
const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8);
|
||||
__m256 acc=_mm256_setzero_ps();
|
||||
for(;i+16<=I;i+=16){ __m128i by=_mm_loadl_epi64((const __m128i*)(w+(i>>1)));
|
||||
__m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
|
||||
__m128i nib=_mm_unpacklo_epi8(lo,hi);
|
||||
__m256 w0=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(nib),b8));
|
||||
__m256 w1=_mm256_cvtepi32_ps(_mm256_sub_epi32(_mm256_cvtepu8_epi32(_mm_srli_si128(nib,8)),b8));
|
||||
acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i),w0,acc);
|
||||
acc=_mm256_fmadd_ps(_mm256_loadu_ps(x+i+8),w1,acc); }
|
||||
a=hsum256(acc);
|
||||
#elif defined(__ARM_NEON)
|
||||
const uint8x8_t m4=vdup_n_u8(0x0F); const int8x8_t b8=vdup_n_s8(8);
|
||||
float32x4_t ac0=vdupq_n_f32(0),ac1=vdupq_n_f32(0);
|
||||
for(;i+16<=I;i+=16){ uint8x8_t by=vld1_u8(w+(i>>1));
|
||||
uint8x8x2_t n=vzip_u8(vand_u8(by,m4),vshr_n_u8(by,4));
|
||||
int16x8_t w0=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[0]),b8));
|
||||
int16x8_t w1=vmovl_s8(vsub_s8(vreinterpret_s8_u8(n.val[1]),b8));
|
||||
ac0=vfmaq_f32(ac0,vld1q_f32(x+i),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w0))));
|
||||
ac1=vfmaq_f32(ac1,vld1q_f32(x+i+4),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w0))));
|
||||
ac0=vfmaq_f32(ac0,vld1q_f32(x+i+8),vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1))));
|
||||
ac1=vfmaq_f32(ac1,vld1q_f32(x+i+12),vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); }
|
||||
a=vaddvq_f32(vaddq_f32(ac0,ac1));
|
||||
#endif
|
||||
for(;i+1<I;i+=2){ uint8_t b=w[i>>1]; a+=x[i]*(float)((b&15)-8)+x[i+1]*(float)((b>>4)-8); }
|
||||
if(i<I) a+=x[i]*(float)((w[i>>1]&15)-8);
|
||||
(z<O?yg:yu)[o]=a*(z<O?sg:su)[o];
|
||||
}
|
||||
}
|
||||
|
||||
static void matmul_qt(float *y,const float *x,QT *w,int S);
|
||||
static void expert_gate_up(float *g,float *u,const float *x,QT *wg,QT *wu,int S){
|
||||
if(S==1&&wg->fmt==2&&wu->fmt==2&&wg->I==wu->I&&wg->O==wu->O)
|
||||
matmul_i4_pair(g,u,x,wg->q4,wg->s,wu->q4,wu->s,wg->I,wg->O);
|
||||
else { matmul_qt(g,x,wg,S); matmul_qt(u,x,wu,S); }
|
||||
}
|
||||
/* y[S,O] = x[S,I] @ W^T con W int2 impacchettato (4 valori/byte) + scala[O]. nibble 2-bit -> [-2,1]. */
|
||||
static void matmul_i2(float *y, const float *x, const uint8_t *q2, const float *scale, int S, int I, int O){
|
||||
int rb=(I+3)/4;
|
||||
@@ -861,6 +926,7 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits
|
||||
m->eroute=calloc(NR,sizeof(int*)); m->enr=calloc(NR,sizeof(int));
|
||||
m->pin=calloc(NR,sizeof(ESlot*)); m->npin=calloc(NR,sizeof(int));
|
||||
m->eusage=calloc(NR,sizeof(uint32_t*)); m->eheat=calloc(NR,sizeof(uint32_t*));
|
||||
m->elast=calloc(NR,sizeof(uint32_t*));
|
||||
m->kv=calloc(1,sizeof(KVState));
|
||||
m->kv_start=m->kv->kv_start=calloc(NR,sizeof(int));
|
||||
for(int i=0;i<c->n_layers;i++){
|
||||
@@ -891,6 +957,7 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits
|
||||
m->eroute[i]=calloc(c->topk,sizeof(int)); /* metodo C: ultimo routing del layer */
|
||||
m->eusage[i]=calloc(c->n_experts,sizeof(uint32_t));
|
||||
m->eheat[i]=calloc(c->n_experts,sizeof(uint32_t));
|
||||
m->elast[i]=calloc(c->n_experts,sizeof(uint32_t));
|
||||
}
|
||||
#undef P
|
||||
}
|
||||
@@ -936,6 +1003,7 @@ static void model_init(Model *m, const char *snap, int cap, int ebits, int dbits
|
||||
m->eroute[i]=calloc(c->topk,sizeof(int));
|
||||
m->eusage[i]=calloc(c->n_experts,sizeof(uint32_t));
|
||||
m->eheat[i]=calloc(c->n_experts,sizeof(uint32_t));
|
||||
m->elast[i]=calloc(c->n_experts,sizeof(uint32_t));
|
||||
m->kv_start[i]=-1; /* KV MTP: parte dalla prima posizione di decode */
|
||||
#undef PM
|
||||
}
|
||||
@@ -1279,6 +1347,24 @@ static inline void pipe_wait(int q){
|
||||
while(!atomic_load_explicit(&g_pp.ready[q],memory_order_acquire)) sched_yield();
|
||||
}
|
||||
|
||||
#ifdef COLI_CUDA
|
||||
static void expert_host_release(Model *m, ESlot *s){
|
||||
if(!s->slab&&!s->fslab) return;
|
||||
#if defined(__APPLE__) || defined(__linux__)
|
||||
if(s->slab) munlock(s->slab,(size_t)s->slab_cap);
|
||||
if(s->fslab) munlock(s->fslab,(size_t)s->fslab_cap*sizeof(float));
|
||||
#endif
|
||||
int64_t bytes=qt_bytes(&s->g)+qt_bytes(&s->u)+qt_bytes(&s->d);
|
||||
free(s->slab); free(s->fslab); s->slab=NULL; s->fslab=NULL; s->slab_cap=s->fslab_cap=0;
|
||||
QT *q[3]={&s->g,&s->u,&s->d};
|
||||
for(int k=0;k<3;k++){ q[k]->qf=NULL; q[k]->q8=NULL; q[k]->q4=NULL; q[k]->s=NULL; }
|
||||
m->resident_bytes-=bytes; if(m->resident_bytes<0) m->resident_bytes=0;
|
||||
}
|
||||
static void expert_host_ensure(Model *m, int layer, ESlot *s){
|
||||
if(!s->slab) expert_load(m,layer,s->eid,s,1);
|
||||
}
|
||||
#endif
|
||||
|
||||
/* prefetch asincrono dei pesi di un expert (e delle sue scale .qs): avvia il readahead
|
||||
* cosi' le letture sincrone successive trovano la page-cache calda. */
|
||||
static void expert_prefetch(Model *m, int layer, int eid){
|
||||
@@ -1324,7 +1410,10 @@ static int cmp_fdesc(const void *a,const void *b){
|
||||
float x=*(const float*)a, y=*(const float*)b; return x<y?1:x>y?-1:0; }
|
||||
|
||||
/* attenzione MLA con KV-cache compressa, su token nuovi x[S,hidden], pos_base = pos del primo */
|
||||
static void attention(Model *m, Layer *l, int layer, float *x, int S, int pos_base, float *out){
|
||||
/* kvs/pos describe a ragged decode batch: each row may belong to a different
|
||||
* sequence. NULL keeps the original contiguous, currently-bound KV path. */
|
||||
static void attention_rows(Model *m, Layer *l, int layer, float *x, int S, int pos_base,
|
||||
KVState *const *kvs, const int *positions, float *out){
|
||||
Cfg *c=&m->c; int H=c->n_heads, D=c->hidden, qh=c->qk_head, vh=c->v_head;
|
||||
int kvb_dim=H*(c->qk_nope+vh), Tk=pos_base+S;
|
||||
double ta0=now_s();
|
||||
@@ -1361,14 +1450,16 @@ static void attention(Model *m, Layer *l, int layer, float *x, int S, int pos_ba
|
||||
/* 1) per ogni token nuovo: query roped + latente normato e k_rot roped -> in cache.
|
||||
* QR tiene il residuo q_a per TUTTE le posizioni: serve anche all'indexer DSA. */
|
||||
for(int s=0;s<S;s++){
|
||||
const float *xs=x+(int64_t)s*D; int pos=pos_base+s;
|
||||
KVState *ks=kvs?kvs[s]:m->kv;
|
||||
const float *xs=x+(int64_t)s*D; int pos=positions?positions[s]:pos_base+s;
|
||||
float *qresid=QR+(int64_t)s*c->q_lora;
|
||||
matmul_qt(qresid, xs, &l->q_a, 1);
|
||||
rmsnorm(qresid, qresid, l->q_a_ln, c->q_lora, c->eps);
|
||||
float *qfull=Q+(int64_t)s*H*qh; matmul_qt(qfull, qresid, &l->q_b, 1);
|
||||
for(int h=0;h<H;h++) rope_interleave(qfull+(int64_t)h*qh+c->qk_nope, pos, c);
|
||||
matmul_qt(comp, xs, &l->kv_a, 1);
|
||||
float *Ldst=m->Lc[layer]+(int64_t)pos*c->kv_lora, *Rdst=m->Rc[layer]+(int64_t)pos*c->qk_rope;
|
||||
float *Ldst=coli_kv_row(ks->Lc[layer],pos,c->kv_lora);
|
||||
float *Rdst=coli_kv_row(ks->Rc[layer],pos,c->qk_rope);
|
||||
memcpy(Ldst, comp, c->kv_lora*sizeof(float));
|
||||
rmsnorm(Ldst, Ldst, l->kv_a_ln, c->kv_lora, c->eps); /* latente normato */
|
||||
memcpy(Rdst, comp+c->kv_lora, c->qk_rope*sizeof(float));
|
||||
@@ -1379,12 +1470,13 @@ static void attention(Model *m, Layer *l, int layer, float *x, int S, int pos_ba
|
||||
* dai layer SHARED successivi). Selezione attiva solo con contesto > index_topk
|
||||
* (o DSA_FORCE=1 per il test: selezionare TUTTO deve dare l'output denso esatto). */
|
||||
const int *dsel=NULL, *dnsel=NULL; int dtopk=0;
|
||||
if(m->has_dsa && layer<c->n_layers && m->kv_start[layer]==0){
|
||||
if(m->has_dsa && layer<c->n_layers && ((!kvs && m->kv_start[layer]==0) || kvs)){
|
||||
int nh=c->index_nh, hd=c->index_hd; dtopk=c->index_topk;
|
||||
if(c->idx_type[layer]){
|
||||
for(int s=0;s<S;s++){
|
||||
const float *xs=x+(int64_t)s*D; int pos=pos_base+s;
|
||||
float *kd=m->Ic[layer]+(int64_t)pos*hd;
|
||||
KVState *ks=kvs?kvs[s]:m->kv;
|
||||
const float *xs=x+(int64_t)s*D; int pos=positions?positions[s]:pos_base+s;
|
||||
float *kd=coli_kv_row(ks->Ic[layer],pos,hd);
|
||||
matmul_qt(kd, xs, &m->ix_wk[layer], 1);
|
||||
layernorm(kd, m->ix_knw[layer], m->ix_knb[layer], hd, 1e-6f);
|
||||
rope_interleave(kd, pos, c); /* primi qk_rope dim, interleaved */
|
||||
@@ -1397,18 +1489,20 @@ static void attention(Model *m, Layer *l, int layer, float *x, int S, int pos_ba
|
||||
}
|
||||
#pragma omp parallel for schedule(dynamic,1)
|
||||
for(int s=0;s<S;s++){
|
||||
int pos=pos_base+s, nk=pos+1;
|
||||
KVState *ks=kvs?kvs[s]:m->kv;
|
||||
int pos=positions?positions[s]:pos_base+s, nk=pos+1;
|
||||
if(ks->kv_start[layer]!=0){ m->dsa_nsel[s]=0; continue; }
|
||||
if(nk<=dtopk && !g_dsa_force){ m->dsa_nsel[s]=0; continue; }
|
||||
int keep = nk<dtopk ? nk : dtopk;
|
||||
float *qi=falloc((int64_t)nh*hd);
|
||||
matmul_qt(qi, QR+(int64_t)s*c->q_lora, &m->ix_wq[layer], 1);
|
||||
for(int h=0;h<nh;h++) rope_interleave(qi+(int64_t)h*hd, pos, c);
|
||||
float w32[64];
|
||||
float *w32=falloc(nh);
|
||||
matmul_qt(w32, x+(int64_t)s*D, &m->ix_wp[layer], 1);
|
||||
float wsc=1.f/sqrtf((float)nh), rs=1.f/sqrtf((float)hd);
|
||||
float *isc=falloc(nk);
|
||||
for(int t=0;t<nk;t++){
|
||||
const float *kt=m->Ic[layer]+(int64_t)t*hd;
|
||||
const float *kt=coli_kv_row(ks->Ic[layer],t,hd);
|
||||
float a=0;
|
||||
for(int h=0;h<nh;h++){ const float *qhp=qi+(int64_t)h*hd;
|
||||
float d0=0; for(int i=0;i<hd;i++) d0+=qhp[i]*kt[i];
|
||||
@@ -1424,7 +1518,7 @@ static void attention(Model *m, Layer *l, int layer, float *x, int S, int pos_ba
|
||||
for(int t=0;t<nk && nd<keep;t++) if(isc[t]>thr) dst[nd++]=t;
|
||||
for(int t=0;t<nk && nd<keep;t++) if(isc[t]==thr) dst[nd++]=t;
|
||||
m->dsa_nsel[s]=nd;
|
||||
free(qi); free(isc); free(tmp);
|
||||
free(qi); free(w32); free(isc); free(tmp);
|
||||
}
|
||||
}
|
||||
if(m->dsa_nsel){ dsel=m->dsa_sel; dnsel=m->dsa_nsel; }
|
||||
@@ -1433,31 +1527,48 @@ static void attention(Model *m, Layer *l, int layer, float *x, int S, int pos_ba
|
||||
* k/v per ogni token del contesto. Per linearita':
|
||||
* q·k_nope_t = (W_K^hT q_nope)·L_t ctx^h = W_V^h (Σ_t a_t L_t)
|
||||
* costo per step ~O(T·kv_lora) invece di O(T·H·(nope+vh)) del matmul kvb_all. */
|
||||
int absorb = g_absorb==1 || (g_absorb<0 && S<=4);
|
||||
int absorb = kvs || g_absorb==1 || (g_absorb<0 && S<=4);
|
||||
if(absorb && c->kv_lora<=512){
|
||||
m->t_aproj+=now_s()-ta0; double tac=now_s();
|
||||
int kvl=c->kv_lora, r0v=c->qk_nope; /* offset righe V dentro il blocco di testa */
|
||||
/* punteggi per-thread sul HEAP: un sc[8192] fisso sullo stack va in overflow quando
|
||||
* il layer attende su tutto il contesto (nessuna selezione DSA: snapshot senza
|
||||
* indexer, o layer MTP) e nt supera 8192 — scrittura oltre lo stack del worker
|
||||
* OMP => segfault (e poco sotto il limite: corruzione SILENZIOSA dello stack). */
|
||||
int64_t sc_cap = Tk - m->kv_start[layer]; /* nt massimo (kv_start=-1 del MTP: +1, ok) */
|
||||
/* punteggi per-thread sul HEAP (vedi dev): cap Tk+1 copre anche il kv_start
|
||||
* per-slot del percorso kvs (MTP: kv_start=-1 -> nt=Tk+1). */
|
||||
int64_t sc_cap = (int64_t)Tk+1;
|
||||
float *sc_all = falloc((int64_t)omp_get_max_threads()*sc_cap);
|
||||
int cuda_core=0;
|
||||
#ifdef COLI_CUDA
|
||||
if(S<=4&&g_cuda_enabled&&getenv("COLI_CUDA_ATTN")&&atoi(getenv("COLI_CUDA_ATTN"))&&
|
||||
l->kv_b.cuda_eligible&&qt_cuda_upload(&l->kv_b)){
|
||||
cuda_core=1;
|
||||
for(int s=0;s<S&&cuda_core;s++){
|
||||
KVState *ks=kvs?kvs[s]:m->kv;int pos=positions?positions[s]:pos_base+s;
|
||||
int st0=ks->kv_start[layer],nt=pos+1-st0;
|
||||
if(dnsel&&dnsel[s]>0){cuda_core=0;break;}
|
||||
cuda_core=coli_cuda_attention_absorb(l->kv_b.cuda,ctx+(int64_t)s*H*vh,
|
||||
Q+(int64_t)s*H*qh,coli_kv_row(ks->Lc[layer],st0,kvl),
|
||||
coli_kv_row(ks->Rc[layer],st0,c->qk_rope),H,c->qk_nope,c->qk_rope,
|
||||
vh,kvl,nt,c->attn_scale);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
if(!cuda_core){
|
||||
#pragma omp parallel for collapse(2) schedule(static)
|
||||
for(int s=0;s<S;s++) for(int h=0;h<H;h++){
|
||||
int pos=pos_base+s;
|
||||
KVState *ks=kvs?kvs[s]:m->kv;
|
||||
int pos=positions?positions[s]:pos_base+s;
|
||||
const float *qp=Q+(int64_t)s*H*qh+(int64_t)h*qh;
|
||||
const float *qr=qp+c->qk_nope;
|
||||
int rbase=h*(c->qk_nope+vh);
|
||||
float qabs[512]; memset(qabs,0,kvl*sizeof(float));
|
||||
for(int d=0;d<c->qk_nope;d++) qt_addrow(&l->kv_b, rbase+d, qp[d], qabs);
|
||||
float *sc = sc_all + (int64_t)omp_get_thread_num()*sc_cap;
|
||||
int st0=m->kv_start[layer];
|
||||
int st0=ks->kv_start[layer];
|
||||
int ns = (dnsel && dnsel[s]>0) ? dnsel[s] : 0; /* DSA: lista top-k o range pieno */
|
||||
const int *tlist = ns ? dsel+(int64_t)s*dtopk : NULL;
|
||||
int nt = ns ? ns : pos+1-st0;
|
||||
for(int jj=0;jj<nt;jj++){ int t = tlist ? tlist[jj] : st0+jj;
|
||||
const float *Lt=m->Lc[layer]+(int64_t)t*kvl;
|
||||
const float *kr=m->Rc[layer]+(int64_t)t*c->qk_rope;
|
||||
const float *Lt=coli_kv_row(ks->Lc[layer],t,kvl);
|
||||
const float *kr=coli_kv_row(ks->Rc[layer],t,c->qk_rope);
|
||||
float a=0; for(int i=0;i<kvl;i++) a+=qabs[i]*Lt[i];
|
||||
for(int d=0;d<c->qk_rope;d++) a+=qr[d]*kr[d];
|
||||
sc[jj]=a*c->attn_scale;
|
||||
@@ -1465,17 +1576,19 @@ static void attention(Model *m, Layer *l, int layer, float *x, int S, int pos_ba
|
||||
softmax(sc,nt);
|
||||
float clat[512]; memset(clat,0,kvl*sizeof(float));
|
||||
for(int jj=0;jj<nt;jj++){ int t = tlist ? tlist[jj] : st0+jj;
|
||||
const float *Lt=m->Lc[layer]+(int64_t)t*kvl;
|
||||
const float *Lt=coli_kv_row(ks->Lc[layer],t,kvl);
|
||||
float a=sc[jj]; for(int i=0;i<kvl;i++) clat[i]+=a*Lt[i]; }
|
||||
qt_matvec_rows(&l->kv_b, rbase+r0v, vh, clat, ctx+((int64_t)s*H+h)*vh);
|
||||
}
|
||||
matmul_qt(out, ctx, &l->o, S);
|
||||
}
|
||||
m->t_acore+=now_s()-tac; double tao=now_s();
|
||||
matmul_qt(out, ctx, &l->o, S); m->t_aout+=now_s()-tao;
|
||||
free(ctx); free(Q); free(QR); free(comp); free(sc_all);
|
||||
m->t_attn += now_s()-ta0;
|
||||
return;
|
||||
}
|
||||
/* 2) ricostruzione di k_nope+value per TUTTI i token 0..Tk-1 (un solo matmul su kv_b) */
|
||||
double tk0=now_s();
|
||||
m->t_aproj+=now_s()-ta0; double tk0=now_s();
|
||||
int stL=m->kv_start[layer];
|
||||
float *kvb_all=falloc((int64_t)Tk*kvb_dim);
|
||||
matmul_qt(kvb_all+(int64_t)stL*kvb_dim, m->Lc[layer]+(int64_t)stL*c->kv_lora, &l->kv_b, Tk-stL);
|
||||
@@ -1484,6 +1597,7 @@ static void attention(Model *m, Layer *l, int layer, float *x, int S, int pos_ba
|
||||
* (punteggi sul heap, per-thread: vedi il commento nel ramo absorb) */
|
||||
int64_t sc_cap = Tk - stL;
|
||||
float *sc_all = falloc((int64_t)omp_get_max_threads()*sc_cap);
|
||||
double tac=now_s();
|
||||
#pragma omp parallel for collapse(2) schedule(static)
|
||||
for(int s=0;s<S;s++) for(int h=0;h<H;h++){
|
||||
int pos=pos_base+s;
|
||||
@@ -1507,11 +1621,16 @@ static void attention(Model *m, Layer *l, int layer, float *x, int S, int pos_ba
|
||||
const float *vv=kvb_all+(int64_t)t*kvb_dim+(int64_t)h*(c->qk_nope+vh)+c->qk_nope;
|
||||
float a=sc[jj]; for(int d=0;d<vh;d++) cx[d]+=a*vv[d]; }
|
||||
}
|
||||
matmul_qt(out, ctx, &l->o, S);
|
||||
m->t_acore+=now_s()-tac; double tao=now_s();
|
||||
matmul_qt(out, ctx, &l->o, S); m->t_aout+=now_s()-tao;
|
||||
free(ctx); free(Q); free(QR); free(comp); free(kvb_all); free(sc_all);
|
||||
m->t_attn += now_s()-ta0;
|
||||
}
|
||||
|
||||
static void attention(Model *m, Layer *l, int layer, float *x, int S, int pos_base, float *out){
|
||||
attention_rows(m,l,layer,x,S,pos_base,NULL,NULL,out);
|
||||
}
|
||||
|
||||
/* MoE GLM su x[S,hidden] -> out (router sigmoid/noaux_tc, n_group=1, + shared expert).
|
||||
* BATCH-UNION: per S>1 (prefill, verifica MTP) ogni expert UNICO del batch viene caricato
|
||||
* una volta sola e moltiplicato per tutte le posizioni che lo usano (pesi letti 1 volta);
|
||||
@@ -1574,6 +1693,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out){
|
||||
for(int kk=0;kk<Ke;kk++){
|
||||
m->eusage[layer][idx[kk]]++;
|
||||
if(m->eheat[layer][idx[kk]]<UINT32_MAX) m->eheat[layer][idx[kk]]++;
|
||||
m->elast[layer][idx[kk]]=++m->eaccess_clock;
|
||||
}
|
||||
if(c->norm_topk){ float sm=0; for(int kk=0;kk<Ke;kk++) sm+=w[kk]; sm+=1e-20f; for(int kk=0;kk<Ke;kk++) w[kk]/=sm; }
|
||||
for(int kk=0;kk<Ke;kk++) w[kk]*=c->routed_scale;
|
||||
@@ -1603,6 +1723,13 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out){
|
||||
/* ---- FASE C/D: risolvi (pin/cache/disco) e calcola, a blocchi di 64 unici ---- */
|
||||
float *xg=falloc((int64_t)S*D), *gg=falloc((int64_t)S*I), *uu=falloc((int64_t)S*I), *hh=falloc((int64_t)S*D);
|
||||
int *rows=malloc(S*sizeof(int)); float *rw=malloc(S*sizeof(float));
|
||||
#ifdef COLI_CUDA
|
||||
int group_enabled=S<=64;
|
||||
float *group_x=group_enabled?falloc((int64_t)S*K*D):NULL;
|
||||
float *group_y=group_enabled?falloc((int64_t)S*K*D):NULL;
|
||||
int *group_row=group_enabled?malloc((size_t)64*S*sizeof(int)):NULL;
|
||||
float *group_weight=group_enabled?malloc((size_t)64*S*sizeof(float)):NULL;
|
||||
#endif
|
||||
int shared_on_gpu=0; (void)shared_on_gpu; /* set by the Metal path when Phase E was fused */
|
||||
for(int base=0;base<nu;base+=64){
|
||||
int nb = nu-base<64 ? nu-base : 64;
|
||||
@@ -1699,6 +1826,9 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out){
|
||||
if(!found) expert_prefetch(m,layer,eid);
|
||||
}
|
||||
}
|
||||
#ifdef COLI_CUDA
|
||||
ESlot *group_e[64]; int group_n[64]; int ngroup=0;
|
||||
#endif
|
||||
#ifdef COLI_METAL
|
||||
if(g_metal_enabled){
|
||||
/* PIPE drain. Two reasons this barrier is mandatory here, and not optional:
|
||||
@@ -1746,17 +1876,78 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out){
|
||||
if(!nr) continue;
|
||||
#ifdef COLI_CUDA
|
||||
if(g_cuda_enabled && e->g.cuda_eligible) m->gpu_expert_calls++;
|
||||
if(group_enabled && g_cuda_enabled && e->g.cuda_eligible && e->u.cuda_eligible && e->d.cuda_eligible &&
|
||||
!omp_in_parallel()){
|
||||
group_e[ngroup]=e; group_n[ngroup]=nr;
|
||||
for(int r=0;r<nr;r++){ group_row[(int64_t)ngroup*S+r]=rows[r]; group_weight[(int64_t)ngroup*S+r]=rw[r]; }
|
||||
ngroup++; continue;
|
||||
}
|
||||
#endif
|
||||
for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D, x+(int64_t)rows[r]*D, D*sizeof(float));
|
||||
double t0=now_s();
|
||||
matmul_qt(gg, xg, &e->g, nr);
|
||||
matmul_qt(uu, xg, &e->u, nr);
|
||||
#ifdef COLI_CUDA
|
||||
if(!group_enabled && g_cuda_enabled && e->g.cuda_eligible && e->u.cuda_eligible &&
|
||||
e->d.cuda_eligible && !omp_in_parallel() &&
|
||||
coli_cuda_expert_mlp(e->g.cuda,e->u.cuda,e->d.cuda,hh,xg,nr)){
|
||||
for(int r=0;r<nr;r++){ float *os=out+(int64_t)rows[r]*D,wgt=rw[r],*hr=hh+(int64_t)r*D;
|
||||
for(int d=0;d<D;d++) os[d]+=wgt*hr[d]; }
|
||||
m->t_emm+=now_s()-t0; continue;
|
||||
}
|
||||
if(!e->slab) expert_host_ensure(m,layer,e);
|
||||
#endif
|
||||
expert_gate_up(gg,uu,xg,&e->g,&e->u,nr);
|
||||
for(int64_t z=0;z<(int64_t)nr*I;z++) gg[z]=siluf(gg[z])*uu[z];
|
||||
matmul_qt(hh, gg, &e->d, nr);
|
||||
for(int r=0;r<nr;r++){ float *os=out+(int64_t)rows[r]*D, wgt=rw[r], *hr=hh+(int64_t)r*D;
|
||||
for(int d=0;d<D;d++) os[d]+=wgt*hr[d]; }
|
||||
m->t_emm += now_s()-t0;
|
||||
}
|
||||
#ifdef COLI_CUDA
|
||||
ColiCudaTensor *dev_g[COLI_CUDA_MAX_DEVICES][64],*dev_u[COLI_CUDA_MAX_DEVICES][64];
|
||||
ColiCudaTensor *dev_d[COLI_CUDA_MAX_DEVICES][64];
|
||||
int dev_rows[COLI_CUDA_MAX_DEVICES][64],dev_which[COLI_CUDA_MAX_DEVICES][64];
|
||||
int dev_nc[COLI_CUDA_MAX_DEVICES]={0},dev_total[COLI_CUDA_MAX_DEVICES]={0};
|
||||
int dev_off[COLI_CUDA_MAX_DEVICES]={0},dev_ok[COLI_CUDA_MAX_DEVICES]={0};
|
||||
for(int di=0;di<g_cuda_ndev;di++) for(int q=0;q<ngroup;q++)
|
||||
if(group_e[q]->g.cuda_device==g_cuda_devices[di]) dev_total[di]+=group_n[q];
|
||||
for(int di=1;di<g_cuda_ndev;di++) dev_off[di]=dev_off[di-1]+dev_total[di-1];
|
||||
for(int di=0;di<g_cuda_ndev;di++){
|
||||
int cursor=0,device=g_cuda_devices[di];
|
||||
for(int q=0;q<ngroup;q++) if(group_e[q]->g.cuda_device==device){
|
||||
int nc=dev_nc[di]++; ESlot *e=group_e[q];
|
||||
dev_g[di][nc]=e->g.cuda; dev_u[di][nc]=e->u.cuda; dev_d[di][nc]=e->d.cuda;
|
||||
dev_rows[di][nc]=group_n[q]; dev_which[di][nc]=q;
|
||||
for(int r=0;r<group_n[q];r++) memcpy(group_x+(int64_t)(dev_off[di]+cursor+r)*D,
|
||||
x+(int64_t)group_row[(int64_t)q*S+r]*D,D*sizeof(float));
|
||||
cursor+=group_n[q];
|
||||
}
|
||||
}
|
||||
double tg=now_s();
|
||||
#pragma omp parallel for if(g_cuda_ndev>1) schedule(static)
|
||||
for(int di=0;di<g_cuda_ndev;di++) if(dev_nc[di])
|
||||
dev_ok[di]=coli_cuda_expert_group(dev_g[di],dev_u[di],dev_d[di],dev_rows[di],dev_nc[di],
|
||||
group_y+(int64_t)dev_off[di]*D,group_x+(int64_t)dev_off[di]*D);
|
||||
for(int di=0;di<g_cuda_ndev;di++){
|
||||
int off=dev_off[di];
|
||||
for(int q=0;q<dev_nc[di];q++){
|
||||
int gi=dev_which[di][q],nr=group_n[gi]; ESlot *e=group_e[gi];
|
||||
if(!dev_ok[di]){
|
||||
for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D,x+(int64_t)group_row[(int64_t)gi*S+r]*D,D*sizeof(float));
|
||||
if(!coli_cuda_expert_mlp(e->g.cuda,e->u.cuda,e->d.cuda,hh,xg,nr)){
|
||||
expert_host_ensure(m,layer,e);
|
||||
expert_gate_up(gg,uu,xg,&e->g,&e->u,nr);
|
||||
for(int64_t z=0;z<(int64_t)nr*I;z++) gg[z]=siluf(gg[z])*uu[z];
|
||||
matmul_qt(hh,gg,&e->d,nr);
|
||||
}
|
||||
}
|
||||
float *src=dev_ok[di]?group_y+(int64_t)off*D:hh;
|
||||
for(int r=0;r<nr;r++){ float *os=out+(int64_t)group_row[(int64_t)gi*S+r]*D,wgt=group_weight[(int64_t)gi*S+r];
|
||||
for(int d=0;d<D;d++) os[d]+=wgt*src[(int64_t)r*D+d]; }
|
||||
off+=nr;
|
||||
}
|
||||
}
|
||||
m->t_emm+=now_s()-tg;
|
||||
#endif
|
||||
/* No drain barrier: the per-expert pipe_wait(qof[j]) above (issued for every
|
||||
* dispatched miss slot, before the nr==0 skip) already waited on all ws[] loads
|
||||
* for this block, so they are complete before the LRU swap — and the gen-tagged
|
||||
@@ -1783,6 +1974,9 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out){
|
||||
}
|
||||
free(logit); free(choice); free(idxs); free(ws); free(keff); free(uniq);
|
||||
free(xg); free(gg); free(uu); free(hh); free(rows); free(rw); free(sg); free(su);
|
||||
#ifdef COLI_CUDA
|
||||
free(group_x); free(group_y); free(group_row); free(group_weight);
|
||||
#endif
|
||||
}
|
||||
|
||||
static void dense_mlp(Layer *l, float *x, int S, int D, int I, float *out){
|
||||
@@ -1914,7 +2108,8 @@ static void pilot_prefetch(Model *m, int lnext, const float *x, int S){
|
||||
}
|
||||
|
||||
/* forward di UN layer (usato dai 78 principali e dal layer MTP) */
|
||||
static void layer_forward(Model *m, Layer *l, int li, float *x, int S, int pos_base, float *nrm, float *tmp){
|
||||
static void layer_forward_rows(Model *m, Layer *l, int li, float *x, int S, int pos_base,
|
||||
KVState *const *kvs, const int *positions, float *nrm, float *tmp){
|
||||
Cfg *c=&m->c; int D=c->hidden;
|
||||
if(g_spec && g_prefetch && l->sparse && m->enr[li]>0)
|
||||
for(int z=0;z<m->enr[li];z++) expert_prefetch(m,li,m->eroute[li][z]);
|
||||
@@ -1974,7 +2169,7 @@ static void layer_forward(Model *m, Layer *l, int li, float *x, int S, int pos_b
|
||||
}
|
||||
#endif
|
||||
for(int s=0;s<S;s++) rmsnorm(nrm+(int64_t)s*D, x+(int64_t)s*D, l->in_ln, D, c->eps);
|
||||
attention(m,l,li,nrm,S,pos_base,tmp);
|
||||
attention_rows(m,l,li,nrm,S,pos_base,kvs,positions,tmp);
|
||||
for(int64_t j=0;j<(int64_t)S*D;j++) x[j]+=tmp[j];
|
||||
if(g_pilot && S<=8 && li+1<c->n_layers && m->L[li+1].sparse) pilot_prefetch(m,li+1,x,S);
|
||||
if(g_looka && S==1 && li+1<c->n_layers && m->L[li+1].sparse) la_predict(m,li+1,x,1);
|
||||
@@ -1982,7 +2177,11 @@ static void layer_forward(Model *m, Layer *l, int li, float *x, int S, int pos_b
|
||||
if(l->sparse) moe(m,l,li,nrm,S,tmp); else dense_mlp(l,nrm,S,D,c->dense_inter,tmp);
|
||||
for(int64_t j=0;j<(int64_t)S*D;j++) x[j]+=tmp[j];
|
||||
}
|
||||
static void layers_forward(Model *m, float *x, int S, int pos_base){
|
||||
static void layer_forward(Model *m, Layer *l, int li, float *x, int S, int pos_base, float *nrm, float *tmp){
|
||||
layer_forward_rows(m,l,li,x,S,pos_base,NULL,NULL,nrm,tmp);
|
||||
}
|
||||
static void layers_forward_rows(Model *m, float *x, int S, int pos_base,
|
||||
KVState *const *kvs, const int *positions){
|
||||
Cfg *c=&m->c; int D=c->hidden;
|
||||
if(g_pilot_real){ /* nuovo forward: il possesso-layer riparte da -1 (i layer si rifanno da 0) */
|
||||
pthread_mutex_lock(&g_pilot_mx);
|
||||
@@ -1995,10 +2194,13 @@ static void layers_forward(Model *m, float *x, int S, int pos_base){
|
||||
* puo' arrivare dopo MINUTI di streaming — al buio sembra un blocco. */
|
||||
if(S>=8 && (i%4==0 || i==c->n_layers-1))
|
||||
fprintf(stderr,"[prefill] layer %d/%d · %d token\n", i+1, c->n_layers, S);
|
||||
layer_forward(m,&m->L[i],i,x,S,pos_base,nrm,tmp);
|
||||
layer_forward_rows(m,&m->L[i],i,x,S,pos_base,kvs,positions,nrm,tmp);
|
||||
}
|
||||
free(nrm); free(tmp);
|
||||
}
|
||||
static void layers_forward(Model *m, float *x, int S, int pos_base){
|
||||
layers_forward_rows(m,x,S,pos_base,NULL,NULL);
|
||||
}
|
||||
|
||||
static void kv_alloc(Model *m, int max_t){
|
||||
Cfg *c=&m->c;
|
||||
@@ -2068,6 +2270,42 @@ static float *step_all(Model *m, const int *ids, int S, int pos_base){
|
||||
free(x); free(row); return lo;
|
||||
}
|
||||
|
||||
/* One decode token from each independent sequence, evaluated as a single MoE
|
||||
* batch. Prefill and speculative batches retain their contiguous-KV path. */
|
||||
static float *step_decode_batch(Model *m, const DecodeRow *rows, int S){
|
||||
Cfg *c=&m->c; int D=c->hidden;
|
||||
/* Ragged KV currently uses MLA absorption; the stack kernel is sized to 512. */
|
||||
if(!rows || S<1 || S>64 || c->kv_lora>512) return NULL;
|
||||
KVState *kvs[64]; int positions[64];
|
||||
float *x=falloc((int64_t)S*D);
|
||||
for(int s=0;s<S;s++){
|
||||
if(!rows[s].kv || !rows[s].kv->Lc || !rows[s].kv->Rc || !rows[s].kv->kv_start ||
|
||||
rows[s].token<0 || rows[s].token>=c->vocab ||
|
||||
rows[s].pos<0 || rows[s].pos>=rows[s].kv->max_t){
|
||||
free(x); return NULL;
|
||||
}
|
||||
for(int l=0;l<c->n_layers;l++){
|
||||
if(!rows[s].kv->Lc[l] || !rows[s].kv->Rc[l] ||
|
||||
rows[s].kv->kv_start[l]<0 || rows[s].kv->kv_start[l]>rows[s].pos ||
|
||||
(m->has_dsa && c->idx_type[l] &&
|
||||
(!rows[s].kv->Ic || !rows[s].kv->Ic[l]))){ free(x); return NULL; }
|
||||
}
|
||||
for(int p=0;p<s;p++) if(rows[p].kv==rows[s].kv){ free(x); return NULL; }
|
||||
kvs[s]=rows[s].kv; positions[s]=rows[s].pos;
|
||||
embed_row(m,rows[s].token,x+(int64_t)s*D);
|
||||
}
|
||||
layers_forward_rows(m,x,S,0,kvs,positions);
|
||||
float *norm=falloc((int64_t)S*D);
|
||||
for(int s=0;s<S;s++)
|
||||
rmsnorm(norm+(int64_t)s*D,x+(int64_t)s*D,m->final_norm,D,c->eps);
|
||||
double th0=now_s();
|
||||
float *logit=falloc((int64_t)S*c->vocab);
|
||||
matmul_qt(logit,norm,&m->lm_head,S);
|
||||
m->t_head+=now_s()-th0;
|
||||
free(x); free(norm);
|
||||
return logit;
|
||||
}
|
||||
|
||||
/* METODO E — prompt-lookup: cerca l'occorrenza piu' recente dell'ultimo bigramma nel
|
||||
* contesto e propone i token che la seguirono. Zero pesi extra, zero costo: e' solo
|
||||
* un'ipotesi che il modello verifichera'. */
|
||||
@@ -2420,10 +2658,12 @@ static void generate(Model *m, const int *prompt, int np, int n_new, int *out){
|
||||
}
|
||||
|
||||
static void profile_print(Model *m, double elapsed){
|
||||
double accounted=m->t_edisk+m->t_emm+m->t_attn+m->t_head;
|
||||
printf("PROFILE: expert-disk %.3fs | expert-matmul %.3fs | attention %.3fs "
|
||||
double accounted=m->t_ewait+m->t_emm+m->t_attn+m->t_head;
|
||||
printf("PROFILE: expert-disk %.3fs service / %.3fs wait | expert-matmul %.3fs | attention %.3fs "
|
||||
"(including kvb %.3fs) | lm_head %.3fs | other %.3fs\n",
|
||||
m->t_edisk,m->t_emm,m->t_attn,m->t_kvb,m->t_head,elapsed-accounted);
|
||||
m->t_edisk,m->t_ewait,m->t_emm,m->t_attn,m->t_kvb,m->t_head,elapsed-accounted);
|
||||
printf("ATTENTION: projection/RoPE %.3fs | score-softmax-value %.3fs | output projection %.3fs\n",
|
||||
m->t_aproj,m->t_acore,m->t_aout);
|
||||
#ifdef COLI_METAL
|
||||
if(g_metal_enabled){ uint64_t ok=0,fb=0,ex=0; double su=0,gp=0,sc=0;
|
||||
coli_metal_moe_counts(&ok,&fb,&ex); coli_metal_moe_times(&su,&gp,&sc);
|
||||
@@ -2443,7 +2683,8 @@ static void run_replay(Model *m, const int *full, int nfull, int np){
|
||||
kv_alloc(m,nfull+2);
|
||||
float *logit=step(m,full,np-1,0); free(logit);
|
||||
m->hits=m->miss=m->ereq=m->gpu_expert_calls=0;
|
||||
m->t_edisk=m->t_emm=m->t_attn=m->t_kvb=m->t_head=0;
|
||||
m->t_edisk=m->t_ewait=m->t_emm=m->t_attn=m->t_kvb=m->t_head=0;
|
||||
m->t_aproj=m->t_acore=m->t_aout=0;
|
||||
double t0=now_s(); int steps=0;
|
||||
for(int i=np-1;i<nfull-1;i++){
|
||||
logit=step(m,full+i,1,i); free(logit); steps++;
|
||||
@@ -2539,7 +2780,8 @@ static int repin_pick(Model *m, RepinCand *out, int maxc){
|
||||
ESlot *P=m->pin[l]; int ids[4096], zp, eu; long g;
|
||||
int np=m->npin[l]; if(np>4096) np=4096;
|
||||
for(int z=0;z<np;z++) ids[z]=P[z].eid;
|
||||
if(!tier_pick_swap(m->eheat[l],c->n_experts,ids,np,&zp,&eu,&g)) continue;
|
||||
if(!tier_pick_lfru(m->eheat[l],m->elast[l],m->eaccess_clock,
|
||||
c->n_experts,ids,np,&zp,&eu,&g)) continue;
|
||||
if(nb<maxc){ out[nb].gain=g; out[nb].l=l; out[nb].slot=zp; out[nb].eid=eu; nb++; }
|
||||
else { int w=0; for(int b=1;b<maxc;b++) if(out[b].gain<out[w].gain) w=b;
|
||||
if(g>out[w].gain){ out[w].gain=g; out[w].l=l; out[w].slot=zp; out[w].eid=eu; } }
|
||||
@@ -2571,6 +2813,7 @@ static void repin_pass(Model *m){
|
||||
+(int64_t)coli_cuda_tensor_bytes(s->u.cuda)
|
||||
+(int64_t)coli_cuda_tensor_bytes(s->d.cuda);
|
||||
m->gpu_expert_bytes+=now_gpu-old_gpu; tier="VRAM";
|
||||
if(g_cuda_release_host) expert_host_release(m,s);
|
||||
} else {
|
||||
qt_cuda_reset(&s->g); qt_cuda_reset(&s->u); qt_cuda_reset(&s->d);
|
||||
s->g.cuda_eligible=s->u.cuda_eligible=s->d.cuda_eligible=0;
|
||||
@@ -2686,6 +2929,141 @@ static void serve_ctx_free(Model *m, ServeCtx *s){
|
||||
free(k->Lc); free(k->Rc); free(k->Ic); free(k->kv_start); free(s->hist);
|
||||
}
|
||||
|
||||
typedef struct {
|
||||
int active, pending, emitted, maximum, prompt_tokens, length_limited;
|
||||
unsigned long long id;
|
||||
float temp, top_p;
|
||||
double started;
|
||||
uint64_t hits0, miss0;
|
||||
} ServeReq;
|
||||
|
||||
static void mux_data(Tok *T, unsigned long long id, int token){
|
||||
char out[256]; int n=tok_decode(T,&token,1,out,sizeof(out));
|
||||
printf("DATA %llu %d\n",id,n); if(n>0) fwrite(out,1,(size_t)n,stdout); putchar('\n');
|
||||
fflush(stdout);
|
||||
}
|
||||
|
||||
static void mux_done(Model *m, ServeCtx *sc, ServeReq *r){
|
||||
double dt=now_s()-r->started; if(dt<1e-6) dt=1e-6;
|
||||
double dh=(double)(m->hits-r->hits0), dm=(double)(m->miss-r->miss0);
|
||||
printf("DONE %llu STAT %d %.2f %.1f %.2f %d %d\n",r->id,r->emitted,
|
||||
r->emitted/dt,(dh+dm)>0?100.0*dh/(dh+dm):0.0,rss_gb(),
|
||||
r->prompt_tokens,r->length_limited);
|
||||
fflush(stdout); kv_bind(m,&sc->kv); kv_disk_append(m,sc->hist,sc->len);
|
||||
r->active=0;
|
||||
}
|
||||
|
||||
/* Read and prefill one request. Returns -1 on EOF, 0 for a rejected frame and
|
||||
* 1 for an accepted request. Prefill deliberately remains serial: continuous
|
||||
* batching starts at decode, where every active slot contributes one row. */
|
||||
static int mux_submit(Model *m, Tok *T, ServeCtx *ctx, ServeReq *req, int nctx,
|
||||
int maxctx, int eos){
|
||||
char *line=NULL; size_t cap=0; ssize_t nr=getline(&line,&cap,stdin);
|
||||
if(nr<0){ free(line); return -1; }
|
||||
if(nr && line[nr-1]=='\n') line[--nr]=0;
|
||||
if(!strncmp(line,"CANCEL ",7)){
|
||||
unsigned long long id=0; char tail;
|
||||
if(sscanf(line+7,"%llu %c",&id,&tail)!=1 || id==0){
|
||||
printf("ERROR 0 BAD_REQUEST\n"); fflush(stdout); free(line); return 0;
|
||||
}
|
||||
for(int i=0;i<nctx;i++) if(req[i].active && req[i].id==id){
|
||||
req[i].active=0; kv_bind(m,&ctx[i].kv);
|
||||
kv_disk_append(m,ctx[i].hist,ctx[i].len);
|
||||
printf("ERROR %llu CANCELLED\n",id); fflush(stdout); free(line); return 0;
|
||||
}
|
||||
printf("ERROR %llu NOT_FOUND\n",id); fflush(stdout); free(line); return 0;
|
||||
}
|
||||
ColiSubmit sub; int valid=coli_submit_parse(line,&sub);
|
||||
if(!valid){ printf("ERROR 0 BAD_REQUEST\n"); fflush(stdout); free(line); return 0; }
|
||||
char *raw=malloc((size_t)sub.bytes+1);
|
||||
if(!raw){ fprintf(stderr,"OOM multiplex payload\n"); exit(1); }
|
||||
if(fread(raw,1,(size_t)sub.bytes,stdin)!=(size_t)sub.bytes){ free(raw); free(line); return -1; }
|
||||
int delim=fgetc(stdin);
|
||||
if(delim!='\n'){
|
||||
printf("ERROR %llu BAD_FRAME\n",sub.id); fflush(stdout);
|
||||
free(raw); free(line); return -1;
|
||||
}
|
||||
raw[sub.bytes]=0;
|
||||
if(sub.slot>=nctx || memchr(raw,0,(size_t)sub.bytes)){
|
||||
printf("ERROR %llu BAD_REQUEST\n",sub.id); fflush(stdout); free(raw); free(line); return 0;
|
||||
}
|
||||
if(req[sub.slot].active){
|
||||
printf("ERROR %llu SLOT_BUSY\n",sub.id); fflush(stdout); free(raw); free(line); return 0;
|
||||
}
|
||||
for(int i=0;i<nctx;i++) if(req[i].active && req[i].id==sub.id){
|
||||
printf("ERROR %llu DUPLICATE_ID\n",sub.id); fflush(stdout); free(raw); free(line); return 0;
|
||||
}
|
||||
ServeCtx *sc=&ctx[sub.slot]; kv_bind(m,&sc->kv);
|
||||
int *tmp=malloc(maxctx*sizeof(int));
|
||||
int nt=tok_encode(T,raw,(int)sub.bytes,tmp,maxctx-2);
|
||||
free(raw); free(line);
|
||||
if(nt<1){ free(tmp); printf("ERROR %llu EMPTY_PROMPT\n",sub.id); fflush(stdout); return 0; }
|
||||
int prefix=0; while(prefix<sc->len && prefix<nt && sc->hist[prefix]==tmp[prefix]) prefix++;
|
||||
if(prefix<sc->len){ sc->len=prefix; if(m->has_mtp) m->kv_start[m->c.n_layers]=-1;
|
||||
kv_disk_truncate(m,sc->len); }
|
||||
int add=nt-sc->len;
|
||||
if(add>0) memcpy(sc->hist+sc->len,tmp+sc->len,(size_t)add*sizeof(int));
|
||||
free(tmp);
|
||||
float *logit = add>0 ? step(m,sc->hist+sc->len,add,sc->len)
|
||||
: step(m,sc->hist+sc->len-1,1,sc->len-1);
|
||||
sc->len+=add; sc->first=0;
|
||||
ServeReq *r=&req[sub.slot]; memset(r,0,sizeof(*r));
|
||||
r->id=sub.id; r->maximum=sub.max_tokens; r->temp=sub.temperature; r->top_p=sub.top_p;
|
||||
r->prompt_tokens=nt; r->started=now_s(); r->hits0=m->hits; r->miss0=m->miss;
|
||||
int room=maxctx-sc->len-1; if(r->maximum>room){r->maximum=room; r->length_limited=1;}
|
||||
g_temp=r->temp; g_nuc=r->top_p;
|
||||
int next=pick_tok(logit,m->c.vocab,-1); free(logit);
|
||||
if(r->maximum<=0 || next==eos || is_stop(next)){ mux_done(m,sc,r); return 1; }
|
||||
r->pending=next; r->emitted=1; r->active=1; sc->hist[sc->len]=next; m->n_emit++;
|
||||
mux_data(T,r->id,next);
|
||||
if(r->emitted>=r->maximum) mux_done(m,sc,r);
|
||||
return 1;
|
||||
}
|
||||
|
||||
static void run_serve_mux(Model *m, const char *snap){
|
||||
char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap);
|
||||
Tok T; tok_load(&T,tkp); int eos=tok_id_of(&T,"<|endoftext|>"); stops_arm(&m->c,eos);
|
||||
g_draft=0; /* one scheduler owns every forward; MTP/speculation is not ragged-safe */
|
||||
int maxctx=getenv("CTX")?atoi(getenv("CTX")):4096;
|
||||
int nctx=getenv("KV_SLOTS")?atoi(getenv("KV_SLOTS")):1;
|
||||
if(nctx<1||nctx>16){fprintf(stderr,"KV_SLOTS deve essere tra 1 e 16\n");exit(2);}
|
||||
g_kvsave=getenv("KVSAVE")?atoi(getenv("KVSAVE")):1;
|
||||
KVState *initial=m->kv; free(initial->kv_start); free(initial);
|
||||
ServeCtx *ctx=calloc(nctx,sizeof(*ctx)); ServeReq *req=calloc(nctx,sizeof(*req));
|
||||
for(int i=0;i<nctx;i++) serve_ctx_init(m,&ctx[i],snap,i,maxctx);
|
||||
setvbuf(stdin,NULL,_IONBF,0);
|
||||
printf("\x01\x01READY\x01\x01\nSTAT 0 0.00 0.0 %.2f\n",rss_gb()); fflush(stdout);
|
||||
int eof=0;
|
||||
for(;;){
|
||||
int active=0; for(int i=0;i<nctx;i++) active+=req[i].active;
|
||||
fd_set rfds; FD_ZERO(&rfds); FD_SET(STDIN_FILENO,&rfds);
|
||||
struct timeval tv={0,0}, *ptv=active?&tv:NULL;
|
||||
int ready=eof?0:select(STDIN_FILENO+1,&rfds,NULL,NULL,ptv);
|
||||
if(ready>0 && FD_ISSET(STDIN_FILENO,&rfds)) if(mux_submit(m,&T,ctx,req,nctx,maxctx,eos)<0) eof=1;
|
||||
active=0; for(int i=0;i<nctx;i++) active+=req[i].active;
|
||||
if(!active){ if(eof) break; continue; }
|
||||
DecodeRow rows[16]; int slots[16], S=0;
|
||||
for(int i=0;i<nctx;i++) if(req[i].active){
|
||||
rows[S]=(DecodeRow){&ctx[i].kv,req[i].pending,ctx[i].len}; slots[S++]=i;
|
||||
}
|
||||
float *lo=step_decode_batch(m,rows,S); if(!lo){fprintf(stderr,"decode batch failed\n");break;}
|
||||
m->n_fw++;
|
||||
for(int s=0;s<S;s++){
|
||||
int i=slots[s]; ServeCtx *sc=&ctx[i]; ServeReq *r=&req[i];
|
||||
sc->len++; g_temp=r->temp; g_nuc=r->top_p;
|
||||
int next=pick_tok(lo+(int64_t)s*m->c.vocab,m->c.vocab,-1);
|
||||
if(next==eos || is_stop(next)){mux_done(m,sc,r);continue;}
|
||||
r->pending=next; sc->hist[sc->len]=next; r->emitted++; m->n_emit++;
|
||||
mux_data(&T,r->id,next);
|
||||
if(r->emitted>=r->maximum) mux_done(m,sc,r);
|
||||
}
|
||||
free(lo);
|
||||
}
|
||||
usage_save(m);
|
||||
for(int i=0;i<nctx;i++) serve_ctx_free(m,&ctx[i]); free(ctx); free(req);
|
||||
m->kv=NULL; m->Lc=m->Rc=m->Ic=NULL; m->kv_start=NULL; m->max_t=0;
|
||||
}
|
||||
|
||||
static void run_serve(Model *m, const char *snap){
|
||||
char tkp[2048]; snprintf(tkp,sizeof(tkp),"%s/tokenizer.json",snap);
|
||||
Tok T; tok_load(&T,tkp);
|
||||
@@ -2811,6 +3189,7 @@ static void run_serve(Model *m, const char *snap){
|
||||
free(raw); g_temp=base_temp; g_nuc=base_nuc;
|
||||
usage_save(m); /* la cache che impara: storia aggiornata a ogni turno */
|
||||
kv_disk_append(m,hist,len); /* KV su disco: il prossimo avvio riparte da qui */
|
||||
repin_pass(m); /* safe request boundary: adapt session-local hot tier */
|
||||
}
|
||||
free(line); free(buf);
|
||||
usage_save(m);
|
||||
@@ -2922,60 +3301,72 @@ static void pin_wire(Model *m){
|
||||
"(no compression) in %.0fs\n", wired/1e9, now_s()-t0);
|
||||
}
|
||||
|
||||
typedef struct { int l,e; uint32_t c; } PinRec;
|
||||
static int pin_rec_cmp(const void *a,const void *b){
|
||||
const PinRec *x=a,*y=b; return x->c<y->c?1:x->c>y->c?-1:0;
|
||||
}
|
||||
static void pin_load(Model *m, const char *statspath, double gb){
|
||||
FILE *f=fopen(statspath,"r"); if(!f){ perror(statspath); return; }
|
||||
typedef struct { int l,e; uint32_t c; } Rec;
|
||||
Cfg *c=&m->c; int cap=(c->n_layers+1)*c->n_experts;
|
||||
Rec *r=malloc((size_t)cap*sizeof(Rec)); int n=0;
|
||||
PinRec *r=malloc((size_t)cap*sizeof(PinRec)); int n=0;
|
||||
unsigned char *seen=calloc((size_t)(c->n_layers+1)*c->n_experts,1);
|
||||
int l,e; uint32_t cnt;
|
||||
while(n<cap && fscanf(f,"%d %d %u",&l,&e,&cnt)==3){
|
||||
int ok = l>=0 && e>=0 && e<c->n_experts &&
|
||||
((l<c->n_layers && m->L[l].sparse) || (l==c->n_layers && m->has_mtp));
|
||||
if(ok) r[n++]=(Rec){l,e,cnt};
|
||||
int64_t key=(int64_t)l*c->n_experts+e;
|
||||
if(ok&&!seen[key]){ r[n++]=(PinRec){l,e,cnt}; seen[key]=1; }
|
||||
}
|
||||
fclose(f);
|
||||
for(int a=0;a<n;a++){ int best=a; /* selection sort parziale, poi taglio */
|
||||
for(int b=a+1;b<n;b++) if(r[b].c>r[best].c) best=b;
|
||||
Rec t=r[a]; r[a]=r[best]; r[best]=t;
|
||||
if(a>4095) break; /* bastano i top ~4k */
|
||||
int fill=getenv("PIN_FILL")?atoi(getenv("PIN_FILL")):0;
|
||||
#ifdef COLI_CUDA
|
||||
if(!getenv("PIN_FILL")&&g_cuda_release_host) fill=1;
|
||||
#endif
|
||||
if(fill) for(int li=0;li<=c->n_layers;li++){
|
||||
int sparse=(li<c->n_layers&&m->L[li].sparse)||(li==c->n_layers&&m->has_mtp);
|
||||
if(sparse) for(int ei=0;ei<c->n_experts;ei++) if(!seen[(int64_t)li*c->n_experts+ei])
|
||||
r[n++]=(PinRec){li,ei,0};
|
||||
}
|
||||
free(seen);
|
||||
qsort(r,(size_t)n,sizeof(*r),pin_rec_cmp);
|
||||
int64_t eb=expert_bytes_probe(m,m->ebits);
|
||||
int npin=(int)(gb*1e9/eb); if(npin>n) npin=n; if(npin>4096) npin=4096;
|
||||
int npin=(int)(gb*1e9/eb); if(npin>n) npin=n;
|
||||
if(npin<1){ free(r); return; }
|
||||
int *cnt_l=calloc(c->n_layers+1,sizeof(int)); /* +1: riga MTP */
|
||||
for(int a=0;a<npin;a++) cnt_l[r[a].l]++;
|
||||
for(int i=0;i<=c->n_layers;i++) if(cnt_l[i]) m->pin[i]=calloc(cnt_l[i],sizeof(ESlot));
|
||||
int *slot_of=malloc((size_t)npin*sizeof(int)), *next=calloc(c->n_layers+1,sizeof(int));
|
||||
for(int a=0;a<npin;a++) slot_of[a]=next[r[a].l]++;
|
||||
for(int i=0;i<=c->n_layers;i++) m->npin[i]=cnt_l[i];
|
||||
double t0=now_s();
|
||||
#pragma omp parallel for schedule(dynamic,1)
|
||||
for(int a=0;a<npin;a++){
|
||||
int li=r[a].l, slot;
|
||||
#pragma omp critical
|
||||
slot=m->npin[li]++;
|
||||
expert_load(m,li,r[a].e,&m->pin[li][slot],1);
|
||||
#ifdef COLI_CUDA
|
||||
double remaining[COLI_CUDA_MAX_DEVICES]={0}, placed_b[COLI_CUDA_MAX_DEVICES]={0};
|
||||
int placed_n[COLI_CUDA_MAX_DEVICES]={0}, gpu_prefix=0;
|
||||
double budget=g_cuda_expert_gb*1e9, safe_total=0;
|
||||
if(g_cuda_enabled&&g_cuda_expert_gb>0) for(int i=0;i<g_cuda_ndev;i++){
|
||||
size_t free_b=0,total_b=0;
|
||||
if(coli_cuda_mem_info(g_cuda_devices[i],&free_b,&total_b)){
|
||||
remaining[i]=(double)free_b-(double)g_cuda_dense_projected[i]-2e9;
|
||||
if(remaining[i]<0) remaining[i]=0; safe_total+=remaining[i];
|
||||
}
|
||||
}
|
||||
m->resident_bytes += (int64_t)npin*eb;
|
||||
fprintf(stderr,"[PIN] hot store: %d experts in RAM (%.1f GB) loaded in %.0fs from %s\n",
|
||||
npin, npin*eb/1e9, now_s()-t0, statspath);
|
||||
if(budget>safe_total) budget=safe_total;
|
||||
if(g_cuda_enabled&&g_cuda_release_host&&budget>0){ gpu_prefix=(int)(budget/eb)+g_cuda_ndev; if(gpu_prefix>npin)gpu_prefix=npin; }
|
||||
#else
|
||||
int gpu_prefix=0;
|
||||
#endif
|
||||
/* Load the VRAM-ranked prefix first. Once uploaded its host backing is
|
||||
* released before the disjoint RAM-ranked suffix is allocated. */
|
||||
#pragma omp parallel for schedule(dynamic,1)
|
||||
for(int a=0;a<(gpu_prefix?gpu_prefix:npin);a++)
|
||||
expert_load(m,r[a].l,r[a].e,&m->pin[r[a].l][slot_of[a]],1);
|
||||
m->resident_bytes+=(int64_t)(gpu_prefix?gpu_prefix:npin)*eb;
|
||||
#ifdef COLI_CUDA
|
||||
if(g_cuda_enabled && g_cuda_expert_gb>0){
|
||||
double remaining[COLI_CUDA_MAX_DEVICES]={0}, placed_b[COLI_CUDA_MAX_DEVICES]={0};
|
||||
int placed_n[COLI_CUDA_MAX_DEVICES]={0};
|
||||
double budget=g_cuda_expert_gb*1e9, safe_total=0;
|
||||
for(int i=0;i<g_cuda_ndev;i++){
|
||||
size_t free_b=0,total_b=0;
|
||||
if(coli_cuda_mem_info(g_cuda_devices[i],&free_b,&total_b)){
|
||||
/* Dense tensors are assigned round-robin and upload lazily.
|
||||
* Reserve their projected footprint plus 2 GB per device. */
|
||||
remaining[i]=(double)free_b-(double)g_cuda_dense_projected[i]-2e9;
|
||||
if(remaining[i]<0) remaining[i]=0;
|
||||
safe_total+=remaining[i];
|
||||
}
|
||||
}
|
||||
if(budget>safe_total) budget=safe_total;
|
||||
for(int a=0;a<npin && m->gpu_expert_bytes<budget;a++){
|
||||
int gpu_limit=gpu_prefix?gpu_prefix:npin;
|
||||
for(int a=0;a<gpu_limit && m->gpu_expert_bytes<budget;a++){
|
||||
int li=r[a].l;
|
||||
for(int z=0;z<m->npin[li];z++) if(m->pin[li][z].eid==r[a].e){
|
||||
ESlot *s=&m->pin[li][z];
|
||||
{ ESlot *s=&m->pin[li][slot_of[a]];
|
||||
int64_t need=qt_bytes(&s->g)+qt_bytes(&s->u)+qt_bytes(&s->d);
|
||||
if(m->gpu_expert_bytes+need>budget) break;
|
||||
int tried[COLI_CUDA_MAX_DEVICES]={0}, placed=0;
|
||||
@@ -2993,6 +3384,7 @@ static void pin_load(Model *m, const char *statspath, double gb){
|
||||
+(int64_t)coli_cuda_tensor_bytes(s->d.cuda);
|
||||
m->gpu_expert_count++; m->gpu_expert_bytes+=actual;
|
||||
remaining[best]-=actual; placed_b[best]+=actual; placed_n[best]++;
|
||||
if(g_cuda_release_host) expert_host_release(m,s);
|
||||
placed=1;
|
||||
} else {
|
||||
qt_cuda_reset(&s->g); qt_cuda_reset(&s->u); qt_cuda_reset(&s->d);
|
||||
@@ -3000,7 +3392,6 @@ static void pin_load(Model *m, const char *statspath, double gb){
|
||||
remaining[best]=0; /* device rejected its projected capacity */
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
fprintf(stderr,"[CUDA] hot expert tier: %d/%d experts, VRAM %.2f GB (total budget %.1f GB)\n",
|
||||
@@ -3009,8 +3400,16 @@ static void pin_load(Model *m, const char *statspath, double gb){
|
||||
g_cuda_devices[i],placed_n[i],placed_b[i]/1e9);
|
||||
}
|
||||
#endif
|
||||
if(gpu_prefix>0&&gpu_prefix<npin){
|
||||
#pragma omp parallel for schedule(dynamic,1)
|
||||
for(int a=gpu_prefix;a<npin;a++)
|
||||
expert_load(m,r[a].l,r[a].e,&m->pin[r[a].l][slot_of[a]],1);
|
||||
m->resident_bytes+=(int64_t)(npin-gpu_prefix)*eb;
|
||||
}
|
||||
fprintf(stderr,"[PIN] placement: %d VRAM + %d RAM expert (%.1f GB warm) in %.0fs da %s\n",
|
||||
m->gpu_expert_count,npin-m->gpu_expert_count,(npin-m->gpu_expert_count)*eb/1e9,now_s()-t0,statspath);
|
||||
pin_wire(m); /* inchioda in RAM (no compressione) / wire in RAM (no compression) */
|
||||
free(r); free(cnt_l);
|
||||
free(r); free(cnt_l); free(slot_of); free(next);
|
||||
}
|
||||
|
||||
static double g_mem_avail_boot=0; /* MemAvailable all'avvio, prima di caricare il modello */
|
||||
@@ -3158,6 +3557,14 @@ int main(int argc, char **argv){
|
||||
if(g_mmap) fprintf(stderr,"[MMAP] expert = viste zero-copy nei file (page cache = cache)\n");
|
||||
g_topk = getenv("TOPK")?atoi(getenv("TOPK")):0;
|
||||
g_topp = getenv("TOPP")?atof(getenv("TOPP")):0;
|
||||
const char *policy=getenv("COLI_POLICY"); if(!policy) policy="quality";
|
||||
int experimental=!strcmp(policy,"experimental-fast");
|
||||
if(strcmp(policy,"quality")&&strcmp(policy,"balanced")&&!experimental){
|
||||
fprintf(stderr,"COLI_POLICY non valida: quality, balanced o experimental-fast\n"); return 2;
|
||||
}
|
||||
if(!experimental&&(g_topk>0||g_topp>0)){
|
||||
fprintf(stderr,"[policy] --topp/--topk drop low-weight experts (~1.6x fewer reads, small quality cost)\n");
|
||||
}
|
||||
g_mlock = getenv("MLOCK")?atoi(getenv("MLOCK")):-1; /* -1 auto (ON macOS), 0 off, 1 force / auto (ON macOS), 0 off, 1 force */
|
||||
g_spec = getenv("SPEC")?atoi(getenv("SPEC")):1;
|
||||
g_draft = getenv("DRAFT")?atoi(getenv("DRAFT")):-1; /* -1 = auto: 3 se MTP, 0 senza */
|
||||
@@ -3203,10 +3610,13 @@ int main(int argc, char **argv){
|
||||
}
|
||||
g_cuda_dense=getenv("CUDA_DENSE")?atoi(getenv("CUDA_DENSE")):0;
|
||||
g_cuda_expert_gb=getenv("CUDA_EXPERT_GB")?atof(getenv("CUDA_EXPERT_GB")):0;
|
||||
g_cuda_release_host=getenv("CUDA_RELEASE_HOST")?atoi(getenv("CUDA_RELEASE_HOST")):(g_cuda_ndev>1);
|
||||
if((getenv("COLI_GPU")||getenv("COLI_GPUS"))&&!g_cuda_enabled){ fprintf(stderr,"COLI_GPU(S) requires COLI_CUDA=1\n"); return 2; }
|
||||
if(g_cuda_dense&&!g_cuda_enabled){ fprintf(stderr,"CUDA_DENSE requires COLI_CUDA=1\n"); return 2; }
|
||||
if(g_cuda_expert_gb>0 && !g_cuda_enabled){ fprintf(stderr,"CUDA_EXPERT_GB requires COLI_CUDA=1\n"); return 2; }
|
||||
if(g_cuda_enabled) fprintf(stderr,"[CUDA] mode: routed experts%s\n",g_cuda_dense?" + resident dense tensors":" only (resident dense on CPU)");
|
||||
if(g_cuda_enabled) fprintf(stderr,"[CUDA] mode: routed experts%s%s\n",
|
||||
g_cuda_dense?" + resident dense tensors":" only (resident dense on CPU)",
|
||||
g_cuda_release_host?"; VRAM experts without host backing":"");
|
||||
#else
|
||||
if((getenv("COLI_CUDA") && atoi(getenv("COLI_CUDA"))) ||
|
||||
getenv("COLI_GPU") || getenv("COLI_GPUS") ||
|
||||
@@ -3233,7 +3643,13 @@ int main(int argc, char **argv){
|
||||
printf("== GLM C engine (glm_moe_dsa), cache=%d experts/layer | experts@%d-bit dense@%d-bit | idot: " IDOT_KERNEL " ==\n", cap, ebits, dbits);
|
||||
g_mem_avail_boot = mem_available_gb();
|
||||
Model m; double t0=now_s(); model_init(&m,snap,cap,ebits,dbits);
|
||||
if(g_draft<0) g_draft = m.has_mtp ? 3 : 0;
|
||||
if(g_draft<0){
|
||||
#ifdef COLI_CUDA
|
||||
g_draft = (m.has_mtp&&!g_cuda_enabled) ? 3 : 0;
|
||||
#else
|
||||
g_draft = m.has_mtp ? 3 : 0;
|
||||
#endif
|
||||
}
|
||||
if(getenv("DSA_TOPK")) m.c.index_topk=atoi(getenv("DSA_TOPK")); /* override per test */
|
||||
printf("loaded in %.2fs | resident dense: %.2f MB | layers=%d experts=%d | MTP %s (draft=%d)\n",
|
||||
now_s()-t0, m.resident_bytes/(1024.0*1024.0), m.c.n_layers, m.c.n_experts,
|
||||
@@ -3272,7 +3688,11 @@ int main(int argc, char **argv){
|
||||
if(getenv("SCORE")){ run_score(&m, getenv("SCORE")); if(stats) stats_dump(&m,stats); return 0; }
|
||||
|
||||
/* modo serve persistente per la CLI 'coli': SERVE=1 */
|
||||
if(getenv("SERVE")){ run_serve(&m, snap); if(stats) stats_dump(&m,stats); return 0; }
|
||||
if(getenv("SERVE")){
|
||||
if(getenv("SERVE_BATCH") && atoi(getenv("SERVE_BATCH"))) run_serve_mux(&m,snap);
|
||||
else run_serve(&m,snap);
|
||||
if(stats) stats_dump(&m,stats); return 0;
|
||||
}
|
||||
|
||||
/* modo testo reale: PROMPT="..." [NGEN=n] -> tokenizza, genera, detokenizza */
|
||||
if(getenv("PROMPT")){
|
||||
|
||||
Reference in New Issue
Block a user