danielkrause/colibri-strix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Unify continuous batching + heterogeneous runtime: decode batching, physical-core planning, disjoint VRAM/RAM placement, topp-policy warning (CPU-validated, CUDA on 6x5090) (#68)

Browse Source 
* Fuse CUDA expert MLP execution

* Group CUDA expert transfers by device

* Instrument grouped CUDA expert execution

* Bound grouped CUDA decode scratch

* Execute expert groups across GPUs in parallel

* Release host backing for multi-GPU experts

* Define quality-preserving memory policies

* Overlap cold expert loading with resident compute

* Adapt expert placement with session LFRU

* Fuse q4 expert gate and up dispatch

* Plan CPU work on physical cores

* Batch grouped expert CUDA kernels

* Separate VRAM and RAM expert placement

* Add ragged multi-sequence decode forward

* feat(runtime): add continuous decode scheduler

* Route concurrent API requests through batch scheduler

* Harden multiplex request lifecycle and framing

* Cancel disconnected multiplex requests

* Bind API port before starting the engine

* fix automatic KV slot allocation

* add native int4 Tensor Core grouped GEMM

* add Tensor Core throughput benchmark

* optimize packed int4 low-row kernels

* add asynchronous CUDA staging streams

* document validated six-GPU dense acceleration

* tune six-GPU expert hot set

* raise validated expert hot-set target

* add CUDA MLA absorption core

* fuse grouped expert gate and up projections

* Warn for explicit lossy routing flags
This commit is contained in:
ZacharyZcR
2026-07-13 20:30:36 +08:00
committed by GitHub
parent 98759bfc40
commit cbd599024e
20 changed files with 1741 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
c/tests/test_backend_cuda.cu
+71 -3
View File
@@ -15,6 +15,12 @@ static int close_enough(const float *got, const float *want, int n) {
return 1;
}
static int relative_rms(const float *got,const float *want,int n,float limit){
double err=0,ref=0; for(int i=0;i<n;i++){double d=got[i]-want[i];err+=d*d;ref+=(double)want[i]*want[i];}
float r=(float)std::sqrt(err/(ref+1e-20));
if(r>limit){std::fprintf(stderr,"relative RMS %.5f exceeds %.5f\n",r,limit);return 0;} return 1;
}
int main(int argc, char **argv) {
int devices[COLI_CUDA_MAX_DEVICES], ndev = argc > 1 ? argc - 1 : 1;
if (ndev > COLI_CUDA_MAX_DEVICES) return 2;
@@ -55,8 +61,67 @@ int main(int argc, char **argv) {
ColiCudaTensor *tf = nullptr;
if (!coli_cuda_matmul(&tf, got, x, wf, nullptr, 0, 1, 4, 2, d0) || !close_enough(got, wantf, 2)) return 1;
const float eg[8] = {1,0,0,0, 0,1,0,0};
const float eu[8] = {1,0,0,0, 0,1,0,0};
const float ed[8] = {1,0, 0,1, 1,1, 1,-1};
ColiCudaTensor *tg=nullptr,*tu=nullptr,*td=nullptr;
if (!coli_cuda_tensor_upload(&tg,eg,nullptr,0,4,2,d0) ||
!coli_cuda_tensor_upload(&tu,eu,nullptr,0,4,2,d0) ||
!coli_cuda_tensor_upload(&td,ed,nullptr,0,2,4,d0)) return 1;
float expert[8], want_expert[8];
for(int s=0;s<2;s++){
float a=x[s*4], b=x[s*4+1];
a=(a/(1.0f+std::exp(-a)))*a; b=(b/(1.0f+std::exp(-b)))*b;
want_expert[s*4]=a; want_expert[s*4+1]=b;
want_expert[s*4+2]=a+b; want_expert[s*4+3]=a-b;
}
if (!coli_cuda_expert_mlp(tg,tu,td,expert,x,2) ||
!close_enough(expert,want_expert,8)) return 1;
ColiCudaTensor *gates[2]={tg,tg},*ups[2]={tu,tu},*downs[2]={td,td};
int group_rows[2]={1,1}; float grouped[8];
if (!coli_cuda_expert_group(gates,ups,downs,group_rows,2,grouped,x) ||
!close_enough(grouped,want_expert,8)) return 1;
const float aw[16]={1,0,0,0, 0,1,0,0, 0,0,1,0, 0,0,0,1};
const float aq[4]={1,2,.5f,-.5f},al[12]={1,0,0,0, 0,1,0,0, 0,0,1,0};
const float ar[6]={1,0, 0,1, 1,1};float actx[2],aref[2];
ColiCudaTensor *at=nullptr;if(!coli_cuda_tensor_upload(&at,aw,nullptr,0,4,4,d0))return 1;
float score[3];for(int t=0;t<3;t++)score[t]=aq[0]*al[t*4]+aq[1]*al[t*4+1]+aq[2]*ar[t*2]+aq[3]*ar[t*2+1];
float mx=score[0],z=0;for(int t=1;t<3;t++)mx=score[t]>mx?score[t]:mx;
for(int t=0;t<3;t++){score[t]=std::exp(score[t]-mx);z+=score[t];}for(int t=0;t<3;t++)score[t]/=z;
for(int v=0;v<2;v++){aref[v]=0;for(int t=0;t<3;t++)aref[v]+=score[t]*al[t*4+2+v];}
if(!coli_cuda_attention_absorb(at,actx,aq,al,ar,1,2,2,2,4,3,1.f)||
!close_enough(actx,aref,2))return 1;
coli_cuda_tensor_free(at);
/* Native s4 WMMA path: compare the quantized-activation result against the
existing FP32-activation/s4-weight grouped implementation. */
uint8_t w4[32*32/2]; float ws4[32], gx4[64], scalar4[64], tensor4[64];
for(int i=0;i<(int)sizeof(w4);i++){
int lo=((i%15)-7)&15,hi=(((i*3)%15)-7)&15;
w4[i]=(uint8_t)(lo|(hi<<4));
}
for(int i=0;i<32;i++)ws4[i]=0.01f+(i%5)*0.002f;
for(int i=0;i<64;i++)gx4[i]=std::sin((float)(i+1)*0.17f)*2.f;
ColiCudaTensor *g4=nullptr,*u4=nullptr,*d4=nullptr;
if(!coli_cuda_tensor_upload(&g4,w4,ws4,2,32,32,d0)||
!coli_cuda_tensor_upload(&u4,w4,ws4,2,32,32,d0)||
!coli_cuda_tensor_upload(&d4,w4,ws4,2,32,32,d0))return 1;
ColiCudaTensor *gg4[2]={g4,g4},*ug4[2]={u4,u4},*dg4[2]={d4,d4};
if(!coli_cuda_expert_group(gg4,ug4,dg4,group_rows,2,scalar4,gx4))return 1;
setenv("COLI_CUDA_TC_INT4","1",1);
setenv("COLI_CUDA_TC_MIN_ROWS","1",1);
if(!coli_cuda_expert_group(gg4,ug4,dg4,group_rows,2,tensor4,gx4)||
!relative_rms(tensor4,scalar4,64,0.30f))return 1;
unsetenv("COLI_CUDA_TC_INT4");
unsetenv("COLI_CUDA_TC_MIN_ROWS");
coli_cuda_tensor_free(g4);coli_cuda_tensor_free(u4);coli_cuda_tensor_free(d4);
uint64_t group_calls=0,group_experts=0,group_total_rows=0;
coli_cuda_group_stats(&group_calls,&group_experts,&group_total_rows,nullptr,nullptr,nullptr);
if(group_calls!=3||group_experts!=6||group_total_rows!=6) return 1;
coli_cuda_stats(-1, &count, &bytes);
if (count != 4 || bytes != 70) {
if (count != 7 || bytes != 166) {
std::fprintf(stderr, "unexpected CUDA stats: %zu tensors, %zu bytes\n", count, bytes);
return 1;
}
@@ -64,15 +129,18 @@ int main(int argc, char **argv) {
coli_cuda_tensor_device(t4) != d1 || coli_cuda_tensor_device(t2) != d1) return 1;
coli_cuda_stats(d0, &count, &bytes);
if (ndev > 1) {
if (count != 2 || bytes != 48) return 1;
if (count != 5 || bytes != 144) return 1;
coli_cuda_stats(d1, &count, &bytes);
if (count != 2 || bytes != 22) return 1;
} else if (count != 4 || bytes != 70) return 1;
} else if (count != 7 || bytes != 166) return 1;
coli_cuda_tensor_free(t8);
coli_cuda_tensor_free(t4);
coli_cuda_tensor_free(t2);
coli_cuda_tensor_free(tf);
coli_cuda_tensor_free(tg);
coli_cuda_tensor_free(tu);
coli_cuda_tensor_free(td);
coli_cuda_stats(-1, &count, &bytes);
if (count || bytes) return 1;
coli_cuda_shutdown();