ZacharyZcR
cbd599024e
* Fuse CUDA expert MLP execution * Group CUDA expert transfers by device * Instrument grouped CUDA expert execution * Bound grouped CUDA decode scratch * Execute expert groups across GPUs in parallel * Release host backing for multi-GPU experts * Define quality-preserving memory policies * Overlap cold expert loading with resident compute * Adapt expert placement with session LFRU * Fuse q4 expert gate and up dispatch * Plan CPU work on physical cores * Batch grouped expert CUDA kernels * Separate VRAM and RAM expert placement * Add ragged multi-sequence decode forward * feat(runtime): add continuous decode scheduler * Route concurrent API requests through batch scheduler * Harden multiplex request lifecycle and framing * Cancel disconnected multiplex requests * Bind API port before starting the engine * fix automatic KV slot allocation * add native int4 Tensor Core grouped GEMM * add Tensor Core throughput benchmark * optimize packed int4 low-row kernels * add asynchronous CUDA staging streams * document validated six-GPU dense acceleration * tune six-GPU expert hot set * raise validated expert hot-set target * add CUDA MLA absorption core * fuse grouped expert gate and up projections * Warn for explicit lossy routing flags
61 lines
2.4 KiB
C
61 lines
2.4 KiB
C
#ifndef COLIBRI_TIER_H
|
|
#define COLIBRI_TIER_H
|
|
|
|
#include <stdint.h>
|
|
|
|
/* Pick one RAM/VRAM hot-store slot to replace from recent routing heat.
|
|
* The fixed margin handles tiny samples; the 25% margin prevents ping-pong. */
|
|
static int tier_pick_swap(const uint32_t *heat, int nexpert,
|
|
const int *pinned, int npin,
|
|
int *slot, int *eid, long *gain){
|
|
if(!heat || !pinned || npin<1 || nexpert<1) return 0;
|
|
int cold=0;
|
|
for(int z=1;z<npin;z++) if(heat[pinned[z]]<heat[pinned[cold]]) cold=z;
|
|
int hot=-1; uint32_t fh=0;
|
|
for(int e=0;e<nexpert;e++){
|
|
int resident=0;
|
|
for(int z=0;z<npin;z++) if(pinned[z]==e){ resident=1; break; }
|
|
if(!resident && heat[e]>fh){ fh=heat[e]; hot=e; }
|
|
}
|
|
if(hot<0) return 0;
|
|
uint32_t fc=heat[pinned[cold]];
|
|
if(fh<=fc+(fc>>2)+4) return 0;
|
|
*slot=cold; *eid=hot; *gain=(long)fh-(long)fc;
|
|
return 1;
|
|
}
|
|
|
|
/* LFRU: frequency is the primary signal; recency breaks close calls. A recent
|
|
* access contributes at most 255 points while one frequency count is worth
|
|
* 256, so a merely recent expert cannot displace a genuinely hotter one. */
|
|
static uint64_t tier_lfru_score(uint32_t heat, uint32_t last, uint32_t clock){
|
|
uint32_t age=clock-last, recent=age<255?255-age:0;
|
|
return ((uint64_t)heat<<8)|recent;
|
|
}
|
|
|
|
static int tier_pick_lfru(const uint32_t *heat, const uint32_t *last, uint32_t clock,
|
|
int nexpert, const int *pinned, int npin,
|
|
int *slot, int *eid, long *gain){
|
|
if(!heat||!last||!pinned||npin<1||nexpert<1) return 0;
|
|
int cold=0;
|
|
for(int z=1;z<npin;z++)
|
|
if(tier_lfru_score(heat[pinned[z]],last[pinned[z]],clock)<
|
|
tier_lfru_score(heat[pinned[cold]],last[pinned[cold]],clock)) cold=z;
|
|
int hot=-1; uint64_t hs=0;
|
|
for(int e=0;e<nexpert;e++){
|
|
int resident=0; for(int z=0;z<npin;z++) if(pinned[z]==e){resident=1;break;}
|
|
uint64_t score=tier_lfru_score(heat[e],last[e],clock);
|
|
if(!resident&&(hot<0||score>hs)){ hot=e; hs=score; }
|
|
}
|
|
if(hot<0) return 0;
|
|
uint64_t cs=tier_lfru_score(heat[pinned[cold]],last[pinned[cold]],clock);
|
|
/* Retain the existing 25%+4-frequency hysteresis in score units. */
|
|
if(hs<=cs+(cs>>2)+(4u<<8)) return 0;
|
|
*slot=cold; *eid=hot; *gain=(long)((hs-cs)>>8); return 1;
|
|
}
|
|
|
|
static void tier_decay(uint32_t *heat, int nexpert){
|
|
for(int e=0;e<nexpert;e++) heat[e]>>=1;
|
|
}
|
|
|
|
#endif
|