ZacharyZcR
cbd599024e
* Fuse CUDA expert MLP execution * Group CUDA expert transfers by device * Instrument grouped CUDA expert execution * Bound grouped CUDA decode scratch * Execute expert groups across GPUs in parallel * Release host backing for multi-GPU experts * Define quality-preserving memory policies * Overlap cold expert loading with resident compute * Adapt expert placement with session LFRU * Fuse q4 expert gate and up dispatch * Plan CPU work on physical cores * Batch grouped expert CUDA kernels * Separate VRAM and RAM expert placement * Add ragged multi-sequence decode forward * feat(runtime): add continuous decode scheduler * Route concurrent API requests through batch scheduler * Harden multiplex request lifecycle and framing * Cancel disconnected multiplex requests * Bind API port before starting the engine * fix automatic KV slot allocation * add native int4 Tensor Core grouped GEMM * add Tensor Core throughput benchmark * optimize packed int4 low-row kernels * add asynchronous CUDA staging streams * document validated six-GPU dense acceleration * tune six-GPU expert hot set * raise validated expert hot-set target * add CUDA MLA absorption core * fuse grouped expert gate and up projections * Warn for explicit lossy routing flags
71 lines
2.9 KiB
C
71 lines
2.9 KiB
C
#ifndef COLIBRI_BACKEND_CUDA_H
|
|
#define COLIBRI_BACKEND_CUDA_H
|
|
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
|
|
#ifdef __cplusplus
|
|
extern "C" {
|
|
#endif
|
|
|
|
#define COLI_CUDA_MAX_DEVICES 16
|
|
|
|
/* Opaque, persistent device copy of one resident quantized tensor. */
|
|
typedef struct ColiCudaTensor ColiCudaTensor;
|
|
|
|
/* Devices are CUDA ordinals, not positions in the input list. */
|
|
int coli_cuda_init(const int *devices, int count);
|
|
void coli_cuda_shutdown(void);
|
|
int coli_cuda_device_count(void);
|
|
int coli_cuda_device_at(int index);
|
|
int coli_cuda_mem_info(int device, size_t *free_bytes, size_t *total_bytes);
|
|
/* device < 0 returns aggregate statistics for all configured devices. */
|
|
void coli_cuda_stats(int device, size_t *tensor_count, size_t *tensor_bytes);
|
|
void coli_cuda_group_stats(uint64_t *calls, uint64_t *experts, uint64_t *rows,
|
|
double *h2d_ms, double *kernel_ms, double *d2h_ms);
|
|
|
|
/* Upload without executing, so capacity failures happen during model startup. */
|
|
int coli_cuda_tensor_upload(ColiCudaTensor **tensor,
|
|
const void *weights, const float *scales,
|
|
int fmt, int I, int O, int device);
|
|
|
|
/*
|
|
* y[S,O] = x[S,I] @ W[O,I]^T.
|
|
* fmt matches QT in glm.c: 0=f32, 1=int8, 2=int4, 3=int2.
|
|
* The first successful call uploads W and its row scales; later calls reuse it.
|
|
* Returns 1 on success and 0 when CUDA is not initialized or the format is invalid.
|
|
*/
|
|
int coli_cuda_matmul(ColiCudaTensor **tensor,
|
|
float *y, const float *x,
|
|
const void *weights, const float *scales,
|
|
int fmt, int S, int I, int O, int device);
|
|
|
|
/* Fused expert pipeline: y = down(silu(gate(x)) * up(x)). All three tensors
|
|
* must already be resident on one device. Activations cross PCIe once in
|
|
* each direction instead of once per matrix. */
|
|
int coli_cuda_expert_mlp(ColiCudaTensor *gate, ColiCudaTensor *up,
|
|
ColiCudaTensor *down, float *y, const float *x, int S);
|
|
|
|
/* Packed group of same-shaped experts. Inputs and outputs contain sum(rows)
|
|
* consecutive [D] rows in call order. */
|
|
int coli_cuda_expert_group(ColiCudaTensor *const *gates,
|
|
ColiCudaTensor *const *ups,
|
|
ColiCudaTensor *const *downs,
|
|
const int *rows, int count,
|
|
float *y, const float *x);
|
|
|
|
/* Decode-only MLA weight-absorption core for one token. kv_b is [H*(Q+V),K]. */
|
|
int coli_cuda_attention_absorb(ColiCudaTensor *kv_b,float *ctx,const float *q,
|
|
const float *latent,const float *rope,int H,int Q,
|
|
int R,int V,int K,int T,float attention_scale);
|
|
|
|
void coli_cuda_tensor_free(ColiCudaTensor *tensor);
|
|
size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor);
|
|
int coli_cuda_tensor_device(const ColiCudaTensor *tensor);
|
|
|
|
#ifdef __cplusplus
|
|
}
|
|
#endif
|
|
|
|
#endif
|