Tiered CUDA acceleration for routed experts (opt-in, CPU default untouched) + REPLAY fixture harness (#16)
* feat: add experimental CUDA backend for resident tensors * feat: promote pinned experts to a bounded VRAM tier * feat: preload the GPU expert tier at startup * fix: harden CUDA backend failure handling * feat: add deterministic multi-GPU tensor placement * test: add deterministic CUDA benchmark fixture * perf: make routed experts the default CUDA path
This commit is contained in:
@@ -0,0 +1,49 @@
|
||||
#ifndef COLIBRI_BACKEND_CUDA_H
|
||||
#define COLIBRI_BACKEND_CUDA_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define COLI_CUDA_MAX_DEVICES 16
|
||||
|
||||
/* Opaque, persistent device copy of one resident quantized tensor. */
|
||||
typedef struct ColiCudaTensor ColiCudaTensor;
|
||||
|
||||
/* Devices are CUDA ordinals, not positions in the input list. */
|
||||
int coli_cuda_init(const int *devices, int count);
|
||||
void coli_cuda_shutdown(void);
|
||||
int coli_cuda_device_count(void);
|
||||
int coli_cuda_device_at(int index);
|
||||
int coli_cuda_mem_info(int device, size_t *free_bytes, size_t *total_bytes);
|
||||
/* device < 0 returns aggregate statistics for all configured devices. */
|
||||
void coli_cuda_stats(int device, size_t *tensor_count, size_t *tensor_bytes);
|
||||
|
||||
/* Upload without executing, so capacity failures happen during model startup. */
|
||||
int coli_cuda_tensor_upload(ColiCudaTensor **tensor,
|
||||
const void *weights, const float *scales,
|
||||
int fmt, int I, int O, int device);
|
||||
|
||||
/*
|
||||
* y[S,O] = x[S,I] @ W[O,I]^T.
|
||||
* fmt matches QT in glm.c: 0=f32, 1=int8, 2=int4, 3=int2.
|
||||
* The first successful call uploads W and its row scales; later calls reuse it.
|
||||
* Returns 1 on success and 0 when CUDA is not initialized or the format is invalid.
|
||||
*/
|
||||
int coli_cuda_matmul(ColiCudaTensor **tensor,
|
||||
float *y, const float *x,
|
||||
const void *weights, const float *scales,
|
||||
int fmt, int S, int I, int O, int device);
|
||||
|
||||
void coli_cuda_tensor_free(ColiCudaTensor *tensor);
|
||||
size_t coli_cuda_tensor_bytes(const ColiCudaTensor *tensor);
|
||||
int coli_cuda_tensor_device(const ColiCudaTensor *tensor);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif
|
||||
Reference in New Issue
Block a user