danielkrause/colibri-strix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Unify continuous batching + heterogeneous runtime: decode batching, physical-core planning, disjoint VRAM/RAM placement, topp-policy warning (CPU-validated, CUDA on 6x5090) (#68)

Browse Source 
* Fuse CUDA expert MLP execution

* Group CUDA expert transfers by device

* Instrument grouped CUDA expert execution

* Bound grouped CUDA decode scratch

* Execute expert groups across GPUs in parallel

* Release host backing for multi-GPU experts

* Define quality-preserving memory policies

* Overlap cold expert loading with resident compute

* Adapt expert placement with session LFRU

* Fuse q4 expert gate and up dispatch

* Plan CPU work on physical cores

* Batch grouped expert CUDA kernels

* Separate VRAM and RAM expert placement

* Add ragged multi-sequence decode forward

* feat(runtime): add continuous decode scheduler

* Route concurrent API requests through batch scheduler

* Harden multiplex request lifecycle and framing

* Cancel disconnected multiplex requests

* Bind API port before starting the engine

* fix automatic KV slot allocation

* add native int4 Tensor Core grouped GEMM

* add Tensor Core throughput benchmark

* optimize packed int4 low-row kernels

* add asynchronous CUDA staging streams

* document validated six-GPU dense acceleration

* tune six-GPU expert hot set

* raise validated expert hot-set target

* add CUDA MLA absorption core

* fuse grouped expert gate and up projections

* Warn for explicit lossy routing flags
This commit is contained in:
ZacharyZcR
2026-07-13 20:30:36 +08:00
committed by GitHub
parent 98759bfc40
commit cbd599024e
20 changed files with 1741 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
c/resource_plan.py
+71 -11
View File
@@ -105,9 +105,33 @@ def discover_gpus():
return devices
def physical_cpu_count():
try:
result = subprocess.run(["lscpu", "-p=core,socket"], text=True,
capture_output=True, check=True, timeout=5)
cores = {tuple(map(int, line.split(","))) for line in result.stdout.splitlines()
if line and not line.startswith("#")}
if cores:
return len(cores)
except (OSError, ValueError, subprocess.SubprocessError):
pass
return os.cpu_count() or 1
POLICIES = {
"quality": {"preserve_quantization": True, "preserve_router": True},
"balanced": {"preserve_quantization": True, "preserve_router": True},
"experimental-fast": {"preserve_quantization": False, "preserve_router": False},
}
def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
available_memory=None, available_disk=None, gpus=None):
available_memory=None, available_disk=None, gpus=None,
policy="quality", physical_cpus=None):
if policy not in POLICIES:
raise ValueError(f"unknown policy: {policy}")
info = analyze_model(model)
physical_cpus = physical_cpu_count() if physical_cpus is None else physical_cpus
cfg = info["config"]
available_memory = memory_available() if available_memory is None else available_memory
if available_disk is None:
@@ -146,8 +170,13 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
safe_vram += usable
gpu_plan.append(dict(gpu, reserve_bytes=reserve, usable_bytes=usable))
requested_vram = int(vram_gb * GB) if vram_gb > 0 else safe_vram
vram_budget = min(requested_vram, safe_vram, cache_bytes)
# VRAM-resident experts do not need duplicate RAM backing: the checkpoint is
# their recovery source. RAM is therefore an independent warm compute tier.
vram_budget = min(requested_vram, safe_vram, info["expert_bytes"])
vram_experts = int(vram_budget // typical) if typical else 0
hot_bytes = min(info["expert_bytes"], vram_experts * typical)
warm_bytes = min(max(0, info["expert_bytes"] - hot_bytes), cache_bytes)
cold_bytes = max(0, info["expert_bytes"] - hot_bytes - warm_bytes)
warnings = []
if cap < 1:
@@ -155,21 +184,41 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
if gpu_indices is not None and len(gpus) != len(set(gpu_indices)):
warnings.append("one or more requested GPUs were not detected")
if gpus and vram_budget < requested_vram:
warnings.append("VRAM tier was clamped by free VRAM or its required RAM backing")
warnings.append("VRAM tier was clamped by free VRAM or model expert size")
if cold_bytes:
warnings.append("cold expert misses may reach disk; normal decode speed depends on hit rate")
if cold_bytes:
bottleneck = "disk expert misses"
elif warm_bytes:
bottleneck = "CPU expert compute and RAM bandwidth"
else:
bottleneck = "GPU compute and interconnect"
return {
"version": 1,
"version": 2,
"policy": {"name": policy, **POLICIES[policy],
"quality_preserving": policy != "experimental-fast"},
"model": {key: value for key, value in info.items() if key != "config"},
"cpu": {"physical_cores": max(1, int(physical_cpus)),
"thread_policy": "physical-cores"},
"tiers": {
"disk": {"role": "backing", "model_bytes": info["model_bytes"],
"available_bytes": available_disk},
"ram": {"role": "resident+cache", "available_bytes": available_memory,
"disk": {"role": "cold-backing", "model_bytes": info["model_bytes"],
"available_bytes": available_disk, "cold_expert_bytes": cold_bytes},
"ram": {"role": "resident+warm-experts", "available_bytes": available_memory,
"budget_bytes": ram_budget, "dense_bytes": info["dense_bytes"],
"runtime_bytes": runtime_bytes, "expert_cache_bytes": cache_bytes,
"cache_slots_per_layer": cap},
"warm_expert_bytes": warm_bytes, "cache_slots_per_layer": cap},
"vram": {"role": "hot-experts", "devices": gpu_plan,
"budget_bytes": vram_budget, "expert_capacity": vram_experts},
"budget_bytes": vram_budget, "hot_expert_bytes": hot_bytes,
"expert_capacity": vram_experts, "requires_host_backing": False},
},
"expected_bottleneck": bottleneck,
"decisions": [
{"target": "VRAM", "reason": "profile-ranked hot experts"},
{"target": "RAM", "reason": "warm experts execute on CPU without quality loss"},
{"target": "Disk", "reason": "immutable recovery source for cold experts"},
],
"warnings": warnings,
}
@@ -177,6 +226,12 @@ def build_plan(model, ram_gb=0, context=4096, gpu_indices=None, vram_gb=0,
def environment_for_plan(plan, env=None, cuda_enabled=True):
"""Apply a plan without overriding explicit user environment settings."""
result = dict(env or {})
result.setdefault("COLI_POLICY", plan["policy"]["name"])
result.setdefault("OMP_NUM_THREADS", str(plan["cpu"]["physical_cores"]))
result.setdefault("OMP_PROC_BIND", "spread")
result.setdefault("OMP_PLACES", "cores")
if plan["policy"]["name"] == "balanced":
result.setdefault("REPIN", "64")
ram = plan["tiers"]["ram"]
result.setdefault("RAM_GB", f"{ram['budget_bytes'] / GB:.3f}")
@@ -203,11 +258,15 @@ def format_bytes(value):
def format_plan(plan):
model, tiers = plan["model"], plan["tiers"]
lines = [f"model {model['shards']} shards · {format_bytes(model['model_bytes'])}",
f"disk backing store · {format_bytes(tiers['disk']['available_bytes'])} free",
policy=plan["policy"]
lines = [f"policy {policy['name']} · quality-preserving {'yes' if policy['quality_preserving'] else 'no'}",
f"model {model['shards']} shards · {format_bytes(model['model_bytes'])}",
f"disk {format_bytes(tiers['disk']['cold_expert_bytes'])} cold experts · "
f"{format_bytes(tiers['disk']['available_bytes'])} free",
f"RAM {format_bytes(tiers['ram']['budget_bytes'])} budget · "
f"{format_bytes(tiers['ram']['dense_bytes'])} dense · "
f"{format_bytes(tiers['ram']['runtime_bytes'])} runtime · "
f"{format_bytes(tiers['ram']['warm_expert_bytes'])} warm experts · "
f"cap {tiers['ram']['cache_slots_per_layer']}/layer"]
vram = tiers["vram"]
if vram["devices"]:
@@ -216,5 +275,6 @@ def format_plan(plan):
f"~{vram['expert_capacity']} experts · {names}")
else:
lines.append("VRAM no NVIDIA device detected · CPU path")
lines.append(f"limit {plan['expected_bottleneck']}")
lines.extend(f"warn {warning}" for warning in plan["warnings"])
return "\n".join(lines)