danielkrause/colibri-strix
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MTP head is int8 by default (int4 head: 0-4% acceptance, unusable — measured in #8); README: honest MTP numbers + M5 Max community datapoint (#4, #5)

Browse Source 
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
JustVugg
2026-07-09 20:34:08 +02:00
parent ab6950876b
commit c035f41778
3 changed files with 15 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
c/coli
+5 -3
View File
@@ -409,9 +409,11 @@ def cmd_convert(a):
rc=subprocess.call(base)
if rc!=0: sys.exit(rc)
if a.no_mtp: sys.exit(0)
# passo 2: testa MTP (layer 78) -> abilita la decodifica speculativa nativa (piu' veloce)
print(f" {C.dim}[2/2] testa MTP (draft speculativi){C.r}")
sys.exit(subprocess.call(base+["--mtp"]))
# passo 2: testa MTP (layer 78). SEMPRE int8: a int4 i draft sbagliano quasi sempre
# (acceptance 0-4% vs 39-59%, misurato — issue #8) e la speculazione non parte mai.
mtp_cmd=list(base); i=mtp_cmd.index("--ebits"); mtp_cmd[i+1]=str(max(8,a.ebits))
print(f" {C.dim}[2/2] testa MTP a int8 (draft speculativi){C.r}")
sys.exit(subprocess.call(mtp_cmd+["--mtp"]))
def main():
common=argparse.ArgumentParser(add_help=False)
c/convert_fp8_to_int4.py
+5 -1
View File
@@ -132,7 +132,7 @@ def main():
ap.add_argument("--repo", default=None)
ap.add_argument("--indir", default=None)
ap.add_argument("--outdir", required=False)
ap.add_argument("--ebits", type=int, default=4) # bit residenti: attn/dense-mlp/shared
ap.add_argument("--ebits", type=int, default=None) # bit residenti (default 4; 8 per --mtp/--indexer)
ap.add_argument("--io-bits", type=int, default=8) # bit di embed/lm_head
ap.add_argument("--xbits", type=int, default=None) # bit degli expert ROUTED (streaming); default=ebits
ap.add_argument("--n-layers", type=int, default=78)
@@ -145,6 +145,10 @@ def main():
"i tensori indexer sono sparsi su ~tutti gli shard: ri-scarica l'intero repo (~756 GB "
"di traffico) per tenerne pochi GB. Resumabile shard per shard. Consigliato --ebits 8.")
a = ap.parse_args()
if a.ebits is None:
# testa MTP a int4 = acceptance ~0-4% (misurato, issue #8): il draft sbaglia sempre
# e la speculazione non parte mai. A int8: 39-59%, 2.2-2.8 token/forward.
a.ebits = 8 if (a.mtp or a.indexer) else 4
if a.xbits is None: a.xbits = a.ebits
if a.selftest: