expert dot: AVX-512 int4→float accumulator, runtime-switchable (I4_ACC512), +7% CPU-heavy decode, more accurate than AVX2 order (#95)
AVX-512F/BW build path only; non-AVX-512 builds bit-identical. Qualified on Xeon Silver 4510: max rel-err 2-6× lower than scalar oracle order, ppl 5.99 vs 5.98 (0.24%), runtime escape hatch I4_ACC512=0. Clears the #80/#94 numerical bar.
This commit is contained in:
ZacharyZcR
+4
-1
@@ -66,7 +66,7 @@ CUDA_ARCH ?= native
|
||||
NVCCFLAGS ?= -O3 -std=c++17 -arch=$(CUDA_ARCH) -Xcompiler=-Wall,-Wextra
|
||||
PYTHON ?= python3
|
||||
CUDA_OBJ =
|
||||
TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE)
|
||||
TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_acc512$(EXE)
|
||||
ifeq ($(CUDA),1)
|
||||
ifeq ($(UNAME_S),Darwin)
|
||||
$(error CUDA=1 is supported only on Linux)
|
||||
@@ -151,6 +151,9 @@ tests/test_decode_batch$(EXE): tests/test_decode_batch.c decode_batch.h
|
||||
tests/test_idot$(EXE): tests/test_idot.c glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h
|
||||
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
|
||||
|
||||
tests/test_i4_acc512$(EXE): tests/test_i4_acc512.c
|
||||
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
|
||||
|
||||
test-c: $(TEST_BINS)
|
||||
@for test in $(TEST_BINS); do ./$$test || exit 1; done
|
||||
|
||||
|
||||
@@ -238,6 +238,49 @@ static float *falloc(int64_t n){
|
||||
if(n<0 || (uint64_t)n > SIZE_MAX/sizeof(float)){ fprintf(stderr,"falloc: n=%lld is out of range\n",(long long)n); exit(1); }
|
||||
float *p=malloc((size_t)n*sizeof(float)); if(!p){fprintf(stderr,"OOM\n");exit(1);} return p; }
|
||||
|
||||
/* ---- Accumulatore int4->float a 512 bit / 512-bit int4->float accumulator ----
|
||||
* Stessa matematica lossless di matmul_i4 (nibble->f32, FMA), ma 32 pesi/iter su
|
||||
* due catene FMA indipendenti. NON bit-identico al vecchio ordine: la riduzione
|
||||
* ad albero accumula MENO errore della somma sequenziale (misurato 2-4x più
|
||||
* vicino all'oracolo double sulle forme reali; perplexity invariata, +4-7% sul
|
||||
* decode con routing CPU-heavy — vedi docs/experiments/glm52-6x5090-2026-07-12.md).
|
||||
* EN: same lossless math as matmul_i4, 32 weights/iter on two independent FMA
|
||||
* chains. Not bit-identical to the old order: tree reduction accumulates LESS
|
||||
* rounding than sequential summation. I4_ACC512=0 restores the old order (A/B). */
|
||||
#if defined(__AVX512F__) && defined(__AVX512BW__)
|
||||
static int g_i4_acc512=1;
|
||||
static inline float dot_i4f_avx512(const uint8_t *w,const float *x,int I){
|
||||
const __m128i m4=_mm_set1_epi8(0x0F); const __m512i b8=_mm512_set1_epi32(8);
|
||||
__m512 acc0=_mm512_setzero_ps(),acc1=_mm512_setzero_ps(); int i=0;
|
||||
for(;i+32<=I;i+=32){ __m128i by=_mm_loadu_si128((const __m128i*)(w+(i>>1)));
|
||||
__m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
|
||||
__m128i n0=_mm_unpacklo_epi8(lo,hi),n1=_mm_unpackhi_epi8(lo,hi);
|
||||
__m512 w0=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n0),b8));
|
||||
__m512 w1=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n1),b8));
|
||||
acc0=_mm512_fmadd_ps(_mm512_loadu_ps(x+i),w0,acc0);
|
||||
acc1=_mm512_fmadd_ps(_mm512_loadu_ps(x+i+16),w1,acc1);
|
||||
}
|
||||
return _mm512_reduce_add_ps(_mm512_add_ps(acc0,acc1));
|
||||
}
|
||||
/* selftest contro il riferimento scalare (I4_ACC512_TEST=1): copre l'ordine dei
|
||||
* nibble e ogni multiplo di 32. / selftest vs the scalar reference. */
|
||||
static int i4_acc512_selftest(void){
|
||||
enum { N=224 }; uint8_t w[(N+1)/2]; float x[N];
|
||||
for(int i=0;i<N;i++){
|
||||
int q=((i*13+5)&15)-8;
|
||||
if(!(i&1)) w[i>>1]=(uint8_t)(q+8);
|
||||
else w[i>>1]|=(uint8_t)((q+8)<<4);
|
||||
x[i]=(float)(((i*29+7)%101)-50)/37.f;
|
||||
}
|
||||
for(int n=32;n<=N;n+=32){
|
||||
float ref=0; for(int i=0;i<n;i++) ref+=x[i]*(float)(((w[i>>1]>>((i&1)*4))&15)-8);
|
||||
float got=dot_i4f_avx512(w,x,n),tol=2e-5f*(1.f+fabsf(ref));
|
||||
if(fabsf(got-ref)>tol){ fprintf(stderr,"AVX512 i4 selftest n=%d: %.9g != %.9g\n",n,got,ref); return 0; }
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* y[S,O] = x[S,I] @ W^T, W[O,I] f32 */
|
||||
static void matmul(float *y, const float *x, const float *W, int S, int I, int O){
|
||||
#pragma omp parallel for schedule(static)
|
||||
@@ -269,6 +312,10 @@ static void matmul_i4(float *y, const float *x, const uint8_t *q4, const float *
|
||||
#pragma omp parallel for schedule(static)
|
||||
for (int o=0;o<O;o++){ const uint8_t *w=q4+(int64_t)o*rb; float sc=scale[o];
|
||||
for (int s=0;s<S;s++){ const float *xs=x+(int64_t)s*I; float a=0; int i=0;
|
||||
#if defined(__AVX512F__) && defined(__AVX512BW__)
|
||||
if(g_i4_acc512){ a=dot_i4f_avx512(w,xs,I); i=I&~31; }
|
||||
else {
|
||||
#endif
|
||||
#ifdef __AVX2__
|
||||
const __m128i m4=_mm_set1_epi8(0x0F); const __m256i b8=_mm256_set1_epi32(8);
|
||||
__m256 acc=_mm256_setzero_ps();
|
||||
@@ -292,6 +339,9 @@ static void matmul_i4(float *y, const float *x, const uint8_t *q4, const float *
|
||||
ac0=vfmaq_f32(ac0, vld1q_f32(xs+i+8), vcvtq_f32_s32(vmovl_s16(vget_low_s16(w1))));
|
||||
ac1=vfmaq_f32(ac1, vld1q_f32(xs+i+12), vcvtq_f32_s32(vmovl_s16(vget_high_s16(w1)))); }
|
||||
a=vaddvq_f32(vaddq_f32(ac0,ac1));
|
||||
#endif
|
||||
#if defined(__AVX512F__) && defined(__AVX512BW__)
|
||||
}
|
||||
#endif
|
||||
for(;i+1<I;i+=2){ uint8_t byte=w[i>>1]; int lo=(int)(byte&0xF)-8, hi=(int)(byte>>4)-8;
|
||||
a += xs[i]*(float)lo + xs[i+1]*(float)hi; }
|
||||
@@ -3549,6 +3599,13 @@ int main(int argc, char **argv){
|
||||
perror("[OMP] execv self-reexec failed, running untuned");
|
||||
#endif
|
||||
}
|
||||
#if defined(__AVX512F__) && defined(__AVX512BW__)
|
||||
if(getenv("I4_ACC512")) g_i4_acc512=atoi(getenv("I4_ACC512"))!=0;
|
||||
if(getenv("I4_ACC512_TEST")){
|
||||
if(!i4_acc512_selftest()) return 1;
|
||||
puts("AVX512 i4 selftest: ok"); return 0;
|
||||
}
|
||||
#endif
|
||||
const char *snap=getenv("SNAP"); if(!snap){fprintf(stderr,"SNAP=<dir>\n");return 1;}
|
||||
g_nopack = getenv("NOPACK")?1:0;
|
||||
g_drop = getenv("DROP")?1:0;
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
/* Numeric regression test for the AVX-512 int4->float accumulator.
|
||||
* Mirrors dot_i4f_avx512 from glm.c (kept in sync by the engine's own
|
||||
* I4_ACC512_TEST selftest) and checks it against a double-precision oracle
|
||||
* over the real GLM-5.2 expert row shapes. The pass criterion is that the
|
||||
* 512-bit tree reduction is at least as accurate as the engine's sequential
|
||||
* scalar-f32 order — the fallback every other platform uses.
|
||||
* On CPUs without AVX-512 the test compiles to a skip. */
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <math.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#if defined(__AVX512F__) && defined(__AVX512BW__)
|
||||
#include <immintrin.h>
|
||||
|
||||
static inline float dot_i4f_avx512(const uint8_t *w,const float *x,int I){
|
||||
const __m128i m4=_mm_set1_epi8(0x0F); const __m512i b8=_mm512_set1_epi32(8);
|
||||
__m512 acc0=_mm512_setzero_ps(),acc1=_mm512_setzero_ps(); int i=0;
|
||||
for(;i+32<=I;i+=32){ __m128i by=_mm_loadu_si128((const __m128i*)(w+(i>>1)));
|
||||
__m128i lo=_mm_and_si128(by,m4),hi=_mm_and_si128(_mm_srli_epi16(by,4),m4);
|
||||
__m128i n0=_mm_unpacklo_epi8(lo,hi),n1=_mm_unpackhi_epi8(lo,hi);
|
||||
__m512 w0=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n0),b8));
|
||||
__m512 w1=_mm512_cvtepi32_ps(_mm512_sub_epi32(_mm512_cvtepu8_epi32(n1),b8));
|
||||
acc0=_mm512_fmadd_ps(_mm512_loadu_ps(x+i),w0,acc0);
|
||||
acc1=_mm512_fmadd_ps(_mm512_loadu_ps(x+i+16),w1,acc1);
|
||||
}
|
||||
float a=_mm512_reduce_add_ps(_mm512_add_ps(acc0,acc1));
|
||||
for(;i<I;i++){ uint8_t b=w[i>>1]; a+=x[i]*(float)(((b>>((i&1)*4))&15)-8); }
|
||||
return a;
|
||||
}
|
||||
static float dot_i4f_scalar(const uint8_t *w,const float *x,int I){
|
||||
float a=0;
|
||||
for(int i=0;i<I;i++){ uint8_t b=w[i>>1]; a+=x[i]*(float)(((b>>((i&1)*4))&15)-8); }
|
||||
return a;
|
||||
}
|
||||
static double dot_i4f_double(const uint8_t *w,const float *x,int I){
|
||||
double a=0;
|
||||
for(int i=0;i<I;i++){ uint8_t b=w[i>>1]; a+=(double)x[i]*(double)(((b>>((i&1)*4))&15)-8); }
|
||||
return a;
|
||||
}
|
||||
static uint64_t rng=0x243F6A8885A308D3ULL;
|
||||
static double rndu(void){ rng^=rng<<13; rng^=rng>>7; rng^=rng<<17;
|
||||
return (double)(rng>>11)*(1.0/9007199254740992.0); }
|
||||
static double rndn(void){ double u1=rndu()+1e-18,u2=rndu();
|
||||
return sqrt(-2.0*log(u1))*cos(6.283185307179586*u2); }
|
||||
|
||||
static int trial(int I, int rows, double xscale, const char *label){
|
||||
uint8_t *w=malloc((size_t)(I+1)/2);
|
||||
float *x=malloc((size_t)I*sizeof(float));
|
||||
double e512=0,esca=0; int bad=0;
|
||||
for(int r=0;r<rows;r++){
|
||||
for(int i=0;i<I;i+=2){ int q0=(int)(rndu()*16),q1=(int)(rndu()*16);
|
||||
w[i>>1]=(uint8_t)(q0|(q1<<4)); }
|
||||
for(int i=0;i<I;i++) x[i]=(float)(rndn()*xscale);
|
||||
double ref=dot_i4f_double(w,x,I);
|
||||
float v512=dot_i4f_avx512(w,x,I), vsca=dot_i4f_scalar(w,x,I);
|
||||
if(!isfinite(v512)) bad++;
|
||||
double den=fabs(ref); if(den<1.0) den=1.0;
|
||||
double e5=fabs((double)v512-ref)/den, es=fabs((double)vsca-ref)/den;
|
||||
if(e5>e512) e512=e5;
|
||||
if(es>esca) esca=es;
|
||||
}
|
||||
free(w); free(x);
|
||||
/* 2x headroom on the scalar bound: the claim is "no worse", not "always 2-4x better" */
|
||||
int ok = !bad && e512 <= esca*2.0 + 1e-7;
|
||||
printf(" %-24s I=%-5d avx512 max %.3e | scalar max %.3e | %s\n",
|
||||
label,I,e512,esca,ok?"ok":"FAIL");
|
||||
return ok;
|
||||
}
|
||||
|
||||
int main(void){
|
||||
int ok=1;
|
||||
ok &= trial(6144, 2000, 1.0, "gate/up rows");
|
||||
ok &= trial(6144, 2000, 30.0, "gate/up large x");
|
||||
ok &= trial(2048, 2000, 1.0, "down rows");
|
||||
ok &= trial(2048, 2000, 0.02, "down small x");
|
||||
ok &= trial(6143, 1000, 1.0, "tail I=6143");
|
||||
ok &= trial(96, 5000, 1.0, "short rows");
|
||||
if(!ok){ puts("test_i4_acc512: FAIL"); return 1; }
|
||||
puts("test_i4_acc512: ok");
|
||||
return 0;
|
||||
}
|
||||
#else
|
||||
int main(void){ puts("test_i4_acc512: skipped (no AVX-512 on this build)"); return 0; }
|
||||
#endif
|
||||
Reference in New Issue
Block a user