From bc6bc9c25093bf50f996265dd91c583c78aead82 Mon Sep 17 00:00:00 2001 From: AutoJanitor Date: Sun, 12 Jul 2026 15:44:56 -0500 Subject: [PATCH] VSX integer-dot kernels for POWER8 (12.8x int8 / 7.6x int4 over scalar, #ifdef-gated, x86 path untouched) (#98) * Makefile: support Linux PowerPC (ppc64le) builds PowerPC GCC uses -mcpu instead of -march, so the Linux branch failed with unrecognized option -march=native on ppc64le. Detect ppc64le and ppc64 via uname -m and use -mcpu=$(ARCH) there. The x86-64 path is unchanged. Validated on an IBM POWER8 S824 (Ubuntu 20.04, gcc 9.4): make test-c passes, teacher forcing 32/32 positions and greedy 20/20 tokens against the transformers oracle, engine reports the scalar idot fallback. Signed-off-by: Scott * VSX integer-dot kernels for POWER8 (12.8x int8, 7.6x int4 over scalar) Adds a VSX path to dot_i8i8 and dot_i4i8 using vec_msum, which sums byte products directly into s32 lanes, so the 16-bit saturation bound of the AVX2 maddubs trick does not apply. abs(w) is built with a modulo-subtract select instead of vec_abs so w=-128 wraps to 128 unsigned instead of saturating to 127. Nibble unpack uses vec_mergeh/vec_mergel, which interleave like x86 unpacklo/unpackhi on ppc64le (verified on hardware). g_i4s=1 on VSX since the f32 fallback is plain scalar there: measured 5.5x for int4 IDOT at S=1. Measured on an IBM POWER8 S824 (gcc 9.4, Ubuntu 20.04 ppc64le), single thread, 1536x6144: dot_i8i8 1.48 -> 18.99 Gops/s (12.8x) dot_i4i8 2.33 -> 17.72 Gops/s (7.6x) S=1 int4 matmul path: 3.925 -> 0.505 ms/call (7.8x vs scalar build) Adds tests/test_idot.c: exactness test of the compiled idot kernels (any arch) against a plain-C reference, covering odd tails and the w=-128 edge. Passes on avx512-vnni (x86) and vsx (POWER8). The tiny oracle stays token-exact on the VSX build: TF 32/32, greedy 20/20. Signed-off-by: Scott --------- Signed-off-by: Scott Co-authored-by: Scott --- c/Makefile | 5 +++- c/glm.c | 56 +++++++++++++++++++++++++++++++++++++++++++++ c/tests/test_idot.c | 44 +++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 c/tests/test_idot.c diff --git a/c/Makefile b/c/Makefile index cd40965..965b506 100644 --- a/c/Makefile +++ b/c/Makefile @@ -66,7 +66,7 @@ CUDA_ARCH ?= native NVCCFLAGS ?= -O3 -std=c++17 -arch=$(CUDA_ARCH) -Xcompiler=-Wall,-Wextra PYTHON ?= python3 CUDA_OBJ = -TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) +TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_idot$(EXE) ifeq ($(CUDA),1) ifeq ($(UNAME_S),Darwin) $(error CUDA=1 is supported only on Linux) @@ -119,6 +119,9 @@ tests/test_tier$(EXE): tests/test_tier.c tier.h tests/test_grammar$(EXE): tests/test_grammar.c grammar.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_idot$(EXE): tests/test_idot.c glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + test-c: $(TEST_BINS) @for test in $(TEST_BINS); do ./$$test || exit 1; done diff --git a/c/glm.c b/c/glm.c index f27ea58..90bbe3c 100644 --- a/c/glm.c +++ b/c/glm.c @@ -48,6 +48,11 @@ static inline float hsum256(__m256 v){ /* somma orizzontale di 8 floa } #elif defined(__ARM_NEON) #include /* Apple Silicon / aarch64: kernel NEON */ +#elif defined(__VSX__) +#include /* POWER8+ (ppc64le): kernel VSX */ +#undef vector /* igiene: si usano __vector/__bool espliciti */ +#undef pixel +#undef bool #endif #ifdef __APPLE__ #include /* host_statistics64: MemAvailable di macOS */ @@ -308,6 +313,8 @@ static void matmul_i2(float *y, const float *x, const uint8_t *q2, const float * #define IDOT_KERNEL "avx2" #elif defined(__ARM_NEON) #define IDOT_KERNEL "neon" +#elif defined(__VSX__) +#define IDOT_KERNEL "vsx" #else #define IDOT_KERNEL "scalar" #endif @@ -316,6 +323,11 @@ static int g_idot=1; static int g_i4s=1; /* SDOT presente: int4 IDOT conviene anche a S=1 (decode). Misurato * su Apple M-series: +14%%, expert-matmul -16%%. EN: with SDOT, int4 * IDOT pays even at S=1 (decode); measured on Apple M-series. */ +#elif defined(__VSX__) +static int g_i4s=1; /* POWER8 vec_msum: qui il fallback f32 e' SCALARE, quindi l'IDOT + * int4 conviene anche a S=1. Misurato su POWER8 S824 (vedi PR). + * EN: on VSX the f32 fallback is plain scalar C, so int4 IDOT + * pays even at S=1. Measured on a POWER8 S824 (see PR). */ #else static int g_i4s=2; /* senza SDOT / altrove: soglia originale (misura AVX2 dell'autore). * EN: without SDOT / elsewhere: original threshold (author's AVX2). */ @@ -375,6 +387,25 @@ static inline int32_t dot_i8i8(const int8_t *w, const int8_t *x, int I){ #endif } sum=vaddvq_s32(acc); +#elif defined(__VSX__) + /* POWER8: vec_msum (s8 x u8 -> s32) somma i prodotti byte DIRETTAMENTE in lane + * s32, 16 byte/iter: il bound anti-saturazione a 16 bit di maddubs qui non serve. + * Stesso trucco del segno (|w| u8 per x*sign(w) s8), ma |w| via select+sub MODULO + * e non vec_abs: -128 deve diventare 128 u8, non saturare a 127. + * EN: vec_msum accumulates byte products straight into s32 lanes; |w| is built + * with a modulo subtract select instead of vec_abs so w=-128 wraps to 128 (u8) + * rather than saturating to 127. |x|<=127 from qrow_i8, so x negation is safe. */ + __vector signed int acc=vec_splats(0); + const __vector signed char vz=vec_splats((signed char)0); + for(;i+16<=I;i+=16){ + __vector signed char wv=vec_xl(0,(const signed char*)(w+i)); + __vector signed char xv=vec_xl(0,(const signed char*)(x+i)); + __vector __bool char neg=vec_cmplt(wv,vz); + __vector signed char xs=vec_sel(xv,vec_sub(vz,xv),neg); + __vector unsigned char wa=(__vector unsigned char)vec_sel(wv,vec_sub(vz,wv),neg); + acc=vec_msum(xs,wa,acc); + } + sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3); #endif for(;i>1)); /* 16 byte = 32 nibble */ + __vector unsigned char lo=vec_and(by,m4v), hi=vec_sr(by,sh4); + __vector signed char w0=vec_sub((__vector signed char)vec_mergeh(lo,hi),b8v); + __vector signed char w1=vec_sub((__vector signed char)vec_mergel(lo,hi),b8v); + __vector signed char x0=vec_xl(0,(const signed char*)(x+i)); + __vector signed char x1=vec_xl(0,(const signed char*)(x+i+16)); + __vector __bool char n0=vec_cmplt(w0,vz), n1=vec_cmplt(w1,vz); + acc=vec_msum(vec_sel(x0,vec_sub(vz,x0),n0), + (__vector unsigned char)vec_sel(w0,vec_sub(vz,w0),n0),acc); + acc=vec_msum(vec_sel(x1,vec_sub(vz,x1),n1), + (__vector unsigned char)vec_sel(w1,vec_sub(vz,w1),n1),acc); + } + sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3); #endif for(;i+1>1]; sum+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; } if(i>1]; sum+=((int)(b&0xF)-8)*x[i]; } diff --git a/c/tests/test_idot.c b/c/tests/test_idot.c new file mode 100644 index 0000000..baacd17 --- /dev/null +++ b/c/tests/test_idot.c @@ -0,0 +1,44 @@ +/* Exactness test for the integer-dot kernels: dot_i8i8 and dot_i4i8 must return + * EXACTLY the same value as a plain-C reference, whatever SIMD path was compiled + * in (avx512-vnni / avx2 / neon / vsx / scalar). Integer arithmetic has no + * rounding, so any mismatch is a kernel bug, not noise. + * + * Covers: odd sizes (scalar tail), sizes below one vector, the w=-128 edge + * (sign-trick kernels must treat |−128| as 128 unsigned, not saturate to 127), + * and random data at qrow_i8's contract (|x| <= 127, w full int8 range). */ +#define main coli_glm_main_unused +#include "../glm.c" +#undef main + +static uint32_t rng_state=0x12345678u; +static uint32_t xr(void){ rng_state^=rng_state<<13; rng_state^=rng_state>>17; rng_state^=rng_state<<5; return rng_state; } + +static int32_t ref_i8i8(const int8_t *w, const int8_t *x, int I){ + int64_t s=0; for(int i=0;i>1]; int v=(i&1)?((int)(b>>4)-8):((int)(b&0xF)-8); s+=v*x[i]; } + return (int32_t)s; +} + +int main(void){ + static const int sizes[]={1,2,15,16,17,31,32,33,63,64,65,100,127,128,1408,4096,4097}; + static int8_t w[8192], x[8192]; static uint8_t w4[4096]; + for(unsigned t=0;t