diff --git a/c/Makefile b/c/Makefile index cd40965..965b506 100644 --- a/c/Makefile +++ b/c/Makefile @@ -66,7 +66,7 @@ CUDA_ARCH ?= native NVCCFLAGS ?= -O3 -std=c++17 -arch=$(CUDA_ARCH) -Xcompiler=-Wall,-Wextra PYTHON ?= python3 CUDA_OBJ = -TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) +TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_idot$(EXE) ifeq ($(CUDA),1) ifeq ($(UNAME_S),Darwin) $(error CUDA=1 is supported only on Linux) @@ -119,6 +119,9 @@ tests/test_tier$(EXE): tests/test_tier.c tier.h tests/test_grammar$(EXE): tests/test_grammar.c grammar.h $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) +tests/test_idot$(EXE): tests/test_idot.c glm.c st.h json.h tok.h tok_unicode.h compat.h grammar.h tier.h + $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS) + test-c: $(TEST_BINS) @for test in $(TEST_BINS); do ./$$test || exit 1; done diff --git a/c/glm.c b/c/glm.c index f27ea58..90bbe3c 100644 --- a/c/glm.c +++ b/c/glm.c @@ -48,6 +48,11 @@ static inline float hsum256(__m256 v){ /* somma orizzontale di 8 floa } #elif defined(__ARM_NEON) #include /* Apple Silicon / aarch64: kernel NEON */ +#elif defined(__VSX__) +#include /* POWER8+ (ppc64le): kernel VSX */ +#undef vector /* igiene: si usano __vector/__bool espliciti */ +#undef pixel +#undef bool #endif #ifdef __APPLE__ #include /* host_statistics64: MemAvailable di macOS */ @@ -308,6 +313,8 @@ static void matmul_i2(float *y, const float *x, const uint8_t *q2, const float * #define IDOT_KERNEL "avx2" #elif defined(__ARM_NEON) #define IDOT_KERNEL "neon" +#elif defined(__VSX__) +#define IDOT_KERNEL "vsx" #else #define IDOT_KERNEL "scalar" #endif @@ -316,6 +323,11 @@ static int g_idot=1; static int g_i4s=1; /* SDOT presente: int4 IDOT conviene anche a S=1 (decode). Misurato * su Apple M-series: +14%%, expert-matmul -16%%. EN: with SDOT, int4 * IDOT pays even at S=1 (decode); measured on Apple M-series. */ +#elif defined(__VSX__) +static int g_i4s=1; /* POWER8 vec_msum: qui il fallback f32 e' SCALARE, quindi l'IDOT + * int4 conviene anche a S=1. Misurato su POWER8 S824 (vedi PR). + * EN: on VSX the f32 fallback is plain scalar C, so int4 IDOT + * pays even at S=1. Measured on a POWER8 S824 (see PR). */ #else static int g_i4s=2; /* senza SDOT / altrove: soglia originale (misura AVX2 dell'autore). * EN: without SDOT / elsewhere: original threshold (author's AVX2). */ @@ -375,6 +387,25 @@ static inline int32_t dot_i8i8(const int8_t *w, const int8_t *x, int I){ #endif } sum=vaddvq_s32(acc); +#elif defined(__VSX__) + /* POWER8: vec_msum (s8 x u8 -> s32) somma i prodotti byte DIRETTAMENTE in lane + * s32, 16 byte/iter: il bound anti-saturazione a 16 bit di maddubs qui non serve. + * Stesso trucco del segno (|w| u8 per x*sign(w) s8), ma |w| via select+sub MODULO + * e non vec_abs: -128 deve diventare 128 u8, non saturare a 127. + * EN: vec_msum accumulates byte products straight into s32 lanes; |w| is built + * with a modulo subtract select instead of vec_abs so w=-128 wraps to 128 (u8) + * rather than saturating to 127. |x|<=127 from qrow_i8, so x negation is safe. */ + __vector signed int acc=vec_splats(0); + const __vector signed char vz=vec_splats((signed char)0); + for(;i+16<=I;i+=16){ + __vector signed char wv=vec_xl(0,(const signed char*)(w+i)); + __vector signed char xv=vec_xl(0,(const signed char*)(x+i)); + __vector __bool char neg=vec_cmplt(wv,vz); + __vector signed char xs=vec_sel(xv,vec_sub(vz,xv),neg); + __vector unsigned char wa=(__vector unsigned char)vec_sel(wv,vec_sub(vz,wv),neg); + acc=vec_msum(xs,wa,acc); + } + sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3); #endif for(;i>1)); /* 16 byte = 32 nibble */ + __vector unsigned char lo=vec_and(by,m4v), hi=vec_sr(by,sh4); + __vector signed char w0=vec_sub((__vector signed char)vec_mergeh(lo,hi),b8v); + __vector signed char w1=vec_sub((__vector signed char)vec_mergel(lo,hi),b8v); + __vector signed char x0=vec_xl(0,(const signed char*)(x+i)); + __vector signed char x1=vec_xl(0,(const signed char*)(x+i+16)); + __vector __bool char n0=vec_cmplt(w0,vz), n1=vec_cmplt(w1,vz); + acc=vec_msum(vec_sel(x0,vec_sub(vz,x0),n0), + (__vector unsigned char)vec_sel(w0,vec_sub(vz,w0),n0),acc); + acc=vec_msum(vec_sel(x1,vec_sub(vz,x1),n1), + (__vector unsigned char)vec_sel(w1,vec_sub(vz,w1),n1),acc); + } + sum=vec_extract(acc,0)+vec_extract(acc,1)+vec_extract(acc,2)+vec_extract(acc,3); #endif for(;i+1>1]; sum+=((int)(b&0xF)-8)*x[i]+((int)(b>>4)-8)*x[i+1]; } if(i>1]; sum+=((int)(b&0xF)-8)*x[i]; } diff --git a/c/tests/test_idot.c b/c/tests/test_idot.c new file mode 100644 index 0000000..baacd17 --- /dev/null +++ b/c/tests/test_idot.c @@ -0,0 +1,44 @@ +/* Exactness test for the integer-dot kernels: dot_i8i8 and dot_i4i8 must return + * EXACTLY the same value as a plain-C reference, whatever SIMD path was compiled + * in (avx512-vnni / avx2 / neon / vsx / scalar). Integer arithmetic has no + * rounding, so any mismatch is a kernel bug, not noise. + * + * Covers: odd sizes (scalar tail), sizes below one vector, the w=-128 edge + * (sign-trick kernels must treat |−128| as 128 unsigned, not saturate to 127), + * and random data at qrow_i8's contract (|x| <= 127, w full int8 range). */ +#define main coli_glm_main_unused +#include "../glm.c" +#undef main + +static uint32_t rng_state=0x12345678u; +static uint32_t xr(void){ rng_state^=rng_state<<13; rng_state^=rng_state>>17; rng_state^=rng_state<<5; return rng_state; } + +static int32_t ref_i8i8(const int8_t *w, const int8_t *x, int I){ + int64_t s=0; for(int i=0;i>1]; int v=(i&1)?((int)(b>>4)-8):((int)(b&0xF)-8); s+=v*x[i]; } + return (int32_t)s; +} + +int main(void){ + static const int sizes[]={1,2,15,16,17,31,32,33,63,64,65,100,127,128,1408,4096,4097}; + static int8_t w[8192], x[8192]; static uint8_t w4[4096]; + for(unsigned t=0;t