Translate user-facing runtime output to English, machine prefixes preserved, + CLI output test (#67, #85)

* feat: standardize runtime output in English

* test: cover English CLI output
This commit is contained in:
Sidd
2026-07-12 17:08:40 +05:30
committed by GitHub
parent 6e7aa6f92e
commit 2416bc9079
24 changed files with 330 additions and 280 deletions
+2 -2
View File
@@ -5,10 +5,10 @@
#include "../tok.h"
int main(int argc, char **argv){
if(argc<2){ fprintf(stderr,"uso: %s tokenizer.json < casi\n",argv[0]); return 1; }
if(argc<2){ fprintf(stderr,"usage: %s tokenizer.json < cases\n",argv[0]); return 1; }
Tok T;
tok_load(&T, argv[1]);
fprintf(stderr,"caricato: vocab_ids=%d specials=%d\n", T.n_ids, T.nsp);
fprintf(stderr,"loaded: vocab_ids=%d specials=%d\n", T.n_ids, T.nsp);
char *line=NULL; size_t cap=0; ssize_t nr;
int pass=0, tot=0, dpass=0;
while((nr=getline(&line,&cap,stdin))>=0){