Hermes Agent
b3b6b702d1
- publish_gatekeeper.py (neues Gatekeeper-System) - 4 Premium-Artikel: Veggie Match, Berlin 1960, Wild Tiled West, Scream Park - Alle kumulierten Artikel, Scripts, Medien und Caches
219 lines
8.1 KiB
Python
219 lines
8.1 KiB
Python
#!/usr/bin/env python3
|
|
"""check_abbreviations.py — Prüft Artikel-HTML auf undefinierte Abkürzungen.
|
|
Exit 0 = alle definiert, Exit 1 = undefinierte gefunden, Exit 2 = Fehler.
|
|
Usage: python3 check_abbreviations.py <artikel.html>
|
|
"""
|
|
import re, sys
|
|
from html import unescape
|
|
from collections import Counter
|
|
|
|
SKIP_WORDS = {
|
|
'UND','ODER','DIE','DER','DAS','IST','VON','MIT','AUF','FÜR','DEN','DEM',
|
|
'DES','EIN','EINE','EINEN','EINER','NACH','ZUM','ZUR','BEI','IM','AM','UM',
|
|
'BIS','ZWEI','DREI','VIER','FÜNF','DANN','NOCH','AUCH','NUN','WIE','WAS',
|
|
'WER','WAR','HAT','HABEN','NICHT','KEIN','ALLE','ALLEN','ALLER','OHNE',
|
|
'ABER','ODER','SEIT','SCHON','MEHR','NUR','ALS','WIRD','WURDE','WORDEN',
|
|
'SIND','SEIN','HIER','DA','DORT','JETZT','HEUTE','MORGEN','GESTERN',
|
|
'IHR','IHRE','IHREN','IHM','IHN','IHRER','SIE','WIR','UNS','EUCH','EUER',
|
|
'MIR','MICH','MEIN','MEINE','DEIN','DEINE','SEINE','SEINER',
|
|
'SPIEL','SPIELE','JAHR','JAHRE','JULI','JUNI','MAI','MÄRZ','APRIL',
|
|
'JANUAR','FEBRUAR','AUGUST','SEPTEMBER','OKTOBER','NOVEMBER','DEZEMBER',
|
|
'EURO','USD','MIN','MAX','CA','ET','AL','US','UK','DE','EN','FR','ES',
|
|
'IT','JP','CN','QR','AI','KI','KW','NR','KG','MB','GB','CM','MM','KM',
|
|
'ML','DL','LLC','AG','GMBH','CO','KGAA','ORF','CD','DVD','PC','TV','VHS',
|
|
'DIY','CEO','CFO','COO','CTO','PR','HR','IT','OG','OGS',
|
|
'SPD','CDU','CSU','FDP','AFD','BSW','ARD','ZDF','RTL','WDR','NDR','SWR',
|
|
'BR','MDR','HRR','RBB','DLF','DPA','AFP','AP','CNN','BBC','NYT','WSJ',
|
|
'SZ','FAZ','SPON','TAG','SON','MONTAG','DIENSTAG','MITTWOCH','DONNERSTAG',
|
|
'FREITAG','SAMSTAG','SONNTAG','AKTION','SALE','BESTE','MONOLOGO','SILOS',
|
|
'UP','BSK','CATAN','PM','SU&SD','SUSD','KOSMOS',
|
|
'SET','BOX','BIG','NEW','OLD','TOP','PRO','PLUS','ONE','TWO',
|
|
'USA','USD','DC','EAN', # Common knowledge / proper names
|
|
'HABA','UVP','HCM','BRD','DDR', # Brand names / Standard-Deutsch-Abkürzungen
|
|
}
|
|
|
|
SKIP_PATTERNS = [
|
|
r'^[A-Z]+\d', # Product codes like S1R
|
|
r'^[A-Z]{2,}_', # Snake-case IDs
|
|
]
|
|
|
|
PROPER_NAME_NEXT = re.compile(
|
|
r'\b(Advisory|Games|Verlag|Group|Entertainment|Productions?|Studios?|Media|'
|
|
r'Press|Publishing|International|Worldwide|Digital|Software|Interactive|'
|
|
r'Awards?|Supplement|Edition|Expansion|League|Legends|Party|Quest)\b'
|
|
)
|
|
|
|
# Known expansion terms for common abbreviations
|
|
KNOWN_EXPANSIONS = {
|
|
'BGG': ['BoardGameGeek', 'Board Game Geek'],
|
|
'TCG': ['Trading Card Game', 'Trading Card Games'],
|
|
'CCG': ['Collectible Card Game', 'Collectible Card Games', 'UniVersus', 'Guilty Gear'],
|
|
'MTG': ['Magic The Gathering', 'Magic: The Gathering'],
|
|
'RPG': ['Roleplaying Game', 'Role Playing Game', 'Rollenspiel'],
|
|
'TTRPG': ['Tabletop Roleplaying', 'Tabletop Rollenspiel', 'Tabletop-Rollenspiel'],
|
|
'VFX': ['Visual Effects', 'Spezialeffekte'],
|
|
'UVP': ['unverbindliche Preisempfehlung', 'unverbindlichen Preisempfehlung'],
|
|
'SHUX': ['Community-Treffen', 'Convention', 'jährliche'],
|
|
'GAMA': ['Tabletop Game Association'],
|
|
'KTBG': ['Kids Table Board Gaming'],
|
|
'CMYK': ['CMYK Games'],
|
|
'SVS': ['Schweizer Spielwarenverband'],
|
|
}
|
|
|
|
|
|
def has_expansion_nearby(ctx, abbr):
|
|
"""Check if known expansion for this abbreviation appears nearby."""
|
|
if abbr not in KNOWN_EXPANSIONS:
|
|
return False
|
|
for term in KNOWN_EXPANSIONS[abbr]:
|
|
if re.search(re.escape(term), ctx, re.IGNORECASE):
|
|
return True
|
|
return False
|
|
|
|
|
|
def is_in_parens_definition(ctx, abbr):
|
|
"""Check for any definition-in-parentheses pattern around the abbreviation."""
|
|
# Strip the base abbr (no plural 's')
|
|
base = abbr.rstrip('s') if abbr.endswith('s') else abbr
|
|
|
|
# Pattern A: "Full Term (ABBR)" — definition BEFORE
|
|
# e.g., "Tabletop-Rollenspiele (TTRPGs)", "Kids Table Board Gaming (KTBG)"
|
|
a = re.search(
|
|
r'([A-ZÄÖÜ][A-Za-zäöüß\-]{3,}(?:\s[A-ZÄÖÜ][A-Za-zäöüß\-]{2,}){0,6})'
|
|
r'\s*\(' + re.escape(base) + r's?\)', ctx)
|
|
if a:
|
|
return True
|
|
|
|
# Pattern B: "ABBR (explanation)" or "ABBR-Compound (explanation)" — definition AFTER
|
|
# e.g., "VFX-Künstler (Visual Effects)", "SHUX-Convention (das jährliche Treffen)"
|
|
b = re.search(
|
|
re.escape(base) + r's?(?:-[A-Za-zäöüß]+)?\s*\(([^)]{3,80})\)', ctx)
|
|
if b:
|
|
return True
|
|
|
|
# Pattern C: "FullTerm(ABBR)" — no space
|
|
# e.g., "BoardGameGeek(BGG)"
|
|
c = re.search(
|
|
r'([A-ZÄÖÜ][A-Za-zäöüß\-]{3,}(?:\s[A-ZÄÖÜ][A-Za-zäöüß\-]{2,}){0,4})'
|
|
r'\(' + re.escape(base) + r's?\)', ctx)
|
|
if c:
|
|
return True
|
|
|
|
return False
|
|
|
|
|
|
def is_full_term_before(ctx, abbr):
|
|
"""Check if full term appears right before abbreviation without parens.
|
|
e.g., "Schweizer Spielwarenverband SVS"
|
|
"""
|
|
base = abbr.rstrip('s') if abbr.endswith('s') else abbr
|
|
m = re.search(
|
|
r'([A-ZÄÖÜ][A-Za-zäöüß\-]{3,}(?:\s[A-ZÄÖÜ][A-Za-zäöüß\-]{2,}){1,6})'
|
|
r'\s+' + re.escape(base) + r's?(?:\s|[,.!?;:\)]|$)', ctx)
|
|
if m:
|
|
full_term = m.group(1)
|
|
if len(full_term) > 10:
|
|
return True
|
|
return False
|
|
|
|
|
|
def main():
|
|
if len(sys.argv) < 2:
|
|
print("Usage: check_abbreviations.py <artikel.html>", file=sys.stderr)
|
|
sys.exit(2)
|
|
|
|
filepath = sys.argv[1]
|
|
|
|
try:
|
|
with open(filepath) as f:
|
|
content = f.read()
|
|
except FileNotFoundError:
|
|
print(f"File not found: {filepath}", file=sys.stderr)
|
|
sys.exit(2)
|
|
|
|
# Extract article body
|
|
m = re.search(r'<article>(.*?)</article>', content, re.DOTALL)
|
|
txt = m.group(1) if m else content
|
|
txt = re.sub(r'<[^>]+>', ' ', txt)
|
|
txt = re.sub(r'\s+', ' ', txt).strip()
|
|
txt = unescape(txt)
|
|
|
|
# Find Quellen section boundary (less strict checks there)
|
|
# There may be multiple Quellen blocks (old + new format).
|
|
# Find the FIRST one and skip everything after it (with some margin)
|
|
quellen_pos = txt.find('Quellen')
|
|
if quellen_pos < 0:
|
|
quellen_pos = len(txt)
|
|
else:
|
|
# Allow 250 chars before the "Quellen" heading for old-style source blocks
|
|
quellen_pos = max(0, quellen_pos - 250)
|
|
|
|
# Find all ALL-CAPS sequences (2+ letters)
|
|
abbrs = re.findall(r'\b[A-Z]{2,}(?:&[A-Z]{2,})?\b', txt)
|
|
counts = Counter(abbrs)
|
|
|
|
# Filter candidates
|
|
candidates = {}
|
|
for abbr, count in counts.items():
|
|
if abbr in SKIP_WORDS:
|
|
continue
|
|
if any(re.match(p, abbr) for p in SKIP_PATTERNS):
|
|
continue
|
|
candidates[abbr] = count
|
|
|
|
issues = []
|
|
for abbr in sorted(candidates.keys()):
|
|
# Match both singular and plural: ABBR or ABBRs
|
|
pattern = re.compile(r'\b' + re.escape(abbr) + r's?\b')
|
|
first = pattern.search(txt)
|
|
if not first:
|
|
continue
|
|
|
|
pos = first.start()
|
|
|
|
# Skip if in Quellen section
|
|
if 0 < quellen_pos < pos:
|
|
continue
|
|
|
|
# Context after the match
|
|
ctx_after = txt[pos:pos+80]
|
|
|
|
# Skip proper names (followed by Advisory, Games, Verlag, Awards, etc.)
|
|
if PROPER_NAME_NEXT.search(ctx_after[:50]):
|
|
continue
|
|
|
|
# Skip slash-separated lists
|
|
before_slash = txt[max(0, pos-15):pos]
|
|
after_slash = txt[pos+len(first.group()):pos+len(first.group())+15]
|
|
if '/' in before_slash or '/' in after_slash:
|
|
continue
|
|
|
|
# Build full context for definition checking
|
|
ctx_start = max(0, pos - 250)
|
|
ctx_end = min(len(txt), pos + 150)
|
|
ctx = txt[ctx_start:ctx_end]
|
|
|
|
# Check definition patterns
|
|
if is_in_parens_definition(ctx, abbr):
|
|
continue
|
|
if is_full_term_before(ctx, abbr):
|
|
continue
|
|
if has_expansion_nearby(ctx, abbr):
|
|
continue
|
|
|
|
# Build issue report
|
|
ctx_before = txt[max(0, pos-40):pos]
|
|
issues.append(
|
|
f'{abbr}: ...{ctx_before[-60:]}«{first.group()}»{ctx_after[:60]}...')
|
|
|
|
if issues:
|
|
for i in issues:
|
|
print(f' UNDEFINIERT: {i}')
|
|
sys.exit(1)
|
|
else:
|
|
print(' Alle Abkürzungen definiert')
|
|
sys.exit(0)
|
|
|
|
if __name__ == '__main__':
|
|
main()
|