#!/usr/bin/env python3 """Rohtext (html2wiki) -> bereinigter englischer Seitenrumpf. - Inhaltsverzeichnis: Überschrift -> fette Zeile, ohne "Follow Elite: Dangerous", Banner bleibt - Links auf andere Newsletter (Mailchimp-IDs) -> interne Links - E-Mail-Adressen nicht fett, kleine Leerzeichen-/Satzzeichenfehler - gezielte Einzelkorrekturen (FIXES) Aufruf: clean_en.py """ import json import os import re import sys FIXES = { '026': [("//''A new sighting", "//'A new sighting")], '015': [('**[[http://elite.frontier.co.uk/|http://elite.frontier.co.u]]k**', '**[[http://elite.frontier.co.uk/|http://elite.frontier.co.uk]]**'), ('**Engines**//: "When', '**Engines:** //"When')], '002': [('but**[[', 'but **[['), (']]**to check', ']]** to check')], '003': [('**[[http://i1305.photobucket.com/albums/s553/FrontierAshley/EmpireshipsketchesCropped_zpsc19947fb.jpg|smaller Imperial Ships]]**', '**[[NL:002|smaller Imperial Ships]]**')], '004': [('**[[http://s1305.photobucket.com/user/FrontierAshley/media/FedandEmpirefighters_zps35212663.jpg.html|early concept thumbnails]]**', '**[[NL:001|early concept thumbnails]]**'), ('\nShield Systems:\n', '\n**Shield Systems:**\n')], '007': [('&e=[UNIQID]', '')], '008': [(' \\\\ --- \\\\ This week’s drabble', '\n\n===== Elite Fiction: Childish Dreams =====\n\nThis week’s drabble'), ('**[[http://www.laveradio.com/|their site]]**now', '**[[http://www.laveradio.com/|their site]]** now')], '009': [('**“**"We don\'t need the whole universe, just the Milky Way." Where to start?', '“We don\'t need the whole universe, just the Milky Way.” Where to start?'), ('ring system(s). **”**', 'ring system(s).'), ('\n- Elsewhere in the Elite galaxy', '\nElsewhere in the Elite galaxy')], # Fußnotensternchen, kein Fettdruck '039': [('****LHS 417:Gernhardt Camp****', '%%**%%LHS 417:Gernhardt Camp%%**%%'), ('** New market to watch**', '%%**%% New market to watch')], # Im Original wurde der ganze Satz als Linkadresse eingetragen '048': [('[[http://caiburn%20told%20a%20touching%20tale%20about%20his%20late%20father%20on%20the%20frontier%20forums%20%E2%80%93%20the%20warm%20response%20to%20his%20story%20can%20be%20seen%20in%20the%20complete%20thread%20-%20https//forums.frontier.co.uk/showthread.php?t=54588&highlight=screen|', '[[https://forums.frontier.co.uk/showthread.php?t=54588|')], '049': [('|become a]]**[[', '|become a]] **[[')], '052': [('|listed as a produc**t**]]', '|listed as a product]]')], '057': [('"f**ck you"', '"f%%**%%ck you"')], } # nr -> [(Anfang, Ende)] – Bereich, in dem alle ** entfernt werden UNBOLD = { '034': [('**GalNet News Transmission 1', 'hard-earned credits.**')], } def main(): rawdir, index, outdir = sys.argv[1:4] idx = json.load(open(index)) by_id = {} for r in idx: m = re.search(r'id=([0-9a-f]{10})', r['url']) if m and re.fullmatch(r'\d+(\.\d)?', r['nr']): # Sonderausgaben wie #51.5 heißen 051_5 whole, _, frac = r['nr'].partition('.') by_id.setdefault(m.group(1), '%03d' % int(whole) + ('_' + frac if frac else '')) os.makedirs(outdir, exist_ok=True) for fn in sorted(os.listdir(rawdir)): m = re.match(r'en_(\d{3}(?:_\d)?)\.txt$', fn) if not m: continue nr = m.group(1) t = open(os.path.join(rawdir, fn)).read() # Inhaltsverzeichnis t = re.sub(r'===== Table of Contents[^=]*=====', '**Contents:**', t) t = re.sub(r'\n \* (\*\*)?Follow Elite: Dangerous(\*\*)?\n', '\n', t) # persönliche Anrede aus dem Mailchimp-Seriendruck (Name des Abonnenten) neutralisieren t = re.sub(r'^Hi (?!Commander)[^,\n]{1,40},', 'Hi Commander,', t) t = re.sub(r'^===== return to contents =====\n', '', t, flags=re.M | re.I) t = re.sub(r'^(back to contents|return to contents)\s*\n', '', t, flags=re.M | re.I) t = re.sub(r'^\{\{[^}]*transparent[^}]*\}\}\n', '', t, flags=re.M) # Platzhalter-GIF # zwei aufeinanderfolgende Links mit gleichem Ziel zusammenfassen # (ohne Leerzeichen, wenn ein Wort mitten im Link getrennt wurde: „S“+„leepless…“) def join_links(m): a, b = m.group(2).strip(), m.group(3).strip() sep = '' if re.search(r'[A-Za-z]$', a) and re.match(r'[a-z]', b) else ' ' return '[[%s|%s%s%s]]' % (m.group(1), a, sep, b) pat = r'\[\[([^|\]]+)\|([^\]]*)\]\]\[\[\1\|([^\]]*)\]\]' while re.search(pat, t): t = re.sub(pat, join_links, t) # Werbe-Parameter (utm_…) aus Links entfernen t = re.sub(r'[?&]utm_[^|\]]*?(?=[|\]])', '', t) t = re.sub(r'&e=[0-9a-f]{6,}', '', t) # Abonnenten-Kennung t = re.sub(r'[?&]_ga=[\d.]+(?=[|\]])', '', t) # Google-Analytics-Kennung t = t.replace('%E2%80%8B', '') # unsichtbares Leerzeichen (Zero Width Space) in Linkadressen # Einzelkorrekturen for a, b in FIXES.get(nr, []): assert a in t, (nr, a[:60]) t = t.replace(a, b) # Abschnitte, die im Original komplett fett und in Stücke zerlegt sind: Fettung entfernen for start, end in UNBOLD.get(nr, []): i = t.index(start); j = t.index(end, i) + len(end) t = t[:i] + t[i:j].replace('**', '') + t[j:] # Links auf andere Newsletter def nl_link(mm): url, text = mm.group(1), mm.group(2) i = re.search(r'id=([0-9a-f]{10})', url) if i and i.group(1) in by_id: target = by_id[i.group(1)] if target != nr: return '[[NL:%s|%s]]' % (target, text) return mm.group(0) t = re.sub(r'\[\[https?://us2\.campaign-archive\d?\.com/home/[^|\]]*\|', '[[NL:start|', t) t = re.sub(r'\[\[(https?://us2\.campaign-archive[^|\]]*)\|([^\]]*)\]\]', nl_link, t) # Fettung um Links mit eigener Fettung: [[url|**x**]] -> **[[url|x]]** t = re.sub(r'\[\[([^|\]]+)\|\*\*([^\]]*?)\*\*\]\]', r'**[[\1|\2]]**', t) # E-Mail-Adresse nicht fett t = re.sub(r'\*\*([\w.]+@frontier\.co\.uk)\*\*', r'\1', t) t = re.sub(r' +([.,!?])', r'\1', t) t = re.sub(r'\]\]\*\*(?=[A-Za-z])', ']]** ', t) t = re.sub(r'(?<=[A-Za-z])\*\*\[\[', ' **[[', t) t = re.sub(r'\]\](?=[A-Za-z])', ']] ', t) t = re.sub(r'(?<=[a-z])\[\[(?=https?:)', ' [[', t) # „>“ am Zeilenanfang wäre in DokuWiki ein Zitat t = re.sub(r'^>', '%%>%%', t, flags=re.M) t = re.sub(r'^If you do not wish to receive any more.*\n', '', t, flags=re.M) t = re.sub(r'\n{3,}', '\n\n', t).strip() + '\n' open(os.path.join(outdir, 'en_%s.txt' % nr), 'w').write(t) if __name__ == '__main__': main()