#!/usr/bin/env python3
"""Rohtext (html2wiki) -> bereinigter englischer Seitenrumpf.
- Inhaltsverzeichnis: Überschrift -> fette Zeile, ohne "Follow Elite: Dangerous", Banner bleibt
- Links auf andere Newsletter (Mailchimp-IDs) -> interne Links
- E-Mail-Adressen nicht fett, kleine Leerzeichen-/Satzzeichenfehler
- gezielte Einzelkorrekturen (FIXES)
Aufruf: clean_en.py
"""
import json
import os
import re
import sys
FIXES = {
'026': [("//''A new sighting", "//'A new sighting")],
'015': [('**[[http://elite.frontier.co.uk/|http://elite.frontier.co.u]]k**', '**[[http://elite.frontier.co.uk/|http://elite.frontier.co.uk]]**'),
('**Engines**//: "When', '**Engines:** //"When')],
'002': [('but**[[', 'but **[['), (']]**to check', ']]** to check')],
'003': [('**[[http://i1305.photobucket.com/albums/s553/FrontierAshley/EmpireshipsketchesCropped_zpsc19947fb.jpg|smaller Imperial Ships]]**',
'**[[NL:002|smaller Imperial Ships]]**')],
'004': [('**[[http://s1305.photobucket.com/user/FrontierAshley/media/FedandEmpirefighters_zps35212663.jpg.html|early concept thumbnails]]**',
'**[[NL:001|early concept thumbnails]]**'),
('\nShield Systems:\n', '\n**Shield Systems:**\n')],
'007': [('&e=[UNIQID]', '')],
'008': [(' \\\\ --- \\\\ This week’s drabble', '\n\n===== Elite Fiction: Childish Dreams =====\n\nThis week’s drabble'),
('**[[http://www.laveradio.com/|their site]]**now', '**[[http://www.laveradio.com/|their site]]** now')],
'009': [('**“**"We don\'t need the whole universe, just the Milky Way." Where to start?',
'“We don\'t need the whole universe, just the Milky Way.” Where to start?'),
('ring system(s). **”**', 'ring system(s).'),
('\n- Elsewhere in the Elite galaxy', '\nElsewhere in the Elite galaxy')],
# Fußnotensternchen, kein Fettdruck
'039': [('****LHS 417:Gernhardt Camp****', '%%**%%LHS 417:Gernhardt Camp%%**%%'),
('** New market to watch**', '%%**%% New market to watch')],
# Im Original wurde der ganze Satz als Linkadresse eingetragen
'048': [('[[http://caiburn%20told%20a%20touching%20tale%20about%20his%20late%20father%20on%20the%20frontier%20forums%20%E2%80%93%20the%20warm%20response%20to%20his%20story%20can%20be%20seen%20in%20the%20complete%20thread%20-%20https//forums.frontier.co.uk/showthread.php?t=54588&highlight=screen|',
'[[https://forums.frontier.co.uk/showthread.php?t=54588|')],
'049': [('|become a]]**[[', '|become a]] **[[')],
'052': [('|listed as a produc**t**]]', '|listed as a product]]')],
'057': [('"f**ck you"', '"f%%**%%ck you"')],
}
# nr -> [(Anfang, Ende)] – Bereich, in dem alle ** entfernt werden
UNBOLD = {
'034': [('**GalNet News Transmission 1', 'hard-earned credits.**')],
}
def main():
rawdir, index, outdir = sys.argv[1:4]
idx = json.load(open(index))
by_id = {}
for r in idx:
m = re.search(r'id=([0-9a-f]{10})', r['url'])
if m and re.fullmatch(r'\d+(\.\d)?', r['nr']):
# Sonderausgaben wie #51.5 heißen 051_5
whole, _, frac = r['nr'].partition('.')
by_id.setdefault(m.group(1), '%03d' % int(whole) + ('_' + frac if frac else ''))
os.makedirs(outdir, exist_ok=True)
for fn in sorted(os.listdir(rawdir)):
m = re.match(r'en_(\d{3}(?:_\d)?)\.txt$', fn)
if not m:
continue
nr = m.group(1)
t = open(os.path.join(rawdir, fn)).read()
# Inhaltsverzeichnis
t = re.sub(r'===== Table of Contents[^=]*=====', '**Contents:**', t)
t = re.sub(r'\n \* (\*\*)?Follow Elite: Dangerous(\*\*)?\n', '\n', t)
# persönliche Anrede aus dem Mailchimp-Seriendruck (Name des Abonnenten) neutralisieren
t = re.sub(r'^Hi (?!Commander)[^,\n]{1,40},', 'Hi Commander,', t)
t = re.sub(r'^===== return to contents =====\n', '', t, flags=re.M | re.I)
t = re.sub(r'^(back to contents|return to contents)\s*\n', '', t, flags=re.M | re.I)
t = re.sub(r'^\{\{[^}]*transparent[^}]*\}\}\n', '', t, flags=re.M) # Platzhalter-GIF
# zwei aufeinanderfolgende Links mit gleichem Ziel zusammenfassen
# (ohne Leerzeichen, wenn ein Wort mitten im Link getrennt wurde: „S“+„leepless…“)
def join_links(m):
a, b = m.group(2).strip(), m.group(3).strip()
sep = '' if re.search(r'[A-Za-z]$', a) and re.match(r'[a-z]', b) else ' '
return '[[%s|%s%s%s]]' % (m.group(1), a, sep, b)
pat = r'\[\[([^|\]]+)\|([^\]]*)\]\]\[\[\1\|([^\]]*)\]\]'
while re.search(pat, t):
t = re.sub(pat, join_links, t)
# Werbe-Parameter (utm_…) aus Links entfernen
t = re.sub(r'[?&]utm_[^|\]]*?(?=[|\]])', '', t)
t = re.sub(r'&e=[0-9a-f]{6,}', '', t) # Abonnenten-Kennung
t = re.sub(r'[?&]_ga=[\d.]+(?=[|\]])', '', t) # Google-Analytics-Kennung
t = t.replace('%E2%80%8B', '') # unsichtbares Leerzeichen (Zero Width Space) in Linkadressen
# Einzelkorrekturen
for a, b in FIXES.get(nr, []):
assert a in t, (nr, a[:60])
t = t.replace(a, b)
# Abschnitte, die im Original komplett fett und in Stücke zerlegt sind: Fettung entfernen
for start, end in UNBOLD.get(nr, []):
i = t.index(start); j = t.index(end, i) + len(end)
t = t[:i] + t[i:j].replace('**', '') + t[j:]
# Links auf andere Newsletter
def nl_link(mm):
url, text = mm.group(1), mm.group(2)
i = re.search(r'id=([0-9a-f]{10})', url)
if i and i.group(1) in by_id:
target = by_id[i.group(1)]
if target != nr:
return '[[NL:%s|%s]]' % (target, text)
return mm.group(0)
t = re.sub(r'\[\[https?://us2\.campaign-archive\d?\.com/home/[^|\]]*\|', '[[NL:start|', t)
t = re.sub(r'\[\[(https?://us2\.campaign-archive[^|\]]*)\|([^\]]*)\]\]', nl_link, t)
# Fettung um Links mit eigener Fettung: [[url|**x**]] -> **[[url|x]]**
t = re.sub(r'\[\[([^|\]]+)\|\*\*([^\]]*?)\*\*\]\]', r'**[[\1|\2]]**', t)
# E-Mail-Adresse nicht fett
t = re.sub(r'\*\*([\w.]+@frontier\.co\.uk)\*\*', r'\1', t)
t = re.sub(r' +([.,!?])', r'\1', t)
t = re.sub(r'\]\]\*\*(?=[A-Za-z])', ']]** ', t)
t = re.sub(r'(?<=[A-Za-z])\*\*\[\[', ' **[[', t)
t = re.sub(r'\]\](?=[A-Za-z])', ']] ', t)
t = re.sub(r'(?<=[a-z])\[\[(?=https?:)', ' [[', t)
# „>“ am Zeilenanfang wäre in DokuWiki ein Zitat
t = re.sub(r'^>', '%%>%%', t, flags=re.M)
t = re.sub(r'^If you do not wish to receive any more.*\n', '', t, flags=re.M)
t = re.sub(r'\n{3,}', '\n\n', t).strip() + '\n'
open(os.path.join(outdir, 'en_%s.txt' % nr), 'w').write(t)
if __name__ == '__main__':
main()