#!/usr/bin/env python3 """ build3300.py - Galnet 3300 (Echtzeit 2014) fuer edwiki: news:galnet:2014:MM:TT_NN python3 -I build3300.py --tools tools --final de_final.json --trans trans.json --tags tags/2014.json \ --meta /pfad/nlwork/meta.json --out paket Quellen de_final.json 56 Blog-Meldungen (EN aus galnetarchive.blogspot.com, DE = Olivers alte Uebersetzung, geprueft) trans.json 11 woechentliche Uebertragungen 03.07.-12.09.3300 aus den Newslettern #30-#40 (DE = KI) tags/2014.json Schlagwoerter je Meldung Seitenaufbau, Uebersichten und Paketteilung kommen aus build_galnet.py (gleiches Layout wie 3301 ff.). """ import argparse, csv, json, sys from pathlib import Path from collections import defaultdict ap = argparse.ArgumentParser() ap.add_argument('--tools', required=True) ap.add_argument('--final', required=True) ap.add_argument('--trans', required=True) ap.add_argument('--tags', required=True) ap.add_argument('--meta', required=True) ap.add_argument('--out', required=True) a = ap.parse_args() sys.path.insert(0, a.tools) import build_galnet as B NS, Y = B.NS, 2014 out = Path(a.out) OLD_IMG = { # alte edwiki.de-Bildnamen -> vorhandene Medien im Wiki 'galnet:community_news_feed_8.jpg': 'news:galnet:communitynewsfeed_8.jpg', 'galnet:community_news_feed_3.jpg': 'news:galnet:communitynewsfeed_3.jpg', 'galnet:community_news_feed_22.jpg': 'news:galnet:community_news_feed_22.jpg', 'galnet:galnet_generic.jpg': 'news:galnet:galnet_generic.jpg', 'galnet:news_image_sp_aisling_duval.jpg': 'news:galnet:news_image_sp_aisling_duval.jpg', 'galnet:news_image_sp_denton_patreus.jpg': 'news:galnet:newsimagepressconferenceimperial.jpg', 'galnet:news_image_sp_zemina_torval.jpg': 'news:galnet:newsimagepressconferenceimperial.jpg', } def picture(old): return OLD_IMG.get(old, 'news:galnet:' + old.split(':')[-1].replace('_', '')) final = json.load(open(a.final, encoding='utf-8')) trans = json.load(open(a.trans, encoding='utf-8')) tags = json.load(open(a.tags, encoding='utf-8')) meta = json.load(open(a.meta, encoding='utf-8')) report = defaultdict(list) rows = [] def emit(pid, date, title_de, body_de, title_en, body_en, note, pic, url, tstate): y, m, d = date.split('-') date_ingame = f'{d}.{m}.{int(y) + B.YEAR_OFFSET}' rel = pid.replace(':', '/') + '.txt' B.write(out / 'pages' / rel, B.article_page(title_de, body_de, date_ingame, note)) B.write(out / 'pages' / 'en' / rel, B.article_page(title_en, body_en, date_ingame)) tg = tags.get(pid.split(':', 2)[2], {}) rows.append(dict(pid=pid, date=date, date_ingame=date_ingame, picture=pic, frontier_id='', url=url, translation=tstate, **{k: ', '.join(tg.get(k, [])) for k in B.TAG_FIELDS}, **{'relations.related': ', '.join(tg.get('related', []))})) for p in final: pid = f"{NS}:{p['id']}" mm, rest = p['id'].split(':')[1:] emit(pid, f'{Y}-{mm}-{rest[:2]}', p['title_de'], p['body_de'], p['title_en'], p['body_en'], '', picture(p['old_img']), p['url'], 'eigene') if p['old_img'] in ('galnet:news_image_sp_denton_patreus.jpg', 'galnet:news_image_sp_zemina_torval.jpg'): report['Porträt fehlt im Wiki, Ersatzbild Pressekonferenz Imperium (altes Bild: ' + p['old_img'].split(':')[-1] + ')'].append(pid) for x in trans: pid = f"{NS}:{x['id']}" nr = x['nl'] link = f"[[news:newsletter:{nr}|Newsletter #{int(nr)}]]" note = f'//Übersetzung aus dem Englischen: Claude. Quelle: {link}.//' if x['note']: note += f'\n\n//{x["note"]}//' emit(pid, f"{Y}-{x['mm']}-{x['dd']}", x['title'], x['body'], x['title_en'], x['body_en'], note, B.GENERIC, meta[nr]['wayback'], 'KI') report['Wöchentliche Übertragung aus dem Newsletter (ohne eigenes Bild, Platzhalter galnet_generic)'].append(pid) report['Englischer Blogtext kürzer als die alte deutsche Fassung (2 Absätze ohne englische Vorlage, DE beibehalten)'].append(f'{NS}:2014:12:29_02') report['Datum im Newsletter falsch (15. statt 22. August)'].append(f'{NS}:2014:08:22_01') report['Neue Handlungsstränge (Seiten fehlen noch)'] += ['stories:storyline:eranin_anniversary_war', 'stories:storyline:sorbago_rebellion'] # Jahres-, Monats-, Uebersichtsseiten B.write(out / 'pages' / 'news/galnet' / str(Y) / 'start.txt', B.year_page(Y)) for m in range(1, 13): B.write(out / 'pages' / 'news/galnet' / str(Y) / f'{m:02d}' / 'start.txt', B.month_page(Y, m)) B.write(out / 'pages' / 'news/galnet' / 'overview.txt', B.overview_page(set(range(2014, 2027)))) rows.sort(key=lambda r: r['pid']) tag = str(Y) with open(out / f'galnet_{tag}.csv', 'w', encoding='utf-8', newline='') as f: w = csv.DictWriter(f, fieldnames=['pid', 'date', 'date_ingame', 'picture', 'frontier_id', 'url', 'translation'] + B.TAG_FIELDS + ['relations.related']) w.writeheader() w.writerows(rows) lines = ['===== Lückenbericht 3300 =====', '', f'Meldungen: {len(rows)} (Blog 56, Newsletter-Übertragungen {len(trans)}) · Sprachseiten: en {len(rows)}', ''] for k, v in report.items(): lines.append(f'==== {k} ({len(v)}) ====') lines += [f' * [[{x}]]' for x in v] lines.append('') B.write(out / f'luecken_{tag}.txt', '\n'.join(lines)) import zipfile zp = out / f'galnet_paket_{tag}.zip' with zipfile.ZipFile(zp, 'w', zipfile.ZIP_DEFLATED) as z: files = sorted((f for f in (out / 'pages').rglob('*') if f.is_file()), key=lambda f: (not f.relative_to(out).as_posix().startswith('pages/news/'), f.as_posix())) for f in files: z.write(f, f.relative_to(out).as_posix()) z.write(out / f'luecken_{tag}.txt', f'luecken_{tag}.txt') z.write(out / f'galnet_{tag}.csv', f'galnet_{tag}.csv') print(f'Paket: {zp} ({zp.stat().st_size / 1e6:.2f} MB), Meldungen {len(rows)}') for k, v in report.items(): print(f' {k}: {len(v)}')