git @ Cat's Eye Technologies Chrysoberyl / b78160d
Attempt to rewrite Feedmark cleanup in Python, for speed. Chris Pressey 2 years ago
2 changed file(s) with 78 addition(s) and 106 deletion(s). Raw diff Collapse all Expand all
+74
-0
script/cleanup.py less more
0 import json
1 import sys
2 from subprocess import check_output
3 import re
4
5 from feedmark.main import read_document_from
6 from feedmark.checkers import Schema
7 from feedmark.formats.markdown import feedmark_htmlize
8 from feedmark.formats.markdown import feedmark_markdownize
9
10
11 ARTICLES = (
12 ('Distribution Organization', None,),
13 ('Electronics Projects', 'Electronics Project',),
14 ('Games', 'Game',),
15 ('Game Implementations', 'Game Implementation',),
16 ('List of Unfinished Interesting Esolangs', 'Unfinished Esolang',),
17 ('Musical Compositions', 'Musical Composition',),
18 ('Pictures', 'Picture',),
19 ('Movies', 'Movie',),
20 ('Retrocomputing', None,),
21 ('Texts', 'Text',),
22 ('Tools', 'Tool',),
23 ('Formats', 'Format',),
24 ('Platforms', 'Platform',),
25 ('Archived', 'Tool',),
26 ('Languages', 'Language',),
27 ('Language Implementations', 'Language Implementation',),
28 ('Automata', 'Language',),
29 ('Gewgaws', 'Gewgaw',),
30 ('Events', 'Event',),
31 ('General Information', None,),
32 ('Project Dependencies', 'Project Dependency',),
33 ('HTML5 Installations', 'HTML5 Installation',),
34 ('News', None,),
35 )
36
37
38 def document_name(title):
39 if title == 'News':
40 return "NEWS.md"
41 else:
42 return "article/{}.md".format(title)
43
44
45 def rewrite_documents(refdex):
46 for title, schema in ARTICLES:
47 print("{}...".format(title))
48 filename = document_name(title)
49 document = read_document_from(filename)
50
51 # TODO FIXME!
52 document.reference_links = rewrite_reference_links(refdex, document.reference_links)
53 for section in document.sections:
54 section.reference_links = rewrite_reference_links(refdex, section.reference_links)
55
56 if schema:
57 schema_document = read_document_from("schema/{}.md".format(schema))
58 schema = Schema(schema_document)
59 results = schema.check_documents([document])
60 if results:
61 raise ValueError(json.dumps(results, indent=4, sort_keys=True))
62
63 s = feedmark_markdownize(document, schema=schema)
64 with open(document.filename, 'w') as f:
65 f.write(s.encode('UTF-8'))
66
67
68 if __name__ == '__main__':
69
70 with codecs.open('refdex.json', 'r', encoding='utf-8') as f:
71 refdex = json.loads(f.read())
72
73 rewrite_documents(refdex)
+4
-106
script/cleanup.sh less more
2222 article/Project?Dependencies.md
2323 "
2424
25 ### rebuild refdexes
26
2527 feedmark --output-refdex $ARTICLES >article-refdex.json || exit 1
26
2728 REFDEXES=misc-refdex/games-refdex.json,misc-refdex/texts-refdex.json,misc-refdex/retrocomputing-refdex.json,misc-refdex/languages-refdex.json,misc-refdex/misc-refdex.json
28
2929 feedmark --input-refdexes=$REFDEXES $ARTICLES --output-refdex >refdex.json || exit 1
3030
31 ## non-lists
32 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
33 "article/Distribution Organization.md" \
34 "article/Retrocomputing.md" \
35 --rewrite-markdown || exit 1
31 ### rewrite documents with new refdexes
3632
37 ## lists
38 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
39 --check-against-schema="schema/Electronics Project.md" \
40 "article/Electronics Projects.md" \
41 --rewrite-markdown || exit 1
42
43 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
44 --check-against-schema="schema/Game.md" \
45 "article/Games.md" \
46 --rewrite-markdown || exit 1
47
48 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
49 --check-against-schema="schema/Game Implementation.md" \
50 "article/Game Implementations.md" \
51 --rewrite-markdown || exit 1
52
53 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
54 --check-against-schema="schema/Musical Composition.md" \
55 "article/Musical Compositions.md" \
56 --rewrite-markdown || exit 1
57
58 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
59 --check-against-schema="schema/Unfinished Esolang.md" \
60 "article/List of Unfinished Interesting Esolangs.md" \
61 --rewrite-markdown || exit 1
62
63 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
64 --check-against-schema="schema/Picture.md" \
65 "article/Pictures.md" \
66 --rewrite-markdown || exit 1
67
68 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
69 --check-against-schema="schema/Movie.md" \
70 "article/Movies.md" \
71 --rewrite-markdown || exit 1
72
73 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
74 --check-against-schema="schema/Text.md" \
75 "article/Texts.md" \
76 --rewrite-markdown || exit 1
77
78 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
79 --check-against-schema="schema/Tool.md" \
80 "article/Tools.md" \
81 --rewrite-markdown || exit 1
82
83 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
84 --check-against-schema="schema/Format.md" \
85 "article/Formats.md" \
86 --rewrite-markdown || exit 1
87
88 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
89 --check-against-schema="schema/Tool.md" \
90 "article/Archived.md" \
91 --rewrite-markdown || exit 1
92
93 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
94 --check-against-schema="schema/Platform.md" \
95 "article/Platforms.md" \
96 --rewrite-markdown || exit 1
97
98 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
99 --check-against-schema="schema/Language.md" \
100 "article/Languages.md" \
101 --rewrite-markdown || exit 1
102
103 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
104 --check-against-schema="schema/Language Implementation.md" \
105 "article/Language Implementations.md" \
106 --rewrite-markdown || exit 1
107
108 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
109 --check-against-schema="schema/Language.md" \
110 "article/Automata.md" \
111 --rewrite-markdown || exit 1
112
113 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
114 --check-against-schema="schema/Gewgaw.md" \
115 "article/Gewgaws.md" \
116 --rewrite-markdown || exit 1
117
118 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
119 --check-against-schema="schema/Event.md" \
120 "article/Events.md" \
121 --rewrite-markdown || exit 1
122
123 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
124 "article/General Information.md" \
125 --rewrite-markdown || exit 1
126
127 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
128 --check-against-schema="schema/HTML5 Installation.md" \
129 "article/HTML5 Installations.md" \
130 --rewrite-markdown || exit 1
131
132 feedmark --input-refdex=refdex.json --input-refdex-filename-prefix="../" \
133 --check-against-schema="schema/Project Dependency.md" \
134 "article/Project Dependencies.md" \
135 --rewrite-markdown || exit 1
33 PYTHONPATH=../Feedmark/src python ./script/cleanup.py || exit 1