git @ Cat's Eye Technologies The-Swallows / 9b1de95
Make sentence deduplication into its own, independent pass. Cat's Eye Technologies 9 years ago
1 changed file(s) with 24 addition(s) and 15 deletion(s). Raw diff Collapse all Expand all
210210 pass
211211
212212
213 class DeduplicateTransformer(Transformer):
214 # check for verbatim repeated. this could be 'dangerous' if, say,
215 # you have two characters, Bob Jones and Bob Smith, and both are
216 # named 'Bob', and they are actually two different events... but...
217 # for now that is an edge case.
218 def transform(self, editor, incoming_events):
219 events = []
220 for event in incoming_events:
221 if events:
222 if str(event) == str(events[-1]):
223 events[-1].phrase = event.phrase + ', twice'
224 elif str(event.rephrase(event.phrase + ', twice')) == str(events[-1]):
225 events[-1].phrase = event.phrase + ', several times'
226 elif str(event.rephrase(event.phrase + ', several times')) == str(events[-1]):
227 pass
228 else:
229 events.append(event)
230 else:
231 events.append(event)
232 return events
233
234
213235 class MegaTransformer(Transformer):
214236 def transform(self, editor, incoming_events):
215237 # TODO: rewrite this to use Python's shift(), whatever that is
216238 incoming_events = list(reversed(incoming_events))
217239 events = []
218240 events.append(incoming_events.pop())
219
220 def dedup_append(event):
221 # check for verbatim repeated. this could be 'dangerous' if, say,
222 # you have two characters, Bob Jones and Bob Smith, and both are
223 # named 'Bob', and they are actually two different events... but...
224 # for now that is an edge case.
225 if str(event) == str(events[-1]):
226 events[-1].phrase = event.phrase + ', twice'
227 elif str(event.rephrase(event.phrase + ', twice')) == str(events[-1]):
228 events[-1].phrase = event.phrase + ', several times'
229 elif str(event.rephrase(event.phrase + ', several times')) == str(events[-1]):
230 pass
231 else:
232 events.append(event)
233241
234242 while incoming_events:
235243 consume_another_event = True
275283 # if editor.character_location[last_character] == events[-1].original_location:
276284 # events[-1].phrase = '<1> wandered around for a bit, then went back to <2>'
277285
278 dedup_append(event)
286 events.append(event)
279287
280288 return events
281289
327335
328336 editor = Editor(collector, self.characters)
329337 editor.add_transformer(MegaTransformer())
338 editor.add_transformer(DeduplicateTransformer())
330339 editor.publish()
331340
332341 def publish(self):