git @ Cat's Eye Technologies yastasoti / 8ba12b0
Accumulate all results, dump only the failures at the end. Chris Pressey 1 year, 10 months ago
1 changed file(s) with 18 addition(s) and 12 deletion(s). Raw diff Collapse all Expand all
8686 raise NotImplementedError
8787
8888 def traverse(self):
89 failures = []
89 self.results = []
9090 for link in tqdm(self.links, total=len(self.links)):
9191 url = link['url']
9292 if url in self.ignore_urls:
108108 response = self.handle_link(url)
109109 if response is None:
110110 continue
111 status = response['status_code']
112111 except Exception as e:
113112 if self.fragile:
114113 raise
115 status = str(e)
116 if status not in (200, 301, 302, 303):
117 failures.append({
118 'status': status,
119 'url': url,
120 'link': link,
121 })
114 response = {
115 "status_code": 600,
116 "error": "{}: {}".format(e.__class__.__name, e)
117 }
118 self.results.append({
119 'response': response,
120 'url': url,
121 'link': link,
122 })
122123 if self.delay_between_requests > 0.0:
123124 sleep(self.delay_between_requests)
124 return failures
125
126 def failures(self):
127 for result in self.results:
128 if result['response']['status_code'] != 200:
129 yield result
125130
126131
127132 class LinkChecker(LinkTraverser):
219224 links, options.archive_to, missing_only=options.archive_missing_only,
220225 **common_kwargs
221226 )
222 elif True:
227 elif False:
223228 traverser = WgetLinkChecker(links, **common_kwargs)
224229 else:
225230 traverser = LinkChecker(links, **common_kwargs)
226231
227 result = traverser.traverse()
232 traverser.traverse()
233 result = list(traverser.failures())
228234 sys.stdout.write(json.dumps(result, indent=4, sort_keys=True))
229235
230236