git @ Cat's Eye Technologies yastasoti / 275e091
Checkpoint an experimental thing, badly. Chris Pressey 1 year, 10 months ago
1 changed file(s) with 14 addition(s) and 1 deletion(s). Raw diff Collapse all Expand all
33 import hashlib
44 import json
55 import os
6 #from subprocess import check_call
6 from subprocess import check_call, CalledProcessError
77 import sys
88 from time import sleep, localtime, strftime
99 import urllib
9797 continue
9898 status = response.status_code
9999 except Exception as e:
100 # TODO: raise
100101 status = str(e)
101102 if status not in (200, 301, 302, 303):
102103 failures.append({
114115 return requests.head(url)
115116
116117
118 class WgetLinkChecker(LinkTraverser):
119 def handle_link(self, url):
120 try:
121 check_call(['wget', '--spider', url])
122 print('OK', url)
123 except CalledProcessError as e:
124 print('BAD {} {}'.format(e.returncode, url))
125 return None
126
127
117128 class LinkArchiver(LinkTraverser):
118129 def __init__(self, links, dest_dir, missing_only=False, **kwargs):
119130 super().__init__(links, **kwargs)
178189 links, options.archive_links_to, missing_only=options.archive_missing_only,
179190 **common_kwargs
180191 )
192 elif True:
193 traverser = WgetLinkChecker(links, **common_kwargs)
181194 else:
182195 traverser = LinkChecker(links, **common_kwargs)
183196