git @ Cat's Eye Technologies yastasoti / b7eea7e
Add --fragile command-line argument. Chris Pressey 1 year, 10 months ago
1 changed file(s) with 11 addition(s) and 5 deletion(s). Raw diff Collapse all Expand all
6363
6464
6565 class LinkTraverser(object):
66 def __init__(self, links, article_root=None, ignore_urls=None, delay_between_requests=0.0):
66 def __init__(self, links, article_root=None, ignore_urls=None, delay_between_requests=0.0, fragile=False):
6767 self.links = links
6868 self.article_root = article_root
6969 self.ignore_urls = ignore_urls or []
7070 self.delay_between_requests = delay_between_requests
71 self.fragile = fragile
7172
7273 def handle_link(self, url):
7374 raise NotImplementedError
9798 continue
9899 status = response.status_code
99100 except Exception as e:
100 # TODO: raise
101 if self.fragile:
102 raise
101103 status = str(e)
102104 if status not in (200, 301, 302, 303):
103105 failures.append({
160162 argparser.add_argument('--article-root', metavar='DIRNAME', type=str, default='.',
161163 help='Directory in which local files found when checking or archiving links should be located'
162164 )
165 argparser.add_argument('--delay-between-requests', metavar='SECONDS', type=float, default=0.0,
166 help='Delay (in seconds, fractions allowed) between successive network requests'
167 )
168 argparser.add_argument('--fragile', action='store_true', default=False,
169 help='Exit on first error of any nature when processing links'
170 )
163171 argparser.add_argument('--ignore-urls', metavar='URLS', type=str, default=None,
164172 help='Comma-separated list of link targets that should not even try to be fetched'
165 )
166 argparser.add_argument('--delay-between-requests', metavar='SECONDS', type=float, default=0.0,
167 help='Delay (in seconds, fractions allowed) between successive network requests'
168173 )
169174
170175 options = argparser.parse_args(sys.argv[1:])
182187 article_root=options.article_root,
183188 ignore_urls=[] if options.ignore_urls is None else options.ignore_urls.split(','),
184189 delay_between_requests=options.delay_between_requests,
190 fragile=options.fragile,
185191 )
186192
187193 if options.archive_links_to: