git @ Cat's Eye Technologies yastasoti / 0f91655
Without a chunk_size, iter_content() is grossly inefficient. Fix. Chris Pressey 1 year, 10 months ago
1 changed file(s) with 5 addition(s) and 2 deletion(s). Raw diff Collapse all Expand all
2828 logger = logging.getLogger("yastasoti")
2929
3030
31 CHUNK_SIZE = 8192
32
33
3134 def url_to_dirname_and_filename(url):
3235 parts = url.split(u'/')
3336 parts = parts[2:]
4548 collector = hashlib.sha1()
4649 with open(filename, 'rb') as f:
4750 while True:
48 data = f.read(1024)
51 data = f.read(CHUNK_SIZE)
4952 if not data:
5053 break
5154 collector.update(data)
5760 partname = os.path.join(dirname, filename + '_part')
5861 logger.info(u"downloading '{}' to {}".format(url, partname).encode('utf-8'))
5962 with open(partname, "wb") as f:
60 for data in response.iter_content():
63 for data in response.iter_content(chunk_size=CHUNK_SIZE):
6164 f.write(data)
6265 destname = os.path.join(dirname, filename)
6366 if os.path.exists(destname):