git @ Cat's Eye Technologies klaus / 4ab4627
Refactor 9dfcf8cfca1d64a15f816033f7aef05eabc233b3 Jonas Haag 10 years ago
2 changed file(s) with 100 addition(s) and 68 deletion(s). Raw diff Collapse all Expand all
+82
-0
klaus/tarutils.py less more
0 import os
1 import stat
2 import tarfile
3 from io import BytesIO
4
5
6 class ListBytesIO(object):
7 """
8 Turns a list of bytestrings into a file-like object.
9
10 This is similar to creating a `BytesIO` from a concatenation of the
11 bytestring list, but saves memory by NOT creating one giant bytestring first::
12
13 BytesIO(b''.join(list_of_bytestrings)) =~= ListBytesIO(list_of_bytestrings)
14 """
15 def __init__(self, contents):
16 self.contents = contents
17 self.pos = (0, 0)
18
19 def read(self, maxbytes=None):
20 if maxbytes < 0:
21 maxbytes = float('inf')
22
23 buf = []
24 chunk, cursor = self.pos
25
26 while chunk < len(self.contents):
27 if maxbytes < len(self.contents[chunk]) - cursor:
28 buf.append(self.contents[chunk][cursor:cursor+maxbytes])
29 cursor += maxbytes
30 self.pos = (chunk, cursor)
31 break
32 else:
33 buf.append(self.contents[chunk][cursor:])
34 maxbytes -= len(self.contents[chunk]) - cursor
35 chunk += 1
36 cursor = 0
37 self.pos = (chunk, cursor)
38 return b''.join(buf)
39
40
41 def tar_stream(repo, tree, mtime):
42 """
43 Returns a generator that lazily assembles a .tar.gz archive, yielding it in
44 pieces (bytestrings). To obtain the complete .tar.gz binary file, simply
45 concatenate these chunks.
46
47 'repo' and 'tree' are the dulwich Repo and Tree objects the archive shall be
48 created from. 'mtime' is a UNIX timestamp that is assigned as the modification
49 time of all files in the resulting .tar.gz archive.
50 """
51 buf = BytesIO()
52 with tarfile.open(None, "w", buf) as tar:
53 for entry_abspath, entry in walk_tree(repo, tree):
54 blob = repo[entry.sha]
55 data = ListBytesIO(blob.chunked)
56
57 info = tarfile.TarInfo()
58 info.name = entry_abspath
59 info.size = blob.raw_length()
60 info.mode = entry.mode
61 info.mtime = mtime
62
63 tar.addfile(info, data)
64 yield buf.getvalue()
65 buf.truncate(0)
66 buf.seek(0)
67 yield buf.getvalue()
68
69
70 def walk_tree(repo, tree, root=''):
71 """
72 Recursively walk a dulwich Tree, yielding tuples of (absolute path,
73 TreeEntry) along the way.
74 """
75 for entry in tree.iteritems():
76 entry_abspath = os.path.join(root, entry.path)
77 if stat.S_ISDIR(entry.mode):
78 for _ in walk_tree(repo, repo[entry.sha], entry_abspath):
79 yield _
80 else:
81 yield (entry_abspath, entry)
+18
-68
klaus/views.py less more
00 import os
11 import stat
22
3 import tarfile
4 from io import BytesIO
5 from time import time
6
73 from flask import request, render_template, current_app
84 from flask.views import View
95
128
139 from dulwich.objects import Blob
1410
15 from klaus import markup
11 from klaus import markup, tarutils
1612 from klaus.utils import parent_directory, subpaths, pygmentize, \
1713 force_unicode, guess_is_binary, guess_is_image
1814
242238 """
243239 def get_response(self):
244240 tarname = "%s@%s.tar" % (self.context['repo'].name, self.context['rev'])
245 resp = Response(self._generator(), mimetype="application/tar")
246 resp.headers['Content-Disposition'] = 'attachment; filename=%s' % tarname
247 resp.headers['Content-Length'] = str(self._size())
248 resp.headers['Cache-Control'] = "no-store" # Disables browser caching
249 return resp
250
251 @staticmethod
252 def _io_len(s):
253 pos = s.tell()
254 s.seek(0, os.SEEK_END)
255 length = s.tell()
256 s.seek(pos)
257 return length
258
259 def _walker(self, directory=''):
260 root_tree = self.context['repo'].get_blob_or_tree(
261 self.context['commit'], directory
262 )
263 for entry in root_tree.iteritems():
264 name, entry = entry.path, entry.in_path(directory)
265 if entry.mode & stat.S_IFDIR:
266 for f in self._walker(entry.path):
267 yield f
268 else:
269 data = self.context['repo'].get_blob_or_tree(
270 self.context['commit'], entry.path
271 ).as_raw_string()
272 yield (entry.path, BytesIO(data), entry.mode)
273
274 def _size(self, directory=''):
275 root_tree = self.context['repo'].get_blob_or_tree(
276 self.context['commit'], directory
277 )
278 size = 0
279 for entry in root_tree.iteritems():
280 name, entry = entry.path, entry.in_path(directory)
281 if entry.mode & stat.S_IFDIR:
282 size += self._size(entry.path)
283 else:
284 size += self.context['repo'].get_blob_or_tree(
285 self.context['commit'], entry.path
286 ).raw_length()
287 #see https://en.wikipedia.org/wiki/Tar_%28file_format%29#File_header
288 size += 512 + 512 # info + padding
289 return size + 512
290
291 def _generator(self):
292 buf = BytesIO()
293 tar = tarfile.open("repo.tar", "w", buf)
294 for fl, f, mode in self._walker():
295 info = tarfile.TarInfo(name=fl)
296 info.size = self._io_len(f)
297 info.mode = mode
298 info.mtime = self.context['commit'].commit_time
299 tar.addfile(info, f)
300 yield buf.getvalue()
301 buf.truncate(0)
302 buf.seek(0)
303 tar.close()
304 yield buf.getvalue()
305
306
307 # TODO v
241 headers = {
242 'Content-Disposition': "attachment; filename=%s" % tarname,
243 'Cache-Control': "no-store", # Disables browser caching
244 }
245
246 tar_stream = tarutils.tar_stream(
247 self.context['repo'],
248 self.context['blob_or_tree'],
249 self.context['commit'].commit_time
250 )
251 return Response(
252 tar_stream,
253 mimetype="application/tar",
254 headers=headers
255 )
256
257
308258 history = HistoryView.as_view('history', 'history', 'history.html')
309259 commit = BaseRepoView.as_view('commit', 'commit', 'view_commit.html')
310260 blob = BlobView.as_view('blob', 'blob', 'view_blob.html')