Commit 28551a2120b2235c1d9a0f1afe8f18769843ab36 - Chrysoberyl

projection_dir, git archive into it, find documents in it. Chris Pressey 8 years ago

5 changed file(s) with 80 addition(s) and 52 deletion(s). Raw diff Collapse all Expand all

-0

.gitignore less more

0	0	*.pyc
1	1	node
2	2	installation
	3	checkout
	4	modules

-0

config.yaml less more

8	8	dist_map: dist_map.json
9	9	catalogue_file: catseye.catalogue
10	10	checkout_dir: checkout
	11	projection_dir: modules
11	12
12	13	installation:
13	14	data_dirs:

19	20	feed_dir: feeds
20	21	dist_map: dist_map.json
21	22	checkout_dir: checkout
	23	projection_dir: modules

+33

-47

src/chrysoberyl/commands.py less more

15	15	from chrysoberyl.loader import (
16	16	load_chrysoberyl_dirs, load_config, overlay_yaml
17	17	)
18		from chrysoberyl.objects import Universe
	18	from chrysoberyl.objects import Universe, get_distname
19	19	from chrysoberyl.renderer import Renderer
20	20	from chrysoberyl.transformer import transform_dates
21	21

30	30	shelf = None
31	31
32	32
33		### helper functions ###
34
35
36		def github_repos(space):
37		"""Generator which yields information about every git repository
38		on Github referenced by some distribution in Chrysoberyl.
39
40		Information is a triple of the distribution key, the Github user
41		(or organization) name, and the repository name.
42
43		"""
44		for key, node in space.iteritems():
45		if node['type'] != 'Distribution':
46		continue
47		if 'github' not in node:
48		continue
49		(user, repo) = node['github'].split('/')
50		yield (key, user, repo)
51
52
53		def get_distname(node):
54		if 'github' in node:
55		match = re.match(r'^catseye/(.*?)$', node['github'])
56		return match.group(1).lower()
57		urls = [release['url'] for release in node['releases']]
58		distnames = set()
59		for url in urls:
60		match = re.match(r'^http:\/\/.\/(.?)\-', url)
61		if not match:
62		raise ValueError(url)
63		distnames.add(match.group(1))
64		if len(distnames) == 1:
65		return distnames.pop()
66		raise ValueError(distnames)
67
68
69	33	### command functions ###
70	34
71	35	def mkdistmap(universe, options, config):
72	36	"""Create a mapping between nodes and distributions."""
73	37	space = universe['node'] # FIXME hardcoded
74	38	dist = {}
75		for (key, user, repo) in github_repos(space):
	39	for (key, user, repo) in space.github_repos():
76	40	dist[key] = (user, repo)
77	41
78	42	repo_to_node = {}

107	71	config[space.name]['template_dirs'],
108	72	config[space.name]['output_dir'],
109	73	config[space.name]['checkout_dir'],
	74	config[space.name]['projection_dir'],
110	75	options.sleek_node_links,
111	76	options.render_nodes,
112	77	)

146	111	"""
147	112	space = universe['node'] # FIXME hardcoded
148	113	lines = []
149		for (key, user, repo) in github_repos(space):
	114	for (key, user, repo) in space.github_repos():
150	115	source = shelf.make_source_from_spec('github.com/%s/%s' % (user, repo))
151	116	tag = source.get_latest_release_tag() or 'tip'
152	117	lines.append('bb:%s/%s@%s' % (user, repo, tag))

162	127
163	128	def checkout(universe, options, config):
164	129	"""Clone (or update, if they already exist) all git repos
165		into a local directory.
	130	into a local directory. Then, project a copy of all the files in
	131	the repository into a plain, non-version-controlled directory in
	132	the projection directory, at (TODO) the latest tag specified in data.
166	133
167	134	"""
168	135	space_key = 'node' # FIXME hardcoded
169	136	space = universe[space_key]
170		lines = []
	137
	138	checkout_dir = os.path.abspath(config[space_key]['checkout_dir'])
171	139	try:
172		os.makedirs(config[space_key]['checkout_dir'])
	140	os.makedirs(checkout_dir)
173	141	except OSError:
174	142	pass
175		for (key, user, repo) in sorted(github_repos(space)):
176		repo_path = os.path.join(config[space_key]['checkout_dir'], repo)
	143
	144	projection_dir = os.path.abspath(config[space_key]['projection_dir'])
	145	try:
	146	os.makedirs(projection_dir)
	147	except OSError:
	148	pass
	149
	150	cwd = os.getcwd()
	151	for (key, user, repo) in sorted(space.github_repos()):
	152	repo_path = os.path.join(checkout_dir, repo)
177	153	if os.path.exists(repo_path):
178		cwd = os.getcwd()
179	154	os.chdir(repo_path)
180	155	command = "git pull origin master"
181	156	print command

187	162	)
188	163	print command
189	164	os.system(command)
	165
	166	os.chdir(repo_path)
	167	distname = get_distname(space[key])
	168	proj_path = os.path.join(projection_dir, distname)
	169	command = "git archive --format=tar --prefix=%s/ HEAD \| (cd %s && tar xf -)" % (
	170	distname, projection_dir
	171	)
	172	print command
	173	os.system(command)
	174
	175	os.chdir(cwd)
190	176
191	177
192	178	def check_releases(universe, options, config):

237	223
238	224	passes = 0
239	225	space = universe['node'] # FIXME hardcoded
240		for (key, user, repo) in sorted(github_repos(space)):
	226	for (key, user, repo) in sorted(space.github_repos()):
241	227	if key in ('The Dipple', 'Illgol: Grand Mal',):
242	228	continue
243	229

325	311	return v_name
326	312
327	313	commands = []
328		for (key, user, repo) in github_repos(space):
	314	for (key, user, repo) in space.github_repos():
329	315	for release in space[key]['releases']:
330	316	url = release['url']
331	317	match = re.match(r'^http\:\/\/catseye\.tc\/distfiles\/(.*?)$', url)

+37

-0

src/chrysoberyl/objects.py less more

70	70	ref_i = ikey
71	71	node['__reference-implementation__'] = ref_i
72	72	return ref_i
	73
	74	def github_repos(self):
	75	"""Generator which yields information about every git repository
	76	on Github referenced by some distribution in Chrysoberyl.
	77
	78	Information is a triple of the distribution key, the Github user
	79	(or organization) name, and the repository name.
	80
	81	"""
	82	for key, node in self.iteritems():
	83	if node['type'] != 'Distribution':
	84	continue
	85	if 'github' not in node:
	86	continue
	87	(user, repo) = node['github'].split('/')
	88	yield (key, user, repo)
73	89
74	90	def convert_chrysoberyl_data(self):
75	91	"""Convert all loaded Chrysoberyl data into a form that can be rendered

213	229
214	230	"""
215	231	return self.year * 10000 + (self.month or 0) * 100 + (self.day or 0)
	232
	233
	234	### helper functions ###
	235
	236
	237	def get_distname(node):
	238	if 'github' in node:
	239	match = re.match(r'^(catseye\|cpressey\|michaelcmartin)/(.*?)$', node['github'])
	240	if match is not None:
	241	return match.group(2).lower()
	242	assert 'releases' in node, str(node)
	243	urls = [release['url'] for release in node['releases']]
	244	distnames = set()
	245	for url in urls:
	246	match = re.match(r'^http:\/\/.\/(.?)\-', url)
	247	if not match:
	248	raise ValueError(url)
	249	distnames.add(match.group(1))
	250	if len(distnames) == 1:
	251	return distnames.pop()
	252	raise ValueError("node %s has bad distnames %s" % (node, distnames))

-5

src/chrysoberyl/renderer.py less more

12	12	import markdown
13	13
14	14	from chrysoberyl import transformer
	15	from chrysoberyl.objects import get_distname
15	16	from chrysoberyl.transformer import (
16	17	filekey, sleek_key, pathname2url, markdown_contents
17	18	)

67	68
68	69	"""
69	70	def __init__(self, universe, space, template_dirs, output_dir, checkout_dir,
70		sleek_node_links, render_nodes):
	71	projection_dir, sleek_node_links, render_nodes):
71	72	self.universe = universe
72	73	self.space = space
73	74	self.template_dirs = template_dirs
74	75	assert isinstance(template_dirs, list)
75	76	self.output_dir = output_dir
76	77	self.checkout_dir = checkout_dir
	78	self.projection_dir = projection_dir
77	79	self.sleek_node_links = sleek_node_links
78	80	self.render_nodes = render_nodes
79	81	self.jinja2_env = Environment(loader=Loader(self.template_dirs))

400	402	@expose
401	403	def documentation(key=key):
402	404	"""Return a list of documentation file names for the given key."""
403		# TODO: this really needs to have a "repo-render dir"
404	405	filenames = []
405	406	node = self.universe.get_node(key)
406	407	if 'github' in node:
407	408	path = os.path.join(
408		self.checkout_dir, node['github'].split('/')[1],
	409	self.projection_dir, get_distname(node),
409	410	)
410	411	for filename in find_likely_documents(path):
411	412	filenames.append(filename)

416	417	def documentation_link(filename, key=key):
417	418	node = self.universe.get_node(key)
418	419	path = os.path.join(
419		self.checkout_dir, 'view',
420		pathname2url(node['github'].split('/')[1]),
	420	self.projection_dir, 'view',
	421	get_distname(node),
421	422	pathname2url(filename)
422	423	)
423	424	# TODO: stat the file, fallback to Github link if not there