Commit d90ac92a334551e5746d9ec74a0e6d442078e25a - SixtyPical

Merge branch 'develop-0.18' into goto-at-end-of-block Chris Pressey 6 years ago

3 changed file(s) with 17 addition(s) and 11 deletion(s). Raw diff Collapse all Expand all

-0

HISTORY.md less more

0	0	History of SixtyPical
1	1	=====================
	2
	3	0.18
	4	----
	5
	6	* Fixed pathological memory use in the lexical scanner - should
	7	be much less inefficient now when parsing large source files.
2	8
3	9	0.17
4	10	----

-1

README.md less more

0	0	SixtyPical
1	1	==========
2	2
3		_Version 0.17. Work-in-progress, everything is subject to change._
	3	_Version 0.18. Work-in-progress, everything is subject to change._
4	4
5	5	SixtyPical is a low-level programming language with advanced
6	6	static analysis. Many of its primitive instructions resemble

+10

-10

src/sixtypical/scanner.py less more

16	16	self.filename = filename
17	17	self.token = None
18	18	self.type = None
	19	self.pos = 0
19	20	self.line_number = 1
20	21	self.scan()
21	22
22		def scan_pattern(self, pattern, type, token_group=1, rest_group=2):
23		pattern = r'^(' + pattern + r')(.*?)$'
24		match = re.match(pattern, self.text, re.DOTALL)
	23	def scan_pattern(self, pattern, type, token_group=1):
	24	pattern = r'(' + pattern + r')'
	25	regexp = re.compile(pattern, flags=re.DOTALL)
	26	match = regexp.match(self.text, pos=self.pos)
25	27	if not match:
26	28	return False
27	29	else:
28	30	self.type = type
29	31	self.token = match.group(token_group)
30		self.text = match.group(rest_group)
	32	self.pos += len(match.group(0))
31	33	self.line_number += self.token.count('\n')
32	34	return True
33	35

35	37	self.scan_pattern(r'[ \t\n\r]*', 'whitespace')
36	38	while self.scan_pattern(r'\/\/.*?[\n\r]', 'comment'):
37	39	self.scan_pattern(r'[ \t\n\r]*', 'whitespace')
38		if not self.text:
	40	if self.pos >= len(self.text):
39	41	self.token = None
40	42	self.type = 'EOF'
41	43	return

43	45	return
44	46	if self.scan_pattern(r'\d+', 'integer literal'):
45	47	return
46		if self.scan_pattern(r'\$([0-9a-fA-F]+)', 'integer literal',
47		token_group=2, rest_group=3):
	48	if self.scan_pattern(r'\$([0-9a-fA-F]+)', 'integer literal', token_group=2):
48	49	# ecch
49	50	self.token = str(eval('0x' + self.token))
50	51	return
51		if self.scan_pattern(r'\"(.*?)\"', 'string literal',
52		token_group=2, rest_group=3):
	52	if self.scan_pattern(r'\"(.*?)\"', 'string literal', token_group=2):
53	53	return
54	54	if self.scan_pattern(r'\w+', 'identifier'):
55	55	return
56	56	if self.scan_pattern(r'.', 'unknown character'):
57	57	return
58	58	else:
59		raise AssertionError("this should never happen, self.text=({})".format(self.text))
	59	raise AssertionError("this should never happen, self.text=({}), self.pos=({})".format(self.text, self.pos))
60	60
61	61	def expect(self, token):
62	62	if self.token == token: