101 | 101 |
return
|
102 | 102 |
end
|
103 | 103 |
|
|
104 |
-- literal decimal number
|
|
105 |
if string:sub(1,1) == "-" or isdigit(string:sub(1,1)) then
|
|
106 |
local len = 0
|
|
107 |
if string:sub(1,1) == "-" then
|
|
108 |
len = len + 1
|
|
109 |
end
|
|
110 |
while isdigit(string:sub(1+len,1+len)) and len <= string:len() do
|
|
111 |
len = len + 1
|
|
112 |
end
|
|
113 |
fields.set_token(string:sub(1, len), "numlit")
|
|
114 |
string = string:sub(len + 1)
|
|
115 |
return
|
|
116 |
end
|
|
117 |
|
104 | 118 |
-- quoted string
|
105 | 119 |
if string:sub(1,1) == "\"" then
|
106 | 120 |
local len = 1
|
|
114 | 128 |
return
|
115 | 129 |
end
|
116 | 130 |
|
117 | |
-- else check for symbols
|
|
131 |
-- symbol
|
118 | 132 |
if issymbolic(string:sub(1,1)) then
|
119 | 133 |
local len = 0
|
120 | 134 |
while issymbolic(string:sub(1+len,1+len)) and len <= string:len() do
|
|
123 | 137 |
local word = string:sub(1, 1+len-1)
|
124 | 138 |
string = string:sub(1+len)
|
125 | 139 |
fields.set_token(word, "symbol")
|
126 | |
return
|
127 | |
end
|
128 | |
|
129 | |
-- else check for literal decimal number
|
130 | |
if string:sub(1,1) == "-" or isdigit(string:sub(1,1)) then
|
131 | |
local len = 0
|
132 | |
if string:sub(1,1) == "-" then
|
133 | |
len = len + 1
|
134 | |
end
|
135 | |
while isdigit(string:sub(1+len,1+len)) and len <= string:len() do
|
136 | |
len = len + 1
|
137 | |
end
|
138 | |
fields.set_token(string:sub(1, len), "numlit")
|
139 | |
string = string:sub(len + 1)
|
140 | 140 |
return
|
141 | 141 |
end
|
142 | 142 |
|