-- Module:PythonScript/Lexer
local Lexer = {}
local byte = string.byte
local sub = string.sub
local KEYWORDS = {
["False"] = true, ["None"] = true, ["True"] = true, ["and"] = true,
["as"] = true, ["assert"] = true, ["async"] = true, ["await"] = true,
["break"] = true, ["class"] = true, ["continue"] = true, ["def"] = true,
["del"] = true, ["elif"] = true, ["else"] = true, ["except"] = true,
["finally"] = true, ["for"] = true, ["from"] = true, ["global"] = true,
["if"] = true, ["import"] = true, ["in"] = true, ["is"] = true,
["lambda"] = true, ["nonlocal"] = true, ["not"] = true, ["or"] = true,
["pass"] = true, ["raise"] = true, ["return"] = true, ["try"] = true,
["while"] = true, ["with"] = true, ["yield"] = true
}
local OP2 = {
["=="] = true, ["!="] = true, ["<="] = true, [">="] = true,
["//"] = true, ["**"] = true, ["<<"] = true, [">>"] = true,
["+="] = true, ["-="] = true, ["*="] = true, ["/="] = true,
["%="] = true, ["&="] = true, ["|="] = true, ["^="] = true,
["@="] = true, ["\\="] = true, [":="] = true
}
-- 預處理器：僅在「非字串」與「非註解」的原始碼區域，還原 HTML 實體 (防維基百科跳脫)
local function unescape_source(src)
local res = {}
local pos, last_pos = 1, 1
local len = #src
local byte = string.byte
local function decode_entity(ent, in_string, is_fstring)
if ent:sub(1,1) == "#" then
local code
if ent:sub(2,2):lower() == "x" then code = tonumber(ent:sub(3), 16)
else code = tonumber(ent:sub(2), 10) end
if code and code > 0 and code < 256 then
-- 一般字串內，為了不傷及無辜，只還原維基易衝突的括號與管線
if in_string and not is_fstring and code ~= 123 and code ~= 125 and code ~= 124 and code ~= 91 and code ~= 93 then
return nil
end
return string.char(code)
end
end
-- 一般字串不還原具名實體 (如 &)，但 f-string 與外部程式碼火力全開
if in_string and not is_fstring then return nil end
if ent == "lt" then return "<"
elseif ent == "gt" then return ">"
elseif ent == "amp" then return "&"
elseif ent == "quot" then return '"'
elseif ent == "apos" then return "'"
end
return nil
end
while pos <= len do
local b = byte(src, pos)
if b == 39 or b == 34 then -- 遇到引號 ' 或 "
-- 偵測是否為 f-string
local is_fstring = false
if pos > 1 then
local p1 = string.lower(string.char(byte(src, pos-1)))
if p1 == "f" then is_fstring = true
elseif pos > 2 then
local p2 = string.lower(string.char(byte(src, pos-2)))
if (p1 == "f" and p2 == "r") or (p1 == "r" and p2 == "f") then is_fstring = true end
end
end
local quote = b
local is_triple = (byte(src, pos+1) == quote and byte(src, pos+2) == quote)
if is_triple then pos = pos + 3 else pos = pos + 1 end
while pos <= len do
local cb = byte(src, pos)
if cb == 92 then pos = pos + 2 -- 略過跳脫字元
elseif cb == quote then
if is_triple then
if byte(src, pos+1) == quote and byte(src, pos+2) == quote then
pos = pos + 3; break
else pos = pos + 1 end
else pos = pos + 1; break end
elseif cb == 38 then -- 字串內部遇到 &
local s_idx, e_idx, m = string.find(src, "&([%a%d#]+);", pos)
if s_idx == pos then
local dec = decode_entity(m, true, is_fstring)
if dec then
if pos > last_pos then table.insert(res, string.sub(src, last_pos, pos - 1)) end
table.insert(res, dec)
pos = e_idx + 1
last_pos = pos
else pos = pos + 1 end
else pos = pos + 1 end
else pos = pos + 1 end
end
elseif b == 35 then -- 遇到 # 註解，略過直到換行
while pos <= len and byte(src, pos) ~= 10 and byte(src, pos) ~= 13 do
pos = pos + 1
end
elseif b == 38 then -- 外部程式碼遇到 &
local s_idx, e_idx, m = string.find(src, "&([%a%d#]+);", pos)
if s_idx == pos then
local dec = decode_entity(m, false, false)
if dec then
if pos > last_pos then table.insert(res, string.sub(src, last_pos, pos - 1)) end
table.insert(res, dec)
pos = e_idx + 1
last_pos = pos
else pos = pos + 1 end
else pos = pos + 1 end
else pos = pos + 1 end
end
if last_pos <= len then table.insert(res, string.sub(src, last_pos, len)) end
return table.concat(res)
end
function Lexer.tokenize(src)
src = unescape_source(src) -- 啟動安全還原
local tokens = {}
local token_count = 0
local pos = 1
local line = 1
local col = 1
local src_len = #src
local indent_stack = {0}
local indent_len = 1
local bracket_level = 0
local at_line_start = true
-- 引入 Runtime 供拋錯使用
local Runtime = require("Module:PythonScript/Runtime")
while pos <= src_len do
if at_line_start then
local current_indent = 0
while pos <= src_len do
local cb = byte(src, pos)
if cb == 32 then current_indent = current_indent + 1; pos = pos + 1; col = col + 1
elseif cb == 9 then current_indent = current_indent + 4; pos = pos + 1; col = col + 1
else break end
end
local peek = byte(src, pos)
if peek == 10 or peek == 13 or peek == 35 or not peek then
else
at_line_start = false
if bracket_level == 0 then
local last_indent = indent_stack[indent_len]
if current_indent > last_indent then
indent_len = indent_len + 1
indent_stack[indent_len] = current_indent
token_count = token_count + 1
tokens[token_count] = {type = "INDENT", value = "", line = line, col = col}
elseif current_indent < last_indent then
while indent_len > 1 and current_indent < indent_stack[indent_len] do
indent_len = indent_len - 1
token_count = token_count + 1
tokens[token_count] = {type = "DEDENT", value = "", line = line, col = col}
end
if current_indent ~= indent_stack[indent_len] then
Runtime.Traceback.current_line = line
Runtime.raise("IndentationError", "unindent does not match any outer indentation level")
end
end
end
end
end
if pos > src_len then break end
local b = byte(src, pos)
if b == 92 and (byte(src, pos+1) == 10 or (byte(src, pos+1) == 13 and byte(src, pos+2) == 10)) then
if byte(src, pos+1) == 13 then pos = pos + 3 else pos = pos + 2 end
line = line + 1; col = 1
elseif b == 10 or b == 13 then
if b == 13 and byte(src, pos+1) == 10 then pos = pos + 1 end
pos = pos + 1
if bracket_level == 0 and not at_line_start then
token_count = token_count + 1
tokens[token_count] = {type = "NEWLINE", value = "\n", line = line, col = col}
at_line_start = true
end
line = line + 1; col = 1
elseif b == 32 or b == 9 then pos = pos + 1; col = col + 1
elseif b == 35 then
while pos <= src_len do
local cb = byte(src, pos)
if cb == 10 or cb == 13 then break end
pos = pos + 1; col = col + 1
end
else
local is_quote = (b == 39 or b == 34)
local prefix_len = 0
if not is_quote and (b == 102 or b == 70 or b == 114 or b == 82 or b == 98 or b == 66 or b == 117 or b == 85 or b == 87 or b == 119) then
local next1 = byte(src, pos+1)
if next1 == 39 or next1 == 34 then prefix_len = 1
elseif next1 then
local next2 = byte(src, pos+2)
if next2 == 39 or next2 == 34 then
local chr_set = {f=true, r=true, b=true, w=true, u=true} -- 補回 u
local _checker = function(chr)
local ch = string.char(chr):lower()
local result = chr_set[ch]
chr_set[ch] = false -- 標記字已經用過 (修復：使用字串 ch 當 key)
return result == true -- 強制轉布林
end
if _checker(b) and _checker(next1) then prefix_len = 2 end
elseif next2 then
local next3 = byte(src, pos+3)
if next3 == 39 or next3 == 34 then
local chr_set = {f=true, r=true, b=true, w=true, u=true}
local _checker = function(chr)
local ch = string.char(chr):lower()
local result = chr_set[ch]
chr_set[ch] = false -- 標記字已經用過
return result == true
end
if _checker(b) and _checker(next1) and _checker(next2) then prefix_len = 3 end
elseif next3 then
-- 為了支援 w 加入後的極端 4 前綴組合 (例如 fwrb"...")，順手補上
local next4 = byte(src, pos+4)
if next4 == 39 or next4 == 34 then
local chr_set = {f=true, r=true, b=true, w=true, u=true}
local _checker = function(chr)
local ch = string.char(chr):lower()
local result = chr_set[ch]
chr_set[ch] = false
return result == true
end
if _checker(b) and _checker(next1) and _checker(next2) and _checker(next3) then prefix_len = 4 end
end
end
end
end
end
if is_quote or prefix_len > 0 then
local start_pos = pos
local is_fstring, is_raw, is_bytes, is_wiki = false, false, false, false
if prefix_len > 0 then
for i = 0, prefix_len - 1 do
local pc = byte(src, pos + i)
if pc == 102 or pc == 70 then is_fstring = true -- 識別 f / F
elseif pc == 114 or pc == 82 then is_raw = true -- 識別 r / R
elseif pc == 98 or pc == 66 then is_bytes = true -- 識別 b / B
elseif pc == 87 or pc == 119 then is_wiki = true end -- 識別 w / W
end
pos = pos + prefix_len; col = col + prefix_len; b = byte(src, pos)
end
local quote_byte = b
local is_triple = false
if byte(src, pos+1) == quote_byte and byte(src, pos+2) == quote_byte then is_triple = true; pos = pos + 3; col = col + 3
else pos = pos + 1; col = col + 1 end
while pos <= src_len do
local cb = byte(src, pos)
if cb == 92 then
if byte(src, pos+1) == 10 then line = line + 1; col = 0 end
pos = pos + 2; col = col + 2
elseif cb == 10 or cb == 13 then
if not is_triple then
Runtime.Traceback.current_line = line
Runtime.raise("SyntaxError", "EOL while scanning string literal")
end
if cb == 13 and byte(src, pos+1) == 10 then pos = pos + 1 end
pos = pos + 1; line = line + 1; col = 1
elseif cb == quote_byte then
if is_triple then
if byte(src, pos+1) == quote_byte and byte(src, pos+2) == quote_byte then pos = pos + 3; col = col + 3; break
else pos = pos + 1; col = col + 1 end
else pos = pos + 1; col = col + 1; break end
else pos = pos + 1; col = col + 1 end
end
token_count = token_count + 1
tokens[token_count] = { type = "STRING", value = sub(src, start_pos, pos - 1), is_fstring = is_fstring, is_raw = is_raw, is_bytes = is_bytes, is_wiki = is_wiki, line = line, col = col }
elseif (b >= 65 and b <= 90) or (b >= 97 and b <= 122) or b == 95 or b >= 128 then
local start_pos = pos
pos = pos + 1; col = col + 1
while pos <= src_len do
local cb = byte(src, pos)
if (cb >= 65 and cb <= 90) or (cb >= 97 and cb <= 122) or (cb >= 48 and cb <= 57) or cb == 95 or cb >= 128 then pos = pos + 1; col = col + 1
else break end
end
local val = sub(src, start_pos, pos - 1)
token_count = token_count + 1
tokens[token_count] = { type = KEYWORDS[val] and "KEYWORD" or "IDENTIFIER", value = val, line = line, col = col }
elseif (b >= 48 and b <= 57) or (b == 46 and byte(src, pos+1) and byte(src, pos+1) >= 48 and byte(src, pos+1) <= 57) then
local start_pos = pos
local is_complex, has_dot, has_e, is_base = false, (b == 46), false, false
if b == 48 then
local nb = byte(src, pos+1)
if nb == 120 or nb == 88 then is_base = true; pos = pos + 2; col = col + 2
while pos <= src_len do
local cb = byte(src, pos)
if (cb >= 48 and cb <= 57) or (cb >= 65 and cb <= 70) or (cb >= 97 and cb <= 102) then pos=pos+1; col=col+1 else break end
end
elseif nb == 111 or nb == 79 then is_base = true; pos = pos + 2; col = col + 2
while pos <= src_len do
local cb = byte(src, pos)
if cb >= 48 and cb <= 55 then pos=pos+1; col=col+1 else break end
end
elseif nb == 98 or nb == 66 then is_base = true; pos = pos + 2; col = col + 2
while pos <= src_len do
local cb = byte(src, pos)
if cb == 48 or cb == 49 then pos=pos+1; col=col+1 else break end
end
end
end
if not is_base then
while pos <= src_len do
local cb = byte(src, pos)
if cb >= 48 and cb <= 57 then pos = pos + 1; col = col + 1
elseif cb == 46 and not has_dot and not has_e then has_dot = true; pos = pos + 1; col = col + 1
elseif (cb == 101 or cb == 69) and not has_e then has_e = true; pos = pos + 1; col = col + 1
local ncb = byte(src, pos)
if ncb == 43 or ncb == 45 then pos = pos + 1; col = col + 1 end
-- 識別 j/J (106/74)、jk/JK 與 k/K (107/75) 四元數/複數後綴
elseif (cb == 106 or cb == 74) then
is_complex = true; pos = pos + 1; col = col + 1
local nb2 = byte(src, pos)
if nb2 == 107 or nb2 == 75 then pos = pos + 1; col = col + 1 end
break
elseif (cb == 107 or cb == 75) then
is_complex = true; pos = pos + 1; col = col + 1; break
else break end
end
end
token_count = token_count + 1
tokens[token_count] = { type = "NUMBER", value = sub(src, start_pos, pos - 1), is_complex = is_complex, line = line, col = col }
else
if b == 40 or b == 91 or b == 123 then bracket_level = bracket_level + 1
elseif b == 41 or b == 93 or b == 125 then bracket_level = bracket_level - 1 end
local is_op = false
if pos + 2 <= src_len then
if (b == 47 and byte(src, pos+1) == 47 and byte(src, pos+2) == 61) then token_count = token_count + 1; tokens[token_count] = {type="PUNCTUATION", value="//=", line=line, col=col}; pos = pos + 3; col = col + 3; is_op = true
elseif (b == 42 and byte(src, pos+1) == 42 and byte(src, pos+2) == 61) then token_count = token_count + 1; tokens[token_count] = {type="PUNCTUATION", value="**=", line=line, col=col}; pos = pos + 3; col = col + 3; is_op = true
elseif (b == 60 and byte(src, pos+1) == 60 and byte(src, pos+2) == 61) then token_count = token_count + 1; tokens[token_count] = {type="PUNCTUATION", value="<<=", line=line, col=col}; pos = pos + 3; col = col + 3; is_op = true
elseif (b == 62 and byte(src, pos+1) == 62 and byte(src, pos+2) == 61) then token_count = token_count + 1; tokens[token_count] = {type="PUNCTUATION", value=">>=", line=line, col=col}; pos = pos + 3; col = col + 3; is_op = true end
end
if not is_op and pos + 1 <= src_len then
local chunk2 = sub(src, pos, pos+1)
if OP2[chunk2] then token_count = token_count + 1; tokens[token_count] = {type="PUNCTUATION", value=chunk2, line=line, col=col}; pos = pos + 2; col = col + 2; is_op = true end
end
if not is_op then token_count = token_count + 1; tokens[token_count] = {type="PUNCTUATION", value=string.char(b), line=line, col=col}; pos = pos + 1; col = col + 1 end
end
end
end
if not at_line_start then token_count = token_count + 1; tokens[token_count] = {type = "NEWLINE", value = "\n", line = line, col = col} end
while indent_len > 1 do indent_len = indent_len - 1; token_count = token_count + 1; tokens[token_count] = {type = "DEDENT", value = "", line = line, col = col} end
token_count = token_count + 1; tokens[token_count] = {type = "EOF", value = "", line = line, col = col}
return tokens
end
return Lexer
