-- Module:PythonScript/Librarys/Codecs
local Codecs = {}
local PyClass = require("Module:沙盒/a2569875/PythonScript/Class")
local Runtime = require("Module:沙盒/a2569875/PythonScript/Runtime")
local function normalize_encoding(enc)
enc = tostring(enc or "utf-8"):lower():gsub("_", "-"):gsub(" ", "-")
if enc == "utf8" or enc == "u8" or enc == "utf" or enc == "utf-8-sig" then return "utf-8" end
if enc == "ascii" or enc == "us-ascii" then return "ascii" end
if enc == "latin1" or enc == "latin-1" or enc == "iso-8859-1" then return "latin-1" end
return enc
end
local error_handlers = {}
function Codecs.register_error(name, handler)
if type(name) ~= "string" then Runtime.raise("TypeError", "register_error() argument 1 must be str") end
error_handlers[name] = handler
end
function Codecs.lookup_error(name)
local h = error_handlers[name]
if not h then Runtime.raise("LookupError", "unknown error handler name '" .. tostring(name) .. "'") end
return h
end
-- ==========================================
-- 1. 內建錯誤處理器 (回傳: 替換字串, 1-based 下一個 Lua 索引)
-- ==========================================
error_handlers["strict"] = function(exc)
Runtime.raise(exc)
end
error_handlers["ignore"] = function(exc)
return "", exc.__dict._lua_end
end
error_handlers["replace"] = function(exc)
if exc.__class.__name == "UnicodeDecodeError" then
return "\239\191\189", exc.__dict._lua_end -- U+FFFD () 的標準 UTF-8 bytes
else
return "?", exc.__dict._lua_end
end
end
error_handlers["backslashreplace"] = function(exc)
local out = {}
local data = exc.__dict._raw_data
if exc.__class.__name == "UnicodeDecodeError" then
for i = exc.__dict._lua_start, exc.__dict._lua_end - 1 do
table.insert(out, string.format("\\x%02x", string.byte(data, i)))
end
else
local cp = mw.ustring.codepoint(data, exc.__dict._lua_start)
if cp <= 0xFF then table.insert(out, string.format("\\x%02x", cp))
elseif cp <= 0xFFFF then table.insert(out, string.format("\\u%04x", cp))
else table.insert(out, string.format("\\U%08x", cp)) end
end
return table.concat(out), exc.__dict._lua_end
end
error_handlers["xmlcharrefreplace"] = function(exc)
if exc.__class.__name == "UnicodeDecodeError" then
Runtime.raise("TypeError", "don't know how to handle UnicodeDecodeError in error callback")
end
local cp = mw.ustring.codepoint(exc.__dict._raw_data, exc.__dict._lua_start)
return string.format("&#%d;", cp), exc.__dict._lua_end
end
error_handlers["surrogateescape"] = function(exc)
local data = exc.__dict._raw_data
if exc.__class.__name == "UnicodeDecodeError" then
local out = {}
for i = exc.__dict._lua_start, exc.__dict._lua_end - 1 do
local b = string.byte(data, i)
local cp = 0xDC00 + b
out[#out + 1] = string.char(0xED, 0xB0 + math.floor((cp - 0xD800) / 64), 0x80 + (cp % 64))
end
return table.concat(out), exc.__dict._lua_end
else
local cp = mw.ustring.codepoint(data, exc.__dict._lua_start)
if cp >= 0xDC80 and cp <= 0xDCFF then
return string.char(cp - 0xDC00), exc.__dict._lua_end
end
Runtime.raise(exc)
end
end
-- ==========================================
-- 2. 例外建構與自定義處理器路由
-- ==========================================
-- 注意：start_idx 與 end_idx 傳入時為 Lua 1-based，對外屬性 (start/end/start_idx/end_idx) 統一轉為 Python 0-based！
local function build_decode_exc(encoding, raw_data, start_idx, end_idx, reason)
local exc = PyClass.make_instance(Runtime.builtins.UnicodeDecodeError)
local py_bytes = { __type = "bytes", _data = raw_data }
local s0, e0 = start_idx - 1, end_idx - 1
exc.__dict.encoding = encoding
exc.__dict.object = py_bytes
exc.__dict.start = s0
exc.__dict["end"] = e0
exc.__dict.start_idx = s0
exc.__dict.end_idx = e0
exc.__dict._raw_data = raw_data
exc.__dict._lua_start = start_idx
exc.__dict._lua_end = end_idx
exc.__dict.reason = reason
local safe_enc = encoding:sub(1, 400)
exc.__dict.msg = string.format("'%s' codec can't decode byte 0x%02x in position %d: %s", safe_enc, string.byte(raw_data, start_idx), s0, reason)
exc.__dict.args = { __type = "tuple", elements = { encoding, py_bytes, s0, e0, reason } }
return exc
end
local function build_encode_exc(encoding, raw_str, start_idx, end_idx, reason)
local exc = PyClass.make_instance(Runtime.builtins.UnicodeEncodeError)
local s0, e0 = start_idx - 1, end_idx - 1
exc.__dict.encoding = encoding
exc.__dict.object = raw_str
exc.__dict.start = s0
exc.__dict["end"] = e0
exc.__dict.start_idx = s0
exc.__dict.end_idx = e0
exc.__dict._raw_data = raw_str
exc.__dict._lua_start = start_idx
exc.__dict._lua_end = end_idx
exc.__dict.reason = reason
local cp = mw.ustring.codepoint(raw_str, start_idx)
local cp_hex = string.format(cp <= 0xFFFF and "\\u%04x" or "\\U%08x", cp)
local safe_enc = encoding:sub(1, 400)
exc.__dict.msg = string.format("'%s' codec can't encode character '%s' in position %d: %s", safe_enc, cp_hex, s0, reason)
exc.__dict.args = { __type = "tuple", elements = { encoding, raw_str, s0, e0, reason } }
return exc
end
local function invoke_error_handler(errors, exc)
local h = error_handlers[errors]
if not h then
Runtime.raise("LookupError", "unknown error handler name '" .. tostring(errors) .. "'")
end
if type(h) == "function" then
return h(exc)
else
local Evaluator = require("Module:沙盒/a2569875/PythonScript/Evaluator")
local ret = Evaluator.invoke(h, { exc }, {})
if type(ret) ~= "table" or ret.__type ~= "tuple" or #ret.elements ~= 2 then
Runtime.raise("TypeError", "encoding/decoding error handler must return (str/bytes, int) tuple")
end
local repl = ret.elements[1]
if type(repl) == "table" and repl.__type == "bytes" then repl = repl._data end
local new_pos_0 = tonumber(ret.elements[2]) or exc.__dict.end_idx
return tostring(repl), new_pos_0 + 1
end
end
-- ==========================================
-- 3. UTF-8 / ASCII / Latin-1 狀態機
-- ==========================================
local function decode_utf8(data, errors, strip_bom)
local i, len = 1, #data
if strip_bom and len >= 3 and data:sub(1, 3) == "\239\187\191" then
i = 4
end
local out = {}
while i <= len do
local b1 = string.byte(data, i)
if b1 <= 0x7F then
out[#out + 1] = string.char(b1); i = i + 1
elseif b1 >= 0xC2 and b1 <= 0xDF and i + 1 <= len then
local b2 = string.byte(data, i + 1)
if b2 >= 0x80 and b2 <= 0xBF then
out[#out + 1] = string.char(b1, b2); i = i + 2
else
local exc = build_decode_exc("utf-8", data, i, i + 1, "invalid continuation byte")
local repl, next_i = invoke_error_handler(errors, exc)
out[#out + 1] = repl; i = next_i
end
elseif b1 >= 0xE0 and b1 <= 0xEF and i + 2 <= len then
local b2, b3 = string.byte(data, i + 1), string.byte(data, i + 2)
local valid = false
if b1 == 0xE0 and b2 >= 0xA0 and b2 <= 0xBF and b3 >= 0x80 and b3 <= 0xBF then valid = true
elseif b1 >= 0xE1 and b1 <= 0xEC and b2 >= 0x80 and b2 <= 0xBF and b3 >= 0x80 and b3 <= 0xBF then valid = true
elseif b1 == 0xED and b2 >= 0x80 and b2 <= 0x9F and b3 >= 0x80 and b3 <= 0xBF then valid = true
elseif b1 >= 0xEE and b1 <= 0xEF and b2 >= 0x80 and b2 <= 0xBF and b3 >= 0x80 and b3 <= 0xBF then valid = true end
if valid then
out[#out + 1] = string.char(b1, b2, b3); i = i + 3
else
local err_len = ((b1 == 0xE0 and b2 >= 0xA0 and b2 <= 0xBF) or (b1 >= 0xE1 and b1 <= 0xEC and b2 >= 0x80 and b2 <= 0xBF) or (b1 == 0xED and b2 >= 0x80 and b2 <= 0x9F) or (b1 >= 0xEE and b1 <= 0xEF and b2 >= 0x80 and b2 <= 0xBF)) and 2 or 1
local exc = build_decode_exc("utf-8", data, i, i + err_len, "invalid continuation byte")
local repl, next_i = invoke_error_handler(errors, exc)
out[#out + 1] = repl; i = next_i
end
elseif b1 >= 0xF0 and b1 <= 0xF4 and i + 3 <= len then
local b2, b3, b4 = string.byte(data, i + 1), string.byte(data, i + 2), string.byte(data, i + 3)
local valid = false
if b1 == 0xF0 and b2 >= 0x90 and b2 <= 0xBF and b3 >= 0x80 and b3 <= 0xBF and b4 >= 0x80 and b4 <= 0xBF then valid = true
elseif b1 >= 0xF1 and b1 <= 0xF3 and b2 >= 0x80 and b2 <= 0xBF and b3 >= 0x80 and b3 <= 0xBF and b4 >= 0x80 and b4 <= 0xBF then valid = true
elseif b1 == 0xF4 and b2 >= 0x80 and b2 <= 0x8F and b3 >= 0x80 and b3 <= 0xBF and b4 >= 0x80 and b4 <= 0xBF then valid = true end
if valid then
out[#out + 1] = string.char(b1, b2, b3, b4); i = i + 4
else
local err_len = 1
if (b1 == 0xF0 and b2 >= 0x90 and b2 <= 0xBF) or (b1 >= 0xF1 and b1 <= 0xF3 and b2 >= 0x80 and b2 <= 0xBF) or (b1 == 0xF4 and b2 >= 0x80 and b2 <= 0x8F) then
err_len = (b3 >= 0x80 and b3 <= 0xBF) and 3 or 2
end
local exc = build_decode_exc("utf-8", data, i, i + err_len, "invalid continuation byte")
local repl, next_i = invoke_error_handler(errors, exc)
out[#out + 1] = repl; i = next_i
end
else
local reason = (b1 >= 0xC2 and b1 <= 0xF4) and "unexpected end of data" or "invalid start byte"
local exc = build_decode_exc("utf-8", data, i, i + 1, reason)
local repl, next_i = invoke_error_handler(errors, exc)
out[#out + 1] = repl; i = next_i
end
end
return table.concat(out)
end
local function encode_utf8(str, errors, add_bom)
local out = add_bom and { "\239\187\191" } or {}
local len, i = mw.ustring.len(str), 1
while i <= len do
local cp = mw.ustring.codepoint(str, i)
if cp >= 0xD800 and cp <= 0xDFFF then
local exc = build_encode_exc("utf-8", str, i, i + 1, "surrogates not allowed")
local repl, next_i = invoke_error_handler(errors, exc)
out[#out + 1] = repl; i = next_i
else
out[#out + 1] = mw.ustring.char(cp); i = i + 1
end
end
return table.concat(out)
end
local function decode_ascii(data, errors)
local out, len, i = {}, #data, 1
while i <= len do
local b = string.byte(data, i)
if b <= 0x7F then
out[#out + 1] = string.char(b); i = i + 1
else
local exc = build_decode_exc("ascii", data, i, i + 1, "ordinal not in range(128)")
local repl, next_i = invoke_error_handler(errors, exc)
out[#out + 1] = repl; i = next_i
end
end
return table.concat(out)
end
local function encode_ascii(str, errors)
local out, len, i = {}, mw.ustring.len(str), 1
while i <= len do
local cp = mw.ustring.codepoint(str, i)
if cp <= 0x7F then
out[#out + 1] = string.char(cp); i = i + 1
else
local exc = build_encode_exc("ascii", str, i, i + 1, "ordinal not in range(128)")
local repl, next_i = invoke_error_handler(errors, exc)
out[#out + 1] = repl; i = next_i
end
end
return table.concat(out)
end
local function decode_latin1(data)
local out = {}
for i = 1, #data do
local b = string.byte(data, i)
if b <= 0x7F then out[i] = string.char(b)
else out[i] = string.char(0xC0 + math.floor(b / 64), 0x80 + (b % 64)) end
end
return table.concat(out)
end
local function encode_latin1(str, errors)
local out, len, i = {}, mw.ustring.len(str), 1
while i <= len do
local cp = mw.ustring.codepoint(str, i)
if cp <= 0xFF then
out[#out + 1] = string.char(cp); i = i + 1
else
local exc = build_encode_exc("latin-1", str, i, i + 1, "ordinal not in range(256)")
local repl, next_i = invoke_error_handler(errors, exc)
out[#out + 1] = repl; i = next_i
end
end
return table.concat(out)
end
-- ==========================================
-- 4. 對外核心 API 與模組建構
-- ==========================================
function Codecs.lua_decode(obj, encoding, errors)
encoding = encoding or "utf-8"
errors = errors or "strict"
if type(obj) == "table" and obj.__type == "instance" and obj.__builtin_value ~= nil then obj = obj.__builtin_value end
if type(obj) == "table" and (obj.__type == "bytes" or obj.__type == "bytearray" or obj.__type == "memoryview") then
if obj.__type == "memoryview" and obj._released then
Runtime.raise("ValueError", "operation forbidden on released memoryview object")
end
obj = obj._data
end
if type(obj) ~= "string" then Runtime.raise("TypeError", "a bytes-like object is required") end
local raw_enc = tostring(encoding):lower():gsub("_", "-")
local is_sig = (raw_enc == "utf-8-sig")
local enc = normalize_encoding(encoding)
if enc == "utf-8" then return decode_utf8(obj, errors, is_sig)
elseif enc == "ascii" then return decode_ascii(obj, errors)
elseif enc == "latin-1" then return decode_latin1(obj)
elseif enc == "base64" or enc == "base64-codec" then
return require("Module:沙盒/a2569875/PythonScript/Librarys/StdAlgorithms").base64().raw_b64decode(obj, nil, false)
elseif enc == "hex" or enc == "hex-codec" then
return require("Module:沙盒/a2569875/PythonScript/Librarys/StdAlgorithms").base64().raw_b16decode(obj, true)
end
Runtime.raise("LookupError", "unknown encoding: " .. tostring(encoding))
end
function Codecs.lua_encode(obj, encoding, errors)
encoding = encoding or "utf-8"
errors = errors or "strict"
if type(obj) == "table" and obj.__type == "instance" and obj.__builtin_value ~= nil then obj = obj.__builtin_value end
if type(obj) ~= "string" then Runtime.raise("TypeError", "encoding with '" .. tostring(encoding) .. "' codec failed") end
local raw_enc = tostring(encoding):lower():gsub("_", "-")
local is_sig = (raw_enc == "utf-8-sig")
local enc, res = normalize_encoding(encoding), nil
if enc == "utf-8" then res = encode_utf8(obj, errors, is_sig)
elseif enc == "ascii" then res = encode_ascii(obj, errors)
elseif enc == "latin-1" then res = encode_latin1(obj, errors)
elseif enc == "base64" or enc == "base64-codec" then
res = require("Module:沙盒/a2569875/PythonScript/Librarys/StdAlgorithms").base64().raw_b64encode(obj)
elseif enc == "hex" or enc == "hex-codec" then
res = require("Module:沙盒/a2569875/PythonScript/Librarys/StdAlgorithms").base64().raw_b16encode(obj):lower()
else
Runtime.raise("LookupError", "unknown encoding: " .. tostring(encoding))
end
return { __type = "bytes", _data = res }
end
function Codecs.get_module()
local mod = PyClass.make_instance(PyClass.make_class("module", {}, {}))
local dict = {
encode = { __type = "builtin_method", __name = "encode", __call = function(a, kw)
local offset = (a[1] == mod) and 1 or 0
local obj = a[offset + 1]
local encoding = (a[offset + 2] ~= nil) and a[offset + 2] or (kw and kw.encoding) or "utf-8"
local errors = (a[offset + 3] ~= nil) and a[offset + 3] or (kw and kw.errors) or "strict"
return Codecs.lua_encode(obj, encoding, errors)
end },
decode = { __type = "builtin_method", __name = "decode", __call = function(a, kw)
local offset = (a[1] == mod) and 1 or 0
local obj = a[offset + 1]
local encoding = (a[offset + 2] ~= nil) and a[offset + 2] or (kw and kw.encoding) or "utf-8"
local errors = (a[offset + 3] ~= nil) and a[offset + 3] or (kw and kw.errors) or "strict"
return Codecs.lua_decode(obj, encoding, errors)
end },
lookup_error = { __type = "builtin_method", __name = "lookup_error", __call = function(a)
local offset = (a[1] == mod) and 1 or 0
return Codecs.lookup_error(a[offset + 1])
end },
register_error = { __type = "builtin_method", __name = "register_error", __call = function(a)
local offset = (a[1] == mod) and 1 or 0
Codecs.register_error(a[offset + 1], a[offset + 2])
return Runtime.builtins.None
end },
BOM_UTF8 = { __type = "bytes", _data = "\239\187\191" },
BOM_LE = { __type = "bytes", _data = "\255\254" },
BOM_BE = { __type = "bytes", _data = "\254\255" }
}
-- 掛載於實例 __dict 上，與 base64 等模組一致
for k, v in pairs(dict) do mod.__dict[k] = v end
mod.__dict["__name__"] = "codecs"
mod.__dict["__all__"] = { __type = "list", elements = { "encode", "decode", "lookup_error", "register_error", "BOM_UTF8", "BOM_LE", "BOM_BE" } }
return mod
end
return { --保留未來其它模組併入的可能性
Codecs=Codecs
}
