[Space Lua] Implement string.pack, string.unpack and string.packsize (#1851)

The three binary-packing functions from Lua are now implemented in
a dedicated `string_pack.ts` module and wired into `stringApi`.

Format parsing follows the exact option set of Lua:

* signed and unsigned integers of configurable width
  (`b`/`B`/`h`/`H`/`l`/`L`/`j`/`J`/`T`/`i`/`I`),
* IEEE 754 single and double precision floats (`f`/`d`/`n`),
* fixed-length strings (`c`),
* length-prefixed strings (`s`),
* zero-terminated strings (`z`),
* padding bytes (`x`),
* alignment padding (`X`), and
* endianness/alignment modifiers (`<`/`>`/`=`/`!`).

`string.packsize` rejects variable-length formats (`s` and `z`) as Lua
requires.

`string.unpack` returns all unpacked values followed by the
next position integer, matching the Lua contract exactly.

Test suite extended in separates `string_pack_test.lua` file. Passes
both Lua and Space Lua execution.

Signed-off-by: Matouš Jan Fialka <mjf@mjf.cz>
This commit is contained in:
Matouš Jan Fialka
2026-02-25 09:29:14 +01:00
committed by GitHub
parent cebc834469
commit fd4ff8518d
4 changed files with 682 additions and 0 deletions
+6
View File
@@ -16,6 +16,7 @@ import {
patternGsub,
patternMatch,
} from "./pattern.ts";
import { strPackFn, strPackSizeFn, strUnpackFn } from "./string_pack.ts";
function capturesToLua(caps: CaptureResult[]): any {
if (caps.length === 0) {return null;}
@@ -162,10 +163,15 @@ export const stringApi = new LuaTable({
}
return "";
}),
split: new LuaBuiltinFunction((_sf, s: string, sep: string) => {
return s.split(sep);
}),
pack: strPackFn,
unpack: strUnpackFn,
packsize: strPackSizeFn,
// Non-standard extensions
startsWith: new LuaBuiltinFunction((_sf, s: string, prefix: string) => {
return s.startsWith(prefix);
+486
View File
@@ -0,0 +1,486 @@
import {
LuaBuiltinFunction,
LuaMultiRes,
LuaRuntimeError,
} from "../runtime.ts";
import { isTaggedFloat } from "../numeric.ts";
function untagN(x: any): number {
if (typeof x === "number") return x;
if (isTaggedFloat(x)) return x.value;
return Number(x);
}
const NATIVE_LITTLE = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1;
const NATIVE_MAXALIGN = 8; // JS doubles are 8-byte aligned
type KOption =
| "int"
| "uint"
| "float"
| "double"
| "number"
| "char"
| "string"
| "zstr"
| "padding"
| "paddalign"
| "nop";
interface ParsedOption {
opt: KOption;
size: number; // byte width
ntoalign: number; // padding bytes before this field
}
interface Header {
islittle: boolean;
maxalign: number;
}
function makeHeader(): Header {
return { islittle: NATIVE_LITTLE, maxalign: NATIVE_MAXALIGN };
}
// Read digits from fmt starting at pos; return [value, newPos]
function readNum(fmt: string, pos: number, dflt: number): [number, number] {
if (pos >= fmt.length || fmt[pos] < "0" || fmt[pos] > "9") return [dflt, pos];
let v = 0;
while (pos < fmt.length && fmt[pos] >= "0" && fmt[pos] <= "9") {
v = v * 10 + (fmt.charCodeAt(pos) - 48);
pos++;
}
return [v, pos];
}
function numLimit(
fmt: string,
pos: number,
dflt: number,
src: string,
): [number, number] {
const [sz, np] = readNum(fmt, pos, dflt);
if (sz < 1 || sz > 16) {
throw new Error(`integral size (${sz}) out of limits [1,16] in '${src}'`);
}
return [sz, np];
}
// Parse one option from fmt[pos], return [parsed, newPos]
// Modifies header in place for '<', '>', '=', '!'
function getOption(
fmt: string,
pos: number,
h: Header,
): [KOption, number, number] { // [opt, size, newPos]
const c = fmt[pos++];
switch (c) {
case "b":
return ["int", 1, pos];
case "B":
return ["uint", 1, pos];
case "h":
return ["int", 2, pos];
case "H":
return ["uint", 2, pos];
case "l":
return ["int", 8, pos];
case "L":
return ["uint", 8, pos];
case "j":
return ["int", 8, pos];
case "J":
return ["uint", 8, pos];
case "T":
return ["uint", 8, pos];
case "f":
return ["float", 4, pos];
case "n":
return ["number", 8, pos];
case "d":
return ["double", 8, pos];
case "i": {
const [sz, np] = numLimit(fmt, pos, 4, "i");
return ["int", sz, np];
}
case "I": {
const [sz, np] = numLimit(fmt, pos, 4, "I");
return ["uint", sz, np];
}
case "s": {
const [sz, np] = numLimit(fmt, pos, 8, "s");
return ["string", sz, np];
}
case "c": {
const [sz, np] = readNum(fmt, pos, -1);
if (sz === -1) throw new Error("missing size for format option 'c'");
return ["char", sz, np];
}
case "z":
return ["zstr", 0, pos];
case "x":
return ["padding", 1, pos];
case "X":
return ["paddalign", 0, pos];
case " ":
return ["nop", 0, pos];
case "<":
h.islittle = true;
return ["nop", 0, pos];
case ">":
h.islittle = false;
return ["nop", 0, pos];
case "=":
h.islittle = NATIVE_LITTLE;
return ["nop", 0, pos];
case "!": {
const [sz, np] = readNum(fmt, pos, NATIVE_MAXALIGN);
h.maxalign = sz;
return ["nop", 0, np];
}
default:
throw new Error(`invalid format option '${c}'`);
}
}
// Compute alignment padding
function getDetails(
fmt: string,
pos: number,
h: Header,
totalsize: number,
): [ParsedOption, number] {
let opt: KOption, size: number;
[opt, size, pos] = getOption(fmt, pos, h);
let align = size;
if (opt === "paddalign") {
if (pos >= fmt.length) {
throw new Error("invalid next option for option 'X'");
}
const hCopy = { ...h };
let nextOpt: KOption, nextSize: number;
[nextOpt, nextSize, pos] = getOption(fmt, pos, hCopy);
if (nextOpt === "char" || nextSize === 0) {
throw new Error("invalid next option for option 'X'");
}
align = nextSize;
}
let ntoalign = 0;
if (
opt !== "char" && opt !== "nop" && opt !== "padding" && opt !== "paddalign"
) {
const realign = Math.min(align, h.maxalign);
if (realign > 0) {
ntoalign = (realign - (totalsize % realign)) % realign;
}
}
return [{ opt, size, ntoalign }, pos];
}
function packInt(v: bigint, size: number, islittle: boolean): Uint8Array {
const buf = new Uint8Array(size);
let val = v;
// Two's complement mask
const mask = (1n << BigInt(size * 8)) - 1n;
val = ((val % (mask + 1n)) + (mask + 1n)) & mask; // normalise to unsigned
for (let i = 0; i < size; i++) {
buf[islittle ? i : size - 1 - i] = Number(val & 0xffn);
val >>= 8n;
}
return buf;
}
function unpackInt(
buf: Uint8Array,
pos: number,
size: number,
islittle: boolean,
issigned: boolean,
): bigint {
let res = 0n;
const limit = Math.min(size, 8);
for (let i = limit - 1; i >= 0; i--) {
res = (res << 8n) | BigInt(buf[pos + (islittle ? i : size - 1 - i)]);
}
if (issigned && size <= 8) {
const mask = 1n << BigInt(size * 8 - 1);
if (res & mask) res -= mask << 1n;
}
return res;
}
function packFloat32(v: number, islittle: boolean): Uint8Array {
const buf = new ArrayBuffer(4);
new DataView(buf).setFloat32(0, v, islittle);
return new Uint8Array(buf);
}
function unpackFloat32(
buf: Uint8Array,
pos: number,
islittle: boolean,
): number {
return new DataView(buf.buffer, buf.byteOffset + pos, 4).getFloat32(
0,
islittle,
);
}
function packFloat64(v: number, islittle: boolean): Uint8Array {
const buf = new ArrayBuffer(8);
new DataView(buf).setFloat64(0, v, islittle);
return new Uint8Array(buf);
}
function unpackFloat64(
buf: Uint8Array,
pos: number,
islittle: boolean,
): number {
return new DataView(buf.buffer, buf.byteOffset + pos, 8).getFloat64(
0,
islittle,
);
}
export const strPackFn = new LuaBuiltinFunction(
(sf, fmt: string, ...args: any[]) => {
const h = makeHeader();
const parts: Uint8Array[] = [];
let totalsize = 0;
let argIdx = 0;
let pos = 0;
while (pos < fmt.length) {
let opt: ParsedOption;
[opt, pos] = getDetails(fmt, pos, h, totalsize);
// alignment padding
if (opt.ntoalign > 0) {
parts.push(new Uint8Array(opt.ntoalign));
totalsize += opt.ntoalign;
}
switch (opt.opt) {
case "nop":
case "paddalign":
break;
case "padding":
parts.push(new Uint8Array(1)); // LUAL_PACKPADBYTE = 0
totalsize += 1;
break;
case "int":
case "uint": {
const v = args[argIdx++];
if (v === undefined || v === null) {
throw new LuaRuntimeError(
`bad argument #${argIdx} to 'pack' (value expected)`,
sf,
);
}
let bi: bigint;
if (typeof v === "bigint") bi = v;
else bi = BigInt(Math.trunc(untagN(v)));
parts.push(packInt(bi, opt.size, h.islittle));
totalsize += opt.size;
break;
}
case "float": {
const v = untagN(args[argIdx++]);
parts.push(packFloat32(v, h.islittle));
totalsize += 4;
break;
}
case "double":
case "number": {
const v = untagN(args[argIdx++]);
parts.push(packFloat64(v, h.islittle));
totalsize += 8;
break;
}
case "char": {
const s: string = String(args[argIdx++]);
const enc = new TextEncoder().encode(s);
const buf = new Uint8Array(opt.size);
buf.set(enc.subarray(0, opt.size));
parts.push(buf);
totalsize += opt.size;
break;
}
case "string": {
const s: string = String(args[argIdx++]);
const enc = new TextEncoder().encode(s);
const lenBuf = packInt(BigInt(enc.length), opt.size, h.islittle);
parts.push(lenBuf);
parts.push(enc);
totalsize += opt.size + enc.length;
break;
}
case "zstr": {
const s: string = String(args[argIdx++]);
if (s.includes("\0")) {
throw new LuaRuntimeError(
"string contains zeros for format 'z'",
sf,
);
}
const enc = new TextEncoder().encode(s);
parts.push(enc);
parts.push(new Uint8Array(1)); // null terminator
totalsize += enc.length + 1;
break;
}
}
}
// Concatenate all parts into one binary string (latin-1 encoding)
let total = 0;
for (const p of parts) total += p.length;
const out = new Uint8Array(total);
let off = 0;
for (const p of parts) {
out.set(p, off);
off += p.length;
}
// Return as a Lua binary string (each byte is a char code 0-255)
let result = "";
for (let i = 0; i < out.length; i++) result += String.fromCharCode(out[i]);
return result;
},
);
export const strUnpackFn = new LuaBuiltinFunction(
(sf, fmt: string, data: string, init?: number) => {
const h = makeHeader();
const buf = new Uint8Array(data.length);
for (let i = 0; i < data.length; i++) {
buf[i] = data.charCodeAt(i) & 0xff;
}
let pos = (init !== undefined && init !== null ? init : 1) - 1;
const results: any[] = [];
let fmtPos = 0;
while (fmtPos < fmt.length) {
let opt: ParsedOption;
[opt, fmtPos] = getDetails(fmt, fmtPos, h, pos);
if (opt.ntoalign + opt.size > buf.length - pos) {
if (
opt.opt !== "nop" && opt.opt !== "paddalign" && opt.opt !== "padding"
) {
throw new LuaRuntimeError("data string too short", sf);
}
}
pos += opt.ntoalign; // skip alignment padding
switch (opt.opt) {
case "nop":
case "paddalign":
break;
case "padding":
pos += 1;
break;
case "int": {
const v = unpackInt(buf, pos, opt.size, h.islittle, true);
const n = Number(v);
results.push(Number.isSafeInteger(n) ? n : v);
pos += opt.size;
break;
}
case "uint": {
const v = unpackInt(buf, pos, opt.size, h.islittle, false);
const n = Number(v);
results.push(Number.isSafeInteger(n) ? n : v);
pos += opt.size;
break;
}
case "float": {
results.push(unpackFloat32(buf, pos, h.islittle));
pos += 4;
break;
}
case "double":
case "number": {
results.push(unpackFloat64(buf, pos, h.islittle));
pos += 8;
break;
}
case "char": {
const s = String.fromCharCode(...buf.subarray(pos, pos + opt.size));
results.push(s);
pos += opt.size;
break;
}
case "string": {
const len = Number(unpackInt(buf, pos, opt.size, h.islittle, false));
if (len > buf.length - pos - opt.size) {
throw new LuaRuntimeError("data string too short", sf);
}
pos += opt.size;
const s = new TextDecoder().decode(buf.subarray(pos, pos + len));
results.push(s);
pos += len;
break;
}
case "zstr": {
let end = pos;
while (end < buf.length && buf[end] !== 0) end++;
if (end >= buf.length) {
throw new LuaRuntimeError("unfinished string for format 'z'", sf);
}
results.push(new TextDecoder().decode(buf.subarray(pos, end)));
pos = end + 1;
break;
}
}
}
results.push(pos + 1);
return new LuaMultiRes(results);
},
);
export const strPackSizeFn = new LuaBuiltinFunction(
(_sf, fmt: string) => {
const h = makeHeader();
let totalsize = 0;
let pos = 0;
while (pos < fmt.length) {
let opt: ParsedOption;
[opt, pos] = getDetails(fmt, pos, h, totalsize);
if (opt.opt === "string" || opt.opt === "zstr") {
throw new LuaRuntimeError("variable-length format", _sf);
}
totalsize += opt.ntoalign + opt.size;
}
return totalsize;
},
);
@@ -0,0 +1,186 @@
local function assertEquals(a, b, msg)
if a ~= b then
error((msg or "assertEquals") .. ": expected " .. tostring(b) .. " got " .. tostring(a))
end
end
local function assertError(fn, msg)
local ok, err = pcall(fn)
if ok then error(msg or "expected error") end
end
-- packsize: fixed formats
do
assertEquals(string.packsize("b"), 1, "packsize b")
assertEquals(string.packsize("B"), 1, "packsize B")
assertEquals(string.packsize("h"), 2, "packsize h")
assertEquals(string.packsize("H"), 2, "packsize H")
assertEquals(string.packsize("i4"), 4, "packsize i4")
assertEquals(string.packsize("I4"), 4, "packsize I4")
assertEquals(string.packsize("i8"), 8, "packsize i8")
assertEquals(string.packsize("f"), 4, "packsize f")
assertEquals(string.packsize("d"), 8, "packsize d")
assertEquals(string.packsize("c10"), 10, "packsize c10")
assertEquals(string.packsize("i4i4"), 8, "packsize i4i4")
-- variable-length formats must error
assertError(function() string.packsize("s4") end, "packsize s4 must error")
assertError(function() string.packsize("z") end, "packsize z must error")
end
-- little-endian integers
do
local s = string.pack("<i4", 1)
assertEquals(#s, 4, "pack i4 length")
assertEquals(s:byte(1), 1, "pack <i4(1) byte 1")
assertEquals(s:byte(2), 0, "pack <i4(1) byte 2")
assertEquals(s:byte(3), 0, "pack <i4(1) byte 3")
assertEquals(s:byte(4), 0, "pack <i4(1) byte 4")
local v = string.unpack("<i4", s)
assertEquals(v, 1, "unpack <i4")
local s2 = string.pack("<i4", -1)
local v2 = string.unpack("<i4", s2)
assertEquals(v2, -1, "unpack <i4 negative")
local s3 = string.pack("<I4", 0xDEADBEEF)
local v3 = string.unpack("<I4", s3)
assertEquals(v3, 0xDEADBEEF, "unpack <I4 0xDEADBEEF")
end
-- big-endian integers
do
local s = string.pack(">i2", 256)
assertEquals(s:byte(1), 1, "pack >i2(256) byte 1")
assertEquals(s:byte(2), 0, "pack >i2(256) byte 2")
local v = string.unpack(">i2", s)
assertEquals(v, 256, "unpack >i2")
local s2 = string.pack(">I2", 0xABCD)
local v2 = string.unpack(">I2", s2)
assertEquals(v2, 0xABCD, "unpack >I2")
end
-- byte / unsigned byte
do
local s = string.pack("BB", 65, 66)
assertEquals(#s, 2, "pack BB length")
local a, b, _ = string.unpack("BB", s)
assertEquals(a, 65, "unpack B first")
assertEquals(b, 66, "unpack B second")
-- signed byte: -1 round-trips
local s2 = string.pack("b", -1)
local v2 = string.unpack("b", s2)
assertEquals(v2, -1, "unpack b -1")
end
-- float (f) and double (d)
do
local s = string.pack("<f", 1.0)
assertEquals(#s, 4, "pack f length")
local v = string.unpack("<f", s)
-- float has less precision; allow small epsilon
local diff = v - 1.0
if diff < 0 then diff = -diff end
assert(diff < 1e-6, "unpack f ~= 1.0")
local s2 = string.pack("<d", 3.14)
local v2 = string.unpack("<d", s2)
local diff2 = v2 - 3.14
if diff2 < 0 then diff2 = -diff2 end
assert(diff2 < 1e-12, "unpack d ~= 3.14")
end
-- fixed-length string (c)
do
local s = string.pack("c5", "hello")
assertEquals(#s, 5, "pack c5 length")
local v, _ = string.unpack("c5", s)
assertEquals(v, "hello", "unpack c5")
-- shorter string: rest is zero-padded
local s2 = string.pack("c5", "hi")
assertEquals(#s2, 5, "pack c5 short length")
local v2, _ = string.unpack("c5", s2)
assertEquals(v2:sub(1,2), "hi", "unpack c5 short prefix")
end
-- length-prefixed string (s)
do
local s = string.pack("<s4", "world")
assertEquals(#s, 4 + 5, "pack s4 length")
local v, nxt = string.unpack("<s4", s)
assertEquals(v, "world", "unpack s4 value")
assertEquals(nxt, #s + 1, "unpack s4 next pos")
end
-- zero-terminated string (z)
do
local s = string.pack("z", "lua")
assertEquals(#s, 4, "pack z length (with null)")
assertEquals(s:byte(4), 0, "pack z null terminator")
local v, nxt = string.unpack("z", s)
assertEquals(v, "lua", "unpack z value")
assertEquals(nxt, 5, "unpack z next pos")
-- z with embedded zero must error
assertError(
function()
string.pack("z", "a" .. string.char(0) .. "b")
end,
"z embedded zero"
)
end
-- multiple values and next-position return
do
local s = string.pack("<i2i2", 10, 20)
assertEquals(#s, 4, "pack <i2i2 length")
local a, b, nxt = string.unpack("<i2i2", s)
assertEquals(a, 10, "unpack multi a")
assertEquals(b, 20, "unpack multi b")
assertEquals(nxt, 5, "unpack multi next pos")
end
-- init (offset) argument of unpack
do
local s = string.pack("<i2i2", 100, 200)
-- skip first field by starting at byte 3
local v, nxt = string.unpack("<i2", s, 3)
assertEquals(v, 200, "unpack with init")
assertEquals(nxt, 5, "unpack with init next pos")
end
-- padding (x) and alignment
do
local s = string.pack("bxi2", 1, 1000)
-- 1 byte b + 1 byte pad + 2 bytes i2 = 4
assertEquals(#s, 4, "pack bxi2 length")
local a, b, _ = string.unpack("bxi2", s)
assertEquals(a, 1, "unpack bxi2 b")
assertEquals(b, 1000, "unpack bxi2 i2")
end
-- endianness switch inside format
do
local s = string.pack("<I2>I2", 0x0102, 0x0304)
-- little: 02 01; big: 03 04
assertEquals(s:byte(1), 0x02, "LE low byte")
assertEquals(s:byte(2), 0x01, "LE high byte")
assertEquals(s:byte(3), 0x03, "BE high byte")
assertEquals(s:byte(4), 0x04, "BE low byte")
end