diff --git a/client/space_lua/aggregates.test.ts b/client/space_lua/aggregates.test.ts new file mode 100644 index 00000000..a684b6cf --- /dev/null +++ b/client/space_lua/aggregates.test.ts @@ -0,0 +1,784 @@ +import { parseExpressionString } from "./parse.ts"; +import { evalExpression } from "./eval.ts"; +import { + type AggregateSpec, + executeAggregate, + getAggregateSpec, +} from "./aggregates.ts"; +import { + applyQuery, + evalExpressionWithAggregates, +} from "./query_collection.ts"; +import { + LuaBuiltinFunction, + LuaEnv, + LuaStackFrame, + LuaTable, + luaValueToJS, +} from "./runtime.ts"; +import { assertEquals } from "@std/assert"; + +function luaArray(items: Record[]): LuaTable { + const t = new LuaTable(); + for (let i = 0; i < items.length; i++) { + t.rawSetArrayIndex(i + 1, new LuaTable(items[i])); + } + return t; +} + +function installFakeConfig( + specs: Record = {}, +): () => void { + const configStore: Record = {}; + for (const [name, spec] of Object.entries(specs)) { + configStore[`aggregates.${name}`] = spec; + } + const prev = (globalThis as any).client; + (globalThis as any).client = { + config: { + get(key: string, fallback: any) { + return configStore[key] ?? fallback; + }, + }, + }; + return () => { + (globalThis as any).client = prev; + }; +} + +function requireSpec(name: string): AggregateSpec { + const cleanup = installFakeConfig(); + const spec = getAggregateSpec(name); + cleanup(); + if (!spec) throw new Error(`builtin aggregate "${name}" not found`); + return spec; +} + +const sumSpec = requireSpec("sum"); +const countSpec = requireSpec("count"); +const minSpec = requireSpec("min"); +const maxSpec = requireSpec("max"); +const avgSpec = requireSpec("avg"); +const arrayAggSpec = requireSpec("array_agg"); + +const sf = LuaStackFrame.lostFrame; + +// Unit tests per builtin + +Deno.test("aggregate: sum", async () => { + const result = await executeAggregate( + sumSpec, + luaArray([{ v: 10 }, { v: 20 }, { v: 30 }]), + parseExpressionString("_.v"), + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, 60); +}); + +Deno.test("aggregate: sum with nils", async () => { + const result = await executeAggregate( + sumSpec, + luaArray([{ v: 5 }, { x: 1 }, { v: 15 }]), + parseExpressionString("_.v"), + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, 20); +}); + +Deno.test("aggregate: count with expression", async () => { + const result = await executeAggregate( + countSpec, + luaArray([{ v: 1 }, { v: 2 }, { v: 3 }]), + parseExpressionString("_.v"), + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, 3); +}); + +Deno.test("aggregate: count with no argument (count(*))", async () => { + const result = await executeAggregate( + countSpec, + luaArray([{ v: 1 }, { v: 2 }, { v: 3 }, { v: 4 }]), + null, + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, 4); +}); + +Deno.test("aggregate: min", async () => { + const result = await executeAggregate( + minSpec, + luaArray([{ v: 30 }, { v: 10 }, { v: 20 }]), + parseExpressionString("_.v"), + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, 10); +}); + +Deno.test("aggregate: max", async () => { + const result = await executeAggregate( + maxSpec, + luaArray([{ v: 30 }, { v: 10 }, { v: 20 }]), + parseExpressionString("_.v"), + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, 30); +}); + +Deno.test("aggregate: avg", async () => { + const result = await executeAggregate( + avgSpec, + luaArray([{ v: 10 }, { v: 20 }, { v: 30 }]), + parseExpressionString("_.v"), + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, 20); +}); + +Deno.test("aggregate: avg empty group", async () => { + const result = await executeAggregate( + avgSpec, + new LuaTable(), + parseExpressionString("_.v"), + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, null); +}); + +Deno.test("aggregate: min/max on empty group", async () => { + const items = new LuaTable(); + const env = new LuaEnv(); + const expr = parseExpressionString("_.v"); + assertEquals( + await executeAggregate( + minSpec, + items, + expr, + undefined, + env, + sf, + evalExpression, + ), + null, + ); + assertEquals( + await executeAggregate( + maxSpec, + items, + expr, + undefined, + env, + sf, + evalExpression, + ), + null, + ); +}); + +Deno.test("aggregate: array_agg", async () => { + const result = await executeAggregate( + arrayAggSpec, + luaArray([{ v: "a" }, { v: "b" }, { v: "c" }]), + parseExpressionString("_.v"), + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result instanceof LuaTable, true); + assertEquals((result as LuaTable).rawGet(1), "a"); + assertEquals((result as LuaTable).rawGet(2), "b"); + assertEquals((result as LuaTable).rawGet(3), "c"); +}); + +Deno.test("aggregate: sum with objectVariable", async () => { + const result = await executeAggregate( + sumSpec, + luaArray([{ v: 3 }, { v: 7 }]), + parseExpressionString("p.v"), + "p", + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, 10); +}); + +Deno.test("aggregate: user-defined overrides builtin", async () => { + const customSum: AggregateSpec = { + name: "sum", + initialize: new LuaBuiltinFunction((_sf) => 100), + iterate: new LuaBuiltinFunction((_sf, state: any, value: any) => { + if (value === null || value === undefined) return state; + return state + value; + }), + }; + const cleanup = installFakeConfig({ sum: customSum }); + try { + const spec = getAggregateSpec("sum")!; + const result = await executeAggregate( + spec, + luaArray([{ v: 1 }, { v: 2 }]), + parseExpressionString("_.v"), + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, 103); + } finally { + cleanup(); + } +}); + +Deno.test("aggregate: builtin available without config", () => { + const cleanup = installFakeConfig(); + try { + const spec = getAggregateSpec("sum"); + assertEquals(spec !== null, true); + assertEquals(spec!.name, "sum"); + + const spec2 = getAggregateSpec("nonexistent"); + assertEquals(spec2, null); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: sum in table constructor", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ age: 10 }, { age: 20 }, { age: 30 }]); + const env = new LuaEnv(); + const expr = parseExpressionString("{ total = sum(_.age) }"); + const result = await evalExpressionWithAggregates( + expr, + env, + sf, + groupItems, + undefined, + env, + ); + assertEquals(result instanceof LuaTable, true); + assertEquals((result as LuaTable).rawGet("total"), 60); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: count() with no args", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ x: 1 }, { x: 2 }, { x: 3 }]); + const env = new LuaEnv(); + const expr = parseExpressionString("count()"); + const result = await evalExpressionWithAggregates( + expr, + env, + sf, + groupItems, + undefined, + env, + ); + assertEquals(result, 3); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: non-aggregate falls through", async () => { + const cleanup = installFakeConfig(); + try { + const env = new LuaEnv(); + env.setLocal("key", "hello"); + const expr = parseExpressionString("key"); + const result = await evalExpressionWithAggregates( + expr, + env, + sf, + new LuaTable(), + undefined, + env, + ); + assertEquals(result, "hello"); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: multiple aggregates in table", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ v: 5 }, { v: 15 }, { v: 10 }]); + const env = new LuaEnv(); + env.setLocal("key", "grp1"); + const expr = parseExpressionString( + "{ k = key, total = sum(_.v), n = count(_.v), smallest = min(_.v) }", + ); + const result = await evalExpressionWithAggregates( + expr, + env, + sf, + groupItems, + undefined, + env, + ); + const tbl = result as LuaTable; + assertEquals(tbl.rawGet("k"), "grp1"); + assertEquals(tbl.rawGet("total"), 30); + assertEquals(tbl.rawGet("n"), 3); + assertEquals(tbl.rawGet("smallest"), 5); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: avg with finish step", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ v: 10 }, { v: 30 }]); + const env = new LuaEnv(); + const expr = parseExpressionString("avg(_.v)"); + const result = await evalExpressionWithAggregates( + expr, + env, + sf, + groupItems, + undefined, + env, + ); + assertEquals(result, 20); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: binary comparison (count > N)", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ v: 1 }, { v: 2 }, { v: 3 }]); + const env = new LuaEnv(); + const result = await evalExpressionWithAggregates( + parseExpressionString("count(_.v) > 2"), + env, + sf, + groupItems, + undefined, + env, + ); + assertEquals(result, true); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: binary arithmetic (sum + sum)", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ a: 10, b: 5 }, { a: 20, b: 15 }]); + const env = new LuaEnv(); + const result = await evalExpressionWithAggregates( + parseExpressionString("sum(_.a) + sum(_.b)"), + env, + sf, + groupItems, + undefined, + env, + ); + assertEquals(result, 50); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: unary minus on aggregate", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ v: 10 }, { v: 20 }]); + const env = new LuaEnv(); + const result = await evalExpressionWithAggregates( + parseExpressionString("-sum(_.v)"), + env, + sf, + groupItems, + undefined, + env, + ); + assertEquals(result, -30); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: parenthesized aggregate", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ v: 7 }, { v: 3 }]); + const env = new LuaEnv(); + const result = await evalExpressionWithAggregates( + parseExpressionString("(sum(_.v))"), + env, + sf, + groupItems, + undefined, + env, + ); + assertEquals(result, 10); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: and short-circuit", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ v: 1 }]); + const env = new LuaEnv(); + assertEquals( + await evalExpressionWithAggregates( + parseExpressionString("count(_.v) and 42"), + env, + sf, + groupItems, + undefined, + env, + ), + 42, + ); + assertEquals( + await evalExpressionWithAggregates( + parseExpressionString("false and count(_.v)"), + env, + sf, + groupItems, + undefined, + env, + ), + false, + ); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: or short-circuit", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ v: 1 }]); + const env = new LuaEnv(); + assertEquals( + await evalExpressionWithAggregates( + parseExpressionString("count(_.v) or 99"), + env, + sf, + groupItems, + undefined, + env, + ), + 1, + ); + assertEquals( + await evalExpressionWithAggregates( + parseExpressionString("nil or count(_.v)"), + env, + sf, + groupItems, + undefined, + env, + ), + 1, + ); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: not aggregate", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ v: 1 }]); + const env = new LuaEnv(); + assertEquals( + await evalExpressionWithAggregates( + parseExpressionString("not count(_.v)"), + env, + sf, + groupItems, + undefined, + env, + ), + false, + ); + } finally { + cleanup(); + } +}); + +Deno.test("applyQuery: group by + select with aggregates", async () => { + const cleanup = installFakeConfig(); + try { + const data = [ + new LuaTable({ dept: "eng", salary: 100 }), + new LuaTable({ dept: "eng", salary: 200 }), + new LuaTable({ dept: "sales", salary: 150 }), + new LuaTable({ dept: "sales", salary: 50 }), + new LuaTable({ dept: "sales", salary: 100 }), + ]; + const results = await applyQuery( + data, + { + objectVariable: "p", + groupBy: [parseExpressionString("p.dept")], + select: parseExpressionString( + "{ dept = key, total = sum(p.salary), n = count(p.salary) }", + ), + }, + new LuaEnv(), + sf, + {}, + ); + assertEquals(results.length, 2); + const eng = results[0] as LuaTable; + const sales = results[1] as LuaTable; + assertEquals(eng.rawGet("dept"), "eng"); + assertEquals(eng.rawGet("total"), 300); + assertEquals(eng.rawGet("n"), 2); + assertEquals(sales.rawGet("dept"), "sales"); + assertEquals(sales.rawGet("total"), 300); + assertEquals(sales.rawGet("n"), 3); + } finally { + cleanup(); + } +}); + +Deno.test("applyQuery: group by + having with aggregate", async () => { + const cleanup = installFakeConfig(); + try { + const data = [ + new LuaTable({ dept: "eng", name: "a" }), + new LuaTable({ dept: "eng", name: "b" }), + new LuaTable({ dept: "sales", name: "c" }), + ]; + const results = await applyQuery( + data, + { + objectVariable: "p", + groupBy: [parseExpressionString("p.dept")], + having: parseExpressionString("count(p.name) > 1"), + }, + new LuaEnv(), + sf, + {}, + ); + assertEquals(results.length, 1); + assertEquals((results[0] as LuaTable).rawGet("key"), "eng"); + } finally { + cleanup(); + } +}); + +Deno.test("applyQuery: group by without aggregates still works", async () => { + const cleanup = installFakeConfig(); + try { + const data = [ + new LuaTable({ dept: "eng", name: "a" }), + new LuaTable({ dept: "eng", name: "b" }), + new LuaTable({ dept: "sales", name: "c" }), + ]; + const results = await applyQuery( + data, + { + objectVariable: "p", + groupBy: [parseExpressionString("p.dept")], + select: parseExpressionString("key"), + }, + new LuaEnv(), + sf, + {}, + ); + assertEquals(results.length, 2); + assertEquals(results[0], "eng"); + assertEquals(results[1], "sales"); + } finally { + cleanup(); + } +}); + +Deno.test("applyQuery: having with compound expression", async () => { + const cleanup = installFakeConfig(); + try { + const data = [ + new LuaTable({ dept: "eng", salary: 100 }), + new LuaTable({ dept: "eng", salary: 200 }), + new LuaTable({ dept: "sales", salary: 50 }), + ]; + const results = await applyQuery( + data, + { + objectVariable: "p", + groupBy: [parseExpressionString("p.dept")], + having: parseExpressionString("sum(p.salary) > 100"), + select: parseExpressionString("{ dept = key, total = sum(p.salary) }"), + }, + new LuaEnv(), + sf, + {}, + ); + assertEquals(results.length, 1); + assertEquals((results[0] as LuaTable).rawGet("dept"), "eng"); + assertEquals((results[0] as LuaTable).rawGet("total"), 300); + } finally { + cleanup(); + } +}); + +Deno.test("applyQuery: implicit _ with group by + multiple aggregates", async () => { + const cleanup = installFakeConfig(); + try { + const data = [ + new LuaTable({ category: "fruit", price: 3 }), + new LuaTable({ category: "fruit", price: 5 }), + new LuaTable({ category: "veg", price: 2 }), + new LuaTable({ category: "veg", price: 7 }), + new LuaTable({ category: "veg", price: 1 }), + ]; + const results = await applyQuery( + data, + { + groupBy: [parseExpressionString("_.category")], + select: parseExpressionString( + "{ cat = key, total = sum(_.price), best = max(_.price) }", + ), + }, + new LuaEnv(), + sf, + {}, + ); + assertEquals(results.length, 2); + const fruit = results[0] as LuaTable; + assertEquals(fruit.rawGet("cat"), "fruit"); + assertEquals(fruit.rawGet("total"), 8); + assertEquals(fruit.rawGet("best"), 5); + const veg = results[1] as LuaTable; + assertEquals(veg.rawGet("cat"), "veg"); + assertEquals(veg.rawGet("total"), 10); + assertEquals(veg.rawGet("best"), 7); + } finally { + cleanup(); + } +}); + +Deno.test("applyQuery: group by + having + order by + limit", async () => { + const cleanup = installFakeConfig(); + try { + const data = [ + new LuaTable({ tag: "a", page: "p1" }), + new LuaTable({ tag: "a", page: "p2" }), + new LuaTable({ tag: "a", page: "p3" }), + new LuaTable({ tag: "b", page: "p1" }), + new LuaTable({ tag: "c", page: "p1" }), + new LuaTable({ tag: "c", page: "p2" }), + ]; + const results = await applyQuery( + data, + { + objectVariable: "t", + groupBy: [parseExpressionString("t.tag")], + having: parseExpressionString("count(t.page) > 1"), + orderBy: [{ expr: parseExpressionString("key"), desc: false }], + select: parseExpressionString("key"), + limit: 1, + }, + new LuaEnv(), + sf, + {}, + ); + assertEquals(results.length, 1); + assertEquals(results[0], "a"); + } finally { + cleanup(); + } +}); + +Deno.test("applyQuery: select with aggregate division (float result)", async () => { + const cleanup = installFakeConfig(); + try { + const data = [ + new LuaTable({ dept: "eng", salary: 100 }), + new LuaTable({ dept: "eng", salary: 200 }), + ]; + const results = await applyQuery( + data, + { + objectVariable: "p", + groupBy: [parseExpressionString("p.dept")], + select: parseExpressionString( + "{ dept = key, avg_salary = sum(p.salary) / count(p.salary) }", + ), + }, + new LuaEnv(), + sf, + {}, + ); + assertEquals(results.length, 1); + const row = results[0] as LuaTable; + assertEquals(row.rawGet("dept"), "eng"); + assertEquals(luaValueToJS(row.rawGet("avg_salary"), sf), 150); + } finally { + cleanup(); + } +}); + +Deno.test("aggregate: custom concat with finish", async () => { + const concatSpec: AggregateSpec = { + name: "concat", + initialize: new LuaBuiltinFunction((_sf) => + new LuaTable({ first: true, s: "" }) + ), + iterate: new LuaBuiltinFunction((_sf, state: any, value: any) => { + if (value === null || value === undefined) return state; + if (state.rawGet("first")) { + state.rawSet("s", String(value)); + state.rawSet("first", false); + } else { + state.rawSet("s", state.rawGet("s") + ", " + String(value)); + } + return state; + }), + finish: new LuaBuiltinFunction((_sf, state: any) => state.rawGet("s")), + }; + const result = await executeAggregate( + concatSpec, + luaArray([{ v: "a" }, { v: "b" }, { v: "c" }]), + parseExpressionString("_.v"), + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, "a, b, c"); +}); diff --git a/client/space_lua/aggregates.ts b/client/space_lua/aggregates.ts new file mode 100644 index 00000000..8311bc31 --- /dev/null +++ b/client/space_lua/aggregates.ts @@ -0,0 +1,201 @@ +/** + * Aggregate function definitions and execution for LIQ. + * + * Built-in aggregates (sum, count, min, max, avg, array_agg) are + * implemented in TypeScript for speed. Users can override any builtin + * via `aggregate.define` or `aggregate.update`. + * + * Builtins implement ILuaFunction via plain objects rather than + * LuaBuiltinFunction instances. This avoids ES module TDZ issues: + * `class` exports are not available during circular module init, + * but `interface`/`type` imports are. + */ + +import type { ILuaFunction, LuaStackFrame } from "./runtime.ts"; +import { + luaCall, + type LuaEnv, + LuaTable, + type LuaValue, +} from "./runtime.ts"; +import type { LuaExpression } from "./ast.ts"; +import { buildItemEnv } from "./query_env.ts"; + +export interface AggregateSpec { + name: string; + description?: string; + initialize: LuaValue; // ILuaFunction + iterate: LuaValue; // ILuaFunction + finish?: LuaValue; // ILuaFunction | undefined +} + +// Helper to build an ILuaFunction from a plain function. Equivalent to +// LuaBuiltinFunction but without referencing the class. +function aggFn( + fn: (sf: LuaStackFrame, ...args: LuaValue[]) => LuaValue, +): ILuaFunction { + return { + call(sf: LuaStackFrame, ...args: LuaValue[]) { + return fn(sf, ...args); + }, + asString() { + return ""; + }, + }; +} + +// Built-in aggregate specs +const builtinAggregates: Record = { + sum: { + name: "sum", + description: "Sum of numeric values", + initialize: aggFn((_sf) => 0), + iterate: aggFn((_sf, state: any, value: any) => { + if (value === null || value === undefined) return state; + return (state as number) + (value as number); + }), + }, + count: { + name: "count", + description: "Count of values; count() with no argument counts all rows", + initialize: aggFn((_sf) => 0), + iterate: aggFn((_sf, state: any, value: any) => { + if (value === null || value === undefined) return state; + return (state as number) + 1; + }), + }, + min: { + name: "min", + description: "Minimum value", + initialize: aggFn((_sf) => null), + iterate: aggFn((_sf, state: any, value: any) => { + if (value === null || value === undefined) return state; + if (state === null || value < state) return value; + return state; + }), + }, + max: { + name: "max", + description: "Maximum value", + initialize: aggFn((_sf) => null), + iterate: aggFn((_sf, state: any, value: any) => { + if (value === null || value === undefined) return state; + if (state === null || value > state) return value; + return state; + }), + }, + avg: { + name: "avg", + description: "Average of numeric values", + initialize: aggFn((_sf) => ({ sum: 0, count: 0 })), + iterate: aggFn((_sf, state: any, value: any) => { + if (value === null || value === undefined) return state; + state.sum += value as number; + state.count += 1; + return state; + }), + finish: aggFn((_sf, state: any) => { + if (state.count === 0) return null; + return state.sum / state.count; + }), + }, + array_agg: { + name: "array_agg", + description: "Collect values into an array", + initialize: aggFn((_sf) => new LuaTable()), + iterate: aggFn((_sf, state: any, value: any) => { + (state as LuaTable).rawSetArrayIndex( + (state as LuaTable).length + 1, + value, + ); + return state; + }), + }, +}; + +const noCtx = {}; + +function buildAggCtx(name: string): LuaTable { + const ctx = new LuaTable(); + ctx.rawSet("name", name); + // @ts-ignore: Hack to access client via the browser + const clientConfig = globalThis.client?.config; + const aggConfig = clientConfig + ? clientConfig.get(`aggregateConfig.${name}`, {}) + : {}; + ctx.rawSet("config", aggConfig); + return ctx; +} + +export function getAggregateSpec(name: string): AggregateSpec | null { + // @ts-ignore: Hack to access client via the browser + const clientConfig = globalThis.client?.config; + if (clientConfig) { + const spec: any = clientConfig.get(`aggregates.${name}`, null); + if (spec) { + let candidate: AggregateSpec | null = null; + if (spec instanceof LuaTable) { + const init = spec.rawGet("initialize"); + const iter = spec.rawGet("iterate"); + if (init && iter) { + candidate = { + name: spec.rawGet("name") ?? name, + description: spec.rawGet("description"), + initialize: init, + iterate: iter, + finish: spec.rawGet("finish"), + }; + } + } else if (spec.initialize && spec.iterate) { + candidate = spec as AggregateSpec; + } + if (candidate) return candidate; + } + } + return builtinAggregates[name] ?? null; +} + +/** + * Execute an aggregate function over a group of items. + * + * `evalExprFn` is passed in to avoid a circular import with eval.ts. + */ +export async function executeAggregate( + spec: AggregateSpec, + items: LuaTable, + valueExpr: LuaExpression | null, + objectVariable: string | undefined, + env: LuaEnv, + sf: LuaStackFrame, + evalExprFn: ( + e: LuaExpression, + env: LuaEnv, + sf: LuaStackFrame, + ) => Promise | LuaValue, +): Promise { + const ctx = buildAggCtx(spec.name); + + // Initialize + let state = await luaCall(spec.initialize, [ctx], noCtx, sf); + + // Iterate + const len = items.length; + for (let i = 1; i <= len; i++) { + const item = items.rawGet(i); + let value: LuaValue; + if (valueExpr === null) { + value = item; + } else { + const itemEnv = buildItemEnv(objectVariable, item, env, sf); + value = await evalExprFn(valueExpr, itemEnv, sf); + } + state = await luaCall(spec.iterate, [state, value, ctx], noCtx, sf); + } + + // Finish + if (spec.finish) { + state = await luaCall(spec.finish, [state, ctx], noCtx, sf); + } + + return state; +} diff --git a/client/space_lua/ast.ts b/client/space_lua/ast.ts index f622197f..49be9293 100644 --- a/client/space_lua/ast.ts +++ b/client/space_lua/ast.ts @@ -292,7 +292,9 @@ export type LuaQueryClause = | LuaWhereClause | LuaLimitClause | LuaOrderByClause - | LuaSelectClause; + | LuaSelectClause + | LuaGroupByClause + | LuaHavingClause; export type LuaFromClause = { type: "From"; @@ -326,3 +328,13 @@ export type LuaSelectClause = { type: "Select"; expression: LuaExpression; } & ASTContext; + +export type LuaGroupByClause = { + type: "GroupBy"; + expressions: LuaExpression[]; +} & ASTContext; + +export type LuaHavingClause = { + type: "Having"; + expression: LuaExpression; +} & ASTContext; diff --git a/client/space_lua/eval.ts b/client/space_lua/eval.ts index a709f30c..6c8ab0d4 100644 --- a/client/space_lua/eval.ts +++ b/client/space_lua/eval.ts @@ -251,7 +251,7 @@ function arithCoercionErrorOrThrow( throw e; } -function luaOp( +export function luaOp( op: string, left: any, right: any, @@ -907,6 +907,14 @@ export function evalExpression( } break; } + case "GroupBy": { + query.groupBy = clause.expressions; + break; + } + case "Having": { + query.having = clause.expression; + break; + } } } diff --git a/client/space_lua/lua.grammar b/client/space_lua/lua.grammar index 99b8435c..36d67a2d 100644 --- a/client/space_lua/lua.grammar +++ b/client/space_lua/lua.grammar @@ -95,7 +95,9 @@ QueryClause { WhereClause | OrderByClause | SelectClause | - LimitClause + LimitClause | + GroupByClause | + HavingClause } FromClause { ckw<"from"> (Name "=")? exp } @@ -104,6 +106,8 @@ LimitClause { ckw<"limit"> exp ("," exp)? } OrderByClause { ckw<"order"> ckw<"by"> list } OrderBy { exp ckw<"desc">? } SelectClause { ckw<"select"> exp } +GroupByClause { ckw<"group"> ckw<"by"> list } +HavingClause { ckw<"having"> exp } field[@isGroup=Field] { diff --git a/client/space_lua/lua.test.ts b/client/space_lua/lua.test.ts index 56275712..d60a531c 100644 --- a/client/space_lua/lua.test.ts +++ b/client/space_lua/lua.test.ts @@ -57,6 +57,10 @@ Deno.test("[Lua] String tests", async () => { await runLuaTest("./stdlib/string_test.lua"); }); +Deno.test("[Lua] String pack/unpack/packsize tests", async () => { + await runLuaTest("./stdlib/string_pack_test.lua"); +}); + Deno.test("[Lua] Space Lua tests", async () => { await runLuaTest("./stdlib/space_lua_test.lua"); }); @@ -89,7 +93,44 @@ Deno.test("[Lua] Lume functions tests", async () => { await runLuaTest("./lume_test.lua"); }); +Deno.test("[Lua] Lua Integrated Query tests", async () => { + await runLuaTest("./query_test.lua"); +}); + async function runLuaTest(luaPath: string) { + if ( + typeof globalThis.client !== "undefined" && + globalThis.client && + typeof globalThis.client === "object" + ) { + if ( + !globalThis.client.config || + typeof globalThis.client.config.get !== "function" + ) { + try { + Object.defineProperty(globalThis.client, "config", { + value: { + get(_key: string, fallback: unknown) { + return fallback ?? {}; + }, + }, + configurable: true, + writable: true, + }); + } catch { + // ignore + } + } + } else { + (globalThis as any).client = { + config: { + get(_key: string, fallback: unknown) { + return fallback ?? {}; + }, + }, + }; + } + const luaFile = await Deno.readTextFile( fileURLToPath(new URL(luaPath, import.meta.url)), ); diff --git a/client/space_lua/parse-lua.js b/client/space_lua/parse-lua.js index 28273805..57f9ce05 100644 --- a/client/space_lua/parse-lua.js +++ b/client/space_lua/parse-lua.js @@ -1,22 +1,22 @@ // This file was generated by lezer-generator. You probably shouldn't edit it. import {LRParser} from "@lezer/lr" -const spec_identifier = {__proto__:null,break:16, goto:20, do:24, end:26, while:30, nil:32, true:34, false:36, or:80, and:82, not:104, function:114, query:122, from:129, where:133, order:137, by:139, desc:143, select:147, limit:151, repeat:154, until:156, if:160, then:162, elseif:164, else:166, for:170, in:178, local:188, return:204} +const spec_identifier = {__proto__:null,break:16, goto:20, do:24, end:26, while:30, nil:32, true:34, false:36, or:80, and:82, not:104, function:114, query:122, from:129, where:133, order:137, by:139, desc:143, select:147, limit:151, group:155, having:159, repeat:162, until:164, if:168, then:170, elseif:172, else:174, for:178, in:186, local:196, return:212} export const parser = LRParser.deserialize({ version: 14, - states: "EUO!ZQPOOOOQO'#Cc'#CcO!UQPO'#CaO!bQPOOOOQO'#E{'#E{O!vQQO'#CwO$]QPO'#EzOOQO'#Ez'#EzO$gQPO'#EzOOQO'#E`'#E`O%vQPO'#E_OOQO'#Ev'#EvOOQO'#Eg'#EgO%{QPO'#C_OOQO'#C_'#C_QOQPOOO!UQPO'#CeO&`QPO'#CgO!vQQO'#CjO&gQPO'#DzO!vQQO'#D}O!UQPO'#ESO!UQPO'#EZO&nQPO'#EaO&vQQO'#EeO'^QPO,58{OOQO'#Cq'#CqO!UQPO,59^O!vQQO,59`O(gQQO'#C|O(nQQO'#FQOOQO'#E|'#E|OOQO,59f,59fO!UQPO,59fO(uQPO'#ExO,eQPO,59cOOQO'#Dc'#DcOOQO'#Dd'#DdOOQO'#De'#DeO!vQQO'#DaOOQO'#Ex'#ExO,lQPO'#DfO,qQSO'#DjO,vQPO'#EpO-OQPO,5SQPO,5:vO!UQPO,5:vOOQO1G0a1G0aO!UQPO'#EdOOQO,5:},5:}O!UQPO'#EqO>_QPO,5uAN>uO&`QPOAN>uO!;OQPO,5;XO!;VQPOAN>uO!;[QPO<YQ`O1G/ZO!>aQ`O1G/ZO!@RQ`O1G/ZO!@YQ`O1G/ZO!AtQ`O1G/ZO!BRQ`O1G/ZO!B`Q`O1G/ZO!DTQ`O1G/ZO!D[Q`O1G/ZO!DcQSO7+%jO!DyQPOG24aO=mQPO1G0sOOQOG24aG24aO!vQQOAN>|OOQO,5;W,5;WOOQO-E8j-E8jOOQOLD){LD){OOQO7+&_7+&_O!EOQPOG24hOAVQQO'#DaO!%YQQO'#DaOAVQQO,59oO!%YQQO,59oOAVQQO,59oO!%YQQO,59oOAVQQO,59oO!%YQQO,59oOAVQQO,59oO!%YQQO,59oOAVQQO,59oO!%YQQO,59oOAVQQO,59oO!%YQQO,59oOAVQQO,59oO!%YQQO,59oOAVQQO,59oO!%YQQO,59oOAVQQO,59oO!%YQQO,59o", - stateData: "!EY~O#hOS#iOSPOS~OSZOUQOWZOY`O[aO_bOlTO!ZfO!ocO!rdO!weO#QgO#YhO#kPO~O#fRP~P]OgkOilOlnOoqOqmO#mjO~O`xOaxObxOcxOdxOlTOqmO!UwO!ZyO!_zO#kPO#mjO#xuO#|tO$SvO~Og#nXi#nXl#nXo#nXq#nX#m#nX~Ov{O#r$YX~P#wOS#jXU#jXW#jXY#jX[#jX_#jXl#jX!Z#jX!o#jX!r#jX!w#jX#Q#jX#Y#jX#f#jX#k#jX]#jX!p#jX!t#jX!u#jX~P#wO#r}O~O#fRX]RX!pRX!tRX!uRX~P]O]RP~P]O!pRP~P]O!Z!`O#kPO~OS!dO#f#XX]#XX!p#XX!t#XX!u#XX~P!vOU!fO~O`xOaxObxOcxOdxOi!kOlTOqmO!U&kO!ZyO!_zO#kPO#mjO#xuO#|tO$SvO~Ou!nO~P'cOm!pO~P!vOm#lXx#lXy#lXz#lX!P#lX#v#lX#w#lX#x#lX#y#lX#z#lX#{#lX#|#lX#}#lX$O#lX$P#lX$Q#lX$R#lX[#lX!s#lXS#lXv#lX#f#lXj#lXU#lXW#lXY#lX_#lX!Z#lX!o#lX!r#lX!w#lX#Q#lX#Y#lX#k#lX]#lX!p#lX!t#lX!u#lX~P!bOx#SOy#TOz!{O!P#PO#v!tO#w!uO#x!vO#y!wO#z!wO#{!xO#|!xO#}!yO$O!yO$P!yO$Q!yO$R!zO~Om!sO~P+aOl#VO~O$U#XO~OlTO#kPO~Ov{O#r$Ya~O]#]O~O[#^O~P+aO!p#_O~O!s#`O~P+aOv#bO#r#aO!{$XX~O!{#dO~O[#eO~Og#fOo#hOl#OX~O$[#jOS#WPU#WPW#WPY#WP[#WP_#WPl#WPv#WP!Z#WP!o#WP!r#WP!w#WP#Q#WP#Y#WP#f#WP#k#WP#r#WP]#WP!p#WP!t#WP!u#WP~Ov#lOS$ZXU$ZXW$ZXY$ZX[$ZX_$ZXl$ZX!Z$ZX!o$ZX!r$ZX!w$ZX#Q$ZX#Y$ZX#f$ZX#k$ZX#r$ZX]$ZX!p$ZX!t$ZX!u$ZX~O#r#oOS#TaU#TaW#TaY#Ta[#Ta_#Tal#Ta!Z#Ta!o#Ta!r#Ta!w#Ta#Q#Ta#Y#Ta#f#Ta#k#Ta]#Ta!p#Ta!t#Ta!u#Ta~Ov#pOS#uX#f#uXm#uXU#uXW#uXY#uX[#uX_#uXl#uX!Z#uX!o#uX!r#uX!w#uX#Q#uX#Y#uX#k#uX]#uX!p#uX!t#uX!u#uX~P+aOS#rO#f#Xa]#Xa!p#Xa!t#Xa!u#Xa~Oj#sO~P+aOu#lXx#lXy#lXz#lX!P#lX#s#lX#v#lX#w#lX#x#lX#y#lX#z#lX#{#lX#|#lX#}#lX$O#lX$P#lX$Q#lX$R#lX!b#lX!d#lX!f#lX!k#lX!m#lX$W#lX~P!bOg#oXi#oXl#oXo#oXq#oXx#oXy#oXz#oX!P#oX#m#oX#v#oX#w#oX#x#oX#y#oX#z#oX#{#oX#|#oX#}#oX$O#oX$P#oX$Q#oX$R#oX~O#r#tOu#oX#s#oX~P6_Ox&{Oy&}Oz&mO!P&uO#v!tO#w!uO#x!vO#y!wO#z!wO#{!xO#|!xO#}!yO$O!yO$P!yO$Q!yO$R!zO~OutX#stX~P8SOu#xO#s#vO~Om#zO~OlnOqmO#mjO~O$R!zOx!Tay!Taz!Ta!P!Ta#v!Ta#w!Ta#x!Ta#y!Ta#z!Ta#{!Ta#|!Ta#}!Ta$O!Ta$P!Ta$Q!Tav!Ta~Om!Ta[!Ta!s!TaS!Ta#f!Taj!TaU!TaW!TaY!Ta_!Tal!Ta!Z!Ta!o!Ta!r!Ta!w!Ta#Q!Ta#Y!Ta#k!Ta]!Ta!p!Ta!t!Ta!u!Ta~P9zOc$WOlTO#kPOm!]P~O!b$^O!d$_O!f$`O!k$aO!m$bO~O$W$cO~PhO#w!uO~P!>hO#}!yO$O!yO$P!yO$Q!yO$R!zOvwixwiywizwi!bwi!dwi!fwi!kwi!mwi#vwi#wwi#xwi#ywi#zwi$Wwi!iwi~O!P&vO#{!xO#|!xO~P!@aO!Pwi#{wi#|wi~P!@aOvwi!iwi~P!/rOz&nO!P&vO#v!tO#w!uO#x!vO#y!wO#z!wO#{!xO#|!xO#}!yO$O!yO$P!yO$Q!yO$R!zOvwixwi!bwi!dwi!fwi!kwi!mwi$Wwi!iwi~Oy'OO~P!BjOywi~P!BjO!b!lq!d!lq!f!lq!k!lq!m!lq$W!lq~P8SO]&hO~O[!x!Z~P+aOP#|~", - goto: "CY$[PPP$]P${P%YP${P${PP${PPPPPP'v)_P)_PP*}PP,mP'vP._._._PP'}PPP.e/[0X1RP2R3U4['}P5h5h5h'}P6|7V'}P7Y7^P7^P7^PP7bP7^P7^P${PP${PPPP${P7h7h7kP7n${7z${P${7}${8[8_8e8hP8w9W9^9d9k9q9w9}:T:Z:aPPPP:gP:tP>`@UA|BVPPB_BfPPPPPPPPPPPPPBoPBrPBuBxCVQ_OQ!RaQ!TcQ$d#^Q$f#`Q$k#eQ%s%YQ&O%hQ&b%|R&i&cgZO]ac#^#`#e%Y%h%|&c$OSOT]abcdhlnw{}!k!{!|!}#O#P#Q#R#S#T#V#^#_#`#a#d#e#o#p#t$_$a$b%W%Y%`%f%h%i%m%t%x%|&U&c&e&k&l&m&n&o&p&q&r&s&t&u&v&w&x&y&z&{&|&}'OQiQQ!Q`Q!VeQ!ZfS!]g#lQ!gkW!jm#v%U%oQ!rqQ#n!`Q$h#bQ$l#fQ$o#hQ$p#jQ%]$^R%k$nYoRr!i!r%b#mxTbdhlmnw}!k!{!|!}#O#P#Q#R#S#T#_#a#d#o#p#t#v$^$_$a$b%U%`%f%i%m%o%t%x&U&e&k&l&m&n&o&p&q&r&s&t&u&v&w&x&y&z&{&|&}'O$ZSOT]abcdhlmnw{}!k!{!|!}#O#P#Q#R#S#T#V#^#_#`#a#d#e#o#p#t#v$^$_$a$b%U%W%Y%`%f%h%i%m%o%t%x%|&U&c&e&k&l&m&n&o&p&q&r&s&t&u&v&w&x&y&z&{&|&}'O$ZVOT]abcdhlmnw{}!k!{!|!}#O#P#Q#R#S#T#V#^#_#`#a#d#e#o#p#t#v$^$_$a$b%U%W%Y%`%f%h%i%m%o%t%x%|&U&c&e&k&l&m&n&o&p&q&r&s&t&u&v&w&x&y&z&{&|&}'O#rVTbdhlmnw{}!k!{!|!}#O#P#Q#R#S#T#V#_#a#d#o#p#t#v$^$_$a$b%U%W%`%f%i%m%o%t%x&U&e&k&l&m&n&o&p&q&r&s&t&u&v&w&x&y&z&{&|&}'OgWO]ac#^#`#e%Y%h%|&cX!mm#v%U%on!|s!S!U!b!h#u#|$T$U$e$g$u%}&P&jf&o!l$w$y%Q%R%^%_%a&Q&S&aZ&p%c%u&W&_&`r!}s!S!U!b!h#u#|#}$P$T$U$e$g$u%}&P&jj&q!l$w$y$z$|%Q%R%^%_%a&Q&S&a_&r%c%u&W&X&Z&_&`p#Os!S!U!b!h#u#|#}$T$U$e$g$u%}&P&jh&s!l$w$y$z%Q%R%^%_%a&Q&S&a]&t%c%u&W&X&_&`t#Ps!S!U!b!h#u#|#}$O$P$T$U$e$g$u%}&P&jl&u!l$w$y$z${$|%Q%R%^%_%a&Q&S&aa&v%c%u&W&X&Y&Z&_&`v#Qs!S!U!b!h#u#|#}$O$P$Q$T$U$e$g$u%}&P&jn&w!l$w$y$z${$|$}%Q%R%^%_%a&Q&S&ac&x%c%u&W&X&Y&Z&[&_&`x#Rs!S!U!b!h#u#|#}$O$P$Q$R$T$U$e$g$u%}&P&jp&y!l$w$y$z${$|$}%O%Q%R%^%_%a&Q&S&ae&z%c%u&W&X&Y&Z&[&]&_&`|#Rs!S!U!b!h#U#u#|#}$O$P$Q$R$S$T$U$e$g$u%}&P&jt&y!l#y$w$y$z${$|$}%O%P%Q%R%^%_%a&Q&S&ai&z%c%u%y&W&X&Y&Z&[&]&^&_&`!UwTbdhlnw}!k!{!|!}#O#P#Q#R#S#T#_#a#d#o#p%f%i&ez&km#t#v$^$_$a%U%m%o%t%x&k&m&o&q&s&u&w&y&{&}k&l$b%`&U&l&n&p&r&t&v&x&z&|'OQ#WyQ#i![R$s#nR$Y#VT$[#X$]T$Z#X$]Q%v%`R&f&UR!YeR!XeQ!ehQ#[}Q$j#dR$t#oR![fgYO]ac#^#`#e%Y%h%|&cR!agQ!^gR$q#lR#k!]d^Oac#^#`#e%Y%h%|&cR!P]d]Oac#^#`#e%Y%h%|&cR!O]Q#w!oR%V#wQ#q!bR$v#qS%X$V$WR%r%XQ$]#XR%Z$]Q&V%vR&g&VQ%e$fR%z%eQ#c!VR$i#cQ#g!ZR$m#gQ|UR#Z|Q#m!^R$r#mg[O]ac#^#`#e%Y%h%|&cQsTQ!SbQ!UdY!bhn}#d#oQ!hlW!lm#v%U%oQ#UwQ#u!kQ#y&kQ#|!{Q#}!|Q$O!}Q$P#OQ$Q#PQ$R#QQ$S#RQ$T#SQ$U#TQ$e#_Q$g#aQ$u#pQ$w#tQ$y&mQ$z&oQ${&qQ$|&sQ$}&uQ%O&wQ%P&yQ%Q&{Q%R&}Q%^$^Q%_$_Q%a$aQ%c$bS%u%`&UQ%y&lQ%}%fQ&P%iQ&Q%mQ&S%tQ&W&nQ&X&pQ&Y&rQ&Z&tQ&[&vQ&]&xQ&^&zQ&_&|Q&`'OQ&a%xR&j&elRO]ac{#V#^#`#e%W%Y%h%|&c!UrTbdhlnw}!k!{!|!}#O#P#Q#R#S#T#_#a#d#o#p%f%i&ez!im#t#v$^$_$a%U%m%o%t%x&k&m&o&q&s&u&w&y&{&}k%b$b%`&U&l&n&p&r&t&v&x&z&|'OfUO]ac#^#`#e%Y%h%|&c#lVTbdhlmnw}!k!{!|!}#O#P#Q#R#S#T#_#a#d#o#p#t#v$^$_$a$b%U%`%f%i%m%o%t%x&U&e&k&l&m&n&o&p&q&r&s&t&u&v&w&x&y&z&{&|&}'OQ#Y{Q$V#VR%p%WWpRr!i%bR#{!rQ!omV%S#v%U%oZoRr!i!r%bW!ch}#d#oR!qnR$X#VR%w%`R!WegXO]ac#^#`#e%Y%h%|&cR!_g", - nodeNames: "⚠ Comment Chunk Block ; Label :: Name break Goto goto Scope do end WhileStatement while nil true false Ellipsis Number LiteralString Property . MemberExpression [ ] Parens ( ) FunctionCall : TableConstructor { FieldDynamic FieldProp FieldExp } , BinaryExpression or and CompareOp BitOp BitOp BitOp BitOp Concat ArithOp ArithOp ArithOp UnaryExpression not ArithOp BitOp LenOp FunctionDef function FuncBody ArgList Query query QueryClause FromClause from WhereClause where OrderByClause order by OrderBy desc SelectClause select LimitClause limit RepeatStatement repeat until IfStatement if then elseif else ForStatement for ForNumeric ForGeneric NameList in ExpList Function FuncName LocalFunction local Assign VarList Local AttNameList AttName Attrib ReturnStatement return", - maxTerm: 151, + states: "F^O!ZQPOOOOQO'#Cc'#CcO!UQPO'#CaO!bQPOOOOQO'#FP'#FPO!vQQO'#CwO$]QPO'#FOOOQO'#FO'#FOO$gQPO'#FOOOQO'#Ed'#EdO%vQPO'#EcOOQO'#Ez'#EzOOQO'#Ek'#EkO%{QPO'#C_OOQO'#C_'#C_QOQPOOO!UQPO'#CeO&`QPO'#CgO!vQQO'#CjO&gQPO'#EOO!vQQO'#ERO!UQPO'#EWO!UQPO'#E_O&nQPO'#EeO&vQQO'#EiO'^QPO,58{OOQO'#Cq'#CqO!UQPO,59^O!vQQO,59`O(gQQO'#C|O(nQQO'#FUOOQO'#FQ'#FQOOQO,59f,59fO!UQPO,59fO(uQPO'#E|O,eQPO,59cOOQO'#Dc'#DcOOQO'#Dd'#DdOOQO'#De'#DeO!vQQO'#DaOOQO'#E|'#E|O,lQPO'#DfO,qQSO'#DjO,vQPO'#EtO-OQPO,5WQPO,5`QPO,5:zO!UQPO,5:zOOQO1G0e1G0eO!UQPO'#EhOOQO,5;R,5;RO!UQPO'#EuO>kQPO,5yAN>yO&`QPOAN>yO!>fQPO,5;]O!>mQPOAN>yO!>rQPO<|QSO<R#Q;'S=o;'S;=`>}<%lO=ok>UVO!_>k!_!`?T!`#P>k#P#Q@X#Q;'S>k;'S;=`Ao<%lO>kk>nTO#P=o#P#Q7z#Q;'S=o;'S;=`>}<%lO=ok?QP;=`<%l=ok?WTO#P?g#P#Q@P#Q;'S?g;'S;=`?y<%lO?gk?jTO#P?g#P#Q7z#Q;'S?g;'S;=`?y<%lO?gk?|P;=`<%l?gk@UP#mk!_!`8Qk@[VO!_=o!_!`@q!`#P=o#P#QAu#Q;'S=o;'S;=`>}<%lO=ok@tTO#P=o#P#QAT#Q;'S=o;'S;=`>}<%lO=okAYV#mkO!_>k!_!`?T!`#P>k#P#Q@X#Q;'S>k;'S;=`Ao<%lO>kkArP;=`<%l>kkAxVO!_?g!_!`?T!`#P?g#P#QAu#Q;'S?g;'S;=`?y<%lO?goBdT$USO#PBs#P#QCV#Q;'SBs;'S;=`Ci<%lOBskBvTO#PBs#P#QCV#Q;'SBs;'S;=`Ci<%lOBskCYTO#PBs#P#Q%b#Q;'SBs;'S;=`Ci<%lOBskClP;=`<%lBsoCtPjP#P#QCwnC|O$Wn~DRO$R~~DWOq~~D]O#v~~DbOu~~DgP#x~!_!`$m", + tokenData: "Dj~RuXY#fYZ$Q[]#f]^$_pq#fqr$grs$rst)`uv)evw)jwx)oxy.Wyz.]z{.b{|.g|}.l}!O.s!O!P/g!P!Q/|!Q!R0^!R![1u![!]3m!]!^3z!^!_4R!_!`4e!`!a4m!c!}5P!}#O5b#O#P#w#P#QCo#Q#RC|#R#S5P#T#o5P#o#pDR#p#qDW#q#rD]#r#sDb~#kS#m~XY#f[]#fpq#f#O#P#w~#zQYZ#f]^#f~$VP#l~]^$Y~$_O#l~~$dP#l~YZ$Ym$jP!_!`$mm$rOzmk$uXOY$rZ]$r^r$rrs%bs#O$r#O#P%g#P;'S$r;'S;=`'}<%lO$rk%gO#qkk%j]rs$rwx$r!Q![&c#O#P$r#T#U$r#U#V$r#Y#Z$r#b#c$r#f#g$r#h#i$r#i#j(T#l#m(v#n#o$rk&fZOY$rZ]$r^r$rrs%bs!Q$r!Q!['X![#O$r#O#P%g#P;'S$r;'S;=`'}<%lO$rk'[ZOY$rZ]$r^r$rrs%bs!Q$r!Q![$r![#O$r#O#P%g#P;'S$r;'S;=`'}<%lO$rk(QP;=`<%l$rk(WP#o#p(Zk(^R!Q![(g!c!i(g#T#Z(gk(jS!Q![(g!c!i(g#T#Z(g#q#r$rk(yR!Q![)S!c!i)S#T#Z)Sk)VR!Q![$r!c!i$r#T#Z$r~)eO$W~~)jO$T~~)oO#{~k)rXOY)oZ])o^w)owx%bx#O)o#O#P*_#P;'S)o;'S;=`,u<%lO)ok*b]rs)owx)o!Q![+Z#O#P)o#T#U)o#U#V)o#Y#Z)o#b#c)o#f#g)o#h#i)o#i#j,{#l#m-n#n#o)ok+^ZOY)oZ])o^w)owx%bx!Q)o!Q![,P![#O)o#O#P*_#P;'S)o;'S;=`,u<%lO)ok,SZOY)oZ])o^w)owx%bx!Q)o!Q![)o![#O)o#O#P*_#P;'S)o;'S;=`,u<%lO)ok,xP;=`<%l)ok-OP#o#p-Rk-UR!Q![-_!c!i-_#T#Z-_k-bS!Q![-_!c!i-_#T#Z-_#q#r)ok-qR!Q![-z!c!i-z#T#Z-zk-}R!Q![)o!c!i)o#T#Z)o~.]Ol~~.bOm~~.gO$R~~.lO$P~o.sOvc#w[~.xP$Q~}!O.{~/QTP~OY.{Z].{^;'S.{;'S;=`/a<%lO.{~/dP;=`<%l.{o/lPgi!O!P/oo/tP!Pm!O!P/wQ/|OcQ~0RQ$S~!P!Q0X!_!`$m~0^O$U~~0cUd~!O!P0u!Q![1u!g!h1Z!z!{2W#X#Y1Z#l#m2W~0xP!Q![0{~1QRd~!Q![0{!g!h1Z#X#Y1Z~1^R{|1g}!O1g!Q![1m~1jP!Q![1m~1rPd~!Q![1m~1zSd~!O!P0u!Q![1u!g!h1Z#X#Y1Z~2ZR!Q![2d!c!i2d#T#Z2d~2iUd~!O!P2{!Q![2d!c!i2d!r!s1Z#T#Z2d#d#e1Z~3OR!Q![3X!c!i3X#T#Z3X~3^Td~!Q![3X!c!i3X!r!s1Z#T#Z3X#d#e1Z~3rPo~![!]3u~3zOU~_4ROSR#w[o4YQ$`Qzm!^!_4`!_!`$mm4eO#}m~4jP#v~!_!`$mo4tQ$aQzm!_!`$m!`!a4zm5PO$Om~5US#o~!Q![5P!c!}5P#R#S5P#T#o5Po5gQik!_!`5m!}#OB_k5pQ!_!`5v!}#O=ok5yP!}#O5|k6PTO#P5|#P#Q6`#Q;'S5|;'S;=`8W<%lO5|k6cVO!_6x!_!`:s!`#P6x#P#Q;w#Q;'S6x;'S;=`=i<%lO6xk6{VO!_7b!_!`8^!`#P7b#P#Q8v#Q;'S7b;'S;=`=c<%lO7bk7eTO#P5|#P#Q7t#Q;'S5|;'S;=`8W<%lO5|k7wP!_!`7zk7}P!_!`8Qk8TP#P#Q%bk8ZP;=`<%l5|k8aTO#P8^#P#Q7t#Q;'S8^;'S;=`8p<%lO8^k8sP;=`<%l8^k8yUO!_5|!_!`9]!`#P5|#Q;'S5|;'S;=`8W<%lO5|k9`VO!_5|!_!`9u!`#P5|#P#Q6`#Q;'S5|;'S;=`8W<%lO5|k9xTO#P5|#P#Q:X#Q;'S5|;'S;=`8W<%lO5|k:^V#qkO!_6x!_!`:s!`#P6x#P#Q;w#Q;'S6x;'S;=`=i<%lO6xk:vVO!_8^!_!`;]!`#P8^#P#Q7t#Q;'S8^;'S;=`8p<%lO8^k;`TO#P8^#P#Q;o#Q;'S8^;'S;=`8p<%lO8^k;tP#qk!_!`7zk;zVO!_7b!_!`:s!`#P7b#P#QR#Q;'S=o;'S;=`>}<%lO=ok>UVO!_>k!_!`?T!`#P>k#P#Q@X#Q;'S>k;'S;=`Ao<%lO>kk>nTO#P=o#P#Q7z#Q;'S=o;'S;=`>}<%lO=ok?QP;=`<%l=ok?WTO#P?g#P#Q@P#Q;'S?g;'S;=`?y<%lO?gk?jTO#P?g#P#Q7z#Q;'S?g;'S;=`?y<%lO?gk?|P;=`<%l?gk@UP#qk!_!`8Qk@[VO!_=o!_!`@q!`#P=o#P#QAu#Q;'S=o;'S;=`>}<%lO=ok@tTO#P=o#P#QAT#Q;'S=o;'S;=`>}<%lO=okAYV#qkO!_>k!_!`?T!`#P>k#P#Q@X#Q;'S>k;'S;=`Ao<%lO>kkArP;=`<%l>kkAxVO!_?g!_!`?T!`#P?g#P#QAu#Q;'S?g;'S;=`?y<%lO?goBdT$YSO#PBs#P#QCV#Q;'SBs;'S;=`Ci<%lOBskBvTO#PBs#P#QCV#Q;'SBs;'S;=`Ci<%lOBskCYTO#PBs#P#Q%b#Q;'SBs;'S;=`Ci<%lOBskClP;=`<%lBsoCtPjP#P#QCwnC|O$[n~DRO$V~~DWOq~~D]O#z~~DbOu~~DgP#|~!_!`$m", tokenizers: [0, 1, 2, 3, 4], topRules: {"Chunk":[0,2]}, - dynamicPrecedences: {"128":1}, - specialized: [{term: 119, get: (value) => spec_identifier[value] || -1}], - tokenPrec: 3779 + dynamicPrecedences: {"132":1}, + specialized: [{term: 123, get: (value) => spec_identifier[value] || -1}], + tokenPrec: 4041 }) diff --git a/client/space_lua/parse-lua.terms.js b/client/space_lua/parse-lua.terms.js index 2c088b0b..ade51d75 100644 --- a/client/space_lua/parse-lua.terms.js +++ b/client/space_lua/parse-lua.terms.js @@ -22,15 +22,17 @@ export const OrderBy = 70, SelectClause = 72, LimitClause = 74, - IfStatement = 79, - ForStatement = 84, - ForNumeric = 86, - ForGeneric = 87, - NameList = 88, - ExpList = 90, - FuncName = 92, - VarList = 96, - AttNameList = 98, - AttName = 99, - Attrib = 100, - ReturnStatement = 101 + GroupByClause = 76, + HavingClause = 78, + IfStatement = 83, + ForStatement = 88, + ForNumeric = 90, + ForGeneric = 91, + NameList = 92, + ExpList = 94, + FuncName = 96, + VarList = 100, + AttNameList = 102, + AttName = 103, + Attrib = 104, + ReturnStatement = 105 diff --git a/client/space_lua/parse.test.ts b/client/space_lua/parse.test.ts index bd4ba299..7792f5d9 100644 --- a/client/space_lua/parse.test.ts +++ b/client/space_lua/parse.test.ts @@ -136,6 +136,22 @@ Deno.test("Test query parsing", () => { `_(query[[from p = index.tag("page") order by p.lastModified desc, p.name]])`, ); parse(`_(query[[from p = index.tag("page") order by p.lastModified]])`); + // group by single key + parse( + `_(query[[from p = index.tag("page") group by p.category]])`, + ); + // group by multiple keys + parse( + `_(query[[from p = index.tag("page") group by p.category, p.status]])`, + ); + // group by + having + parse( + `_(query[[from p = index.tag("page") group by p.category having #group > 1]])`, + ); + // group by + having + select + parse( + `_(query[[from p = index.tag("page") group by p.category, p.status having #group > 2 select { key = key, count = #group }]])`, + ); }); Deno.test("Test numeric constant parsing", () => { diff --git a/client/space_lua/parse.ts b/client/space_lua/parse.ts index 16a48cdb..723d87fd 100644 --- a/client/space_lua/parse.ts +++ b/client/space_lua/parse.ts @@ -35,7 +35,7 @@ const luaStyleTags = styleTags({ CompareOp: t.operator, "true false": t.bool, Comment: t.lineComment, - "return break goto do end while repeat until function local if then else elseif in for nil or and not query from where limit select order by desc": + "return break goto do end while repeat until function local if then else elseif in for nil or and not query from where limit select order by desc group having": t.keyword, }); @@ -131,6 +131,7 @@ function expressionHasFunctionDef(e: LuaExpression): boolean { break; case "Where": case "Select": + case "Having": if (expressionHasFunctionDef(c.expression)) return true; break; case "Limit": @@ -144,6 +145,11 @@ function expressionHasFunctionDef(e: LuaExpression): boolean { } } break; + case "GroupBy": + for (let j = 0; j < c.expressions.length; j++) { + if (expressionHasFunctionDef(c.expressions[j])) return true; + } + break; } } return false; @@ -213,6 +219,7 @@ function exprReferencesNames(e: LuaExpression, names: Set): boolean { break; case "Where": case "Select": + case "Having": if (exprReferencesNames(c.expression, names)) return true; break; case "Limit": @@ -226,6 +233,11 @@ function exprReferencesNames(e: LuaExpression, names: Set): boolean { } } break; + case "GroupBy": + for (let j = 0; j < c.expressions.length; j++) { + if (exprReferencesNames(c.expressions[j], names)) return true; + } + break; } } return false; @@ -512,6 +524,7 @@ function exprCapturesNames(e: LuaExpression, names: Set): boolean { break; case "Where": case "Select": + case "Having": if (exprCapturesNames(c.expression, names)) return true; break; case "Limit": @@ -525,6 +538,11 @@ function exprCapturesNames(e: LuaExpression, names: Set): boolean { } } break; + case "GroupBy": + for (let j = 0; j < c.expressions.length; j++) { + if (exprCapturesNames(c.expressions[j], names)) return true; + } + break; } } return false; @@ -1261,6 +1279,27 @@ function parseQueryClause(t: ParseTree, ctx: ASTCtx): LuaQueryClause { ctx: context(t, ctx), }; } + case "GroupByClause": { + // children: ckw<"group">, ckw<"by">, exp, ",", exp, ... + const expressions: LuaExpression[] = t.children! + .filter((c) => + c.type !== undefined && c.type !== "group" && c.type !== "by" && + c.type !== "," + ) + .map((c) => parseExpression(c, ctx)); + return { + type: "GroupBy", + expressions, + ctx: context(t, ctx), + }; + } + case "HavingClause": { + return { + type: "Having", + expression: parseExpression(t.children![1], ctx), + ctx: context(t, ctx), + }; + } default: console.error(t); throw new Error(`Unknown query clause type: ${t.type}`); diff --git a/client/space_lua/query_collection.ts b/client/space_lua/query_collection.ts index 266672e0..0e197302 100644 --- a/client/space_lua/query_collection.ts +++ b/client/space_lua/query_collection.ts @@ -1,6 +1,24 @@ -import type { LuaExpression } from "./ast.ts"; -import { LuaEnv, luaGet, luaKeys, LuaStackFrame, LuaTable } from "./runtime.ts"; -import { evalExpression } from "./eval.ts"; +import type { + LuaBinaryExpression, + LuaDynamicField, + LuaExpression, + LuaExpressionField, + LuaFunctionCallExpression, + LuaParenthesizedExpression, + LuaPropField, + LuaUnaryExpression, +} from "./ast.ts"; +import { + LuaEnv, + luaGet, + luaKeys, + LuaStackFrame, + LuaTable, + luaTruthy, + type LuaValue, + singleResult, +} from "./runtime.ts"; +import { evalExpression, luaOp } from "./eval.ts"; import { asyncQuickSort } from "./util.ts"; import type { DataStore } from "../data/datastore.ts"; import type { KvPrimitives } from "../data/kv_primitives.ts"; @@ -9,22 +27,65 @@ import type { QueryCollationConfig } from "../../plug-api/types/config.ts"; import type { KvKey } from "../../plug-api/types/datastore.ts"; -export function buildItemEnv( +import { executeAggregate, getAggregateSpec } from "./aggregates.ts"; + +export { buildItemEnv } from "./query_env.ts"; +import { buildItemEnv } from "./query_env.ts"; + +// Build environment for post-`group by` clauses (`having`, `select`, +// `order by`). Injects `key` and `group` as top-level variables. Also +// unpacks the `group by` key fields as locals so that `group by name, +// tag` makes `name` and `tag` accessible. +function buildGroupItemEnv( objectVariable: string | undefined, + groupByNames: string[] | undefined, item: any, env: LuaEnv, sf: LuaStackFrame, ): LuaEnv { const itemEnv = new LuaEnv(env); - if (!objectVariable) { - // Inject all item keys as variables - for (const key of luaKeys(item)) { - itemEnv.setLocal(key, luaGet(item, key, sf.astCtx ?? null, sf)); - } - // As well as _ - itemEnv.setLocal("_", item); - } else { + if (objectVariable) { itemEnv.setLocal(objectVariable, item); + } else { + itemEnv.setLocal("_", item); + } + if (item instanceof LuaTable) { + const keyVal = item.rawGet("key"); + const groupVal = item.rawGet("group"); + + if (!objectVariable && groupVal instanceof LuaTable) { + const firstItem = groupVal.rawGet(1); + if (firstItem) { + for (const k of luaKeys(firstItem)) { + itemEnv.setLocal( + k, + luaGet(firstItem, k, sf.astCtx ?? null, sf), + ); + } + } + } + + if (keyVal !== undefined) { + itemEnv.setLocal("key", keyVal); + } + if (groupVal !== undefined) { + itemEnv.setLocal("group", groupVal); + } + + if (keyVal instanceof LuaTable) { + for (const k of luaKeys(keyVal)) { + itemEnv.setLocal( + k, + luaGet(keyVal, k, sf.astCtx ?? null, sf), + ); + } + } + if ( + !(keyVal instanceof LuaTable) && groupByNames && + groupByNames.length === 1 + ) { + itemEnv.setLocal(groupByNames[0], keyVal); + } } return itemEnv; } @@ -51,6 +112,10 @@ export type LuaCollectionQuery = { offset?: number; // Whether to return only distinct values distinct?: boolean; + // The group by expressions evaluated with Lua + groupBy?: LuaExpression[]; + // The having expression evaluated with Lua + having?: LuaExpression; }; export interface LuaQueryCollection { @@ -78,6 +143,194 @@ export class ArrayQueryCollection implements LuaQueryCollection { } } +// Check whether an expression tree contains any aggregate function call. +function containsAggregate(expr: LuaExpression): boolean { + switch (expr.type) { + case "FunctionCall": { + const fc = expr as LuaFunctionCallExpression; + if (fc.prefix.type === "Variable" && getAggregateSpec(fc.prefix.name)) { + return true; + } + return fc.args.some(containsAggregate); + } + case "Binary": { + const bin = expr as LuaBinaryExpression; + return containsAggregate(bin.left) || containsAggregate(bin.right); + } + case "Unary": { + const un = expr as LuaUnaryExpression; + return containsAggregate(un.argument); + } + case "Parenthesized": { + const p = expr as LuaParenthesizedExpression; + return containsAggregate(p.expression); + } + case "TableConstructor": + return expr.fields.some((f) => { + switch (f.type) { + case "PropField": + return containsAggregate((f as LuaPropField).value); + case "DynamicField": { + const df = f as LuaDynamicField; + return containsAggregate(df.key) || containsAggregate(df.value); + } + case "ExpressionField": + return containsAggregate((f as LuaExpressionField).value); + default: + return false; + } + }); + default: + return false; + } +} + +/** + * Evaluate an expression in aggregate-aware mode. + * + * When a FunctionCall matches a registered aggregate name, the aggregate + * protocol is executed instead of normal call semantics. All other + * expressions fall through to normal evalExpression. + */ +export async function evalExpressionWithAggregates( + expr: LuaExpression, + env: LuaEnv, + sf: LuaStackFrame, + groupItems: LuaTable, + objectVariable: string | undefined, + outerEnv: LuaEnv, +): Promise { + // Fast path: no aggregate calls in tree -> normal eval + if (!containsAggregate(expr)) { + return evalExpression(expr, env, sf); + } + + const recurse = (e: LuaExpression) => + evalExpressionWithAggregates( + e, + env, + sf, + groupItems, + objectVariable, + outerEnv, + ); + + // Direct aggregate call + if (expr.type === "FunctionCall") { + const fc = expr as LuaFunctionCallExpression; + if (fc.prefix.type === "Variable") { + const name = fc.prefix.name; + const spec = getAggregateSpec(name); + if (spec) { + const valueExpr = fc.args.length > 0 ? fc.args[0] : null; + return executeAggregate( + spec, + groupItems, + valueExpr, + objectVariable, + outerEnv, + sf, + evalExpression, + ); + } + } + } + + // Recurse into field values + if (expr.type === "TableConstructor") { + const table = new LuaTable(); + let nextArrayIndex = 1; + for (const field of expr.fields) { + switch (field.type) { + case "PropField": { + const pf = field as LuaPropField; + const value = await recurse(pf.value); + table.set(pf.key, value, sf); + break; + } + case "DynamicField": { + const df = field as LuaDynamicField; + const key = await evalExpression(df.key, env, sf); + const value = await recurse(df.value); + table.set(key, value, sf); + break; + } + case "ExpressionField": { + const ef = field as LuaExpressionField; + const value = await recurse(ef.value); + table.rawSetArrayIndex(nextArrayIndex, value); + nextArrayIndex++; + break; + } + } + } + return table; + } + + // Recurse operands, apply via luaOp + if (expr.type === "Binary") { + const bin = expr as LuaBinaryExpression; + + if (bin.operator === "and") { + const left = singleResult(await recurse(bin.left)); + if (!luaTruthy(left)) return left; + return singleResult(await recurse(bin.right)); + } + if (bin.operator === "or") { + const left = singleResult(await recurse(bin.left)); + if (luaTruthy(left)) return left; + return singleResult(await recurse(bin.right)); + } + + const left = singleResult(await recurse(bin.left)); + const right = singleResult(await recurse(bin.right)); + return luaOp( + bin.operator, + left, + right, + undefined, + undefined, + expr.ctx, + sf, + ); + } + + // Recurse argument, apply operator + if (expr.type === "Unary") { + const un = expr as LuaUnaryExpression; + const arg = singleResult(await recurse(un.argument)); + switch (un.operator) { + case "-": + return typeof arg === "number" ? -arg : luaOp( + "-", + 0, + arg, + undefined, + undefined, + expr.ctx, + sf, + ); + case "not": + return !luaTruthy(arg); + case "#": + return evalExpression(expr, env, sf); + case "~": + if (typeof arg === "number") return ~arg; + throw new Error("attempt to perform bitwise operation on a non-number"); + default: + return evalExpression(expr, env, sf); + } + } + + // Unwrap + if (expr.type === "Parenthesized") { + const paren = expr as LuaParenthesizedExpression; + return singleResult(await recurse(paren.expression)); + } + + return evalExpression(expr, env, sf); +} + /** * Applies a given query (where, order by, limit etc.) to a set of results */ @@ -104,6 +357,106 @@ export async function applyQuery( results = filteredResults; } + const grouped = !!query.groupBy; + + // Collect `group by` key names for unpacking into the environment. + let groupByNames: string[] | undefined; + + // Apply `group by` + if (query.groupBy) { + groupByNames = query.groupBy.map((expr) => { + if (expr.type === "Variable") { + return expr.name; + } + if (expr.type === "PropertyAccess") { + return expr.property; + } + return undefined as unknown as string; + }).filter(Boolean); + + const groups = new Map(); + for (const item of results) { + const itemEnv = buildItemEnv(query.objectVariable, item, env, sf); + // Evaluate all `group by` expressions to form a composite key + const keyParts: any[] = []; + for (const expr of query.groupBy) { + keyParts.push(await evalExpression(expr, itemEnv, sf)); + } + const compositeKey = keyParts.length === 1 + ? generateKey(keyParts[0]) + : JSON.stringify(keyParts.map(generateKey)); + let entry = groups.get(compositeKey); + if (!entry) { + // Unwrap single key; multi-key to `LuaTable` with named fields + let keyVal: any; + if (keyParts.length === 1) { + keyVal = keyParts[0]; + } else { + const kt = new LuaTable(); + for (let i = 0; i < keyParts.length; i++) { + kt.rawSetArrayIndex(i + 1, keyParts[i]); + if (groupByNames && groupByNames[i]) { + kt.rawSet(groupByNames[i], keyParts[i]); + } + } + keyVal = kt; + } + entry = { key: keyVal, items: [] }; + groups.set(compositeKey, entry); + } + entry.items.push(item); + } + // Convert groups to result rows with `key` and `group` + results = []; + for (const { key, items } of groups.values()) { + const groupTable = new LuaTable(); + for (let i = 0; i < items.length; i++) { + groupTable.rawSetArrayIndex(i + 1, items[i]); + } + const row = new LuaTable(); + row.rawSet("key", key); + row.rawSet("group", groupTable); + results.push(row); + } + } + + // Apply `having` -- aggregate-aware when grouped + if (query.having) { + const filteredResults = []; + for (const value of results) { + const itemEnv = buildGroupItemEnv( + query.objectVariable, + groupByNames, + value, + env, + sf, + ); + let condResult; + if (grouped) { + const groupTable = (value as LuaTable).rawGet("group"); + condResult = await evalExpressionWithAggregates( + query.having, + itemEnv, + sf, + groupTable, + query.objectVariable, + env, + ); + } else { + condResult = await evalExpression(query.having, itemEnv, sf); + } + if (condResult) { + filteredResults.push(value); + } + } + results = filteredResults; + } + + const mkEnv = grouped + ? (ov: string | undefined, item: any, e: LuaEnv, s: LuaStackFrame) => + buildGroupItemEnv(ov, groupByNames, item, e, s) + : buildItemEnv; + // Apply `order by` next if (query.orderBy) { // Retrieve from config API if not passed @@ -119,8 +472,8 @@ export async function applyQuery( results = await asyncQuickSort(results, async (a, b) => { // Compare each orderBy clause until we find a difference for (const { expr, desc } of query.orderBy!) { - const aEnv = buildItemEnv(query.objectVariable, a, env, sf); - const bEnv = buildItemEnv(query.objectVariable, b, env, sf); + const aEnv = mkEnv(query.objectVariable, a, env, sf); + const bEnv = mkEnv(query.objectVariable, b, env, sf); const aVal = await evalExpression(expr, aEnv, sf); const bVal = await evalExpression(expr, bEnv, sf); @@ -145,12 +498,26 @@ export async function applyQuery( }); } - // Apply the select + // Apply the select -- aggregate-aware when grouped if (query.select) { const newResult = []; for (const item of results) { - const itemEnv = buildItemEnv(query.objectVariable, item, env, sf); - newResult.push(await evalExpression(query.select, itemEnv, sf)); + const itemEnv = mkEnv(query.objectVariable, item, env, sf); + if (grouped) { + const groupTable = (item as LuaTable).rawGet("group"); + newResult.push( + await evalExpressionWithAggregates( + query.select, + itemEnv, + sf, + groupTable, + query.objectVariable, + env, + ), + ); + } else { + newResult.push(await evalExpression(query.select, itemEnv, sf)); + } } results = newResult; } @@ -215,6 +582,7 @@ function generateKey(value: any) { ? JSON.stringify(value) : value; } + export class DataStoreQueryCollection implements LuaQueryCollection { constructor( private readonly dataStore: DataStore, diff --git a/client/space_lua/query_env.ts b/client/space_lua/query_env.ts new file mode 100644 index 00000000..81197138 --- /dev/null +++ b/client/space_lua/query_env.ts @@ -0,0 +1,26 @@ +import { LuaEnv, luaGet, luaKeys, type LuaStackFrame } from "./runtime.ts"; + +/** + * Build an environment for evaluating per-item expressions in queries. + * Extracted to its own module to avoid circular imports between + * query_collection.ts and aggregates.ts. + */ +export function buildItemEnv( + objectVariable: string | undefined, + item: any, + env: LuaEnv, + sf: LuaStackFrame, +): LuaEnv { + const itemEnv = new LuaEnv(env); + if (!objectVariable) { + // Inject all item keys as variables + for (const key of luaKeys(item)) { + itemEnv.setLocal(key, luaGet(item, key, sf.astCtx ?? null, sf)); + } + // As well as _ + itemEnv.setLocal("_", item); + } else { + itemEnv.setLocal(objectVariable, item); + } + return itemEnv; +} diff --git a/client/space_lua/query_test.lua b/client/space_lua/query_test.lua index f60107a5..e8c3bf67 100644 --- a/client/space_lua/query_test.lua +++ b/client/space_lua/query_test.lua @@ -1,47 +1,510 @@ local function assertEquals(a, b) - if a ~= b then - error("Assertion failed: " .. a .. " is not equal to " .. b) - end + if a ~= b then + error("Assertion failed: " + .. tostring(a) .. " is not equal to " + .. tostring(b)) + end end --- Test query functionality -local data = { { name = "John", lastModified = 1, age = 20 }, { name = "Jane", lastModified = 2, age = 21 } } +-- Dataset +local pages = { + { name = "Alice", tags = {"work", "urgent"}, size = 10, age = 31 }, + { name = "Bob", tags = {"work"}, size = 20, age = 25 }, + { name = "Carol", tags = {"personal", "urgent"}, size = 5, age = 41 }, + { name = "Dave", tags = {"personal"}, size = 15, age = 52 }, + { name = "Ed", tags = {}, size = 3, age = 19 }, + { name = "Fran", tags = {"random"}, size = 1, age = 55 }, + { name = "Greg", tags = {"work", "fun"}, size = 2, age = 63 }, +} --- Basic limit query -local r = query [[from p = data limit 1]] -assertEquals(#r, 1) -assertEquals(r[1].name, "John") -assertEquals(r[1].lastModified, 1) +-- 1. Basic `from` +do + local r = query [[ + from + pages + ]] --- Order by descending -local r = query [[from p = data order by p.lastModified desc]] -assertEquals(#r, 2) -assertEquals(r[1].name, "Jane") -assertEquals(r[1].lastModified, 2) -assertEquals(r[2].name, "John") -assertEquals(r[2].lastModified, 1) + assertEquals(#r, #pages) +end --- Order by ascending -local r = query [[from p = data order by p.lastModified]] -assertEquals(#r, 2) -assertEquals(r[1].name, "John") -assertEquals(r[1].lastModified, 1) -assertEquals(r[2].name, "Jane") -assertEquals(r[2].lastModified, 2) +do + local r = query [[ + from + p = pages + ]] --- Select specific fields -local r = query [[from p = data order by p.age select {name=p.name, age=p.age}]] -assertEquals(#r, 2) -assertEquals(r[1].name, "John") -assertEquals(r[1].age, 20) -assertEquals(r[2].name, "Jane") -assertEquals(r[2].age, 21) -assertEquals(r[1].lastModified, nil) -assertEquals(r[2].lastModified, nil) + assertEquals(r[1].name, "Alice") +end --- Array transformation -local r = query [[from {1, 2, 3} select _ + 1]] -assertEquals(#r, 3) -assertEquals(r[1], 2) -assertEquals(r[2], 3) -assertEquals(r[3], 4) \ No newline at end of file +-- 2. Select/projection: direct, binding, mixed +do + local r = query [[ + from + pages + select { + n = name, + t = tags[1], + } + ]] + + assertEquals(r[1].n, "Alice") +end + +do + local r = query [[ + from + p = pages + select { + n = p.name, + t = p.tags[1], + } + ]] + + assertEquals(r[2].n, "Bob") +end + +do + local r = query [[ + from + pages + select { + a = name, + b = p and p.tags[1], + } + ]] + + assertEquals(r[1].a, "Alice") +end + +-- 3. Select field, select as array, order/limit/offset, where +do + local r1 = query [[ + from + pages + select { + value = size, + } + ]] + + assertEquals(r1[1].value, 10) + + local r2 = query [[ + from + p = pages + select { + value = p.size, + } + ]] + + assertEquals(r2[1].value, 10) +end + +do + local r1 = query [[ + from + pages + limit 2 + select { + name = name, + } + ]] + + assertEquals(#r1, 2) + + local r2 = query [[ + from + p = pages + limit 3, 2 + select { + name = p.name, + } + ]] + + assertEquals(r2[1].name, "Carol") +end + +do + local r1 = query [[ + from + pages + order by + size desc + select { + name = name, + } + ]] + + assert(r1[1].name == "Bob") + + local r2 = query [[ + from + p = pages + order by + p.age + select { + age = p.age, + } + ]] + + assert(r2[1].age == 19) +end + +do + local r1 = query [[ + from + pages + where + size > 10 + select { + name = name, + } + ]] + + assert(r1[1].name == "Bob") + + local r2 = query [[ + from + p = pages + where + p.age < 30 + select { + name = p.name, + } + ]] + + assert(r2[1].name == "Bob", + "Expected Bob in r2[1], got " .. tostring(r2[1] and r2[1].name)) + + assert(r2[2].name == "Ed", + "Expected Ed in r2[2], got " .. tostring(r2[2] and r2[2].name)) +end + +-- 4. Grouping/group select + +do + local r1 = query [[ + from + pages + where + tags[1] ~= nil + group by + tags[1] + select { + key = key, + } + ]] + local r2 = query [[ + from + p = pages + where + p.tags[1] ~= nil + group by + p.tags[1] + select { + key = key, + } + ]] + + assert(type(r1[1].key) == "string" or r1[1].key == nil) + assert(#r1 == #r2) +end + +do + local r1 = query [[ + from + pages + where + tags[1] ~= nil and tags[2] ~= nil + group by + tags[1], tags[2] + select { + k1 = key[1], + k2 = key[2], + } + ]] + local r2 = query [[ + from + p = pages + where + p.tags[1] ~= nil and p.tags[2] ~= nil + group by + p.tags[1], p.tags[2] + select { + k1 = key[1], + k2 = key[2], + } + ]] + + assert(#r1 == #r2) +end + +do + local r = query [[ + from + pages + group by + tags[1] + select { + k = key, + gc = #group, + first = group[1].name, + f2 = group[1].tags[2], + } + ]] + + assert(type(r[1].k) == "string" or r[1].k == nil) + assert(type(r[1].gc) == "number") +end + +do + local r = query [[ + from + p = pages + group by + p.tags[1] + select { + k = key, + gc = #group, + n = group[1].name, + } + ]] + + assert(type(r[1].gc) == "number") +end + +do + local r = query [[ + from + pages + group by + tags[1] + select { + k = key, + n = group[1].name, + t = group[1].tags[1], + } + ]] + + assert(type(r[1].k) == "string" or r[1].k == nil) +end + +-- 5. Aggregation/builtins + +do + local r1 = query [[ + from + pages + group by + tags[1] + select { + count = count(name), + } + ]] + local r2 = query [[ + from + p = pages + group by + p.tags[1] + select { + count = count(p.name), + } + ]] + + assert(type(r1[1].count) == "number") + assert(#r1 == #r2) +end + +do + local r1 = query [[ + from + pages + group by + tags[1] + select { + min = min(age), + max = max(age), + avg = avg(age), + sum = sum(age), + } + ]] + local r2 = query [[ + from + p = pages + group by + p.tags[1] + select { + min = min(p.size), + max = max(p.size), + avg = avg(p.size), + sum = sum(p.size), + } + ]] + + for _, row in ipairs(r2) do + assert(type(row.min) == "number" or row.min == nil) + assert(type(row.avg) == "number" or row.avg == nil) + end +end + +do + local r1 = query [[ + from + pages + group by + tags[1] + select { + arr = array_agg(name), + } + ]] + local r2 = query [[ + from + p = pages + group by + p.tags[1] + select { + arr = array_agg(p.name), + } + ]] + + assert(type(r1[1].arr) == "table" or r1[1].arr == nil) + assert(#r1 == #r2) +end + +do + local r = query [[ + from + pages + group by + tags[1] + select { + c = count(name), + v = min(size), + x = p and count(p.name), + } + ]] + + assert(type(r[1].c) == "number") +end + +-- 6. `having` + +do + local r1 = query [[ + from + pages + group by + tags[1] + having + count(name) > 1 + select { + key = key, + } + ]] + local r2 = query [[ + from + p = pages + group by + p.tags[1] + having + count(p.name) > 1 + select { + key = key, + } + ]] + + assertEquals(#r1, #r2) +end + +do + local r = query [[ + from + p = pages + group by + p.tags[1] + having + sum(p.size) > 15 + select { + key = key, + } + ]] + + assert((#r > 0), "expected some groups") +end + +-- 7. Full pipeline + +do + local r = query [[ + from + p = pages + where + p.age > 20 + group by + p.tags[1] + having + min(p.age) > 25 + select { + tag = key, + top = max(p.name), + total = count(p.name), + sum_size = sum(p.size), + min_age = min(age), + avg_size = avg(p.size), + } + order by + avg_size desc + limit 2 + ]] + + assert(#r <= 2) +end + +do + local r = query [[ + from + pages + where + age > 20 + group by + tags[1] + having + min(age) > 25 + select { + tag = key, + top = max(name), + total = count(name), + sum_size = sum(size), + min_age = min(age), + avg_size = avg(size), + } + order by + avg_size desc + limit 2 + ]] + + assert(#r <= 2) +end + +do + local r = query [[ + from + pages + where + age > 20 + group by + tags[1] + having + min(age) > 25 + select { + tag = key, + p = p and p.name, + total = count(name), + avg_size = avg(size), + } + order by + avg_size desc + limit 2 + ]] + + assert(#r <= 2) +end diff --git a/client/space_lua/stdlib/string.ts b/client/space_lua/stdlib/string.ts index 10839ac1..1b1736da 100644 --- a/client/space_lua/stdlib/string.ts +++ b/client/space_lua/stdlib/string.ts @@ -16,6 +16,7 @@ import { patternGsub, patternMatch, } from "./pattern.ts"; +import { strPackFn, strPackSizeFn, strUnpackFn } from "./string_pack.ts"; function capturesToLua(caps: CaptureResult[]): any { if (caps.length === 0) return null; @@ -164,10 +165,15 @@ export const stringApi = new LuaTable({ } return ""; }), + split: new LuaBuiltinFunction((_sf, s: string, sep: string) => { return s.split(sep); }), + pack: strPackFn, + unpack: strUnpackFn, + packsize: strPackSizeFn, + // Non-standard extensions startsWith: new LuaBuiltinFunction((_sf, s: string, prefix: string) => { return s.startsWith(prefix); diff --git a/client/space_lua/stdlib/string_pack.ts b/client/space_lua/stdlib/string_pack.ts new file mode 100644 index 00000000..13d91e21 --- /dev/null +++ b/client/space_lua/stdlib/string_pack.ts @@ -0,0 +1,486 @@ +import { + LuaBuiltinFunction, + LuaMultiRes, + LuaRuntimeError, +} from "../runtime.ts"; + +import { isTaggedFloat } from "../numeric.ts"; + +function untagN(x: any): number { + if (typeof x === "number") return x; + if (isTaggedFloat(x)) return x.value; + return Number(x); +} + +const NATIVE_LITTLE = new Uint8Array(new Uint16Array([1]).buffer)[0] === 1; +const NATIVE_MAXALIGN = 8; // JS doubles are 8-byte aligned + +type KOption = + | "int" + | "uint" + | "float" + | "double" + | "number" + | "char" + | "string" + | "zstr" + | "padding" + | "paddalign" + | "nop"; + +interface ParsedOption { + opt: KOption; + size: number; // byte width + ntoalign: number; // padding bytes before this field +} + +interface Header { + islittle: boolean; + maxalign: number; +} + +function makeHeader(): Header { + return { islittle: NATIVE_LITTLE, maxalign: NATIVE_MAXALIGN }; +} + +// Read digits from fmt starting at pos; return [value, newPos] +function readNum(fmt: string, pos: number, dflt: number): [number, number] { + if (pos >= fmt.length || fmt[pos] < "0" || fmt[pos] > "9") return [dflt, pos]; + let v = 0; + while (pos < fmt.length && fmt[pos] >= "0" && fmt[pos] <= "9") { + v = v * 10 + (fmt.charCodeAt(pos) - 48); + pos++; + } + return [v, pos]; +} + +function numLimit( + fmt: string, + pos: number, + dflt: number, + src: string, +): [number, number] { + const [sz, np] = readNum(fmt, pos, dflt); + if (sz < 1 || sz > 16) { + throw new Error(`integral size (${sz}) out of limits [1,16] in '${src}'`); + } + return [sz, np]; +} + +// Parse one option from fmt[pos], return [parsed, newPos] +// Modifies header in place for '<', '>', '=', '!' +function getOption( + fmt: string, + pos: number, + h: Header, +): [KOption, number, number] { // [opt, size, newPos] + const c = fmt[pos++]; + switch (c) { + case "b": + return ["int", 1, pos]; + case "B": + return ["uint", 1, pos]; + case "h": + return ["int", 2, pos]; + case "H": + return ["uint", 2, pos]; + case "l": + return ["int", 8, pos]; + case "L": + return ["uint", 8, pos]; + case "j": + return ["int", 8, pos]; + case "J": + return ["uint", 8, pos]; + case "T": + return ["uint", 8, pos]; + case "f": + return ["float", 4, pos]; + case "n": + return ["number", 8, pos]; + case "d": + return ["double", 8, pos]; + case "i": { + const [sz, np] = numLimit(fmt, pos, 4, "i"); + return ["int", sz, np]; + } + case "I": { + const [sz, np] = numLimit(fmt, pos, 4, "I"); + return ["uint", sz, np]; + } + case "s": { + const [sz, np] = numLimit(fmt, pos, 8, "s"); + return ["string", sz, np]; + } + case "c": { + const [sz, np] = readNum(fmt, pos, -1); + if (sz === -1) throw new Error("missing size for format option 'c'"); + return ["char", sz, np]; + } + case "z": + return ["zstr", 0, pos]; + case "x": + return ["padding", 1, pos]; + case "X": + return ["paddalign", 0, pos]; + case " ": + return ["nop", 0, pos]; + case "<": + h.islittle = true; + return ["nop", 0, pos]; + case ">": + h.islittle = false; + return ["nop", 0, pos]; + case "=": + h.islittle = NATIVE_LITTLE; + return ["nop", 0, pos]; + case "!": { + const [sz, np] = readNum(fmt, pos, NATIVE_MAXALIGN); + h.maxalign = sz; + return ["nop", 0, np]; + } + default: + throw new Error(`invalid format option '${c}'`); + } +} + +// Compute alignment padding +function getDetails( + fmt: string, + pos: number, + h: Header, + totalsize: number, +): [ParsedOption, number] { + let opt: KOption, size: number; + [opt, size, pos] = getOption(fmt, pos, h); + + let align = size; + + if (opt === "paddalign") { + if (pos >= fmt.length) { + throw new Error("invalid next option for option 'X'"); + } + const hCopy = { ...h }; + let nextOpt: KOption, nextSize: number; + [nextOpt, nextSize, pos] = getOption(fmt, pos, hCopy); + if (nextOpt === "char" || nextSize === 0) { + throw new Error("invalid next option for option 'X'"); + } + align = nextSize; + } + + let ntoalign = 0; + if ( + opt !== "char" && opt !== "nop" && opt !== "padding" && opt !== "paddalign" + ) { + const realign = Math.min(align, h.maxalign); + if (realign > 0) { + ntoalign = (realign - (totalsize % realign)) % realign; + } + } + + return [{ opt, size, ntoalign }, pos]; +} + +function packInt(v: bigint, size: number, islittle: boolean): Uint8Array { + const buf = new Uint8Array(size); + let val = v; + // Two's complement mask + const mask = (1n << BigInt(size * 8)) - 1n; + val = ((val % (mask + 1n)) + (mask + 1n)) & mask; // normalise to unsigned + for (let i = 0; i < size; i++) { + buf[islittle ? i : size - 1 - i] = Number(val & 0xffn); + val >>= 8n; + } + return buf; +} + +function unpackInt( + buf: Uint8Array, + pos: number, + size: number, + islittle: boolean, + issigned: boolean, +): bigint { + let res = 0n; + const limit = Math.min(size, 8); + for (let i = limit - 1; i >= 0; i--) { + res = (res << 8n) | BigInt(buf[pos + (islittle ? i : size - 1 - i)]); + } + if (issigned && size <= 8) { + const mask = 1n << BigInt(size * 8 - 1); + if (res & mask) res -= mask << 1n; + } + return res; +} + +function packFloat32(v: number, islittle: boolean): Uint8Array { + const buf = new ArrayBuffer(4); + new DataView(buf).setFloat32(0, v, islittle); + return new Uint8Array(buf); +} + +function unpackFloat32( + buf: Uint8Array, + pos: number, + islittle: boolean, +): number { + return new DataView(buf.buffer, buf.byteOffset + pos, 4).getFloat32( + 0, + islittle, + ); +} + +function packFloat64(v: number, islittle: boolean): Uint8Array { + const buf = new ArrayBuffer(8); + new DataView(buf).setFloat64(0, v, islittle); + return new Uint8Array(buf); +} + +function unpackFloat64( + buf: Uint8Array, + pos: number, + islittle: boolean, +): number { + return new DataView(buf.buffer, buf.byteOffset + pos, 8).getFloat64( + 0, + islittle, + ); +} + +export const strPackFn = new LuaBuiltinFunction( + (sf, fmt: string, ...args: any[]) => { + const h = makeHeader(); + const parts: Uint8Array[] = []; + let totalsize = 0; + let argIdx = 0; + + let pos = 0; + while (pos < fmt.length) { + let opt: ParsedOption; + [opt, pos] = getDetails(fmt, pos, h, totalsize); + + // alignment padding + if (opt.ntoalign > 0) { + parts.push(new Uint8Array(opt.ntoalign)); + totalsize += opt.ntoalign; + } + + switch (opt.opt) { + case "nop": + case "paddalign": + break; + + case "padding": + parts.push(new Uint8Array(1)); // LUAL_PACKPADBYTE = 0 + totalsize += 1; + break; + + case "int": + case "uint": { + const v = args[argIdx++]; + if (v === undefined || v === null) { + throw new LuaRuntimeError( + `bad argument #${argIdx} to 'pack' (value expected)`, + sf, + ); + } + let bi: bigint; + if (typeof v === "bigint") bi = v; + else bi = BigInt(Math.trunc(untagN(v))); + parts.push(packInt(bi, opt.size, h.islittle)); + totalsize += opt.size; + break; + } + + case "float": { + const v = untagN(args[argIdx++]); + parts.push(packFloat32(v, h.islittle)); + totalsize += 4; + break; + } + + case "double": + case "number": { + const v = untagN(args[argIdx++]); + parts.push(packFloat64(v, h.islittle)); + totalsize += 8; + break; + } + + case "char": { + const s: string = String(args[argIdx++]); + const enc = new TextEncoder().encode(s); + const buf = new Uint8Array(opt.size); + buf.set(enc.subarray(0, opt.size)); + parts.push(buf); + totalsize += opt.size; + break; + } + + case "string": { + const s: string = String(args[argIdx++]); + const enc = new TextEncoder().encode(s); + const lenBuf = packInt(BigInt(enc.length), opt.size, h.islittle); + parts.push(lenBuf); + parts.push(enc); + totalsize += opt.size + enc.length; + break; + } + + case "zstr": { + const s: string = String(args[argIdx++]); + if (s.includes("\0")) { + throw new LuaRuntimeError( + "string contains zeros for format 'z'", + sf, + ); + } + const enc = new TextEncoder().encode(s); + parts.push(enc); + parts.push(new Uint8Array(1)); // null terminator + totalsize += enc.length + 1; + break; + } + } + } + + // Concatenate all parts into one binary string (latin-1 encoding) + let total = 0; + for (const p of parts) total += p.length; + const out = new Uint8Array(total); + let off = 0; + for (const p of parts) { + out.set(p, off); + off += p.length; + } + + // Return as a Lua binary string (each byte is a char code 0-255) + let result = ""; + for (let i = 0; i < out.length; i++) result += String.fromCharCode(out[i]); + return result; + }, +); + +export const strUnpackFn = new LuaBuiltinFunction( + (sf, fmt: string, data: string, init?: number) => { + const h = makeHeader(); + + const buf = new Uint8Array(data.length); + for (let i = 0; i < data.length; i++) { + buf[i] = data.charCodeAt(i) & 0xff; + } + + let pos = (init !== undefined && init !== null ? init : 1) - 1; + const results: any[] = []; + + let fmtPos = 0; + while (fmtPos < fmt.length) { + let opt: ParsedOption; + [opt, fmtPos] = getDetails(fmt, fmtPos, h, pos); + + if (opt.ntoalign + opt.size > buf.length - pos) { + if ( + opt.opt !== "nop" && opt.opt !== "paddalign" && opt.opt !== "padding" + ) { + throw new LuaRuntimeError("data string too short", sf); + } + } + + pos += opt.ntoalign; // skip alignment padding + + switch (opt.opt) { + case "nop": + case "paddalign": + break; + + case "padding": + pos += 1; + break; + + case "int": { + const v = unpackInt(buf, pos, opt.size, h.islittle, true); + const n = Number(v); + results.push(Number.isSafeInteger(n) ? n : v); + pos += opt.size; + break; + } + + case "uint": { + const v = unpackInt(buf, pos, opt.size, h.islittle, false); + const n = Number(v); + results.push(Number.isSafeInteger(n) ? n : v); + pos += opt.size; + break; + } + + case "float": { + results.push(unpackFloat32(buf, pos, h.islittle)); + pos += 4; + break; + } + + case "double": + case "number": { + results.push(unpackFloat64(buf, pos, h.islittle)); + pos += 8; + break; + } + + case "char": { + const s = String.fromCharCode(...buf.subarray(pos, pos + opt.size)); + results.push(s); + pos += opt.size; + break; + } + + case "string": { + const len = Number(unpackInt(buf, pos, opt.size, h.islittle, false)); + if (len > buf.length - pos - opt.size) { + throw new LuaRuntimeError("data string too short", sf); + } + pos += opt.size; + const s = new TextDecoder().decode(buf.subarray(pos, pos + len)); + results.push(s); + pos += len; + break; + } + + case "zstr": { + let end = pos; + while (end < buf.length && buf[end] !== 0) end++; + if (end >= buf.length) { + throw new LuaRuntimeError("unfinished string for format 'z'", sf); + } + results.push(new TextDecoder().decode(buf.subarray(pos, end))); + pos = end + 1; + break; + } + } + } + + results.push(pos + 1); + return new LuaMultiRes(results); + }, +); + +export const strPackSizeFn = new LuaBuiltinFunction( + (_sf, fmt: string) => { + const h = makeHeader(); + let totalsize = 0; + let pos = 0; + + while (pos < fmt.length) { + let opt: ParsedOption; + [opt, pos] = getDetails(fmt, pos, h, totalsize); + + if (opt.opt === "string" || opt.opt === "zstr") { + throw new LuaRuntimeError("variable-length format", _sf); + } + + totalsize += opt.ntoalign + opt.size; + } + + return totalsize; + }, +); diff --git a/client/space_lua/stdlib/string_pack_test.lua b/client/space_lua/stdlib/string_pack_test.lua new file mode 100644 index 00000000..71790b20 --- /dev/null +++ b/client/space_lua/stdlib/string_pack_test.lua @@ -0,0 +1,186 @@ +local function assertEquals(a, b, msg) + if a ~= b then + error((msg or "assertEquals") .. ": expected " .. tostring(b) .. " got " .. tostring(a)) + end +end + +local function assertError(fn, msg) + local ok, err = pcall(fn) + if ok then error(msg or "expected error") end +end + +-- packsize: fixed formats +do + assertEquals(string.packsize("b"), 1, "packsize b") + assertEquals(string.packsize("B"), 1, "packsize B") + assertEquals(string.packsize("h"), 2, "packsize h") + assertEquals(string.packsize("H"), 2, "packsize H") + assertEquals(string.packsize("i4"), 4, "packsize i4") + assertEquals(string.packsize("I4"), 4, "packsize I4") + assertEquals(string.packsize("i8"), 8, "packsize i8") + assertEquals(string.packsize("f"), 4, "packsize f") + assertEquals(string.packsize("d"), 8, "packsize d") + assertEquals(string.packsize("c10"), 10, "packsize c10") + assertEquals(string.packsize("i4i4"), 8, "packsize i4i4") + + -- variable-length formats must error + assertError(function() string.packsize("s4") end, "packsize s4 must error") + assertError(function() string.packsize("z") end, "packsize z must error") +end + +-- little-endian integers +do + local s = string.pack("i2", 256) + assertEquals(s:byte(1), 1, "pack >i2(256) byte 1") + assertEquals(s:byte(2), 0, "pack >i2(256) byte 2") + + local v = string.unpack(">i2", s) + assertEquals(v, 256, "unpack >i2") + + local s2 = string.pack(">I2", 0xABCD) + local v2 = string.unpack(">I2", s2) + assertEquals(v2, 0xABCD, "unpack >I2") +end + +-- byte / unsigned byte +do + local s = string.pack("BB", 65, 66) + assertEquals(#s, 2, "pack BB length") + + local a, b, _ = string.unpack("BB", s) + assertEquals(a, 65, "unpack B first") + assertEquals(b, 66, "unpack B second") + + -- signed byte: -1 round-trips + local s2 = string.pack("b", -1) + local v2 = string.unpack("b", s2) + assertEquals(v2, -1, "unpack b -1") +end + +-- float (f) and double (d) +do + local s = string.pack("I2", 0x0102, 0x0304) + -- little: 02 01; big: 03 04 + assertEquals(s:byte(1), 0x02, "LE low byte") + assertEquals(s:byte(2), 0x01, "LE high byte") + assertEquals(s:byte(3), 0x03, "BE high byte") + assertEquals(s:byte(4), 0x04, "BE low byte") +end diff --git a/libraries/Library/Std/APIs/Aggregate.md b/libraries/Library/Std/APIs/Aggregate.md new file mode 100644 index 00000000..33ca0512 --- /dev/null +++ b/libraries/Library/Std/APIs/Aggregate.md @@ -0,0 +1,122 @@ +--- +description: APIs to define custom aggregate functions for LIQ +tags: meta/api +--- + +APIs to define and override aggregate functions used in [[Space Lua/Lua Integrated Query|LIQ]] `select` and `having` clauses after `group by`. + +Built-in aggregates: `count`, `sum`, `min`, `max`, `avg` and `array_agg`. + +# API + +## aggregate.define(spec) + +Defines a new aggregate function. Required keys: + +* `name`: name of the aggregate (used in queries as `name(expr)`) +* `initialize`: function that returns the initial state +* `iterate`: function(state, value) that returns updated state + +Optional keys: + +* `description`: description of the aggregate +* `finish`: `function(state)` that transforms the final state into the result + +## aggregate.update(spec) + +Updates an existing aggregate definition. Same keys as `aggregate.define`. Only the provided keys are overwritten. + +# Examples + +## Define a custom aggregate + +Define a custom aggregate `concat` that concatenates strings. + +```lua +aggregate.define { + name = 'concat', + + initialize = function() + return '' + end, + + iterate = function(state, value) + if state == '' then + return tostring(value) + end + return state .. ', ' .. tostring(value) + end, +} +``` + +## Update an existing aggregate + +```lua +aggregate.update { + name = 'count', + description = 'Custom count aggregate that counts even nils', + + iterate = function(state, value) + return state + 1 + end, +} +``` + +# Implementation + +```space-lua +-- priority: 50 +aggregate = aggregate or {} + +local aggregateSchema = { + type = 'object', + + required = { + 'name', + 'initialize', + 'iterate' + }, + + properties = { + name = schema.string(), + description = schema.string(), + initialize = schema.func(), + iterate = schema.func(), + finish = schema.func(), + } +} + +function aggregate.define(spec) + local validationResult = jsonschema.validateObject(aggregateSchema, spec) + + if validationResult then + error('aggregate.define: ' .. validationResult) + end + + config.set({'aggregates', spec.name}, spec) +end + +function aggregate.update(spec) + if not spec.name then + error('aggregate.update: name is required') + end + + local existing = config.get({'aggregates', spec.name}, {}) + + for k, v in pairs(spec) do + existing[k] = v + end + + if not existing.initialize then + error('aggregate.update: aggregate ' + .. spec.name .. ' has no initialize after merge') + end + + if not existing.iterate then + error('aggregate.update: aggregate ' + .. spec.name .. ' has no iterate after merge') + end + + config.set({'aggregates', spec.name}, existing) +end +``` diff --git a/plugs/editor/editor.ts b/plugs/editor/editor.ts index 266798f7..26e8faf3 100644 --- a/plugs/editor/editor.ts +++ b/plugs/editor/editor.ts @@ -109,8 +109,13 @@ export async function centerCursorCommand() { } export async function moveToPosCommand() { - const posString = await editor.prompt("Move to position:"); - if (!posString) { + let posString = await editor.prompt("Move to position:"); + if (posString === undefined) { + return; + } + posString = posString.trim(); + if (posString === "") { + editor.flashNotification("Must provide a position.", "error"); return; } const pos = +posString; @@ -132,10 +137,15 @@ export async function copyLinkCommand() { } export async function moveToLineCommand() { - const lineString = await editor.prompt( + let lineString = await editor.prompt( "Move to line (and optionally column):", ); - if (!lineString) { + if (lineString === undefined) { + return; + } + lineString = lineString.trim(); + if (lineString === "") { + editor.flashNotification("Must provide a line number.", "error"); return; } // Match sequence of digits at the start, optionally another sequence diff --git a/plugs/editor/page.ts b/plugs/editor/page.ts index cd349f2f..d4a0f9a6 100644 --- a/plugs/editor/page.ts +++ b/plugs/editor/page.ts @@ -31,9 +31,13 @@ export async function copyPage( const fromName = sourcePage || currentPage; const suggestedName = toName || fromName; - const newName = await editor.prompt(`Copy to page:`, suggestedName); - - if (!newName) { + let newName = await editor.prompt(`Copy to page:`, suggestedName); + if (newName === undefined) { + return; + } + newName = newName.trim(); + if (newName === "") { + editor.flashNotification("Must provide a non-empty page name.", "error"); return; } diff --git a/plugs/index/refactor.ts b/plugs/index/refactor.ts index 9e4f8b52..4bc8411d 100644 --- a/plugs/index/refactor.ts +++ b/plugs/index/refactor.ts @@ -33,9 +33,14 @@ import { notFoundError } from "@silverbulletmd/silverbullet/constants"; */ export async function renamePageCommand(cmdDef: any) { const oldName: string = cmdDef.oldPage || await editor.getCurrentPage(); - const newName: string = cmdDef.page || + let newName: string = cmdDef.page || await editor.prompt(`Rename ${oldName} to:`, oldName); - if (!newName) { + if (newName === undefined) { + return false; + } + newName = newName.trim(); + if (newName === "") { + editor.flashNotification("Must provide a non-empty page title.", "error"); return false; } const pageList: [string, string][] = [[oldName + ".md", newName + ".md"]]; @@ -66,8 +71,13 @@ export async function renamePageLinkCommand() { } const oldName = wikiLinkPage.children![0].text!; - const newName = await editor.prompt(`Rename ${oldName} to:`, oldName); - if (!newName) { + let newName = await editor.prompt(`Rename ${oldName} to:`, oldName); + if (newName === undefined) { + return false; + } + newName = newName.trim(); + if (newName === "") { + editor.flashNotification("Must provide a non-empty page title.", "error"); return false; } const pageList: [string, string][] = [[oldName + ".md", newName + ".md"]]; @@ -84,9 +94,17 @@ export async function renamePageLinkCommand() { */ export async function renameDocumentCommand(cmdDef: any) { const oldName: string = cmdDef.oldDocument || await editor.getCurrentPath(); - const newName: string = cmdDef.document || + let newName: string = cmdDef.document || await editor.prompt(`Rename ${oldName} to:`, oldName); - if (!newName) { + if (newName === undefined) { + return false; + } + newName = newName.trim(); + if (newName === "") { + editor.flashNotification( + "Must provide a non-empty document name.", + "error", + ); return false; } const pageList: [string, string][] = [[oldName, newName]]; @@ -300,13 +318,19 @@ async function renameDocument( export async function renamePrefixCommand(cmdDef: any) { const oldPrefix = cmdDef.oldPrefix ?? await editor.prompt("Prefix to rename:", ""); - if (!oldPrefix) { + if (oldPrefix === undefined) { + return false; + } + // Note, we do *not* trim the old or new prefix input as the user may + // actually want to add or remove white space. They can also input an empty + // string for the new prefix to remove the old prefix. + if (oldPrefix === "") { + editor.flashNotification("Must provide a non-empty prefix.", "error"); return false; } - const newPrefix = cmdDef.newPrefix ?? await editor.prompt("New prefix:", oldPrefix); - if (!newPrefix) { + if (newPrefix === undefined) { return false; } @@ -348,8 +372,12 @@ export async function extractToPageCommand() { newName = "new page"; } newName = await editor.prompt(`New page title:`, newName); - if (!newName) { - return; + if (newName === undefined) { + return false; + } + newName = newName.trim(); + if (newName === "") { + editor.flashNotification("Must provide a non-empty page title.", "error"); } try { diff --git a/website/Space Lua/Lua Integrated Query.md b/website/Space Lua/Lua Integrated Query.md index 045f3bd7..28bfc031 100644 --- a/website/Space Lua/Lua Integrated Query.md +++ b/website/Space Lua/Lua Integrated Query.md @@ -2,13 +2,15 @@ Lua Integrated Query (LIQ) is a SilverBullet specific Lua extension. It adds a c The syntax for LIQ is `query[[my query]]`. In regular Lua `[[my query]]` is just another way of writing `"my query"` (it is an alternative string syntax). Function calls that only take a string argument can omit parentheses, therefore `query[[my query]]` is equivalent to `query("my query")`. -However, in [[Space Lua]] it interpreted as an SQL (and [LINQ](https://learn.microsoft.com/en-us/dotnet/csharp/linq/))-inspired integrated query language. +However, in [[Space Lua]] it is interpreted as an SQL (and [LINQ](https://learn.microsoft.com/en-us/dotnet/csharp/linq/))-inspired integrated query language. General syntax: query[[ from in where + group by [, , ...] + having order by limit , select @@ -54,8 +56,8 @@ However, there is also the more concise: implicitly binding each item to the variable `_` as well as making all attributes directly available as variables. The latter, while shorter, is less performant and will block future optimizations, so the variable-binding variant is preferred. > **warning** Warning -> When you use a `from` clause without explicit variable binding (so without thee `v in` syntax), note that any attribute of the object you’re iterating over will shadow global variables. For instance, if you have an object with a `table` attribute, regular `table` APIs will become inaccessible within the query. -> +> When you use a `from` clause without explicit variable binding (so without the `v in` syntax), note that any attribute of the object you’re iterating over will shadow global variables. For instance, if you have an object with a `table` attribute, regular `table` APIs will become inaccessible within the query. +> > **Recommendation:** Use the explicit variable binding syntax Example without variable binding: @@ -78,10 +80,47 @@ Or to select 5 pages tagged with `#meta`: ${query[[from p = index.tag "page" where table.includes(p.tags, "meta") limit 5]]} -Or select based on name (including folder) and a [[API/string|string function]] +Or select based on name (including folder) and a [[API/string|string function]]: ${query[[from p = index.tag "page" where p.name:startsWith("Person")]]} +## group by [, , ...] +The `group by` clause groups results by one or more key expressions. After grouping, each result row becomes a table with two fields: + +- `key` — the group key value (single value for one key, table for multi-key) +- `group` — a table (array) of all original items in that group + +The `group by` field names are also available as bare variables in `having`, `select`, and `order by`. Use `#group` to get the count of items in a group. + +Example: + +${query[[ + from p = index.tag "tag" + group by p.name + select { name = p.name, count = #p.group } + limit 5 +]]} + +See [[Space Lua/Lua Integrated Query/Grouping]] for detailed examples. + +## having +The `having` clause filters groups **after** `group by`. It follows SQL semantics: only group key fields, `key`, and `group` are accessible — use `where` to filter individual rows before grouping. + +Aggregate functions like `count()`, `sum()`, `min()`, `max()`, and `avg()` can be used in `having` expressions. See [[Space Lua/Lua Integrated Query/Aggregating]] for details. + +Example: + +${query[[ + from p = index.tag "tag" + group by p.name + having #group > 2 + select { name = name, count = #group } + order by count desc + limit 5 +]]} + +See [[Space Lua/Lua Integrated Query/Grouping]] for detailed examples. + ## order by [desc] The `order by` clause allows you to sort data, when `desc` is specified it reverts the sort order. @@ -118,6 +157,8 @@ ${query[[from {1, 2, 3, 4, 5} limit 3, 2]]} ## select The `select` clause allows you to transform each item in the result set. If omitted, it defaults to returning the item itself. +When used with `group by`, aggregate functions like `sum()`, `count()`, `min()`, `max()`, and `avg()` can be used in the `select` expression to compute values across each group. See [[Space Lua/Lua Integrated Query/Aggregating]] for details. + Some examples: Double each number: @@ -125,12 +166,24 @@ ${query[[from n = {1, 2, 3} select n * 2]]} It is convenient to combine it with the [[API/table#table.select(table, keys...)]] API: ${query[[ - from p = index.tag "page" + from p = index.tag "page" select table.select(p, "name", "lastModified") limit 3 ]]} -## Rendering the output +# Listing available aggregate functions + +You can list all available aggregator functions (built-ins and custom) using LIQ. + +Example: + +${query [[ + from k, v in pairs(config.get("aggregates", {})) + select { name = k, desc = v.description } + order by name +]]} + +# Rendering the output To render the output as a template, you can rely on the fact that queries return Lua tables. For example, to apply a template to render every page as a link: ${template.each(query[[ diff --git a/website/Space Lua/Lua Integrated Query/Aggregating.md b/website/Space Lua/Lua Integrated Query/Aggregating.md new file mode 100644 index 00000000..aa74cca7 --- /dev/null +++ b/website/Space Lua/Lua Integrated Query/Aggregating.md @@ -0,0 +1,181 @@ +#maturity/experimental + +The `group by` and `having` clauses of [[Space Lua/Lua Integrated Query]] support aggregate functions for grouped analysis, following SQL-style semantics. + +After `group by`, each result row contains: + +- `key`: the group key (a single value or, for multi-key grouping, a table) +- `group`: a Lua table containing all items in that group + +All aggregate functions (such as `count`, `sum`, `min`, `max`, `avg`, and custom aggregates) can be applied in `select` and `having` clauses. Aggregate expressions are available in both forms: with or without a variable binding in the `from` clause. The variable `_` always refers to the current item. + +Field names used in `group by` are exposed as locals in `having`, `select`, and `order by`. Use `#group` to obtain the item count per group. + +> **note** Note +> The `having` clause acts only on grouped output. For filtering individual items, use `where` prior to grouping. + +# Examples + +All queries operate on `index.tag 'page'`. + +## Counting with and without binding + +Grouping pages by their first tag, and computing the count and aggregate statistics: + +**Without binding variable:** + +```lua +query[[from + index.tag 'page' +group by + tags[1] +select { + tag = key, + total = count(name), + min_size = min(size), + max_size = max(size), + avg_size = avg(size) +}]] +``` +${query [[ + from + index.tag 'page' + group by + tags[1] + select { + tag = key, + total = count(name), + min_size = min(size), + max_size = max(size), + avg_size = avg(size) + } +]]} + +**With binding variable:** + +```lua +query[[ + from + p = index.tag 'page' + group by + p.tags[1] + select { + tag = key, + total = count(p.name), + min_size = min(p.size), + max_size = max(p.size), + avg_size = avg(p.size) + } +]] +``` +${query [[ + from + p = index.tag 'page' + group by + p.tags[1] + select { + tag = key, + total = count(p.name), + min_size = min(p.size), + max_size = max(p.size), + avg_size = avg(p.size) + } +]]} + +## Multi-key grouping and aggregate + +```lua +query[[ + from + index.tag 'page' + group by + tags[1], + tags[2] + select { + first = key[1], + second = key[2], + count = count(name) + } +]] +``` +${query [[ + from + index.tag 'page' + group by + tags[1], + tags[2] + select { + first = key[1], + second = key[2], + count = count(name) + } +]]} + +## Group filtering with `having` and aggregates + +Only groups with more than two items: + +```lua +query[[ + from + index.tag 'page' + group by + tags[1] + having + count(name) > 2 + select { + tag = key, + total = count(name) + } +]] +``` +${query [[ + from + index.tag 'page' + group by + tags[1] + having + count(name) > 2 + select { + tag = key, + total = count(name) + } +]]} + +## Field access after grouping + +Non-aggregated field references, such as `name` in `select`, refer to the first item in the group, matching common SQL and MySQL semantics. + +```lua +query[[ + from + index.tag 'page' + group by + tags[1] + select { + tag = key, + first_page = name, + n = count(name) + } +]] +``` +${query [[ + from + index.tag 'page' + group by + tags[1] + select { + tag = key, + first_page = name, + n = count(name) + } +]]} + +## Custom aggregators + +Custom aggregator functions may be defined by the user using [[Library/Std/APIs/Aggregate|dedicated API]]. + +# See also + +- [[Space Lua/Lua Integrated Query/Grouping]] — grouping queries without aggregation +- [[Space Lua/Lua Integrated Query]] — full LIQ language reference and listing available aggregators diff --git a/website/Space Lua/Lua Integrated Query/Grouping.md b/website/Space Lua/Lua Integrated Query/Grouping.md new file mode 100644 index 00000000..9e8ae8a3 --- /dev/null +++ b/website/Space Lua/Lua Integrated Query/Grouping.md @@ -0,0 +1,565 @@ +#maturity/experimental + +The `group by` and `having` clauses extend [[Space Lua/Lua Integrated Query]] with SQL-style grouping and aggregate filtering. + +After `group by`, each result row has two fields: + +- **`key`** - the group key (single value or table for multi-key) +- **`group`** - a table (array) of all items in that group + +The field names used in `group by` are also available as bare variables in `having`, `select`, and `order by`. Use `#group` to count items per group. + +> **note** Note +> `having` can only reference group key fields, `key`, `group`, and aggregates like `#group`. To filter individual rows, use `where`. + +# Examples + +All examples below use `index.tag 'tag'`. + +## Group by single key + +Group all tags by `name`: + +```lua +query [[ + from + index.tag 'tag' + group by + name +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name +]]} + +## Group by multiple keys + +Group tags by `name` and `parent` together: + +```lua +query [[ + from + index.tag 'tag' + group by + name, + parent +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name, + parent +]]} + +## Filter groups by count + +Only show tags that appear more than 2 times: + +```lua +query [[ + from + index.tag 'tag' + group by + name + having + #group > 2 +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name + having + #group > 2 +]]} + +## Find unique tags + +Tags appearing exactly once: + +```lua +query [[ + from + index.tag 'tag' + group by + name + having + #group == 1 +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name + having + #group == 1 +]]} + +## Filter groups by key value + +Only show the group where `name` is "meta": + +```lua +query [[ + from + index.tag 'tag' + group by + name + having + name == 'meta' +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name + having + name == 'meta' +]]} + +## Multi-key having + +Groups by `name` and `parent`, keep only page-level tags with more than 1 entry: + +```lua +query [[ + from + index.tag 'tag' + group by + name, + parent + having + parent == 'page' and + #group > 1 +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name, + parent + having + parent == 'page' and + #group > 1 +]]} + +## `where` before `group by` + +Filter to page parents first, then group by `name`: + +```lua +query [[ + from + index.tag 'tag' + where + parent == 'page' + group by + name +]] +``` + +${query [[ + from + index.tag 'tag' + where + parent == 'page' + group by + name +]]} + +## `where`, `group by` and `having` combined + +Filter to page parents, group by `name`, keep groups with 2+ items: + +```lua +query [[ + from + index.tag 'tag' + where + parent == 'page' + group by + name + having + #group >= 2 +]] +``` + +${query [[ + from + index.tag 'tag' + where + parent == 'page' + group by + name + having + #group >= 2 +]]} + +## `select` name and count + +Project each group into a table with `name` and `count`: + +```lua +query [[ + from + index.tag 'tag' + group by + name + select { + name = name, + count = #group + } +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name + select { + name = name, + count = #group + } +]]} + +## `select` with multi-key + +Project both key parts and count: + +```lua +query [[ + from + index.tag 'tag' + group by + name, + parent + select { + name = name, + parent = parent, + count = #group + } +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name, + parent + select { + name = name, + parent = parent, + count = #group + } +]]} + +## Full pipeline: `where`, `group by`, `having` and `select` + +Filter, group, filter groups, then project: + +```lua +query [[ + from + index.tag 'tag' + where + parent == 'page' or + parent == 'task' + group by + name + having + #group > 1 + select { + tag = name, + total = #group + } +]] +``` + +${query [[ + from + index.tag 'tag' + where + parent == 'page' or + parent == 'task' + group by + name + having + #group > 1 + select { + tag = name, + total = #group + } +]]} + +## Order groups by count + +Sort groups by size, largest first: + +```lua +query [[ + from + index.tag 'tag' + group by + name + order by + #group desc +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name + order by + #group desc +]]} + +## Top tags with `having`, `order by`, and `select` + +Tags with 2+ occurrences, sorted by count, projected: + +```lua +query [[ + from + index.tag 'tag' + group by + name + having + #group >= 2 + order by + #group desc + select { + tag = name, + count = #group + } +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name + having + #group >= 2 + order by + #group desc + select { + tag = name, + count = #group + } +]]} + +## Top N groups with `limit` + +Top 3 most used tags: + +```lua +query [[ + from + index.tag 'tag' + group by + name + order by + #group desc + limit + 3 +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name + order by + #group desc + limit + 3 +]]} + +## Full pipeline with `limit` + +Top 5 tags with 2+ uses, showing name and count: + +```lua +query [[ + from + index.tag 'tag' + group by + name + having + #group >= 2 + order by + #group desc + select { + tag = name, + count = #group + } + limit + 5 +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name + having + #group >= 2 + order by + #group desc + select { + tag = name, + count = #group + } + limit + 5 +]]} + +## With explicit object variable + +The same works with `p =` binding: + +```lua +query [[ + from + p = index.tag 'tag' + group by + p.name + having + #group > 1 + select { + tag = name, + count = #group + } +]] +``` + +${query [[ + from + p = index.tag 'tag' + group by + p.name + having + #group > 1 + select { + tag = name, + count = #group + } +]]} + +## Multi-key with explicit object variable + +Full pipeline with `p =` binding and two group keys: + +```lua +query [[ + from + p = index.tag 'tag' + where + p.parent == 'page' + group by + p.name, + p.parent + having + #group >= 2 + order by + #group desc + select { + tag = name, + parent = parent, + count = #group + } +]] +``` + +${query [[ + from + p = index.tag 'tag' + where + p.parent == 'page' + group by + p.name, + p.parent + having + #group >= 2 + order by + #group desc + select { + tag = name, + parent = parent, + count = #group + } +]]} + +## Access `key` directly + +For single-key grouping, `key` holds the value directly: + +```lua +query [[ + from + index.tag 'tag' + group by + name + having + key == 'meta' +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name + having + key == 'meta' +]]} + +## Access `key` table for multi-key + +For multi-key grouping, `key` is a table indexed from 1: + +```lua +query [[ + from + index.tag 'tag' + group by + name, + parent + having + key[1] == 'meta' and + key[2] == 'page' +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name, + parent + having + key[1] == 'meta' and + key[2] == 'page' +]]}