diff --git a/client/space_lua/aggregates.test.ts b/client/space_lua/aggregates.test.ts new file mode 100644 index 00000000..a684b6cf --- /dev/null +++ b/client/space_lua/aggregates.test.ts @@ -0,0 +1,784 @@ +import { parseExpressionString } from "./parse.ts"; +import { evalExpression } from "./eval.ts"; +import { + type AggregateSpec, + executeAggregate, + getAggregateSpec, +} from "./aggregates.ts"; +import { + applyQuery, + evalExpressionWithAggregates, +} from "./query_collection.ts"; +import { + LuaBuiltinFunction, + LuaEnv, + LuaStackFrame, + LuaTable, + luaValueToJS, +} from "./runtime.ts"; +import { assertEquals } from "@std/assert"; + +function luaArray(items: Record[]): LuaTable { + const t = new LuaTable(); + for (let i = 0; i < items.length; i++) { + t.rawSetArrayIndex(i + 1, new LuaTable(items[i])); + } + return t; +} + +function installFakeConfig( + specs: Record = {}, +): () => void { + const configStore: Record = {}; + for (const [name, spec] of Object.entries(specs)) { + configStore[`aggregates.${name}`] = spec; + } + const prev = (globalThis as any).client; + (globalThis as any).client = { + config: { + get(key: string, fallback: any) { + return configStore[key] ?? fallback; + }, + }, + }; + return () => { + (globalThis as any).client = prev; + }; +} + +function requireSpec(name: string): AggregateSpec { + const cleanup = installFakeConfig(); + const spec = getAggregateSpec(name); + cleanup(); + if (!spec) throw new Error(`builtin aggregate "${name}" not found`); + return spec; +} + +const sumSpec = requireSpec("sum"); +const countSpec = requireSpec("count"); +const minSpec = requireSpec("min"); +const maxSpec = requireSpec("max"); +const avgSpec = requireSpec("avg"); +const arrayAggSpec = requireSpec("array_agg"); + +const sf = LuaStackFrame.lostFrame; + +// Unit tests per builtin + +Deno.test("aggregate: sum", async () => { + const result = await executeAggregate( + sumSpec, + luaArray([{ v: 10 }, { v: 20 }, { v: 30 }]), + parseExpressionString("_.v"), + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, 60); +}); + +Deno.test("aggregate: sum with nils", async () => { + const result = await executeAggregate( + sumSpec, + luaArray([{ v: 5 }, { x: 1 }, { v: 15 }]), + parseExpressionString("_.v"), + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, 20); +}); + +Deno.test("aggregate: count with expression", async () => { + const result = await executeAggregate( + countSpec, + luaArray([{ v: 1 }, { v: 2 }, { v: 3 }]), + parseExpressionString("_.v"), + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, 3); +}); + +Deno.test("aggregate: count with no argument (count(*))", async () => { + const result = await executeAggregate( + countSpec, + luaArray([{ v: 1 }, { v: 2 }, { v: 3 }, { v: 4 }]), + null, + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, 4); +}); + +Deno.test("aggregate: min", async () => { + const result = await executeAggregate( + minSpec, + luaArray([{ v: 30 }, { v: 10 }, { v: 20 }]), + parseExpressionString("_.v"), + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, 10); +}); + +Deno.test("aggregate: max", async () => { + const result = await executeAggregate( + maxSpec, + luaArray([{ v: 30 }, { v: 10 }, { v: 20 }]), + parseExpressionString("_.v"), + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, 30); +}); + +Deno.test("aggregate: avg", async () => { + const result = await executeAggregate( + avgSpec, + luaArray([{ v: 10 }, { v: 20 }, { v: 30 }]), + parseExpressionString("_.v"), + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, 20); +}); + +Deno.test("aggregate: avg empty group", async () => { + const result = await executeAggregate( + avgSpec, + new LuaTable(), + parseExpressionString("_.v"), + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, null); +}); + +Deno.test("aggregate: min/max on empty group", async () => { + const items = new LuaTable(); + const env = new LuaEnv(); + const expr = parseExpressionString("_.v"); + assertEquals( + await executeAggregate( + minSpec, + items, + expr, + undefined, + env, + sf, + evalExpression, + ), + null, + ); + assertEquals( + await executeAggregate( + maxSpec, + items, + expr, + undefined, + env, + sf, + evalExpression, + ), + null, + ); +}); + +Deno.test("aggregate: array_agg", async () => { + const result = await executeAggregate( + arrayAggSpec, + luaArray([{ v: "a" }, { v: "b" }, { v: "c" }]), + parseExpressionString("_.v"), + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result instanceof LuaTable, true); + assertEquals((result as LuaTable).rawGet(1), "a"); + assertEquals((result as LuaTable).rawGet(2), "b"); + assertEquals((result as LuaTable).rawGet(3), "c"); +}); + +Deno.test("aggregate: sum with objectVariable", async () => { + const result = await executeAggregate( + sumSpec, + luaArray([{ v: 3 }, { v: 7 }]), + parseExpressionString("p.v"), + "p", + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, 10); +}); + +Deno.test("aggregate: user-defined overrides builtin", async () => { + const customSum: AggregateSpec = { + name: "sum", + initialize: new LuaBuiltinFunction((_sf) => 100), + iterate: new LuaBuiltinFunction((_sf, state: any, value: any) => { + if (value === null || value === undefined) return state; + return state + value; + }), + }; + const cleanup = installFakeConfig({ sum: customSum }); + try { + const spec = getAggregateSpec("sum")!; + const result = await executeAggregate( + spec, + luaArray([{ v: 1 }, { v: 2 }]), + parseExpressionString("_.v"), + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, 103); + } finally { + cleanup(); + } +}); + +Deno.test("aggregate: builtin available without config", () => { + const cleanup = installFakeConfig(); + try { + const spec = getAggregateSpec("sum"); + assertEquals(spec !== null, true); + assertEquals(spec!.name, "sum"); + + const spec2 = getAggregateSpec("nonexistent"); + assertEquals(spec2, null); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: sum in table constructor", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ age: 10 }, { age: 20 }, { age: 30 }]); + const env = new LuaEnv(); + const expr = parseExpressionString("{ total = sum(_.age) }"); + const result = await evalExpressionWithAggregates( + expr, + env, + sf, + groupItems, + undefined, + env, + ); + assertEquals(result instanceof LuaTable, true); + assertEquals((result as LuaTable).rawGet("total"), 60); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: count() with no args", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ x: 1 }, { x: 2 }, { x: 3 }]); + const env = new LuaEnv(); + const expr = parseExpressionString("count()"); + const result = await evalExpressionWithAggregates( + expr, + env, + sf, + groupItems, + undefined, + env, + ); + assertEquals(result, 3); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: non-aggregate falls through", async () => { + const cleanup = installFakeConfig(); + try { + const env = new LuaEnv(); + env.setLocal("key", "hello"); + const expr = parseExpressionString("key"); + const result = await evalExpressionWithAggregates( + expr, + env, + sf, + new LuaTable(), + undefined, + env, + ); + assertEquals(result, "hello"); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: multiple aggregates in table", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ v: 5 }, { v: 15 }, { v: 10 }]); + const env = new LuaEnv(); + env.setLocal("key", "grp1"); + const expr = parseExpressionString( + "{ k = key, total = sum(_.v), n = count(_.v), smallest = min(_.v) }", + ); + const result = await evalExpressionWithAggregates( + expr, + env, + sf, + groupItems, + undefined, + env, + ); + const tbl = result as LuaTable; + assertEquals(tbl.rawGet("k"), "grp1"); + assertEquals(tbl.rawGet("total"), 30); + assertEquals(tbl.rawGet("n"), 3); + assertEquals(tbl.rawGet("smallest"), 5); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: avg with finish step", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ v: 10 }, { v: 30 }]); + const env = new LuaEnv(); + const expr = parseExpressionString("avg(_.v)"); + const result = await evalExpressionWithAggregates( + expr, + env, + sf, + groupItems, + undefined, + env, + ); + assertEquals(result, 20); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: binary comparison (count > N)", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ v: 1 }, { v: 2 }, { v: 3 }]); + const env = new LuaEnv(); + const result = await evalExpressionWithAggregates( + parseExpressionString("count(_.v) > 2"), + env, + sf, + groupItems, + undefined, + env, + ); + assertEquals(result, true); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: binary arithmetic (sum + sum)", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ a: 10, b: 5 }, { a: 20, b: 15 }]); + const env = new LuaEnv(); + const result = await evalExpressionWithAggregates( + parseExpressionString("sum(_.a) + sum(_.b)"), + env, + sf, + groupItems, + undefined, + env, + ); + assertEquals(result, 50); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: unary minus on aggregate", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ v: 10 }, { v: 20 }]); + const env = new LuaEnv(); + const result = await evalExpressionWithAggregates( + parseExpressionString("-sum(_.v)"), + env, + sf, + groupItems, + undefined, + env, + ); + assertEquals(result, -30); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: parenthesized aggregate", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ v: 7 }, { v: 3 }]); + const env = new LuaEnv(); + const result = await evalExpressionWithAggregates( + parseExpressionString("(sum(_.v))"), + env, + sf, + groupItems, + undefined, + env, + ); + assertEquals(result, 10); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: and short-circuit", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ v: 1 }]); + const env = new LuaEnv(); + assertEquals( + await evalExpressionWithAggregates( + parseExpressionString("count(_.v) and 42"), + env, + sf, + groupItems, + undefined, + env, + ), + 42, + ); + assertEquals( + await evalExpressionWithAggregates( + parseExpressionString("false and count(_.v)"), + env, + sf, + groupItems, + undefined, + env, + ), + false, + ); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: or short-circuit", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ v: 1 }]); + const env = new LuaEnv(); + assertEquals( + await evalExpressionWithAggregates( + parseExpressionString("count(_.v) or 99"), + env, + sf, + groupItems, + undefined, + env, + ), + 1, + ); + assertEquals( + await evalExpressionWithAggregates( + parseExpressionString("nil or count(_.v)"), + env, + sf, + groupItems, + undefined, + env, + ), + 1, + ); + } finally { + cleanup(); + } +}); + +Deno.test("evalExpressionWithAggregates: not aggregate", async () => { + const cleanup = installFakeConfig(); + try { + const groupItems = luaArray([{ v: 1 }]); + const env = new LuaEnv(); + assertEquals( + await evalExpressionWithAggregates( + parseExpressionString("not count(_.v)"), + env, + sf, + groupItems, + undefined, + env, + ), + false, + ); + } finally { + cleanup(); + } +}); + +Deno.test("applyQuery: group by + select with aggregates", async () => { + const cleanup = installFakeConfig(); + try { + const data = [ + new LuaTable({ dept: "eng", salary: 100 }), + new LuaTable({ dept: "eng", salary: 200 }), + new LuaTable({ dept: "sales", salary: 150 }), + new LuaTable({ dept: "sales", salary: 50 }), + new LuaTable({ dept: "sales", salary: 100 }), + ]; + const results = await applyQuery( + data, + { + objectVariable: "p", + groupBy: [parseExpressionString("p.dept")], + select: parseExpressionString( + "{ dept = key, total = sum(p.salary), n = count(p.salary) }", + ), + }, + new LuaEnv(), + sf, + {}, + ); + assertEquals(results.length, 2); + const eng = results[0] as LuaTable; + const sales = results[1] as LuaTable; + assertEquals(eng.rawGet("dept"), "eng"); + assertEquals(eng.rawGet("total"), 300); + assertEquals(eng.rawGet("n"), 2); + assertEquals(sales.rawGet("dept"), "sales"); + assertEquals(sales.rawGet("total"), 300); + assertEquals(sales.rawGet("n"), 3); + } finally { + cleanup(); + } +}); + +Deno.test("applyQuery: group by + having with aggregate", async () => { + const cleanup = installFakeConfig(); + try { + const data = [ + new LuaTable({ dept: "eng", name: "a" }), + new LuaTable({ dept: "eng", name: "b" }), + new LuaTable({ dept: "sales", name: "c" }), + ]; + const results = await applyQuery( + data, + { + objectVariable: "p", + groupBy: [parseExpressionString("p.dept")], + having: parseExpressionString("count(p.name) > 1"), + }, + new LuaEnv(), + sf, + {}, + ); + assertEquals(results.length, 1); + assertEquals((results[0] as LuaTable).rawGet("key"), "eng"); + } finally { + cleanup(); + } +}); + +Deno.test("applyQuery: group by without aggregates still works", async () => { + const cleanup = installFakeConfig(); + try { + const data = [ + new LuaTable({ dept: "eng", name: "a" }), + new LuaTable({ dept: "eng", name: "b" }), + new LuaTable({ dept: "sales", name: "c" }), + ]; + const results = await applyQuery( + data, + { + objectVariable: "p", + groupBy: [parseExpressionString("p.dept")], + select: parseExpressionString("key"), + }, + new LuaEnv(), + sf, + {}, + ); + assertEquals(results.length, 2); + assertEquals(results[0], "eng"); + assertEquals(results[1], "sales"); + } finally { + cleanup(); + } +}); + +Deno.test("applyQuery: having with compound expression", async () => { + const cleanup = installFakeConfig(); + try { + const data = [ + new LuaTable({ dept: "eng", salary: 100 }), + new LuaTable({ dept: "eng", salary: 200 }), + new LuaTable({ dept: "sales", salary: 50 }), + ]; + const results = await applyQuery( + data, + { + objectVariable: "p", + groupBy: [parseExpressionString("p.dept")], + having: parseExpressionString("sum(p.salary) > 100"), + select: parseExpressionString("{ dept = key, total = sum(p.salary) }"), + }, + new LuaEnv(), + sf, + {}, + ); + assertEquals(results.length, 1); + assertEquals((results[0] as LuaTable).rawGet("dept"), "eng"); + assertEquals((results[0] as LuaTable).rawGet("total"), 300); + } finally { + cleanup(); + } +}); + +Deno.test("applyQuery: implicit _ with group by + multiple aggregates", async () => { + const cleanup = installFakeConfig(); + try { + const data = [ + new LuaTable({ category: "fruit", price: 3 }), + new LuaTable({ category: "fruit", price: 5 }), + new LuaTable({ category: "veg", price: 2 }), + new LuaTable({ category: "veg", price: 7 }), + new LuaTable({ category: "veg", price: 1 }), + ]; + const results = await applyQuery( + data, + { + groupBy: [parseExpressionString("_.category")], + select: parseExpressionString( + "{ cat = key, total = sum(_.price), best = max(_.price) }", + ), + }, + new LuaEnv(), + sf, + {}, + ); + assertEquals(results.length, 2); + const fruit = results[0] as LuaTable; + assertEquals(fruit.rawGet("cat"), "fruit"); + assertEquals(fruit.rawGet("total"), 8); + assertEquals(fruit.rawGet("best"), 5); + const veg = results[1] as LuaTable; + assertEquals(veg.rawGet("cat"), "veg"); + assertEquals(veg.rawGet("total"), 10); + assertEquals(veg.rawGet("best"), 7); + } finally { + cleanup(); + } +}); + +Deno.test("applyQuery: group by + having + order by + limit", async () => { + const cleanup = installFakeConfig(); + try { + const data = [ + new LuaTable({ tag: "a", page: "p1" }), + new LuaTable({ tag: "a", page: "p2" }), + new LuaTable({ tag: "a", page: "p3" }), + new LuaTable({ tag: "b", page: "p1" }), + new LuaTable({ tag: "c", page: "p1" }), + new LuaTable({ tag: "c", page: "p2" }), + ]; + const results = await applyQuery( + data, + { + objectVariable: "t", + groupBy: [parseExpressionString("t.tag")], + having: parseExpressionString("count(t.page) > 1"), + orderBy: [{ expr: parseExpressionString("key"), desc: false }], + select: parseExpressionString("key"), + limit: 1, + }, + new LuaEnv(), + sf, + {}, + ); + assertEquals(results.length, 1); + assertEquals(results[0], "a"); + } finally { + cleanup(); + } +}); + +Deno.test("applyQuery: select with aggregate division (float result)", async () => { + const cleanup = installFakeConfig(); + try { + const data = [ + new LuaTable({ dept: "eng", salary: 100 }), + new LuaTable({ dept: "eng", salary: 200 }), + ]; + const results = await applyQuery( + data, + { + objectVariable: "p", + groupBy: [parseExpressionString("p.dept")], + select: parseExpressionString( + "{ dept = key, avg_salary = sum(p.salary) / count(p.salary) }", + ), + }, + new LuaEnv(), + sf, + {}, + ); + assertEquals(results.length, 1); + const row = results[0] as LuaTable; + assertEquals(row.rawGet("dept"), "eng"); + assertEquals(luaValueToJS(row.rawGet("avg_salary"), sf), 150); + } finally { + cleanup(); + } +}); + +Deno.test("aggregate: custom concat with finish", async () => { + const concatSpec: AggregateSpec = { + name: "concat", + initialize: new LuaBuiltinFunction((_sf) => + new LuaTable({ first: true, s: "" }) + ), + iterate: new LuaBuiltinFunction((_sf, state: any, value: any) => { + if (value === null || value === undefined) return state; + if (state.rawGet("first")) { + state.rawSet("s", String(value)); + state.rawSet("first", false); + } else { + state.rawSet("s", state.rawGet("s") + ", " + String(value)); + } + return state; + }), + finish: new LuaBuiltinFunction((_sf, state: any) => state.rawGet("s")), + }; + const result = await executeAggregate( + concatSpec, + luaArray([{ v: "a" }, { v: "b" }, { v: "c" }]), + parseExpressionString("_.v"), + undefined, + new LuaEnv(), + sf, + evalExpression, + ); + assertEquals(result, "a, b, c"); +}); diff --git a/client/space_lua/aggregates.ts b/client/space_lua/aggregates.ts new file mode 100644 index 00000000..8311bc31 --- /dev/null +++ b/client/space_lua/aggregates.ts @@ -0,0 +1,201 @@ +/** + * Aggregate function definitions and execution for LIQ. + * + * Built-in aggregates (sum, count, min, max, avg, array_agg) are + * implemented in TypeScript for speed. Users can override any builtin + * via `aggregate.define` or `aggregate.update`. + * + * Builtins implement ILuaFunction via plain objects rather than + * LuaBuiltinFunction instances. This avoids ES module TDZ issues: + * `class` exports are not available during circular module init, + * but `interface`/`type` imports are. + */ + +import type { ILuaFunction, LuaStackFrame } from "./runtime.ts"; +import { + luaCall, + type LuaEnv, + LuaTable, + type LuaValue, +} from "./runtime.ts"; +import type { LuaExpression } from "./ast.ts"; +import { buildItemEnv } from "./query_env.ts"; + +export interface AggregateSpec { + name: string; + description?: string; + initialize: LuaValue; // ILuaFunction + iterate: LuaValue; // ILuaFunction + finish?: LuaValue; // ILuaFunction | undefined +} + +// Helper to build an ILuaFunction from a plain function. Equivalent to +// LuaBuiltinFunction but without referencing the class. +function aggFn( + fn: (sf: LuaStackFrame, ...args: LuaValue[]) => LuaValue, +): ILuaFunction { + return { + call(sf: LuaStackFrame, ...args: LuaValue[]) { + return fn(sf, ...args); + }, + asString() { + return ""; + }, + }; +} + +// Built-in aggregate specs +const builtinAggregates: Record = { + sum: { + name: "sum", + description: "Sum of numeric values", + initialize: aggFn((_sf) => 0), + iterate: aggFn((_sf, state: any, value: any) => { + if (value === null || value === undefined) return state; + return (state as number) + (value as number); + }), + }, + count: { + name: "count", + description: "Count of values; count() with no argument counts all rows", + initialize: aggFn((_sf) => 0), + iterate: aggFn((_sf, state: any, value: any) => { + if (value === null || value === undefined) return state; + return (state as number) + 1; + }), + }, + min: { + name: "min", + description: "Minimum value", + initialize: aggFn((_sf) => null), + iterate: aggFn((_sf, state: any, value: any) => { + if (value === null || value === undefined) return state; + if (state === null || value < state) return value; + return state; + }), + }, + max: { + name: "max", + description: "Maximum value", + initialize: aggFn((_sf) => null), + iterate: aggFn((_sf, state: any, value: any) => { + if (value === null || value === undefined) return state; + if (state === null || value > state) return value; + return state; + }), + }, + avg: { + name: "avg", + description: "Average of numeric values", + initialize: aggFn((_sf) => ({ sum: 0, count: 0 })), + iterate: aggFn((_sf, state: any, value: any) => { + if (value === null || value === undefined) return state; + state.sum += value as number; + state.count += 1; + return state; + }), + finish: aggFn((_sf, state: any) => { + if (state.count === 0) return null; + return state.sum / state.count; + }), + }, + array_agg: { + name: "array_agg", + description: "Collect values into an array", + initialize: aggFn((_sf) => new LuaTable()), + iterate: aggFn((_sf, state: any, value: any) => { + (state as LuaTable).rawSetArrayIndex( + (state as LuaTable).length + 1, + value, + ); + return state; + }), + }, +}; + +const noCtx = {}; + +function buildAggCtx(name: string): LuaTable { + const ctx = new LuaTable(); + ctx.rawSet("name", name); + // @ts-ignore: Hack to access client via the browser + const clientConfig = globalThis.client?.config; + const aggConfig = clientConfig + ? clientConfig.get(`aggregateConfig.${name}`, {}) + : {}; + ctx.rawSet("config", aggConfig); + return ctx; +} + +export function getAggregateSpec(name: string): AggregateSpec | null { + // @ts-ignore: Hack to access client via the browser + const clientConfig = globalThis.client?.config; + if (clientConfig) { + const spec: any = clientConfig.get(`aggregates.${name}`, null); + if (spec) { + let candidate: AggregateSpec | null = null; + if (spec instanceof LuaTable) { + const init = spec.rawGet("initialize"); + const iter = spec.rawGet("iterate"); + if (init && iter) { + candidate = { + name: spec.rawGet("name") ?? name, + description: spec.rawGet("description"), + initialize: init, + iterate: iter, + finish: spec.rawGet("finish"), + }; + } + } else if (spec.initialize && spec.iterate) { + candidate = spec as AggregateSpec; + } + if (candidate) return candidate; + } + } + return builtinAggregates[name] ?? null; +} + +/** + * Execute an aggregate function over a group of items. + * + * `evalExprFn` is passed in to avoid a circular import with eval.ts. + */ +export async function executeAggregate( + spec: AggregateSpec, + items: LuaTable, + valueExpr: LuaExpression | null, + objectVariable: string | undefined, + env: LuaEnv, + sf: LuaStackFrame, + evalExprFn: ( + e: LuaExpression, + env: LuaEnv, + sf: LuaStackFrame, + ) => Promise | LuaValue, +): Promise { + const ctx = buildAggCtx(spec.name); + + // Initialize + let state = await luaCall(spec.initialize, [ctx], noCtx, sf); + + // Iterate + const len = items.length; + for (let i = 1; i <= len; i++) { + const item = items.rawGet(i); + let value: LuaValue; + if (valueExpr === null) { + value = item; + } else { + const itemEnv = buildItemEnv(objectVariable, item, env, sf); + value = await evalExprFn(valueExpr, itemEnv, sf); + } + state = await luaCall(spec.iterate, [state, value, ctx], noCtx, sf); + } + + // Finish + if (spec.finish) { + state = await luaCall(spec.finish, [state, ctx], noCtx, sf); + } + + return state; +} diff --git a/client/space_lua/ast.ts b/client/space_lua/ast.ts index f622197f..49be9293 100644 --- a/client/space_lua/ast.ts +++ b/client/space_lua/ast.ts @@ -292,7 +292,9 @@ export type LuaQueryClause = | LuaWhereClause | LuaLimitClause | LuaOrderByClause - | LuaSelectClause; + | LuaSelectClause + | LuaGroupByClause + | LuaHavingClause; export type LuaFromClause = { type: "From"; @@ -326,3 +328,13 @@ export type LuaSelectClause = { type: "Select"; expression: LuaExpression; } & ASTContext; + +export type LuaGroupByClause = { + type: "GroupBy"; + expressions: LuaExpression[]; +} & ASTContext; + +export type LuaHavingClause = { + type: "Having"; + expression: LuaExpression; +} & ASTContext; diff --git a/client/space_lua/eval.ts b/client/space_lua/eval.ts index 18e9928b..03dc8301 100644 --- a/client/space_lua/eval.ts +++ b/client/space_lua/eval.ts @@ -250,7 +250,7 @@ function arithCoercionErrorOrThrow( throw e; } -function luaOp( +export function luaOp( op: string, left: any, right: any, @@ -906,6 +906,14 @@ export function evalExpression( } break; } + case "GroupBy": { + query.groupBy = clause.expressions; + break; + } + case "Having": { + query.having = clause.expression; + break; + } } } diff --git a/client/space_lua/lua.grammar b/client/space_lua/lua.grammar index 99b8435c..36d67a2d 100644 --- a/client/space_lua/lua.grammar +++ b/client/space_lua/lua.grammar @@ -95,7 +95,9 @@ QueryClause { WhereClause | OrderByClause | SelectClause | - LimitClause + LimitClause | + GroupByClause | + HavingClause } FromClause { ckw<"from"> (Name "=")? exp } @@ -104,6 +106,8 @@ LimitClause { ckw<"limit"> exp ("," exp)? } OrderByClause { ckw<"order"> ckw<"by"> list } OrderBy { exp ckw<"desc">? } SelectClause { ckw<"select"> exp } +GroupByClause { ckw<"group"> ckw<"by"> list } +HavingClause { ckw<"having"> exp } field[@isGroup=Field] { diff --git a/client/space_lua/lua.test.ts b/client/space_lua/lua.test.ts index d4b6c96a..367d21fe 100644 --- a/client/space_lua/lua.test.ts +++ b/client/space_lua/lua.test.ts @@ -89,7 +89,44 @@ Deno.test("[Lua] Lume functions tests", async () => { await runLuaTest("./lume_test.lua"); }); +Deno.test("[Lua] Lua Integrated Query tests", async () => { + await runLuaTest("./query_test.lua"); +}); + async function runLuaTest(luaPath: string) { + if ( + typeof globalThis.client !== "undefined" && + globalThis.client && + typeof globalThis.client === "object" + ) { + if ( + !globalThis.client.config || + typeof globalThis.client.config.get !== "function" + ) { + try { + Object.defineProperty(globalThis.client, "config", { + value: { + get(_key: string, fallback: unknown) { + return fallback ?? {}; + }, + }, + configurable: true, + writable: true, + }); + } catch { + // ignore + } + } + } else { + (globalThis as any).client = { + config: { + get(_key: string, fallback: unknown) { + return fallback ?? {}; + }, + }, + }; + } + const luaFile = await Deno.readTextFile( fileURLToPath(new URL(luaPath, import.meta.url)), ); diff --git a/client/space_lua/parse-lua.js b/client/space_lua/parse-lua.js index 28273805..57f9ce05 100644 --- a/client/space_lua/parse-lua.js +++ b/client/space_lua/parse-lua.js @@ -1,22 +1,22 @@ // This file was generated by lezer-generator. You probably shouldn't edit it. import {LRParser} from "@lezer/lr" -const spec_identifier = {__proto__:null,break:16, goto:20, do:24, end:26, while:30, nil:32, true:34, false:36, or:80, and:82, not:104, function:114, query:122, from:129, where:133, order:137, by:139, desc:143, select:147, limit:151, repeat:154, until:156, if:160, then:162, elseif:164, else:166, for:170, in:178, local:188, return:204} +const spec_identifier = {__proto__:null,break:16, goto:20, do:24, end:26, while:30, nil:32, true:34, false:36, or:80, and:82, not:104, function:114, query:122, from:129, where:133, order:137, by:139, desc:143, select:147, limit:151, group:155, having:159, repeat:162, until:164, if:168, then:170, elseif:172, else:174, for:178, in:186, local:196, return:212} export const parser = LRParser.deserialize({ version: 14, - states: "EUO!ZQPOOOOQO'#Cc'#CcO!UQPO'#CaO!bQPOOOOQO'#E{'#E{O!vQQO'#CwO$]QPO'#EzOOQO'#Ez'#EzO$gQPO'#EzOOQO'#E`'#E`O%vQPO'#E_OOQO'#Ev'#EvOOQO'#Eg'#EgO%{QPO'#C_OOQO'#C_'#C_QOQPOOO!UQPO'#CeO&`QPO'#CgO!vQQO'#CjO&gQPO'#DzO!vQQO'#D}O!UQPO'#ESO!UQPO'#EZO&nQPO'#EaO&vQQO'#EeO'^QPO,58{OOQO'#Cq'#CqO!UQPO,59^O!vQQO,59`O(gQQO'#C|O(nQQO'#FQOOQO'#E|'#E|OOQO,59f,59fO!UQPO,59fO(uQPO'#ExO,eQPO,59cOOQO'#Dc'#DcOOQO'#Dd'#DdOOQO'#De'#DeO!vQQO'#DaOOQO'#Ex'#ExO,lQPO'#DfO,qQSO'#DjO,vQPO'#EpO-OQPO,5SQPO,5:vO!UQPO,5:vOOQO1G0a1G0aO!UQPO'#EdOOQO,5:},5:}O!UQPO'#EqO>_QPO,5uAN>uO&`QPOAN>uO!;OQPO,5;XO!;VQPOAN>uO!;[QPO<YQ`O1G/ZO!>aQ`O1G/ZO!@RQ`O1G/ZO!@YQ`O1G/ZO!AtQ`O1G/ZO!BRQ`O1G/ZO!B`Q`O1G/ZO!DTQ`O1G/ZO!D[Q`O1G/ZO!DcQSO7+%jO!DyQPOG24aO=mQPO1G0sOOQOG24aG24aO!vQQOAN>|OOQO,5;W,5;WOOQO-E8j-E8jOOQOLD){LD){OOQO7+&_7+&_O!EOQPOG24hOAVQQO'#DaO!%YQQO'#DaOAVQQO,59oO!%YQQO,59oOAVQQO,59oO!%YQQO,59oOAVQQO,59oO!%YQQO,59oOAVQQO,59oO!%YQQO,59oOAVQQO,59oO!%YQQO,59oOAVQQO,59oO!%YQQO,59oOAVQQO,59oO!%YQQO,59oOAVQQO,59oO!%YQQO,59oOAVQQO,59oO!%YQQO,59o", - stateData: "!EY~O#hOS#iOSPOS~OSZOUQOWZOY`O[aO_bOlTO!ZfO!ocO!rdO!weO#QgO#YhO#kPO~O#fRP~P]OgkOilOlnOoqOqmO#mjO~O`xOaxObxOcxOdxOlTOqmO!UwO!ZyO!_zO#kPO#mjO#xuO#|tO$SvO~Og#nXi#nXl#nXo#nXq#nX#m#nX~Ov{O#r$YX~P#wOS#jXU#jXW#jXY#jX[#jX_#jXl#jX!Z#jX!o#jX!r#jX!w#jX#Q#jX#Y#jX#f#jX#k#jX]#jX!p#jX!t#jX!u#jX~P#wO#r}O~O#fRX]RX!pRX!tRX!uRX~P]O]RP~P]O!pRP~P]O!Z!`O#kPO~OS!dO#f#XX]#XX!p#XX!t#XX!u#XX~P!vOU!fO~O`xOaxObxOcxOdxOi!kOlTOqmO!U&kO!ZyO!_zO#kPO#mjO#xuO#|tO$SvO~Ou!nO~P'cOm!pO~P!vOm#lXx#lXy#lXz#lX!P#lX#v#lX#w#lX#x#lX#y#lX#z#lX#{#lX#|#lX#}#lX$O#lX$P#lX$Q#lX$R#lX[#lX!s#lXS#lXv#lX#f#lXj#lXU#lXW#lXY#lX_#lX!Z#lX!o#lX!r#lX!w#lX#Q#lX#Y#lX#k#lX]#lX!p#lX!t#lX!u#lX~P!bOx#SOy#TOz!{O!P#PO#v!tO#w!uO#x!vO#y!wO#z!wO#{!xO#|!xO#}!yO$O!yO$P!yO$Q!yO$R!zO~Om!sO~P+aOl#VO~O$U#XO~OlTO#kPO~Ov{O#r$Ya~O]#]O~O[#^O~P+aO!p#_O~O!s#`O~P+aOv#bO#r#aO!{$XX~O!{#dO~O[#eO~Og#fOo#hOl#OX~O$[#jOS#WPU#WPW#WPY#WP[#WP_#WPl#WPv#WP!Z#WP!o#WP!r#WP!w#WP#Q#WP#Y#WP#f#WP#k#WP#r#WP]#WP!p#WP!t#WP!u#WP~Ov#lOS$ZXU$ZXW$ZXY$ZX[$ZX_$ZXl$ZX!Z$ZX!o$ZX!r$ZX!w$ZX#Q$ZX#Y$ZX#f$ZX#k$ZX#r$ZX]$ZX!p$ZX!t$ZX!u$ZX~O#r#oOS#TaU#TaW#TaY#Ta[#Ta_#Tal#Ta!Z#Ta!o#Ta!r#Ta!w#Ta#Q#Ta#Y#Ta#f#Ta#k#Ta]#Ta!p#Ta!t#Ta!u#Ta~Ov#pOS#uX#f#uXm#uXU#uXW#uXY#uX[#uX_#uXl#uX!Z#uX!o#uX!r#uX!w#uX#Q#uX#Y#uX#k#uX]#uX!p#uX!t#uX!u#uX~P+aOS#rO#f#Xa]#Xa!p#Xa!t#Xa!u#Xa~Oj#sO~P+aOu#lXx#lXy#lXz#lX!P#lX#s#lX#v#lX#w#lX#x#lX#y#lX#z#lX#{#lX#|#lX#}#lX$O#lX$P#lX$Q#lX$R#lX!b#lX!d#lX!f#lX!k#lX!m#lX$W#lX~P!bOg#oXi#oXl#oXo#oXq#oXx#oXy#oXz#oX!P#oX#m#oX#v#oX#w#oX#x#oX#y#oX#z#oX#{#oX#|#oX#}#oX$O#oX$P#oX$Q#oX$R#oX~O#r#tOu#oX#s#oX~P6_Ox&{Oy&}Oz&mO!P&uO#v!tO#w!uO#x!vO#y!wO#z!wO#{!xO#|!xO#}!yO$O!yO$P!yO$Q!yO$R!zO~OutX#stX~P8SOu#xO#s#vO~Om#zO~OlnOqmO#mjO~O$R!zOx!Tay!Taz!Ta!P!Ta#v!Ta#w!Ta#x!Ta#y!Ta#z!Ta#{!Ta#|!Ta#}!Ta$O!Ta$P!Ta$Q!Tav!Ta~Om!Ta[!Ta!s!TaS!Ta#f!Taj!TaU!TaW!TaY!Ta_!Tal!Ta!Z!Ta!o!Ta!r!Ta!w!Ta#Q!Ta#Y!Ta#k!Ta]!Ta!p!Ta!t!Ta!u!Ta~P9zOc$WOlTO#kPOm!]P~O!b$^O!d$_O!f$`O!k$aO!m$bO~O$W$cO~PhO#w!uO~P!>hO#}!yO$O!yO$P!yO$Q!yO$R!zOvwixwiywizwi!bwi!dwi!fwi!kwi!mwi#vwi#wwi#xwi#ywi#zwi$Wwi!iwi~O!P&vO#{!xO#|!xO~P!@aO!Pwi#{wi#|wi~P!@aOvwi!iwi~P!/rOz&nO!P&vO#v!tO#w!uO#x!vO#y!wO#z!wO#{!xO#|!xO#}!yO$O!yO$P!yO$Q!yO$R!zOvwixwi!bwi!dwi!fwi!kwi!mwi$Wwi!iwi~Oy'OO~P!BjOywi~P!BjO!b!lq!d!lq!f!lq!k!lq!m!lq$W!lq~P8SO]&hO~O[!x!Z~P+aOP#|~", - goto: "CY$[PPP$]P${P%YP${P${PP${PPPPPP'v)_P)_PP*}PP,mP'vP._._._PP'}PPP.e/[0X1RP2R3U4['}P5h5h5h'}P6|7V'}P7Y7^P7^P7^PP7bP7^P7^P${PP${PPPP${P7h7h7kP7n${7z${P${7}${8[8_8e8hP8w9W9^9d9k9q9w9}:T:Z:aPPPP:gP:tP>`@UA|BVPPB_BfPPPPPPPPPPPPPBoPBrPBuBxCVQ_OQ!RaQ!TcQ$d#^Q$f#`Q$k#eQ%s%YQ&O%hQ&b%|R&i&cgZO]ac#^#`#e%Y%h%|&c$OSOT]abcdhlnw{}!k!{!|!}#O#P#Q#R#S#T#V#^#_#`#a#d#e#o#p#t$_$a$b%W%Y%`%f%h%i%m%t%x%|&U&c&e&k&l&m&n&o&p&q&r&s&t&u&v&w&x&y&z&{&|&}'OQiQQ!Q`Q!VeQ!ZfS!]g#lQ!gkW!jm#v%U%oQ!rqQ#n!`Q$h#bQ$l#fQ$o#hQ$p#jQ%]$^R%k$nYoRr!i!r%b#mxTbdhlmnw}!k!{!|!}#O#P#Q#R#S#T#_#a#d#o#p#t#v$^$_$a$b%U%`%f%i%m%o%t%x&U&e&k&l&m&n&o&p&q&r&s&t&u&v&w&x&y&z&{&|&}'O$ZSOT]abcdhlmnw{}!k!{!|!}#O#P#Q#R#S#T#V#^#_#`#a#d#e#o#p#t#v$^$_$a$b%U%W%Y%`%f%h%i%m%o%t%x%|&U&c&e&k&l&m&n&o&p&q&r&s&t&u&v&w&x&y&z&{&|&}'O$ZVOT]abcdhlmnw{}!k!{!|!}#O#P#Q#R#S#T#V#^#_#`#a#d#e#o#p#t#v$^$_$a$b%U%W%Y%`%f%h%i%m%o%t%x%|&U&c&e&k&l&m&n&o&p&q&r&s&t&u&v&w&x&y&z&{&|&}'O#rVTbdhlmnw{}!k!{!|!}#O#P#Q#R#S#T#V#_#a#d#o#p#t#v$^$_$a$b%U%W%`%f%i%m%o%t%x&U&e&k&l&m&n&o&p&q&r&s&t&u&v&w&x&y&z&{&|&}'OgWO]ac#^#`#e%Y%h%|&cX!mm#v%U%on!|s!S!U!b!h#u#|$T$U$e$g$u%}&P&jf&o!l$w$y%Q%R%^%_%a&Q&S&aZ&p%c%u&W&_&`r!}s!S!U!b!h#u#|#}$P$T$U$e$g$u%}&P&jj&q!l$w$y$z$|%Q%R%^%_%a&Q&S&a_&r%c%u&W&X&Z&_&`p#Os!S!U!b!h#u#|#}$T$U$e$g$u%}&P&jh&s!l$w$y$z%Q%R%^%_%a&Q&S&a]&t%c%u&W&X&_&`t#Ps!S!U!b!h#u#|#}$O$P$T$U$e$g$u%}&P&jl&u!l$w$y$z${$|%Q%R%^%_%a&Q&S&aa&v%c%u&W&X&Y&Z&_&`v#Qs!S!U!b!h#u#|#}$O$P$Q$T$U$e$g$u%}&P&jn&w!l$w$y$z${$|$}%Q%R%^%_%a&Q&S&ac&x%c%u&W&X&Y&Z&[&_&`x#Rs!S!U!b!h#u#|#}$O$P$Q$R$T$U$e$g$u%}&P&jp&y!l$w$y$z${$|$}%O%Q%R%^%_%a&Q&S&ae&z%c%u&W&X&Y&Z&[&]&_&`|#Rs!S!U!b!h#U#u#|#}$O$P$Q$R$S$T$U$e$g$u%}&P&jt&y!l#y$w$y$z${$|$}%O%P%Q%R%^%_%a&Q&S&ai&z%c%u%y&W&X&Y&Z&[&]&^&_&`!UwTbdhlnw}!k!{!|!}#O#P#Q#R#S#T#_#a#d#o#p%f%i&ez&km#t#v$^$_$a%U%m%o%t%x&k&m&o&q&s&u&w&y&{&}k&l$b%`&U&l&n&p&r&t&v&x&z&|'OQ#WyQ#i![R$s#nR$Y#VT$[#X$]T$Z#X$]Q%v%`R&f&UR!YeR!XeQ!ehQ#[}Q$j#dR$t#oR![fgYO]ac#^#`#e%Y%h%|&cR!agQ!^gR$q#lR#k!]d^Oac#^#`#e%Y%h%|&cR!P]d]Oac#^#`#e%Y%h%|&cR!O]Q#w!oR%V#wQ#q!bR$v#qS%X$V$WR%r%XQ$]#XR%Z$]Q&V%vR&g&VQ%e$fR%z%eQ#c!VR$i#cQ#g!ZR$m#gQ|UR#Z|Q#m!^R$r#mg[O]ac#^#`#e%Y%h%|&cQsTQ!SbQ!UdY!bhn}#d#oQ!hlW!lm#v%U%oQ#UwQ#u!kQ#y&kQ#|!{Q#}!|Q$O!}Q$P#OQ$Q#PQ$R#QQ$S#RQ$T#SQ$U#TQ$e#_Q$g#aQ$u#pQ$w#tQ$y&mQ$z&oQ${&qQ$|&sQ$}&uQ%O&wQ%P&yQ%Q&{Q%R&}Q%^$^Q%_$_Q%a$aQ%c$bS%u%`&UQ%y&lQ%}%fQ&P%iQ&Q%mQ&S%tQ&W&nQ&X&pQ&Y&rQ&Z&tQ&[&vQ&]&xQ&^&zQ&_&|Q&`'OQ&a%xR&j&elRO]ac{#V#^#`#e%W%Y%h%|&c!UrTbdhlnw}!k!{!|!}#O#P#Q#R#S#T#_#a#d#o#p%f%i&ez!im#t#v$^$_$a%U%m%o%t%x&k&m&o&q&s&u&w&y&{&}k%b$b%`&U&l&n&p&r&t&v&x&z&|'OfUO]ac#^#`#e%Y%h%|&c#lVTbdhlmnw}!k!{!|!}#O#P#Q#R#S#T#_#a#d#o#p#t#v$^$_$a$b%U%`%f%i%m%o%t%x&U&e&k&l&m&n&o&p&q&r&s&t&u&v&w&x&y&z&{&|&}'OQ#Y{Q$V#VR%p%WWpRr!i%bR#{!rQ!omV%S#v%U%oZoRr!i!r%bW!ch}#d#oR!qnR$X#VR%w%`R!WegXO]ac#^#`#e%Y%h%|&cR!_g", - nodeNames: "⚠ Comment Chunk Block ; Label :: Name break Goto goto Scope do end WhileStatement while nil true false Ellipsis Number LiteralString Property . MemberExpression [ ] Parens ( ) FunctionCall : TableConstructor { FieldDynamic FieldProp FieldExp } , BinaryExpression or and CompareOp BitOp BitOp BitOp BitOp Concat ArithOp ArithOp ArithOp UnaryExpression not ArithOp BitOp LenOp FunctionDef function FuncBody ArgList Query query QueryClause FromClause from WhereClause where OrderByClause order by OrderBy desc SelectClause select LimitClause limit RepeatStatement repeat until IfStatement if then elseif else ForStatement for ForNumeric ForGeneric NameList in ExpList Function FuncName LocalFunction local Assign VarList Local AttNameList AttName Attrib ReturnStatement return", - maxTerm: 151, + states: "F^O!ZQPOOOOQO'#Cc'#CcO!UQPO'#CaO!bQPOOOOQO'#FP'#FPO!vQQO'#CwO$]QPO'#FOOOQO'#FO'#FOO$gQPO'#FOOOQO'#Ed'#EdO%vQPO'#EcOOQO'#Ez'#EzOOQO'#Ek'#EkO%{QPO'#C_OOQO'#C_'#C_QOQPOOO!UQPO'#CeO&`QPO'#CgO!vQQO'#CjO&gQPO'#EOO!vQQO'#ERO!UQPO'#EWO!UQPO'#E_O&nQPO'#EeO&vQQO'#EiO'^QPO,58{OOQO'#Cq'#CqO!UQPO,59^O!vQQO,59`O(gQQO'#C|O(nQQO'#FUOOQO'#FQ'#FQOOQO,59f,59fO!UQPO,59fO(uQPO'#E|O,eQPO,59cOOQO'#Dc'#DcOOQO'#Dd'#DdOOQO'#De'#DeO!vQQO'#DaOOQO'#E|'#E|O,lQPO'#DfO,qQSO'#DjO,vQPO'#EtO-OQPO,5WQPO,5`QPO,5:zO!UQPO,5:zOOQO1G0e1G0eO!UQPO'#EhOOQO,5;R,5;RO!UQPO'#EuO>kQPO,5yAN>yO&`QPOAN>yO!>fQPO,5;]O!>mQPOAN>yO!>rQPO<|QSO<R#Q;'S=o;'S;=`>}<%lO=ok>UVO!_>k!_!`?T!`#P>k#P#Q@X#Q;'S>k;'S;=`Ao<%lO>kk>nTO#P=o#P#Q7z#Q;'S=o;'S;=`>}<%lO=ok?QP;=`<%l=ok?WTO#P?g#P#Q@P#Q;'S?g;'S;=`?y<%lO?gk?jTO#P?g#P#Q7z#Q;'S?g;'S;=`?y<%lO?gk?|P;=`<%l?gk@UP#mk!_!`8Qk@[VO!_=o!_!`@q!`#P=o#P#QAu#Q;'S=o;'S;=`>}<%lO=ok@tTO#P=o#P#QAT#Q;'S=o;'S;=`>}<%lO=okAYV#mkO!_>k!_!`?T!`#P>k#P#Q@X#Q;'S>k;'S;=`Ao<%lO>kkArP;=`<%l>kkAxVO!_?g!_!`?T!`#P?g#P#QAu#Q;'S?g;'S;=`?y<%lO?goBdT$USO#PBs#P#QCV#Q;'SBs;'S;=`Ci<%lOBskBvTO#PBs#P#QCV#Q;'SBs;'S;=`Ci<%lOBskCYTO#PBs#P#Q%b#Q;'SBs;'S;=`Ci<%lOBskClP;=`<%lBsoCtPjP#P#QCwnC|O$Wn~DRO$R~~DWOq~~D]O#v~~DbOu~~DgP#x~!_!`$m", + tokenData: "Dj~RuXY#fYZ$Q[]#f]^$_pq#fqr$grs$rst)`uv)evw)jwx)oxy.Wyz.]z{.b{|.g|}.l}!O.s!O!P/g!P!Q/|!Q!R0^!R![1u![!]3m!]!^3z!^!_4R!_!`4e!`!a4m!c!}5P!}#O5b#O#P#w#P#QCo#Q#RC|#R#S5P#T#o5P#o#pDR#p#qDW#q#rD]#r#sDb~#kS#m~XY#f[]#fpq#f#O#P#w~#zQYZ#f]^#f~$VP#l~]^$Y~$_O#l~~$dP#l~YZ$Ym$jP!_!`$mm$rOzmk$uXOY$rZ]$r^r$rrs%bs#O$r#O#P%g#P;'S$r;'S;=`'}<%lO$rk%gO#qkk%j]rs$rwx$r!Q![&c#O#P$r#T#U$r#U#V$r#Y#Z$r#b#c$r#f#g$r#h#i$r#i#j(T#l#m(v#n#o$rk&fZOY$rZ]$r^r$rrs%bs!Q$r!Q!['X![#O$r#O#P%g#P;'S$r;'S;=`'}<%lO$rk'[ZOY$rZ]$r^r$rrs%bs!Q$r!Q![$r![#O$r#O#P%g#P;'S$r;'S;=`'}<%lO$rk(QP;=`<%l$rk(WP#o#p(Zk(^R!Q![(g!c!i(g#T#Z(gk(jS!Q![(g!c!i(g#T#Z(g#q#r$rk(yR!Q![)S!c!i)S#T#Z)Sk)VR!Q![$r!c!i$r#T#Z$r~)eO$W~~)jO$T~~)oO#{~k)rXOY)oZ])o^w)owx%bx#O)o#O#P*_#P;'S)o;'S;=`,u<%lO)ok*b]rs)owx)o!Q![+Z#O#P)o#T#U)o#U#V)o#Y#Z)o#b#c)o#f#g)o#h#i)o#i#j,{#l#m-n#n#o)ok+^ZOY)oZ])o^w)owx%bx!Q)o!Q![,P![#O)o#O#P*_#P;'S)o;'S;=`,u<%lO)ok,SZOY)oZ])o^w)owx%bx!Q)o!Q![)o![#O)o#O#P*_#P;'S)o;'S;=`,u<%lO)ok,xP;=`<%l)ok-OP#o#p-Rk-UR!Q![-_!c!i-_#T#Z-_k-bS!Q![-_!c!i-_#T#Z-_#q#r)ok-qR!Q![-z!c!i-z#T#Z-zk-}R!Q![)o!c!i)o#T#Z)o~.]Ol~~.bOm~~.gO$R~~.lO$P~o.sOvc#w[~.xP$Q~}!O.{~/QTP~OY.{Z].{^;'S.{;'S;=`/a<%lO.{~/dP;=`<%l.{o/lPgi!O!P/oo/tP!Pm!O!P/wQ/|OcQ~0RQ$S~!P!Q0X!_!`$m~0^O$U~~0cUd~!O!P0u!Q![1u!g!h1Z!z!{2W#X#Y1Z#l#m2W~0xP!Q![0{~1QRd~!Q![0{!g!h1Z#X#Y1Z~1^R{|1g}!O1g!Q![1m~1jP!Q![1m~1rPd~!Q![1m~1zSd~!O!P0u!Q![1u!g!h1Z#X#Y1Z~2ZR!Q![2d!c!i2d#T#Z2d~2iUd~!O!P2{!Q![2d!c!i2d!r!s1Z#T#Z2d#d#e1Z~3OR!Q![3X!c!i3X#T#Z3X~3^Td~!Q![3X!c!i3X!r!s1Z#T#Z3X#d#e1Z~3rPo~![!]3u~3zOU~_4ROSR#w[o4YQ$`Qzm!^!_4`!_!`$mm4eO#}m~4jP#v~!_!`$mo4tQ$aQzm!_!`$m!`!a4zm5PO$Om~5US#o~!Q![5P!c!}5P#R#S5P#T#o5Po5gQik!_!`5m!}#OB_k5pQ!_!`5v!}#O=ok5yP!}#O5|k6PTO#P5|#P#Q6`#Q;'S5|;'S;=`8W<%lO5|k6cVO!_6x!_!`:s!`#P6x#P#Q;w#Q;'S6x;'S;=`=i<%lO6xk6{VO!_7b!_!`8^!`#P7b#P#Q8v#Q;'S7b;'S;=`=c<%lO7bk7eTO#P5|#P#Q7t#Q;'S5|;'S;=`8W<%lO5|k7wP!_!`7zk7}P!_!`8Qk8TP#P#Q%bk8ZP;=`<%l5|k8aTO#P8^#P#Q7t#Q;'S8^;'S;=`8p<%lO8^k8sP;=`<%l8^k8yUO!_5|!_!`9]!`#P5|#Q;'S5|;'S;=`8W<%lO5|k9`VO!_5|!_!`9u!`#P5|#P#Q6`#Q;'S5|;'S;=`8W<%lO5|k9xTO#P5|#P#Q:X#Q;'S5|;'S;=`8W<%lO5|k:^V#qkO!_6x!_!`:s!`#P6x#P#Q;w#Q;'S6x;'S;=`=i<%lO6xk:vVO!_8^!_!`;]!`#P8^#P#Q7t#Q;'S8^;'S;=`8p<%lO8^k;`TO#P8^#P#Q;o#Q;'S8^;'S;=`8p<%lO8^k;tP#qk!_!`7zk;zVO!_7b!_!`:s!`#P7b#P#QR#Q;'S=o;'S;=`>}<%lO=ok>UVO!_>k!_!`?T!`#P>k#P#Q@X#Q;'S>k;'S;=`Ao<%lO>kk>nTO#P=o#P#Q7z#Q;'S=o;'S;=`>}<%lO=ok?QP;=`<%l=ok?WTO#P?g#P#Q@P#Q;'S?g;'S;=`?y<%lO?gk?jTO#P?g#P#Q7z#Q;'S?g;'S;=`?y<%lO?gk?|P;=`<%l?gk@UP#qk!_!`8Qk@[VO!_=o!_!`@q!`#P=o#P#QAu#Q;'S=o;'S;=`>}<%lO=ok@tTO#P=o#P#QAT#Q;'S=o;'S;=`>}<%lO=okAYV#qkO!_>k!_!`?T!`#P>k#P#Q@X#Q;'S>k;'S;=`Ao<%lO>kkArP;=`<%l>kkAxVO!_?g!_!`?T!`#P?g#P#QAu#Q;'S?g;'S;=`?y<%lO?goBdT$YSO#PBs#P#QCV#Q;'SBs;'S;=`Ci<%lOBskBvTO#PBs#P#QCV#Q;'SBs;'S;=`Ci<%lOBskCYTO#PBs#P#Q%b#Q;'SBs;'S;=`Ci<%lOBskClP;=`<%lBsoCtPjP#P#QCwnC|O$[n~DRO$V~~DWOq~~D]O#z~~DbOu~~DgP#|~!_!`$m", tokenizers: [0, 1, 2, 3, 4], topRules: {"Chunk":[0,2]}, - dynamicPrecedences: {"128":1}, - specialized: [{term: 119, get: (value) => spec_identifier[value] || -1}], - tokenPrec: 3779 + dynamicPrecedences: {"132":1}, + specialized: [{term: 123, get: (value) => spec_identifier[value] || -1}], + tokenPrec: 4041 }) diff --git a/client/space_lua/parse-lua.terms.js b/client/space_lua/parse-lua.terms.js index 2c088b0b..ade51d75 100644 --- a/client/space_lua/parse-lua.terms.js +++ b/client/space_lua/parse-lua.terms.js @@ -22,15 +22,17 @@ export const OrderBy = 70, SelectClause = 72, LimitClause = 74, - IfStatement = 79, - ForStatement = 84, - ForNumeric = 86, - ForGeneric = 87, - NameList = 88, - ExpList = 90, - FuncName = 92, - VarList = 96, - AttNameList = 98, - AttName = 99, - Attrib = 100, - ReturnStatement = 101 + GroupByClause = 76, + HavingClause = 78, + IfStatement = 83, + ForStatement = 88, + ForNumeric = 90, + ForGeneric = 91, + NameList = 92, + ExpList = 94, + FuncName = 96, + VarList = 100, + AttNameList = 102, + AttName = 103, + Attrib = 104, + ReturnStatement = 105 diff --git a/client/space_lua/parse.test.ts b/client/space_lua/parse.test.ts index bd4ba299..7792f5d9 100644 --- a/client/space_lua/parse.test.ts +++ b/client/space_lua/parse.test.ts @@ -136,6 +136,22 @@ Deno.test("Test query parsing", () => { `_(query[[from p = index.tag("page") order by p.lastModified desc, p.name]])`, ); parse(`_(query[[from p = index.tag("page") order by p.lastModified]])`); + // group by single key + parse( + `_(query[[from p = index.tag("page") group by p.category]])`, + ); + // group by multiple keys + parse( + `_(query[[from p = index.tag("page") group by p.category, p.status]])`, + ); + // group by + having + parse( + `_(query[[from p = index.tag("page") group by p.category having #group > 1]])`, + ); + // group by + having + select + parse( + `_(query[[from p = index.tag("page") group by p.category, p.status having #group > 2 select { key = key, count = #group }]])`, + ); }); Deno.test("Test numeric constant parsing", () => { diff --git a/client/space_lua/parse.ts b/client/space_lua/parse.ts index 16a48cdb..723d87fd 100644 --- a/client/space_lua/parse.ts +++ b/client/space_lua/parse.ts @@ -35,7 +35,7 @@ const luaStyleTags = styleTags({ CompareOp: t.operator, "true false": t.bool, Comment: t.lineComment, - "return break goto do end while repeat until function local if then else elseif in for nil or and not query from where limit select order by desc": + "return break goto do end while repeat until function local if then else elseif in for nil or and not query from where limit select order by desc group having": t.keyword, }); @@ -131,6 +131,7 @@ function expressionHasFunctionDef(e: LuaExpression): boolean { break; case "Where": case "Select": + case "Having": if (expressionHasFunctionDef(c.expression)) return true; break; case "Limit": @@ -144,6 +145,11 @@ function expressionHasFunctionDef(e: LuaExpression): boolean { } } break; + case "GroupBy": + for (let j = 0; j < c.expressions.length; j++) { + if (expressionHasFunctionDef(c.expressions[j])) return true; + } + break; } } return false; @@ -213,6 +219,7 @@ function exprReferencesNames(e: LuaExpression, names: Set): boolean { break; case "Where": case "Select": + case "Having": if (exprReferencesNames(c.expression, names)) return true; break; case "Limit": @@ -226,6 +233,11 @@ function exprReferencesNames(e: LuaExpression, names: Set): boolean { } } break; + case "GroupBy": + for (let j = 0; j < c.expressions.length; j++) { + if (exprReferencesNames(c.expressions[j], names)) return true; + } + break; } } return false; @@ -512,6 +524,7 @@ function exprCapturesNames(e: LuaExpression, names: Set): boolean { break; case "Where": case "Select": + case "Having": if (exprCapturesNames(c.expression, names)) return true; break; case "Limit": @@ -525,6 +538,11 @@ function exprCapturesNames(e: LuaExpression, names: Set): boolean { } } break; + case "GroupBy": + for (let j = 0; j < c.expressions.length; j++) { + if (exprCapturesNames(c.expressions[j], names)) return true; + } + break; } } return false; @@ -1261,6 +1279,27 @@ function parseQueryClause(t: ParseTree, ctx: ASTCtx): LuaQueryClause { ctx: context(t, ctx), }; } + case "GroupByClause": { + // children: ckw<"group">, ckw<"by">, exp, ",", exp, ... + const expressions: LuaExpression[] = t.children! + .filter((c) => + c.type !== undefined && c.type !== "group" && c.type !== "by" && + c.type !== "," + ) + .map((c) => parseExpression(c, ctx)); + return { + type: "GroupBy", + expressions, + ctx: context(t, ctx), + }; + } + case "HavingClause": { + return { + type: "Having", + expression: parseExpression(t.children![1], ctx), + ctx: context(t, ctx), + }; + } default: console.error(t); throw new Error(`Unknown query clause type: ${t.type}`); diff --git a/client/space_lua/query_collection.ts b/client/space_lua/query_collection.ts index f6a2890f..0e197302 100644 --- a/client/space_lua/query_collection.ts +++ b/client/space_lua/query_collection.ts @@ -1,6 +1,24 @@ -import type { LuaExpression } from "./ast.ts"; -import { LuaEnv, luaGet, luaKeys, LuaStackFrame, LuaTable } from "./runtime.ts"; -import { evalExpression } from "./eval.ts"; +import type { + LuaBinaryExpression, + LuaDynamicField, + LuaExpression, + LuaExpressionField, + LuaFunctionCallExpression, + LuaParenthesizedExpression, + LuaPropField, + LuaUnaryExpression, +} from "./ast.ts"; +import { + LuaEnv, + luaGet, + luaKeys, + LuaStackFrame, + LuaTable, + luaTruthy, + type LuaValue, + singleResult, +} from "./runtime.ts"; +import { evalExpression, luaOp } from "./eval.ts"; import { asyncQuickSort } from "./util.ts"; import type { DataStore } from "../data/datastore.ts"; import type { KvPrimitives } from "../data/kv_primitives.ts"; @@ -9,22 +27,65 @@ import type { QueryCollationConfig } from "../../plug-api/types/config.ts"; import type { KvKey } from "../../plug-api/types/datastore.ts"; -export function buildItemEnv( +import { executeAggregate, getAggregateSpec } from "./aggregates.ts"; + +export { buildItemEnv } from "./query_env.ts"; +import { buildItemEnv } from "./query_env.ts"; + +// Build environment for post-`group by` clauses (`having`, `select`, +// `order by`). Injects `key` and `group` as top-level variables. Also +// unpacks the `group by` key fields as locals so that `group by name, +// tag` makes `name` and `tag` accessible. +function buildGroupItemEnv( objectVariable: string | undefined, + groupByNames: string[] | undefined, item: any, env: LuaEnv, sf: LuaStackFrame, ): LuaEnv { const itemEnv = new LuaEnv(env); - if (!objectVariable) { - // Inject all item keys as variables - for (const key of luaKeys(item)) { - itemEnv.setLocal(key, luaGet(item, key, sf.astCtx ?? null, sf)); - } - // As well as _ - itemEnv.setLocal("_", item); - } else { + if (objectVariable) { itemEnv.setLocal(objectVariable, item); + } else { + itemEnv.setLocal("_", item); + } + if (item instanceof LuaTable) { + const keyVal = item.rawGet("key"); + const groupVal = item.rawGet("group"); + + if (!objectVariable && groupVal instanceof LuaTable) { + const firstItem = groupVal.rawGet(1); + if (firstItem) { + for (const k of luaKeys(firstItem)) { + itemEnv.setLocal( + k, + luaGet(firstItem, k, sf.astCtx ?? null, sf), + ); + } + } + } + + if (keyVal !== undefined) { + itemEnv.setLocal("key", keyVal); + } + if (groupVal !== undefined) { + itemEnv.setLocal("group", groupVal); + } + + if (keyVal instanceof LuaTable) { + for (const k of luaKeys(keyVal)) { + itemEnv.setLocal( + k, + luaGet(keyVal, k, sf.astCtx ?? null, sf), + ); + } + } + if ( + !(keyVal instanceof LuaTable) && groupByNames && + groupByNames.length === 1 + ) { + itemEnv.setLocal(groupByNames[0], keyVal); + } } return itemEnv; } @@ -51,6 +112,10 @@ export type LuaCollectionQuery = { offset?: number; // Whether to return only distinct values distinct?: boolean; + // The group by expressions evaluated with Lua + groupBy?: LuaExpression[]; + // The having expression evaluated with Lua + having?: LuaExpression; }; export interface LuaQueryCollection { @@ -78,6 +143,194 @@ export class ArrayQueryCollection implements LuaQueryCollection { } } +// Check whether an expression tree contains any aggregate function call. +function containsAggregate(expr: LuaExpression): boolean { + switch (expr.type) { + case "FunctionCall": { + const fc = expr as LuaFunctionCallExpression; + if (fc.prefix.type === "Variable" && getAggregateSpec(fc.prefix.name)) { + return true; + } + return fc.args.some(containsAggregate); + } + case "Binary": { + const bin = expr as LuaBinaryExpression; + return containsAggregate(bin.left) || containsAggregate(bin.right); + } + case "Unary": { + const un = expr as LuaUnaryExpression; + return containsAggregate(un.argument); + } + case "Parenthesized": { + const p = expr as LuaParenthesizedExpression; + return containsAggregate(p.expression); + } + case "TableConstructor": + return expr.fields.some((f) => { + switch (f.type) { + case "PropField": + return containsAggregate((f as LuaPropField).value); + case "DynamicField": { + const df = f as LuaDynamicField; + return containsAggregate(df.key) || containsAggregate(df.value); + } + case "ExpressionField": + return containsAggregate((f as LuaExpressionField).value); + default: + return false; + } + }); + default: + return false; + } +} + +/** + * Evaluate an expression in aggregate-aware mode. + * + * When a FunctionCall matches a registered aggregate name, the aggregate + * protocol is executed instead of normal call semantics. All other + * expressions fall through to normal evalExpression. + */ +export async function evalExpressionWithAggregates( + expr: LuaExpression, + env: LuaEnv, + sf: LuaStackFrame, + groupItems: LuaTable, + objectVariable: string | undefined, + outerEnv: LuaEnv, +): Promise { + // Fast path: no aggregate calls in tree -> normal eval + if (!containsAggregate(expr)) { + return evalExpression(expr, env, sf); + } + + const recurse = (e: LuaExpression) => + evalExpressionWithAggregates( + e, + env, + sf, + groupItems, + objectVariable, + outerEnv, + ); + + // Direct aggregate call + if (expr.type === "FunctionCall") { + const fc = expr as LuaFunctionCallExpression; + if (fc.prefix.type === "Variable") { + const name = fc.prefix.name; + const spec = getAggregateSpec(name); + if (spec) { + const valueExpr = fc.args.length > 0 ? fc.args[0] : null; + return executeAggregate( + spec, + groupItems, + valueExpr, + objectVariable, + outerEnv, + sf, + evalExpression, + ); + } + } + } + + // Recurse into field values + if (expr.type === "TableConstructor") { + const table = new LuaTable(); + let nextArrayIndex = 1; + for (const field of expr.fields) { + switch (field.type) { + case "PropField": { + const pf = field as LuaPropField; + const value = await recurse(pf.value); + table.set(pf.key, value, sf); + break; + } + case "DynamicField": { + const df = field as LuaDynamicField; + const key = await evalExpression(df.key, env, sf); + const value = await recurse(df.value); + table.set(key, value, sf); + break; + } + case "ExpressionField": { + const ef = field as LuaExpressionField; + const value = await recurse(ef.value); + table.rawSetArrayIndex(nextArrayIndex, value); + nextArrayIndex++; + break; + } + } + } + return table; + } + + // Recurse operands, apply via luaOp + if (expr.type === "Binary") { + const bin = expr as LuaBinaryExpression; + + if (bin.operator === "and") { + const left = singleResult(await recurse(bin.left)); + if (!luaTruthy(left)) return left; + return singleResult(await recurse(bin.right)); + } + if (bin.operator === "or") { + const left = singleResult(await recurse(bin.left)); + if (luaTruthy(left)) return left; + return singleResult(await recurse(bin.right)); + } + + const left = singleResult(await recurse(bin.left)); + const right = singleResult(await recurse(bin.right)); + return luaOp( + bin.operator, + left, + right, + undefined, + undefined, + expr.ctx, + sf, + ); + } + + // Recurse argument, apply operator + if (expr.type === "Unary") { + const un = expr as LuaUnaryExpression; + const arg = singleResult(await recurse(un.argument)); + switch (un.operator) { + case "-": + return typeof arg === "number" ? -arg : luaOp( + "-", + 0, + arg, + undefined, + undefined, + expr.ctx, + sf, + ); + case "not": + return !luaTruthy(arg); + case "#": + return evalExpression(expr, env, sf); + case "~": + if (typeof arg === "number") return ~arg; + throw new Error("attempt to perform bitwise operation on a non-number"); + default: + return evalExpression(expr, env, sf); + } + } + + // Unwrap + if (expr.type === "Parenthesized") { + const paren = expr as LuaParenthesizedExpression; + return singleResult(await recurse(paren.expression)); + } + + return evalExpression(expr, env, sf); +} + /** * Applies a given query (where, order by, limit etc.) to a set of results */ @@ -104,6 +357,106 @@ export async function applyQuery( results = filteredResults; } + const grouped = !!query.groupBy; + + // Collect `group by` key names for unpacking into the environment. + let groupByNames: string[] | undefined; + + // Apply `group by` + if (query.groupBy) { + groupByNames = query.groupBy.map((expr) => { + if (expr.type === "Variable") { + return expr.name; + } + if (expr.type === "PropertyAccess") { + return expr.property; + } + return undefined as unknown as string; + }).filter(Boolean); + + const groups = new Map(); + for (const item of results) { + const itemEnv = buildItemEnv(query.objectVariable, item, env, sf); + // Evaluate all `group by` expressions to form a composite key + const keyParts: any[] = []; + for (const expr of query.groupBy) { + keyParts.push(await evalExpression(expr, itemEnv, sf)); + } + const compositeKey = keyParts.length === 1 + ? generateKey(keyParts[0]) + : JSON.stringify(keyParts.map(generateKey)); + let entry = groups.get(compositeKey); + if (!entry) { + // Unwrap single key; multi-key to `LuaTable` with named fields + let keyVal: any; + if (keyParts.length === 1) { + keyVal = keyParts[0]; + } else { + const kt = new LuaTable(); + for (let i = 0; i < keyParts.length; i++) { + kt.rawSetArrayIndex(i + 1, keyParts[i]); + if (groupByNames && groupByNames[i]) { + kt.rawSet(groupByNames[i], keyParts[i]); + } + } + keyVal = kt; + } + entry = { key: keyVal, items: [] }; + groups.set(compositeKey, entry); + } + entry.items.push(item); + } + // Convert groups to result rows with `key` and `group` + results = []; + for (const { key, items } of groups.values()) { + const groupTable = new LuaTable(); + for (let i = 0; i < items.length; i++) { + groupTable.rawSetArrayIndex(i + 1, items[i]); + } + const row = new LuaTable(); + row.rawSet("key", key); + row.rawSet("group", groupTable); + results.push(row); + } + } + + // Apply `having` -- aggregate-aware when grouped + if (query.having) { + const filteredResults = []; + for (const value of results) { + const itemEnv = buildGroupItemEnv( + query.objectVariable, + groupByNames, + value, + env, + sf, + ); + let condResult; + if (grouped) { + const groupTable = (value as LuaTable).rawGet("group"); + condResult = await evalExpressionWithAggregates( + query.having, + itemEnv, + sf, + groupTable, + query.objectVariable, + env, + ); + } else { + condResult = await evalExpression(query.having, itemEnv, sf); + } + if (condResult) { + filteredResults.push(value); + } + } + results = filteredResults; + } + + const mkEnv = grouped + ? (ov: string | undefined, item: any, e: LuaEnv, s: LuaStackFrame) => + buildGroupItemEnv(ov, groupByNames, item, e, s) + : buildItemEnv; + // Apply `order by` next if (query.orderBy) { // Retrieve from config API if not passed @@ -119,8 +472,8 @@ export async function applyQuery( results = await asyncQuickSort(results, async (a, b) => { // Compare each orderBy clause until we find a difference for (const { expr, desc } of query.orderBy!) { - const aEnv = buildItemEnv(query.objectVariable, a, env, sf); - const bEnv = buildItemEnv(query.objectVariable, b, env, sf); + const aEnv = mkEnv(query.objectVariable, a, env, sf); + const bEnv = mkEnv(query.objectVariable, b, env, sf); const aVal = await evalExpression(expr, aEnv, sf); const bVal = await evalExpression(expr, bEnv, sf); @@ -145,15 +498,25 @@ export async function applyQuery( }); } - // Apply the select + // Apply the select -- aggregate-aware when grouped if (query.select) { const newResult = []; for (const item of results) { - const itemEnv = buildItemEnv(query.objectVariable, item, env, sf); - const val = await evalExpression(query.select, itemEnv, sf); - // Skip nil results - selecting a missing field produces no row - if (val !== null && val !== undefined) { - newResult.push(val); + const itemEnv = mkEnv(query.objectVariable, item, env, sf); + if (grouped) { + const groupTable = (item as LuaTable).rawGet("group"); + newResult.push( + await evalExpressionWithAggregates( + query.select, + itemEnv, + sf, + groupTable, + query.objectVariable, + env, + ), + ); + } else { + newResult.push(await evalExpression(query.select, itemEnv, sf)); } } results = newResult; @@ -219,6 +582,7 @@ function generateKey(value: any) { ? JSON.stringify(value) : value; } + export class DataStoreQueryCollection implements LuaQueryCollection { constructor( private readonly dataStore: DataStore, diff --git a/client/space_lua/query_env.ts b/client/space_lua/query_env.ts new file mode 100644 index 00000000..81197138 --- /dev/null +++ b/client/space_lua/query_env.ts @@ -0,0 +1,26 @@ +import { LuaEnv, luaGet, luaKeys, type LuaStackFrame } from "./runtime.ts"; + +/** + * Build an environment for evaluating per-item expressions in queries. + * Extracted to its own module to avoid circular imports between + * query_collection.ts and aggregates.ts. + */ +export function buildItemEnv( + objectVariable: string | undefined, + item: any, + env: LuaEnv, + sf: LuaStackFrame, +): LuaEnv { + const itemEnv = new LuaEnv(env); + if (!objectVariable) { + // Inject all item keys as variables + for (const key of luaKeys(item)) { + itemEnv.setLocal(key, luaGet(item, key, sf.astCtx ?? null, sf)); + } + // As well as _ + itemEnv.setLocal("_", item); + } else { + itemEnv.setLocal(objectVariable, item); + } + return itemEnv; +} diff --git a/client/space_lua/query_test.lua b/client/space_lua/query_test.lua index f60107a5..e8c3bf67 100644 --- a/client/space_lua/query_test.lua +++ b/client/space_lua/query_test.lua @@ -1,47 +1,510 @@ local function assertEquals(a, b) - if a ~= b then - error("Assertion failed: " .. a .. " is not equal to " .. b) - end + if a ~= b then + error("Assertion failed: " + .. tostring(a) .. " is not equal to " + .. tostring(b)) + end end --- Test query functionality -local data = { { name = "John", lastModified = 1, age = 20 }, { name = "Jane", lastModified = 2, age = 21 } } +-- Dataset +local pages = { + { name = "Alice", tags = {"work", "urgent"}, size = 10, age = 31 }, + { name = "Bob", tags = {"work"}, size = 20, age = 25 }, + { name = "Carol", tags = {"personal", "urgent"}, size = 5, age = 41 }, + { name = "Dave", tags = {"personal"}, size = 15, age = 52 }, + { name = "Ed", tags = {}, size = 3, age = 19 }, + { name = "Fran", tags = {"random"}, size = 1, age = 55 }, + { name = "Greg", tags = {"work", "fun"}, size = 2, age = 63 }, +} --- Basic limit query -local r = query [[from p = data limit 1]] -assertEquals(#r, 1) -assertEquals(r[1].name, "John") -assertEquals(r[1].lastModified, 1) +-- 1. Basic `from` +do + local r = query [[ + from + pages + ]] --- Order by descending -local r = query [[from p = data order by p.lastModified desc]] -assertEquals(#r, 2) -assertEquals(r[1].name, "Jane") -assertEquals(r[1].lastModified, 2) -assertEquals(r[2].name, "John") -assertEquals(r[2].lastModified, 1) + assertEquals(#r, #pages) +end --- Order by ascending -local r = query [[from p = data order by p.lastModified]] -assertEquals(#r, 2) -assertEquals(r[1].name, "John") -assertEquals(r[1].lastModified, 1) -assertEquals(r[2].name, "Jane") -assertEquals(r[2].lastModified, 2) +do + local r = query [[ + from + p = pages + ]] --- Select specific fields -local r = query [[from p = data order by p.age select {name=p.name, age=p.age}]] -assertEquals(#r, 2) -assertEquals(r[1].name, "John") -assertEquals(r[1].age, 20) -assertEquals(r[2].name, "Jane") -assertEquals(r[2].age, 21) -assertEquals(r[1].lastModified, nil) -assertEquals(r[2].lastModified, nil) + assertEquals(r[1].name, "Alice") +end --- Array transformation -local r = query [[from {1, 2, 3} select _ + 1]] -assertEquals(#r, 3) -assertEquals(r[1], 2) -assertEquals(r[2], 3) -assertEquals(r[3], 4) \ No newline at end of file +-- 2. Select/projection: direct, binding, mixed +do + local r = query [[ + from + pages + select { + n = name, + t = tags[1], + } + ]] + + assertEquals(r[1].n, "Alice") +end + +do + local r = query [[ + from + p = pages + select { + n = p.name, + t = p.tags[1], + } + ]] + + assertEquals(r[2].n, "Bob") +end + +do + local r = query [[ + from + pages + select { + a = name, + b = p and p.tags[1], + } + ]] + + assertEquals(r[1].a, "Alice") +end + +-- 3. Select field, select as array, order/limit/offset, where +do + local r1 = query [[ + from + pages + select { + value = size, + } + ]] + + assertEquals(r1[1].value, 10) + + local r2 = query [[ + from + p = pages + select { + value = p.size, + } + ]] + + assertEquals(r2[1].value, 10) +end + +do + local r1 = query [[ + from + pages + limit 2 + select { + name = name, + } + ]] + + assertEquals(#r1, 2) + + local r2 = query [[ + from + p = pages + limit 3, 2 + select { + name = p.name, + } + ]] + + assertEquals(r2[1].name, "Carol") +end + +do + local r1 = query [[ + from + pages + order by + size desc + select { + name = name, + } + ]] + + assert(r1[1].name == "Bob") + + local r2 = query [[ + from + p = pages + order by + p.age + select { + age = p.age, + } + ]] + + assert(r2[1].age == 19) +end + +do + local r1 = query [[ + from + pages + where + size > 10 + select { + name = name, + } + ]] + + assert(r1[1].name == "Bob") + + local r2 = query [[ + from + p = pages + where + p.age < 30 + select { + name = p.name, + } + ]] + + assert(r2[1].name == "Bob", + "Expected Bob in r2[1], got " .. tostring(r2[1] and r2[1].name)) + + assert(r2[2].name == "Ed", + "Expected Ed in r2[2], got " .. tostring(r2[2] and r2[2].name)) +end + +-- 4. Grouping/group select + +do + local r1 = query [[ + from + pages + where + tags[1] ~= nil + group by + tags[1] + select { + key = key, + } + ]] + local r2 = query [[ + from + p = pages + where + p.tags[1] ~= nil + group by + p.tags[1] + select { + key = key, + } + ]] + + assert(type(r1[1].key) == "string" or r1[1].key == nil) + assert(#r1 == #r2) +end + +do + local r1 = query [[ + from + pages + where + tags[1] ~= nil and tags[2] ~= nil + group by + tags[1], tags[2] + select { + k1 = key[1], + k2 = key[2], + } + ]] + local r2 = query [[ + from + p = pages + where + p.tags[1] ~= nil and p.tags[2] ~= nil + group by + p.tags[1], p.tags[2] + select { + k1 = key[1], + k2 = key[2], + } + ]] + + assert(#r1 == #r2) +end + +do + local r = query [[ + from + pages + group by + tags[1] + select { + k = key, + gc = #group, + first = group[1].name, + f2 = group[1].tags[2], + } + ]] + + assert(type(r[1].k) == "string" or r[1].k == nil) + assert(type(r[1].gc) == "number") +end + +do + local r = query [[ + from + p = pages + group by + p.tags[1] + select { + k = key, + gc = #group, + n = group[1].name, + } + ]] + + assert(type(r[1].gc) == "number") +end + +do + local r = query [[ + from + pages + group by + tags[1] + select { + k = key, + n = group[1].name, + t = group[1].tags[1], + } + ]] + + assert(type(r[1].k) == "string" or r[1].k == nil) +end + +-- 5. Aggregation/builtins + +do + local r1 = query [[ + from + pages + group by + tags[1] + select { + count = count(name), + } + ]] + local r2 = query [[ + from + p = pages + group by + p.tags[1] + select { + count = count(p.name), + } + ]] + + assert(type(r1[1].count) == "number") + assert(#r1 == #r2) +end + +do + local r1 = query [[ + from + pages + group by + tags[1] + select { + min = min(age), + max = max(age), + avg = avg(age), + sum = sum(age), + } + ]] + local r2 = query [[ + from + p = pages + group by + p.tags[1] + select { + min = min(p.size), + max = max(p.size), + avg = avg(p.size), + sum = sum(p.size), + } + ]] + + for _, row in ipairs(r2) do + assert(type(row.min) == "number" or row.min == nil) + assert(type(row.avg) == "number" or row.avg == nil) + end +end + +do + local r1 = query [[ + from + pages + group by + tags[1] + select { + arr = array_agg(name), + } + ]] + local r2 = query [[ + from + p = pages + group by + p.tags[1] + select { + arr = array_agg(p.name), + } + ]] + + assert(type(r1[1].arr) == "table" or r1[1].arr == nil) + assert(#r1 == #r2) +end + +do + local r = query [[ + from + pages + group by + tags[1] + select { + c = count(name), + v = min(size), + x = p and count(p.name), + } + ]] + + assert(type(r[1].c) == "number") +end + +-- 6. `having` + +do + local r1 = query [[ + from + pages + group by + tags[1] + having + count(name) > 1 + select { + key = key, + } + ]] + local r2 = query [[ + from + p = pages + group by + p.tags[1] + having + count(p.name) > 1 + select { + key = key, + } + ]] + + assertEquals(#r1, #r2) +end + +do + local r = query [[ + from + p = pages + group by + p.tags[1] + having + sum(p.size) > 15 + select { + key = key, + } + ]] + + assert((#r > 0), "expected some groups") +end + +-- 7. Full pipeline + +do + local r = query [[ + from + p = pages + where + p.age > 20 + group by + p.tags[1] + having + min(p.age) > 25 + select { + tag = key, + top = max(p.name), + total = count(p.name), + sum_size = sum(p.size), + min_age = min(age), + avg_size = avg(p.size), + } + order by + avg_size desc + limit 2 + ]] + + assert(#r <= 2) +end + +do + local r = query [[ + from + pages + where + age > 20 + group by + tags[1] + having + min(age) > 25 + select { + tag = key, + top = max(name), + total = count(name), + sum_size = sum(size), + min_age = min(age), + avg_size = avg(size), + } + order by + avg_size desc + limit 2 + ]] + + assert(#r <= 2) +end + +do + local r = query [[ + from + pages + where + age > 20 + group by + tags[1] + having + min(age) > 25 + select { + tag = key, + p = p and p.name, + total = count(name), + avg_size = avg(size), + } + order by + avg_size desc + limit 2 + ]] + + assert(#r <= 2) +end diff --git a/client/space_lua/stdlib/string.ts b/client/space_lua/stdlib/string.ts index 9281f2c2..56946bfe 100644 --- a/client/space_lua/stdlib/string.ts +++ b/client/space_lua/stdlib/string.ts @@ -19,7 +19,7 @@ import { import { strPackFn, strPackSizeFn, strUnpackFn } from "./string_pack.ts"; function capturesToLua(caps: CaptureResult[]): any { - if (caps.length === 0) {return null;} + if (caps.length === 0) return null; if (caps.length === 1) { const c = caps[0]; return "s" in c ? c.s : c.position; @@ -33,8 +33,8 @@ export const stringApi = new LuaTable({ byte: new LuaBuiltinFunction((_sf, s: string, i?: number, j?: number) => { i = i ?? 1; j = j ?? i; - if (j > s.length) {j = s.length;} - if (i < 1) {i = 1;} + if (j > s.length) j = s.length; + if (i < 1) i = 1; const result = []; for (let k = i; k <= j; k++) { result.push(s.charCodeAt(k - 1)); @@ -47,7 +47,7 @@ export const stringApi = new LuaTable({ find: new LuaBuiltinFunction( (_sf, s: string, pattern: string, init = 1, plain = false) => { const r = patternFind(s, pattern, init, plain); - if (!r) {return null;} + if (!r) return null; const result: any[] = [r.start, r.end]; for (const c of r.captures) { result.push("s" in c ? c.s : c.position); @@ -66,7 +66,7 @@ export const stringApi = new LuaTable({ const iter = patternGmatch(s, pattern, init); return () => { const caps = iter(); - if (!caps) {return;} + if (!caps) return; return capturesToLua(caps); }; }, @@ -85,7 +85,7 @@ export const stringApi = new LuaTable({ } else if (repl instanceof LuaTable) { callbacks.replTable = (key: string) => { const v = repl.get(key); - if (v === null || v === undefined || v === false) {return null;} + if (v === null || v === undefined || v === false) return null; return String(v); }; } else if (repl.call) { @@ -122,12 +122,12 @@ export const stringApi = new LuaTable({ match: new LuaBuiltinFunction( (_sf, s: string, pattern: string, init = 1) => { const caps = patternMatch(s, pattern, init); - if (!caps) {return null;} + if (!caps) return null; return capturesToLua(caps); }, ), rep: new LuaBuiltinFunction((_sf, s: string, n: number, sep?: string) => { - if (n <= 0) {return "";} + if (n <= 0) return ""; sep = sep ?? ""; const parts: string[] = []; for (let i = 0; i < n; i++) { diff --git a/libraries/Library/Std/APIs/Aggregate.md b/libraries/Library/Std/APIs/Aggregate.md new file mode 100644 index 00000000..33ca0512 --- /dev/null +++ b/libraries/Library/Std/APIs/Aggregate.md @@ -0,0 +1,122 @@ +--- +description: APIs to define custom aggregate functions for LIQ +tags: meta/api +--- + +APIs to define and override aggregate functions used in [[Space Lua/Lua Integrated Query|LIQ]] `select` and `having` clauses after `group by`. + +Built-in aggregates: `count`, `sum`, `min`, `max`, `avg` and `array_agg`. + +# API + +## aggregate.define(spec) + +Defines a new aggregate function. Required keys: + +* `name`: name of the aggregate (used in queries as `name(expr)`) +* `initialize`: function that returns the initial state +* `iterate`: function(state, value) that returns updated state + +Optional keys: + +* `description`: description of the aggregate +* `finish`: `function(state)` that transforms the final state into the result + +## aggregate.update(spec) + +Updates an existing aggregate definition. Same keys as `aggregate.define`. Only the provided keys are overwritten. + +# Examples + +## Define a custom aggregate + +Define a custom aggregate `concat` that concatenates strings. + +```lua +aggregate.define { + name = 'concat', + + initialize = function() + return '' + end, + + iterate = function(state, value) + if state == '' then + return tostring(value) + end + return state .. ', ' .. tostring(value) + end, +} +``` + +## Update an existing aggregate + +```lua +aggregate.update { + name = 'count', + description = 'Custom count aggregate that counts even nils', + + iterate = function(state, value) + return state + 1 + end, +} +``` + +# Implementation + +```space-lua +-- priority: 50 +aggregate = aggregate or {} + +local aggregateSchema = { + type = 'object', + + required = { + 'name', + 'initialize', + 'iterate' + }, + + properties = { + name = schema.string(), + description = schema.string(), + initialize = schema.func(), + iterate = schema.func(), + finish = schema.func(), + } +} + +function aggregate.define(spec) + local validationResult = jsonschema.validateObject(aggregateSchema, spec) + + if validationResult then + error('aggregate.define: ' .. validationResult) + end + + config.set({'aggregates', spec.name}, spec) +end + +function aggregate.update(spec) + if not spec.name then + error('aggregate.update: name is required') + end + + local existing = config.get({'aggregates', spec.name}, {}) + + for k, v in pairs(spec) do + existing[k] = v + end + + if not existing.initialize then + error('aggregate.update: aggregate ' + .. spec.name .. ' has no initialize after merge') + end + + if not existing.iterate then + error('aggregate.update: aggregate ' + .. spec.name .. ' has no iterate after merge') + end + + config.set({'aggregates', spec.name}, existing) +end +``` diff --git a/website/Space Lua/Lua Integrated Query.md b/website/Space Lua/Lua Integrated Query.md index 045f3bd7..28bfc031 100644 --- a/website/Space Lua/Lua Integrated Query.md +++ b/website/Space Lua/Lua Integrated Query.md @@ -2,13 +2,15 @@ Lua Integrated Query (LIQ) is a SilverBullet specific Lua extension. It adds a c The syntax for LIQ is `query[[my query]]`. In regular Lua `[[my query]]` is just another way of writing `"my query"` (it is an alternative string syntax). Function calls that only take a string argument can omit parentheses, therefore `query[[my query]]` is equivalent to `query("my query")`. -However, in [[Space Lua]] it interpreted as an SQL (and [LINQ](https://learn.microsoft.com/en-us/dotnet/csharp/linq/))-inspired integrated query language. +However, in [[Space Lua]] it is interpreted as an SQL (and [LINQ](https://learn.microsoft.com/en-us/dotnet/csharp/linq/))-inspired integrated query language. General syntax: query[[ from in where + group by [, , ...] + having order by limit , select @@ -54,8 +56,8 @@ However, there is also the more concise: implicitly binding each item to the variable `_` as well as making all attributes directly available as variables. The latter, while shorter, is less performant and will block future optimizations, so the variable-binding variant is preferred. > **warning** Warning -> When you use a `from` clause without explicit variable binding (so without thee `v in` syntax), note that any attribute of the object you’re iterating over will shadow global variables. For instance, if you have an object with a `table` attribute, regular `table` APIs will become inaccessible within the query. -> +> When you use a `from` clause without explicit variable binding (so without the `v in` syntax), note that any attribute of the object you’re iterating over will shadow global variables. For instance, if you have an object with a `table` attribute, regular `table` APIs will become inaccessible within the query. +> > **Recommendation:** Use the explicit variable binding syntax Example without variable binding: @@ -78,10 +80,47 @@ Or to select 5 pages tagged with `#meta`: ${query[[from p = index.tag "page" where table.includes(p.tags, "meta") limit 5]]} -Or select based on name (including folder) and a [[API/string|string function]] +Or select based on name (including folder) and a [[API/string|string function]]: ${query[[from p = index.tag "page" where p.name:startsWith("Person")]]} +## group by [, , ...] +The `group by` clause groups results by one or more key expressions. After grouping, each result row becomes a table with two fields: + +- `key` — the group key value (single value for one key, table for multi-key) +- `group` — a table (array) of all original items in that group + +The `group by` field names are also available as bare variables in `having`, `select`, and `order by`. Use `#group` to get the count of items in a group. + +Example: + +${query[[ + from p = index.tag "tag" + group by p.name + select { name = p.name, count = #p.group } + limit 5 +]]} + +See [[Space Lua/Lua Integrated Query/Grouping]] for detailed examples. + +## having +The `having` clause filters groups **after** `group by`. It follows SQL semantics: only group key fields, `key`, and `group` are accessible — use `where` to filter individual rows before grouping. + +Aggregate functions like `count()`, `sum()`, `min()`, `max()`, and `avg()` can be used in `having` expressions. See [[Space Lua/Lua Integrated Query/Aggregating]] for details. + +Example: + +${query[[ + from p = index.tag "tag" + group by p.name + having #group > 2 + select { name = name, count = #group } + order by count desc + limit 5 +]]} + +See [[Space Lua/Lua Integrated Query/Grouping]] for detailed examples. + ## order by [desc] The `order by` clause allows you to sort data, when `desc` is specified it reverts the sort order. @@ -118,6 +157,8 @@ ${query[[from {1, 2, 3, 4, 5} limit 3, 2]]} ## select The `select` clause allows you to transform each item in the result set. If omitted, it defaults to returning the item itself. +When used with `group by`, aggregate functions like `sum()`, `count()`, `min()`, `max()`, and `avg()` can be used in the `select` expression to compute values across each group. See [[Space Lua/Lua Integrated Query/Aggregating]] for details. + Some examples: Double each number: @@ -125,12 +166,24 @@ ${query[[from n = {1, 2, 3} select n * 2]]} It is convenient to combine it with the [[API/table#table.select(table, keys...)]] API: ${query[[ - from p = index.tag "page" + from p = index.tag "page" select table.select(p, "name", "lastModified") limit 3 ]]} -## Rendering the output +# Listing available aggregate functions + +You can list all available aggregator functions (built-ins and custom) using LIQ. + +Example: + +${query [[ + from k, v in pairs(config.get("aggregates", {})) + select { name = k, desc = v.description } + order by name +]]} + +# Rendering the output To render the output as a template, you can rely on the fact that queries return Lua tables. For example, to apply a template to render every page as a link: ${template.each(query[[ diff --git a/website/Space Lua/Lua Integrated Query/Aggregating.md b/website/Space Lua/Lua Integrated Query/Aggregating.md new file mode 100644 index 00000000..aa74cca7 --- /dev/null +++ b/website/Space Lua/Lua Integrated Query/Aggregating.md @@ -0,0 +1,181 @@ +#maturity/experimental + +The `group by` and `having` clauses of [[Space Lua/Lua Integrated Query]] support aggregate functions for grouped analysis, following SQL-style semantics. + +After `group by`, each result row contains: + +- `key`: the group key (a single value or, for multi-key grouping, a table) +- `group`: a Lua table containing all items in that group + +All aggregate functions (such as `count`, `sum`, `min`, `max`, `avg`, and custom aggregates) can be applied in `select` and `having` clauses. Aggregate expressions are available in both forms: with or without a variable binding in the `from` clause. The variable `_` always refers to the current item. + +Field names used in `group by` are exposed as locals in `having`, `select`, and `order by`. Use `#group` to obtain the item count per group. + +> **note** Note +> The `having` clause acts only on grouped output. For filtering individual items, use `where` prior to grouping. + +# Examples + +All queries operate on `index.tag 'page'`. + +## Counting with and without binding + +Grouping pages by their first tag, and computing the count and aggregate statistics: + +**Without binding variable:** + +```lua +query[[from + index.tag 'page' +group by + tags[1] +select { + tag = key, + total = count(name), + min_size = min(size), + max_size = max(size), + avg_size = avg(size) +}]] +``` +${query [[ + from + index.tag 'page' + group by + tags[1] + select { + tag = key, + total = count(name), + min_size = min(size), + max_size = max(size), + avg_size = avg(size) + } +]]} + +**With binding variable:** + +```lua +query[[ + from + p = index.tag 'page' + group by + p.tags[1] + select { + tag = key, + total = count(p.name), + min_size = min(p.size), + max_size = max(p.size), + avg_size = avg(p.size) + } +]] +``` +${query [[ + from + p = index.tag 'page' + group by + p.tags[1] + select { + tag = key, + total = count(p.name), + min_size = min(p.size), + max_size = max(p.size), + avg_size = avg(p.size) + } +]]} + +## Multi-key grouping and aggregate + +```lua +query[[ + from + index.tag 'page' + group by + tags[1], + tags[2] + select { + first = key[1], + second = key[2], + count = count(name) + } +]] +``` +${query [[ + from + index.tag 'page' + group by + tags[1], + tags[2] + select { + first = key[1], + second = key[2], + count = count(name) + } +]]} + +## Group filtering with `having` and aggregates + +Only groups with more than two items: + +```lua +query[[ + from + index.tag 'page' + group by + tags[1] + having + count(name) > 2 + select { + tag = key, + total = count(name) + } +]] +``` +${query [[ + from + index.tag 'page' + group by + tags[1] + having + count(name) > 2 + select { + tag = key, + total = count(name) + } +]]} + +## Field access after grouping + +Non-aggregated field references, such as `name` in `select`, refer to the first item in the group, matching common SQL and MySQL semantics. + +```lua +query[[ + from + index.tag 'page' + group by + tags[1] + select { + tag = key, + first_page = name, + n = count(name) + } +]] +``` +${query [[ + from + index.tag 'page' + group by + tags[1] + select { + tag = key, + first_page = name, + n = count(name) + } +]]} + +## Custom aggregators + +Custom aggregator functions may be defined by the user using [[Library/Std/APIs/Aggregate|dedicated API]]. + +# See also + +- [[Space Lua/Lua Integrated Query/Grouping]] — grouping queries without aggregation +- [[Space Lua/Lua Integrated Query]] — full LIQ language reference and listing available aggregators diff --git a/website/Space Lua/Lua Integrated Query/Grouping.md b/website/Space Lua/Lua Integrated Query/Grouping.md new file mode 100644 index 00000000..9e8ae8a3 --- /dev/null +++ b/website/Space Lua/Lua Integrated Query/Grouping.md @@ -0,0 +1,565 @@ +#maturity/experimental + +The `group by` and `having` clauses extend [[Space Lua/Lua Integrated Query]] with SQL-style grouping and aggregate filtering. + +After `group by`, each result row has two fields: + +- **`key`** - the group key (single value or table for multi-key) +- **`group`** - a table (array) of all items in that group + +The field names used in `group by` are also available as bare variables in `having`, `select`, and `order by`. Use `#group` to count items per group. + +> **note** Note +> `having` can only reference group key fields, `key`, `group`, and aggregates like `#group`. To filter individual rows, use `where`. + +# Examples + +All examples below use `index.tag 'tag'`. + +## Group by single key + +Group all tags by `name`: + +```lua +query [[ + from + index.tag 'tag' + group by + name +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name +]]} + +## Group by multiple keys + +Group tags by `name` and `parent` together: + +```lua +query [[ + from + index.tag 'tag' + group by + name, + parent +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name, + parent +]]} + +## Filter groups by count + +Only show tags that appear more than 2 times: + +```lua +query [[ + from + index.tag 'tag' + group by + name + having + #group > 2 +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name + having + #group > 2 +]]} + +## Find unique tags + +Tags appearing exactly once: + +```lua +query [[ + from + index.tag 'tag' + group by + name + having + #group == 1 +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name + having + #group == 1 +]]} + +## Filter groups by key value + +Only show the group where `name` is "meta": + +```lua +query [[ + from + index.tag 'tag' + group by + name + having + name == 'meta' +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name + having + name == 'meta' +]]} + +## Multi-key having + +Groups by `name` and `parent`, keep only page-level tags with more than 1 entry: + +```lua +query [[ + from + index.tag 'tag' + group by + name, + parent + having + parent == 'page' and + #group > 1 +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name, + parent + having + parent == 'page' and + #group > 1 +]]} + +## `where` before `group by` + +Filter to page parents first, then group by `name`: + +```lua +query [[ + from + index.tag 'tag' + where + parent == 'page' + group by + name +]] +``` + +${query [[ + from + index.tag 'tag' + where + parent == 'page' + group by + name +]]} + +## `where`, `group by` and `having` combined + +Filter to page parents, group by `name`, keep groups with 2+ items: + +```lua +query [[ + from + index.tag 'tag' + where + parent == 'page' + group by + name + having + #group >= 2 +]] +``` + +${query [[ + from + index.tag 'tag' + where + parent == 'page' + group by + name + having + #group >= 2 +]]} + +## `select` name and count + +Project each group into a table with `name` and `count`: + +```lua +query [[ + from + index.tag 'tag' + group by + name + select { + name = name, + count = #group + } +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name + select { + name = name, + count = #group + } +]]} + +## `select` with multi-key + +Project both key parts and count: + +```lua +query [[ + from + index.tag 'tag' + group by + name, + parent + select { + name = name, + parent = parent, + count = #group + } +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name, + parent + select { + name = name, + parent = parent, + count = #group + } +]]} + +## Full pipeline: `where`, `group by`, `having` and `select` + +Filter, group, filter groups, then project: + +```lua +query [[ + from + index.tag 'tag' + where + parent == 'page' or + parent == 'task' + group by + name + having + #group > 1 + select { + tag = name, + total = #group + } +]] +``` + +${query [[ + from + index.tag 'tag' + where + parent == 'page' or + parent == 'task' + group by + name + having + #group > 1 + select { + tag = name, + total = #group + } +]]} + +## Order groups by count + +Sort groups by size, largest first: + +```lua +query [[ + from + index.tag 'tag' + group by + name + order by + #group desc +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name + order by + #group desc +]]} + +## Top tags with `having`, `order by`, and `select` + +Tags with 2+ occurrences, sorted by count, projected: + +```lua +query [[ + from + index.tag 'tag' + group by + name + having + #group >= 2 + order by + #group desc + select { + tag = name, + count = #group + } +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name + having + #group >= 2 + order by + #group desc + select { + tag = name, + count = #group + } +]]} + +## Top N groups with `limit` + +Top 3 most used tags: + +```lua +query [[ + from + index.tag 'tag' + group by + name + order by + #group desc + limit + 3 +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name + order by + #group desc + limit + 3 +]]} + +## Full pipeline with `limit` + +Top 5 tags with 2+ uses, showing name and count: + +```lua +query [[ + from + index.tag 'tag' + group by + name + having + #group >= 2 + order by + #group desc + select { + tag = name, + count = #group + } + limit + 5 +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name + having + #group >= 2 + order by + #group desc + select { + tag = name, + count = #group + } + limit + 5 +]]} + +## With explicit object variable + +The same works with `p =` binding: + +```lua +query [[ + from + p = index.tag 'tag' + group by + p.name + having + #group > 1 + select { + tag = name, + count = #group + } +]] +``` + +${query [[ + from + p = index.tag 'tag' + group by + p.name + having + #group > 1 + select { + tag = name, + count = #group + } +]]} + +## Multi-key with explicit object variable + +Full pipeline with `p =` binding and two group keys: + +```lua +query [[ + from + p = index.tag 'tag' + where + p.parent == 'page' + group by + p.name, + p.parent + having + #group >= 2 + order by + #group desc + select { + tag = name, + parent = parent, + count = #group + } +]] +``` + +${query [[ + from + p = index.tag 'tag' + where + p.parent == 'page' + group by + p.name, + p.parent + having + #group >= 2 + order by + #group desc + select { + tag = name, + parent = parent, + count = #group + } +]]} + +## Access `key` directly + +For single-key grouping, `key` holds the value directly: + +```lua +query [[ + from + index.tag 'tag' + group by + name + having + key == 'meta' +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name + having + key == 'meta' +]]} + +## Access `key` table for multi-key + +For multi-key grouping, `key` is a table indexed from 1: + +```lua +query [[ + from + index.tag 'tag' + group by + name, + parent + having + key[1] == 'meta' and + key[2] == 'page' +]] +``` + +${query [[ + from + index.tag 'tag' + group by + name, + parent + having + key[1] == 'meta' and + key[2] == 'page' +]]}