From 9170627d27e7644d337c9774b4dea8e11e5584e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matou=C5=A1=20Jan=20Fialka?= Date: Thu, 19 Mar 2026 12:34:58 +0100 Subject: [PATCH] Add tests for aggregates and few useful aggregates (#1897) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [LIQ] Add more tests for builtin aggregates Signed-off-by: Matouš Jan Fialka * [LIQ] Add `first`, `last`, `mode` and `median` aggregates and `group_concat` alias Signed-off-by: Matouš Jan Fialka --------- Signed-off-by: Matouš Jan Fialka --- client/space_lua/aggregates.test.ts | 1107 +++++++++++++++++++++++ client/space_lua/aggregates.ts | 63 ++ client/space_lua/query_test.lua | 396 ++++++++ libraries/Library/Std/APIs/Aggregate.md | 1 + 4 files changed, 1567 insertions(+) diff --git a/client/space_lua/aggregates.test.ts b/client/space_lua/aggregates.test.ts index dc1ac8c4..5b15288a 100644 --- a/client/space_lua/aggregates.test.ts +++ b/client/space_lua/aggregates.test.ts @@ -56,6 +56,29 @@ const minSpec = requireSpec("min"); const maxSpec = requireSpec("max"); const avgSpec = requireSpec("avg"); const arrayAggSpec = requireSpec("array_agg"); +const productSpec = requireSpec("product"); +const stringAggSpec = requireSpec("string_agg"); +const yamlAggSpec = requireSpec("yaml_agg"); +const jsonAggSpec = requireSpec("json_agg"); +const bitAndSpec = requireSpec("bit_and"); +const bitOrSpec = requireSpec("bit_or"); +const bitXorSpec = requireSpec("bit_xor"); +const boolAndSpec = requireSpec("bool_and"); +const boolOrSpec = requireSpec("bool_or"); +const stddevPopSpec = requireSpec("stddev_pop"); +const stddevSampSpec = requireSpec("stddev_samp"); +const varPopSpec = requireSpec("var_pop"); +const varSampSpec = requireSpec("var_samp"); +const covarPopSpec = requireSpec("covar_pop"); +const covarSampSpec = requireSpec("covar_samp"); +const corrSpec = requireSpec("corr"); +const quantileSpec = requireSpec("quantile"); +const percentileContSpec = requireSpec("percentile_cont"); +const percentileDiscSpec = requireSpec("percentile_disc"); +const modeSpec = requireSpec("mode"); +const firstSpec = requireSpec("first"); +const lastSpec = requireSpec("last"); +const medianSpec = requireSpec("median"); const sf = LuaStackFrame.lostFrame; const emptyConfig = new Config(); @@ -92,6 +115,21 @@ test("aggregate: sum with nils", async () => { expect(result).toBe(20); }); +test("aggregate: sum empty group", async () => { + const result = await executeAggregate( + sumSpec, + new LuaTable(), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeNull(); +}); + test("aggregate: count with expression", async () => { const result = await executeAggregate( countSpec, @@ -232,6 +270,799 @@ test("aggregate: array_agg", async () => { expect((result as LuaTable).rawGet(3)).toBe("c"); }); +test("aggregate: product", async () => { + const result = await executeAggregate( + productSpec, + jsToLuaValue([{ v: 2 }, { v: 3 }, { v: 5 }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe(30); +}); + +test("aggregate: product with nils", async () => { + const result = await executeAggregate( + productSpec, + jsToLuaValue([{ v: 4 }, { x: 1 }, { v: 5 }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe(20); +}); + +test("aggregate: product empty group", async () => { + const result = await executeAggregate( + productSpec, + new LuaTable(), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeNull(); +}); + +test("aggregate: string_agg", async () => { + const result = await executeAggregate( + stringAggSpec, + jsToLuaValue([{ v: "a" }, { v: "b" }, { v: "c" }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe("a,b,c"); +}); + +test("aggregate: string_agg with custom separator", async () => { + const result = await executeAggregate( + stringAggSpec, + jsToLuaValue([{ v: "x" }, { v: "y" }]), + parseExpressionString("_.v"), + [parseExpressionString("' | '")], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe("x | y"); +}); + +test("aggregate: string_agg skips nils", async () => { + const result = await executeAggregate( + stringAggSpec, + jsToLuaValue([{ v: "a" }, { x: 1 }, { v: "b" }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe("a,b"); +}); + +test("aggregate: yaml_agg", async () => { + const result = await executeAggregate( + yamlAggSpec, + jsToLuaValue([{ v: 1 }, { v: 2 }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(typeof result).toBe("string"); + expect((result as string).trim()).toBe("- 1\n- 2"); +}); + +test("aggregate: json_agg", async () => { + const result = await executeAggregate( + jsonAggSpec, + jsToLuaValue([{ v: 1 }, { v: "hello" }, { v: true }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe('[1,"hello",true]'); +}); + +test("aggregate: bit_and", async () => { + const result = await executeAggregate( + bitAndSpec, + jsToLuaValue([{ v: 0b1111 }, { v: 0b1010 }, { v: 0b1100 }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe(0b1000); +}); + +test("aggregate: bit_and empty group", async () => { + const result = await executeAggregate( + bitAndSpec, + new LuaTable(), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeNull(); +}); + +test("aggregate: bit_or", async () => { + const result = await executeAggregate( + bitOrSpec, + jsToLuaValue([{ v: 0b0001 }, { v: 0b0010 }, { v: 0b0100 }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe(0b0111); +}); + +test("aggregate: bit_or empty group", async () => { + const result = await executeAggregate( + bitOrSpec, + new LuaTable(), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeNull(); +}); + +test("aggregate: bit_xor", async () => { + const result = await executeAggregate( + bitXorSpec, + jsToLuaValue([{ v: 0b1010 }, { v: 0b1100 }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe(0b0110); +}); + +test("aggregate: bit_xor empty group", async () => { + const result = await executeAggregate( + bitXorSpec, + new LuaTable(), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeNull(); +}); + +test("aggregate: bool_and all true", async () => { + const result = await executeAggregate( + boolAndSpec, + jsToLuaValue([{ v: true }, { v: true }, { v: true }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe(true); +}); + +test("aggregate: bool_and with false", async () => { + const result = await executeAggregate( + boolAndSpec, + jsToLuaValue([{ v: true }, { v: false }, { v: true }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe(false); +}); + +test("aggregate: bool_and empty group", async () => { + const result = await executeAggregate( + boolAndSpec, + new LuaTable(), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeNull(); +}); + +test("aggregate: bool_or all false", async () => { + const result = await executeAggregate( + boolOrSpec, + jsToLuaValue([{ v: false }, { v: false }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe(false); +}); + +test("aggregate: bool_or with true", async () => { + const result = await executeAggregate( + boolOrSpec, + jsToLuaValue([{ v: false }, { v: true }, { v: false }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe(true); +}); + +test("aggregate: bool_or empty group", async () => { + const result = await executeAggregate( + boolOrSpec, + new LuaTable(), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeNull(); +}); + +test("aggregate: stddev_pop", async () => { + // values: 2, 4, 4, 4, 5, 5, 7, 9 yields: mean=5, var_pop=4, stddev_pop=2 + const result = await executeAggregate( + stddevPopSpec, + jsToLuaValue([ + { v: 2 }, + { v: 4 }, + { v: 4 }, + { v: 4 }, + { v: 5 }, + { v: 5 }, + { v: 7 }, + { v: 9 }, + ]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeCloseTo(2, 10); +}); + +test("aggregate: stddev_pop empty group", async () => { + const result = await executeAggregate( + stddevPopSpec, + new LuaTable(), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeNull(); +}); + +test("aggregate: stddev_samp", async () => { + // values: 2, 4, 4, 4, 5, 5, 7, 9 yields: var_samp = 32/7, stddev_samp = sqrt(32/7) + const result = await executeAggregate( + stddevSampSpec, + jsToLuaValue([ + { v: 2 }, + { v: 4 }, + { v: 4 }, + { v: 4 }, + { v: 5 }, + { v: 5 }, + { v: 7 }, + { v: 9 }, + ]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeCloseTo(Math.sqrt(32 / 7), 10); +}); + +test("aggregate: stddev_samp single element", async () => { + const result = await executeAggregate( + stddevSampSpec, + jsToLuaValue([{ v: 42 }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeNull(); +}); + +test("aggregate: var_pop", async () => { + // values: 2, 4, 4, 4, 5, 5, 7, 9 yields: var_pop = 4 + const result = await executeAggregate( + varPopSpec, + jsToLuaValue([ + { v: 2 }, + { v: 4 }, + { v: 4 }, + { v: 4 }, + { v: 5 }, + { v: 5 }, + { v: 7 }, + { v: 9 }, + ]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeCloseTo(4, 10); +}); + +test("aggregate: var_pop empty group", async () => { + const result = await executeAggregate( + varPopSpec, + new LuaTable(), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeNull(); +}); + +test("aggregate: var_samp", async () => { + // values: 2, 4, 4, 4, 5, 5, 7, 9 yields: var_samp = 32/7 + const result = await executeAggregate( + varSampSpec, + jsToLuaValue([ + { v: 2 }, + { v: 4 }, + { v: 4 }, + { v: 4 }, + { v: 5 }, + { v: 5 }, + { v: 7 }, + { v: 9 }, + ]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeCloseTo(32 / 7, 10); +}); + +test("aggregate: var_samp single element", async () => { + const result = await executeAggregate( + varSampSpec, + jsToLuaValue([{ v: 42 }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeNull(); +}); + +test("aggregate: covar_pop", async () => { + // x: 1,2,3,4,5 y: 2,4,5,4,5 yields: covar_pop = 6/5 = 1.2 + const result = await executeAggregate( + covarPopSpec, + jsToLuaValue([ + { x: 1, y: 2 }, + { x: 2, y: 4 }, + { x: 3, y: 5 }, + { x: 4, y: 4 }, + { x: 5, y: 5 }, + ]), + parseExpressionString("_.y"), + [parseExpressionString("_.x")], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeCloseTo(1.2, 10); +}); + +test("aggregate: covar_pop empty group", async () => { + const result = await executeAggregate( + covarPopSpec, + new LuaTable(), + parseExpressionString("_.y"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeNull(); +}); + +test("aggregate: covar_pop skips null pairs", async () => { + const result = await executeAggregate( + covarPopSpec, + jsToLuaValue([ + { x: 1, y: 2 }, + { x: 2, y: 4 }, + { x: 3, y: 5 }, + { x: 4, y: 4 }, + { x: 5, y: 5 }, + { y: 99 }, // x is nil + { x: 6 }, // y is nil + ]), + parseExpressionString("_.y"), + [parseExpressionString("_.x")], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + // Same 5 valid pairs yields: covar_pop = 6/5 = 1.2 + expect(result).toBeCloseTo(1.2, 10); +}); + +test("aggregate: covar_samp", async () => { + // x: 1,2,3,4,5 y: 2,4,5,4,5 yields: covar_samp = 6/4 = 1.5 + const result = await executeAggregate( + covarSampSpec, + jsToLuaValue([ + { x: 1, y: 2 }, + { x: 2, y: 4 }, + { x: 3, y: 5 }, + { x: 4, y: 4 }, + { x: 5, y: 5 }, + ]), + parseExpressionString("_.y"), + [parseExpressionString("_.x")], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeCloseTo(1.5, 10); +}); + +test("aggregate: covar_samp single element", async () => { + const result = await executeAggregate( + covarSampSpec, + jsToLuaValue([{ x: 1, y: 2 }]), + parseExpressionString("_.y"), + [parseExpressionString("_.x")], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeNull(); +}); + +test("aggregate: corr", async () => { + // x: 1,2,3,4,5 y: 2,4,6,8,10 (perfect positive linear) + const result = await executeAggregate( + corrSpec, + jsToLuaValue([ + { x: 1, y: 2 }, + { x: 2, y: 4 }, + { x: 3, y: 6 }, + { x: 4, y: 8 }, + { x: 5, y: 10 }, + ]), + parseExpressionString("_.y"), + [parseExpressionString("_.x")], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeCloseTo(1.0, 10); +}); + +test("aggregate: corr single element", async () => { + const result = await executeAggregate( + corrSpec, + jsToLuaValue([{ x: 1, y: 2 }]), + parseExpressionString("_.y"), + [parseExpressionString("_.x")], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeNull(); +}); + +test("aggregate: corr constant x returns null", async () => { + // All x identical yields: zero variance in x yields: denom = 0 yields: null + const result = await executeAggregate( + corrSpec, + jsToLuaValue([ + { x: 3, y: 1 }, + { x: 3, y: 2 }, + { x: 3, y: 3 }, + ]), + parseExpressionString("_.y"), + [parseExpressionString("_.x")], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeNull(); +}); + +test("aggregate: quantile (linear, default)", async () => { + // Pre-sorted input; median of [1,2,3,4,5] at q=0.5 yields: 3 + const result = await executeAggregate( + quantileSpec, + jsToLuaValue([{ v: 1 }, { v: 2 }, { v: 3 }, { v: 4 }, { v: 5 }]), + parseExpressionString("_.v"), + [parseExpressionString("0.5")], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + undefined, + [makeOrderBy("_.v", "asc")], + ); + expect(result).toBe(3); +}); + +test("aggregate: quantile empty group", async () => { + const result = await executeAggregate( + quantileSpec, + new LuaTable(), + parseExpressionString("_.v"), + [parseExpressionString("0.5")], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeNull(); +}); + +test("aggregate: quantile with lower method", async () => { + // [10, 20, 30, 40] q=0.3 yields: idx=0.9, lower yields: values[0] = 10 + const result = await executeAggregate( + quantileSpec, + jsToLuaValue([{ v: 10 }, { v: 20 }, { v: 30 }, { v: 40 }]), + parseExpressionString("_.v"), + [parseExpressionString("0.3"), parseExpressionString("'lower'")], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + undefined, + [makeOrderBy("_.v", "asc")], + ); + expect(result).toBe(10); +}); + +test("aggregate: quantile with higher method", async () => { + // [10, 20, 30, 40] q=0.3 yields: idx=0.9, higher yields: values[1] = 20 + const result = await executeAggregate( + quantileSpec, + jsToLuaValue([{ v: 10 }, { v: 20 }, { v: 30 }, { v: 40 }]), + parseExpressionString("_.v"), + [parseExpressionString("0.3"), parseExpressionString("'higher'")], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + undefined, + [makeOrderBy("_.v", "asc")], + ); + expect(result).toBe(20); +}); + +test("aggregate: quantile with midpoint method", async () => { + // [10, 20, 30, 40] q=0.3 yields: idx=0.9, midpoint yields: (10+20)/2 = 15 + const result = await executeAggregate( + quantileSpec, + jsToLuaValue([{ v: 10 }, { v: 20 }, { v: 30 }, { v: 40 }]), + parseExpressionString("_.v"), + [parseExpressionString("0.3"), parseExpressionString("'midpoint'")], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + undefined, + [makeOrderBy("_.v", "asc")], + ); + expect(result).toBe(15); +}); + +test("aggregate: quantile with nearest method", async () => { + // [10, 20, 30, 40] q=0.3 yields: idx=0.9 (0.9-0=0.9 > 0.5 yields: higher) yields: 20 + const result = await executeAggregate( + quantileSpec, + jsToLuaValue([{ v: 10 }, { v: 20 }, { v: 30 }, { v: 40 }]), + parseExpressionString("_.v"), + [parseExpressionString("0.3"), parseExpressionString("'nearest'")], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + undefined, + [makeOrderBy("_.v", "asc")], + ); + expect(result).toBe(20); +}); + +test("aggregate: percentile_cont", async () => { + // [1,2,3,4,5] q=0.25 yields: idx=1.0 yields: values[1]=2 (linear, exact) + const result = await executeAggregate( + percentileContSpec, + jsToLuaValue([{ v: 1 }, { v: 2 }, { v: 3 }, { v: 4 }, { v: 5 }]), + parseExpressionString("_.v"), + [parseExpressionString("0.25")], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + undefined, + [makeOrderBy("_.v", "asc")], + ); + expect(result).toBe(2); +}); + +test("aggregate: percentile_cont interpolation", async () => { + // [10, 20, 30, 40] q=0.5 yields: idx=1.5 yields: 20 + 0.5*(30-20) = 25 + const result = await executeAggregate( + percentileContSpec, + jsToLuaValue([{ v: 10 }, { v: 20 }, { v: 30 }, { v: 40 }]), + parseExpressionString("_.v"), + [parseExpressionString("0.5")], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + undefined, + [makeOrderBy("_.v", "asc")], + ); + expect(result).toBe(25); +}); + +test("aggregate: percentile_disc", async () => { + // [1,2,3,4,5] q=0.4 yields: idx=1.6, lower yields: values[1]=2 + const result = await executeAggregate( + percentileDiscSpec, + jsToLuaValue([{ v: 1 }, { v: 2 }, { v: 3 }, { v: 4 }, { v: 5 }]), + parseExpressionString("_.v"), + [parseExpressionString("0.4")], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + undefined, + [makeOrderBy("_.v", "asc")], + ); + expect(result).toBe(2); +}); + +test("aggregate: percentile_disc at boundary", async () => { + // [10, 20, 30] q=0 yields: idx=0, lower yields: values[0]=10 + const result = await executeAggregate( + percentileDiscSpec, + jsToLuaValue([{ v: 10 }, { v: 20 }, { v: 30 }]), + parseExpressionString("_.v"), + [parseExpressionString("0")], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + undefined, + [makeOrderBy("_.v", "asc")], + ); + expect(result).toBe(10); +}); + test("aggregate: sum with objectVariable", async () => { const result = await executeAggregate( sumSpec, @@ -1264,3 +2095,279 @@ test("aggregate: extra args forwarded to finish", async () => { ); expect(result2).toBe("[x, y]"); }); + +test("aggregate: mode returns most frequent value", async () => { + const result = await executeAggregate( + modeSpec, + jsToLuaValue([{ v: "a" }, { v: "b" }, { v: "a" }, { v: "c" }, { v: "a" }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe("a"); +}); + +test("aggregate: mode with tie returns first to reach max", async () => { + // a appears at index 1,3 ; b at index 2,4 — a reaches count=2 first + const result = await executeAggregate( + modeSpec, + jsToLuaValue([{ v: "a" }, { v: "b" }, { v: "a" }, { v: "b" }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe("a"); +}); + +test("aggregate: mode on empty group", async () => { + const result = await executeAggregate( + modeSpec, + new LuaTable(), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeNull(); +}); + +test("aggregate: mode skips nulls", async () => { + const result = await executeAggregate( + modeSpec, + jsToLuaValue([{ v: "x" }, { z: 1 }, { z: 2 }, { v: "x" }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe("x"); +}); + +test("aggregate: mode with numbers", async () => { + const result = await executeAggregate( + modeSpec, + jsToLuaValue([{ v: 3 }, { v: 1 }, { v: 3 }, { v: 2 }, { v: 1 }, { v: 3 }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe(3); +}); + +// first + +test("aggregate: first returns first non-null value", async () => { + const result = await executeAggregate( + firstSpec, + jsToLuaValue([{ v: "a" }, { v: "b" }, { v: "c" }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe("a"); +}); + +test("aggregate: first skips leading nulls", async () => { + const result = await executeAggregate( + firstSpec, + jsToLuaValue([{ z: 1 }, { z: 2 }, { v: "found" }, { v: "skip" }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe("found"); +}); + +test("aggregate: first on empty group", async () => { + const result = await executeAggregate( + firstSpec, + new LuaTable(), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeNull(); +}); + +test("aggregate: first with order by", async () => { + const result = await executeAggregate( + firstSpec, + jsToLuaValue([ + { v: "c", k: 3 }, + { v: "a", k: 1 }, + { v: "b", k: 2 }, + ]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + undefined, + [makeOrderBy("_.k", "asc")], + ); + expect(result).toBe("a"); +}); + +test("aggregate: last returns last non-null value", async () => { + const result = await executeAggregate( + lastSpec, + jsToLuaValue([{ v: "a" }, { v: "b" }, { v: "c" }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe("c"); +}); + +test("aggregate: last skips trailing nulls", async () => { + const result = await executeAggregate( + lastSpec, + jsToLuaValue([{ v: "a" }, { v: "last" }, { z: 1 }, { z: 2 }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe("last"); +}); + +test("aggregate: last on empty group", async () => { + const result = await executeAggregate( + lastSpec, + new LuaTable(), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeNull(); +}); + +test("aggregate: last with order by desc", async () => { + const result = await executeAggregate( + lastSpec, + jsToLuaValue([ + { v: "c", k: 3 }, + { v: "a", k: 1 }, + { v: "b", k: 2 }, + ]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + undefined, + [makeOrderBy("_.k", "desc")], + ); + // desc order: c(3), b(2), a(1) -> last = "a" + expect(result).toBe("a"); +}); + +test("aggregate: median of odd count", async () => { + const result = await executeAggregate( + medianSpec, + jsToLuaValue([{ v: 30 }, { v: 10 }, { v: 20 }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + undefined, + [makeOrderBy("_.v", "asc")], + ); + expect(result).toBe(20); +}); + +test("aggregate: median of even count (interpolated)", async () => { + const result = await executeAggregate( + medianSpec, + jsToLuaValue([{ v: 10 }, { v: 20 }, { v: 30 }, { v: 40 }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + undefined, + [makeOrderBy("_.v", "asc")], + ); + // [10,20,30,40] q=0.5 -> idx=1.5 -> 20 + 0.5*(30-20) = 25 + expect(result).toBe(25); +}); + +test("aggregate: median on empty group", async () => { + const result = await executeAggregate( + medianSpec, + new LuaTable(), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBeNull(); +}); + +test("aggregate: median single element", async () => { + const result = await executeAggregate( + medianSpec, + jsToLuaValue([{ v: 42 }]), + parseExpressionString("_.v"), + [], + undefined, + new LuaEnv(), + sf, + evalExpression, + emptyConfig, + ); + expect(result).toBe(42); +}); diff --git a/client/space_lua/aggregates.ts b/client/space_lua/aggregates.ts index 4e3fd5a7..87661850 100644 --- a/client/space_lua/aggregates.ts +++ b/client/space_lua/aggregates.ts @@ -256,6 +256,30 @@ const builtinAggregates: Record = { return state.sum / state.count; }), }, + first: { + name: "first", + description: "First non-null input value (iteration order)", + initialize: aggFn((_sf) => ({ value: null, found: false })), + iterate: aggFn((_sf, state: any, value: any) => { + if (state.found) return state; + if (value === null || value === undefined || isSqlNull(value)) + return state; + state.value = value; + state.found = true; + return state; + }), + finish: aggFn((_sf, state: any) => state.value), + }, + last: { + name: "last", + description: "Last non-null input value (iteration order)", + initialize: aggFn((_sf) => null), + iterate: aggFn((_sf, state: any, value: any) => { + if (value === null || value === undefined || isSqlNull(value)) + return state; + return value; + }), + }, // Collection and format array_agg: { name: "array_agg", @@ -486,6 +510,29 @@ const builtinAggregates: Record = { return state.c / denom; }), }, + mode: { + name: "mode", + description: "Most frequent non-null input value", + initialize: aggFn((_sf) => ({ + freq: new Map(), + best: null as LuaValue, + bestCount: 0, + })), + iterate: aggFn((_sf, state: any, value: any) => { + if (value === null || value === undefined || isSqlNull(value)) + return state; + const c = (state.freq.get(value) ?? 0) + 1; + state.freq.set(value, c); + if (c > state.bestCount) { + state.bestCount = c; + state.best = value; + } + return state; + }), + finish: aggFn((_sf, state: any) => { + return state.bestCount > 0 ? state.best : null; + }), + }, // Quantile and percentile quantile: makeQuantileSpec( "quantile", @@ -499,6 +546,22 @@ const builtinAggregates: Record = { "percentile_disc", "Discrete percentile (nearest lower value) on ordered set of non-null inputs; arguments: value, fraction (0-1)", ), + median: { + name: "median", + description: "Median of non-null inputs (continuous percentile at 0.5)", + initialize: aggFn((_sf) => ({ + values: [] as number[], + q: 0.5, + method: "linear" as QuantileMethod, + })), + iterate: aggFn((_sf, state: any, value: any) => { + if (value === null || value === undefined || isSqlNull(value)) + return state; + state.values.push(value as number); + return state; + }), + finish: aggFn((_sf, state: any) => quantileFinish(state as QuantileState)), + }, }; const noCtx = {}; diff --git a/client/space_lua/query_test.lua b/client/space_lua/query_test.lua index eaa893f4..c684da65 100644 --- a/client/space_lua/query_test.lua +++ b/client/space_lua/query_test.lua @@ -620,6 +620,402 @@ do assertTrue(type(r[1].arr) == "table" or r[1].arr == nil) end +-- 8f. product +do + local r = query [[ + from + p = pages + where + p.tags[1] ~= nil + group by + p.tags[1] + select { + prod = product(p.size), + } + ]] + for _, row in ipairs(r) do + assertTrue(type(row.prod) == "number" or row.prod == nil) + end +end + +-- 8g. string_agg +do + local r = query [[ + from + p = pages + where + p.tags[1] ~= nil + group by + p.tags[1] + select { + names = string_agg(p.name), + } + ]] + for _, row in ipairs(r) do + assertTrue(type(row.names) == "string") + end +end + +-- 8h. string_agg with custom separator +do + local r = query [[ + from + p = pages + where + p.tags[1] ~= nil + group by + p.tags[1] + select { + names = string_agg(p.name, " | "), + } + ]] + for _, row in ipairs(r) do + assertTrue(type(row.names) == "string") + end +end + +-- 8i. yaml_agg +do + local r = query [[ + from + p = pages + where + p.tags[1] ~= nil + group by + p.tags[1] + select { + y = yaml_agg(p.name), + } + ]] + for _, row in ipairs(r) do + assertTrue(type(row.y) == "string") + end +end + +-- 8j. json_agg +do + local r = query [[ + from + p = pages + where + p.tags[1] ~= nil + group by + p.tags[1] + select { + j = json_agg(p.name), + } + ]] + for _, row in ipairs(r) do + assertTrue(type(row.j) == "string") + end +end + +-- 8k. bool_and, bool_or +do + local r = query [[ + from + p = pages + where + p.tags[1] ~= nil + group by + p.tags[1] + select { + all_big = bool_and(p.size > 5), + any_big = bool_or(p.size > 5), + } + ]] + for _, row in ipairs(r) do + assertTrue( + type(row.all_big) == "boolean" or row.all_big == nil, + "bool_and must return boolean or nil" + ) + assertTrue( + type(row.any_big) == "boolean" or row.any_big == nil, + "bool_or must return boolean or nil" + ) + end +end + +-- 8l. stddev_pop, var_pop +do + local r = query [[ + from + p = pages + where + p.tags[1] ~= nil + group by + p.tags[1] + select { + sd = stddev_pop(p.size), + vr = var_pop(p.size), + } + ]] + for _, row in ipairs(r) do + assertTrue(type(row.sd) == "number" or row.sd == nil) + assertTrue(type(row.vr) == "number" or row.vr == nil) + end +end + +-- 8m. stddev_samp, var_samp (nil for single-element groups) +do + local r = query [[ + from + p = pages + where + p.tags[1] ~= nil + group by + p.tags[1] + select { + k = key, + sd = stddev_samp(p.size), + vr = var_samp(p.size), + n = count(p.name), + } + ]] + for _, row in ipairs(r) do + if row.n >= 2 then + assertTrue(type(row.sd) == "number", "stddev_samp >= 2 items") + assertTrue(type(row.vr) == "number", "var_samp >= 2 items") + else + -- single-element group -> nil + assertEquals(row.sd, nil) + assertEquals(row.vr, nil) + end + end +end + +-- 8n. percentile_cont (needs order by ... asc inside aggregate) +do + local r = query [[ + from + p = pages + where + p.tags[1] ~= nil + group by + p.tags[1] + select { + k = key, + med = percentile_cont(p.size, 0.5 order by p.size asc), + } + ]] + for _, row in ipairs(r) do + assertTrue(type(row.med) == "number" or row.med == nil) + end +end + +-- 8o. percentile_disc (needs order by ... asc inside aggregate) +do + local r = query [[ + from + p = pages + where + p.tags[1] ~= nil + group by + p.tags[1] + select { + k = key, + pd = percentile_disc(p.size, 0.5 order by p.size asc), + } + ]] + for _, row in ipairs(r) do + assertTrue(type(row.pd) == "number" or row.pd == nil) + end +end + +-- 8p. quantile with explicit method (needs order by ... asc) +do + local r = query [[ + from + p = pages + where + p.tags[1] ~= nil + group by + p.tags[1] + select { + k = key, + q = quantile(p.size, 0.25, "lower" order by p.size asc), + } + ]] + for _, row in ipairs(r) do + assertTrue(type(row.q) == "number" or row.q == nil) + end +end + +-- 8q. percentile_cont on known data for exact value check +do + local scores = { + { g = "a", v = 10 }, + { g = "a", v = 20 }, + { g = "a", v = 30 }, + { g = "a", v = 40 }, + } + local r = query [[ + from + s = scores + group by + s.g + select { + med = percentile_cont(s.v, 0.5 order by s.v asc), + } + ]] + -- [10,20,30,40] q=0.5 -> idx=1.5 -> 20 + 0.5*(30-20) = 25 + assertEquals(r[1].med, 25) +end + +-- 8r. percentile_disc on known data for exact value check +do + local scores = { + { g = "a", v = 10 }, + { g = "a", v = 20 }, + { g = "a", v = 30 }, + { g = "a", v = 40 }, + { g = "a", v = 50 }, + } + local r = query [[ + from + s = scores + group by + s.g + select { + p25 = percentile_disc(s.v, 0.25 order by s.v asc), + } + ]] + -- [10,20,30,40,50] q=0.25 -> idx=1.0 -> lower -> values[1] = 20 + assertEquals(r[1].p25, 20) +end + +-- 8s. mode: most frequent tag in dataset +do + local data = { + { g = "a", v = "x" }, + { g = "a", v = "y" }, + { g = "a", v = "x" }, + { g = "a", v = "x" }, + { g = "a", v = "y" }, + { g = "b", v = "q" }, + } + local r = query [[ + from + d = data + group by + d.g + select { + k = key, + m = mode(d.v), + } + order by + k + ]] + -- group "a": x=3, y=2 -> mode = "x" + assertEquals(r[1].m, "x") + -- group "b": q=1 -> mode = "q" + assertEquals(r[2].m, "q") +end + +-- 8t. first / last with intra-aggregate order by +do + local data = { + { g = "a", v = "c", k = 3 }, + { g = "a", v = "a", k = 1 }, + { g = "a", v = "b", k = 2 }, + } + local r = query [[ + from + d = data + group by + d.g + select { + f = first(d.v order by d.k asc), + l = last(d.v order by d.k asc), + } + ]] + assertEquals(r[1].f, "a") + assertEquals(r[1].l, "c") +end + +-- 8u. first / last without order by (iteration order) +do + local data = { + { g = "x", v = 10 }, + { g = "x", v = 20 }, + { g = "x", v = 30 }, + } + local r = query [[ + from + d = data + group by + d.g + select { + f = first(d.v), + l = last(d.v), + } + ]] + assertEquals(r[1].f, 10) + assertEquals(r[1].l, 30) +end + +-- 8v. median on known data (odd) +do + local data = { + { g = "a", v = 30 }, + { g = "a", v = 10 }, + { g = "a", v = 20 }, + } + local r = query [[ + from + d = data + group by + d.g + select { + med = median(d.v order by d.v asc), + } + ]] + assertEquals(r[1].med, 20) +end + +-- 8w. median on known data (even, interpolated) +do + local data = { + { g = "a", v = 10 }, + { g = "a", v = 20 }, + { g = "a", v = 30 }, + { g = "a", v = 40 }, + } + local r = query [[ + from + d = data + group by + d.g + select { + med = median(d.v order by d.v asc), + } + ]] + -- [10,20,30,40] -> 25 + assertEquals(r[1].med, 25) +end + +-- 8x. first / last with filter +do + local data = { + { g = "a", v = 1, big = false }, + { g = "a", v = 10, big = true }, + { g = "a", v = 2, big = false }, + { g = "a", v = 20, big = true }, + } + local r = query [[ + from + d = data + group by + d.g + select { + fb = first(d.v order by d.v asc) filter(where d.big), + lb = last(d.v order by d.v asc) filter(where d.big), + } + ]] + assertEquals(r[1].fb, 10) + assertEquals(r[1].lb, 20) +end + -- 9. Having -- 9a. Having with aggregate, unbound diff --git a/libraries/Library/Std/APIs/Aggregate.md b/libraries/Library/Std/APIs/Aggregate.md index 7d057b3a..10ea9ff3 100644 --- a/libraries/Library/Std/APIs/Aggregate.md +++ b/libraries/Library/Std/APIs/Aggregate.md @@ -252,4 +252,5 @@ aggregate.alias('every', 'bool_and') aggregate.alias('std', 'stddev_pop') aggregate.alias('stddev', 'stddev_pop') aggregate.alias('variance', 'var_pop') +aggregate.alias('group_concat', 'string_agg') ```