From 341c07049f7e8b1ac3bdbc4b0f706a180a4d1a08 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Fri, 2 Feb 2024 19:19:07 +0100 Subject: [PATCH] Custom template parser (#672) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New template engine 🤯 --- Dockerfile | 2 +- common/PAGE_TEMPLATES.ts | 4 +- common/languages.ts | 9 +- common/markdown_parser/customtags.ts | 3 + common/markdown_parser/parse-expression.js | 26 +- .../markdown_parser/parse-expression.terms.js | 55 ++-- common/markdown_parser/parse-query.js | 24 +- common/markdown_parser/parse-query.terms.js | 68 +++-- common/markdown_parser/parser.test.ts | 33 ++- common/markdown_parser/parser.ts | 155 ++++++++-- common/markdown_parser/query.grammar | 64 +++- common/query_functions.ts | 42 ++- common/syscalls/handlebar_helpers.ts | 65 ----- common/syscalls/handlebars.ts | 31 -- common/syscalls/template.ts | 34 +++ common/template/parse-template.js | 16 + common/template/parse-template.terms.js | 13 + common/template/render.test.ts | 142 +++++++++ common/template/render.ts | 192 ++++++++++++ common/template/template.grammar | 36 +++ common/template/template_parser.test.ts | 56 ++++ common/template/template_parser.ts | 175 +++++++++++ deno.jsonc | 4 +- import_map.json | 1 - plug-api/app_event.ts | 3 +- plug-api/lib/builtin_query_functions.ts | 85 +++--- plug-api/lib/parse-query.ts | 44 +++ plug-api/lib/parser-query.test.ts | 152 +++++++++- plug-api/lib/query.ts | 191 ++++-------- plug-api/lib/query_expression.test.ts | 45 +++ plug-api/lib/query_expression.ts | 274 ++++++++++++++++++ plug-api/lib/tree.ts | 12 +- plug-api/plugos-syscall/datastore.ts | 10 +- plug-api/silverbullet-syscall/mod.ts | 2 +- .../{handlebars.ts => template.ts} | 8 +- plug-api/types.ts | 16 +- plugos/lib/datastore.test.ts | 18 ++ plugos/lib/datastore.ts | 38 ++- plugos/syscalls/datastore.ts | 20 +- plugs/editor/complete.ts | 41 ++- plugs/editor/editor.plug.yaml | 5 + plugs/editor/navigate.ts | 6 +- plugs/index/api.ts | 6 +- plugs/index/attachment.ts | 7 - plugs/index/index.plug.yaml | 6 - plugs/index/lint.ts | 2 +- plugs/index/widget.ts | 6 +- plugs/query/api.ts | 70 +++++ plugs/query/complete.ts | 34 ++- plugs/query/lint.ts | 1 - plugs/query/query.plug.yaml | 8 + plugs/query/widget.ts | 47 +-- plugs/search/search.ts | 11 +- plugs/template/api.ts | 59 ++-- plugs/template/lint.ts | 81 +++++- plugs/template/page.ts | 4 +- plugs/template/plug_api.ts | 5 +- plugs/template/snippet.ts | 2 +- plugs/template/template.plug.yaml | 14 +- plugs/template/types.ts | 1 + plugs/template/util.ts | 25 +- plugs/template/var.ts | 69 +++-- .../template/{template_block.ts => widget.ts} | 59 +++- scripts/generate.sh | 4 +- server/server_system.ts | 6 +- web/client.ts | 31 +- web/client_system.ts | 4 +- web/components/fuse_search.ts | 3 + web/editor_state.ts | 1 - web/hooks/slash_command.ts | 3 +- web/navigator.ts | 15 +- web/style.ts | 6 +- web/styles/colors.scss | 18 ++ web/styles/editor.scss | 6 +- web/styles/theme.scss | 10 +- web/syscalls/datastore.proxy.ts | 1 + website/Attributes.md | 2 +- website/Blocks.md | 2 +- website/CHANGELOG.md | 33 ++- website/Expression Language.md | 111 +++++++ website/Functions.md | 62 ++++ website/Install/Configuration.md | 3 +- website/Install/Deno.md | 2 +- website/Library/Core/Page/Maintenance.md | 47 +-- website/Library/Core/Page/Template Index.md | 19 +- website/Library/Core/Query/Template.md | 5 +- .../Library/Core/Snippet/each-directive.md | 10 + website/Library/Core/Snippet/if-directive.md | 10 + .../Library/Core/Snippet/if-else-directive.md | 12 + website/Library/Core/Snippet/let-directive.md | 10 + website/Library/Core/Snippet/task.md | 2 +- website/Library/Core/Snippet/template.md | 4 +- .../Library/Core/Widget/Linked Mentions.md | 21 +- website/Library/Core/Widget/Linked Tasks.md | 2 +- website/Library/Core/Widget/Tagged Tasks.md | 4 +- website/Live Queries.md | 117 +------- website/Live Templates.md | 68 +++-- website/Objects.md | 16 +- website/Plugs/Tasks.md | 4 +- website/Query Language.md | 78 +++++ website/SETTINGS.md | 3 + website/Snippets.md | 1 + website/Template Language.md | 75 +++++ website/Templates.md | 34 +-- website/internal/test page.md | 2 +- 105 files changed, 2762 insertions(+), 842 deletions(-) delete mode 100644 common/syscalls/handlebar_helpers.ts delete mode 100644 common/syscalls/handlebars.ts create mode 100644 common/syscalls/template.ts create mode 100644 common/template/parse-template.js create mode 100644 common/template/parse-template.terms.js create mode 100644 common/template/render.test.ts create mode 100644 common/template/render.ts create mode 100644 common/template/template.grammar create mode 100644 common/template/template_parser.test.ts create mode 100644 common/template/template_parser.ts create mode 100644 plug-api/lib/query_expression.test.ts create mode 100644 plug-api/lib/query_expression.ts rename plug-api/silverbullet-syscall/{handlebars.ts => template.ts} (57%) delete mode 100644 plugs/index/attachment.ts create mode 100644 plugs/query/api.ts rename plugs/template/{template_block.ts => widget.ts} (61%) create mode 100644 website/Expression Language.md create mode 100644 website/Functions.md create mode 100644 website/Library/Core/Snippet/each-directive.md create mode 100644 website/Library/Core/Snippet/if-directive.md create mode 100644 website/Library/Core/Snippet/if-else-directive.md create mode 100644 website/Library/Core/Snippet/let-directive.md create mode 100644 website/Query Language.md create mode 100644 website/Template Language.md diff --git a/Dockerfile b/Dockerfile index e9ffa991..2b2610b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM lukechannings/deno:v1.40.2 +FROM lukechannings/deno:v1.40.3 # The volume that will keep the space data # Either create a volume: diff --git a/common/PAGE_TEMPLATES.ts b/common/PAGE_TEMPLATES.ts index 01a89def..cbfc2f20 100644 --- a/common/PAGE_TEMPLATES.ts +++ b/common/PAGE_TEMPLATES.ts @@ -12,7 +12,7 @@ export const INDEX_TEMPLATE = For your convenience we're including an on-boarding live template below. Enjoy! -\`\`\`template -page: "[[!silverbullet.md/Getting Started]]" +\`\`\`include +raw: "[[!silverbullet.md/Getting Started]]" \`\`\` `; diff --git a/common/languages.ts b/common/languages.ts index 7888eee0..7dba113a 100644 --- a/common/languages.ts +++ b/common/languages.ts @@ -26,15 +26,15 @@ import { yamlLanguage, } from "./deps.ts"; import { - expressionParser, + extendedMarkdownLanguage, + highlightingExpressionParser, highlightingQueryParser, } from "./markdown_parser/parser.ts"; export const builtinLanguages: Record = { "meta": StreamLanguage.define(yamlLanguage), "yaml": StreamLanguage.define(yamlLanguage), - "template": StreamLanguage.define(yamlLanguage), - "block": StreamLanguage.define(yamlLanguage), + "include": StreamLanguage.define(yamlLanguage), "embed": StreamLanguage.define(yamlLanguage), "data": StreamLanguage.define(yamlLanguage), "toc": StreamLanguage.define(yamlLanguage), @@ -86,9 +86,10 @@ export const builtinLanguages: Record = { name: "query", parser: highlightingQueryParser, }), + "template": extendedMarkdownLanguage, "expression": LRLanguage.define({ name: "expression", - parser: expressionParser, + parser: highlightingExpressionParser, }), }; diff --git a/common/markdown_parser/customtags.ts b/common/markdown_parser/customtags.ts index a733ca7e..bdde3070 100644 --- a/common/markdown_parser/customtags.ts +++ b/common/markdown_parser/customtags.ts @@ -26,3 +26,6 @@ export const TaskDeadlineTag = Tag.define(); export const HashtagTag = Tag.define(); export const NakedURLTag = Tag.define(); + +export const DirectiveMarkTag = Tag.define(); +export const DirectiveTag = Tag.define(); diff --git a/common/markdown_parser/parse-expression.js b/common/markdown_parser/parse-expression.js index 15d26cf9..496e16a9 100644 --- a/common/markdown_parser/parse-expression.js +++ b/common/markdown_parser/parse-expression.js @@ -1,18 +1,18 @@ // This file was generated by lezer-generator. You probably shouldn't edit it. import {LRParser} from "@lezer/lr" -const spec_Identifier = {__proto__:null,null:20, and:32, or:34} +const spec_Identifier = {__proto__:null,null:22, and:36, or:38, where:48, limit:52, select:66, as:70, render:74, each:76, all:78} export const parser = LRParser.deserialize({ version: 14, - states: "%jOVQPOOOOQO'#Cb'#CbOVQPO'#CgOOQO'#C_'#C_OqQQO'#C^O#UQQO'#ChOOQO'#Ch'#ChOVQPO'#CjQ$lQQOOOOQO'#C^'#C^O%pQQO'#CvO%}QPO,59RO&SQPO,59TO&XQPO,59[O&`QQO,59UOVQPO,59YOVQPO,59YOVQPO,59YOVQPO,59VOVQPO,59VOVQPO'#CqO&gQPO,59bOOQO1G.m1G.mOOQO1G.o1G.oO&rQPO1G.vOOQO1G.v1G.vOOQO1G.p1G.pO(RQQO1G.tOOQO1G.t1G.tO(]QQO1G.tO)tQQO1G.qO){QQO1G.qO*SQQO,59]OOQO-E6o-E6oOOQO7+$b7+$b", - stateData: "*g~OhOS~OSROTROVPOWROXTOYROiQOnVO~Om[O`QXaQXcQXfQXpQXqQXrQXsQXtQXuQXvQXwQXxQXyQXzQX{QX|QXkQXlQXoQX~On]O`[Xa[Xc[Xf[Xm[Xp[Xq[Xr[Xs[Xt[Xu[Xv[Xw[Xx[Xy[Xz[X{[X|[Xk[Xl[Xo[X~O`bOacOc_Op_Oq_Or_Os_Ot_Ou_Ov_Ow_Ox`Oy`Oz`O{aO|aO~OkdOljXojX~P$lOlfO~OXgO~OoiO~PVOojO~P$lOkdOljaoja~OorO~Ox`Oy`Oz`O`biabicbifbipbiqbirbisbitbiubivbiwbikbilbiobi~O{aO|aO~P&wO{bi|bi~P&wOc_Op_Oq_Or_Os_Ot_Ou_Ov_Ow_Ox`Oy`Oz`O{aO|aOa_if_ik_il_io_i~O`_i~P(gO`bO~P(gOkealeaoea~P$lOVcXSX~", - goto: "#hkPPl!YPP!fPPPP!f!r#O!Y!YPP!YP!Y#[PPPP#bQWOSYQ]Q^VQk_Ql`QmaQnbQocRpdeXOQV]_`abcdeROQV]_`abcdeSOQV]_`abcdeUOQV]_`abcdQeYRqeQZQRh]", - nodeNames: "⚠ Program Expression Value Number String Bool BooleanKW Regex Identifier null List LVal Attribute ParenthesizedExpression LogicalExpression and or BinExpression InKW Call", - maxTerm: 44, - skippedNodes: [0], - repeatNodeCount: 1, - tokenData: "/j~RuX^#fpq#fqr$Zrs$nuv%fxy%kyz%pz{%u{|%z|}&P}!O&U!O!P&Z!P!Q&`!Q![)Q!^!_)Y!_!`)g!`!a)t!c!}*R!}#O*g#P#Q*l#T#Y*R#Y#Z*q#Z#]*R#]#^-b#^#h*R#h#i.d#i#o*R#y#z#f$f$g#f#BY#BZ#f$IS$I_#f$Ip$Iq$n$Iq$Ir$n$I|$JO#f$JT$JU#f$KV$KW#f&FU&FV#f~#kYh~X^#fpq#f#y#z#f$f$g#f#BY#BZ#f$IS$I_#f$I|$JO#f$JT$JU#f$KV$KW#f&FU&FV#f~$^P!_!`$a~$fPs~#r#s$i~$nOw~~$qWOr$nrs%Zs$Ip$n$Ip$Iq%Z$Iq$Ir%Z$Ir;'S$n;'S;=`%`<%lO$n~%`OT~~%cP;=`<%l$n~%kOz~~%pOn~~%uOo~~%zOx~~&PO{~~&UOk~~&ZO|~~&`Om~R&gXyQWPOY'SZ]'S^!P'S!P!Q't!Q#O'S#O#P'y#P;'S'S;'S;=`(z<%lO'SP'XXWPOY'SZ]'S^!P'S!P!Q't!Q#O'S#O#P'y#P;'S'S;'S;=`(z<%lO'SP'yOWPP'|RO;'S'S;'S;=`(V;=`O'SP([YWPOY'SZ]'S^!P'S!P!Q't!Q#O'S#O#P'y#P;'S'S;'S;=`(z;=`<%l'S<%lO'SP(}P;=`<%l'S~)VPS~!Q![)Q~)_Pp~!_!`)b~)gOq~~)lPr~#r#s)o~)tOv~~)yPu~!_!`)|~*ROt~~*WTX~}!O*R!Q![*R!c!}*R#R#S*R#T#o*R~*lOi~~*qOl~~*vUX~}!O*R!Q![*R!c!}*R#R#S*R#T#U+Y#U#o*R~+_VX~}!O*R!Q![*R!c!}*R#R#S*R#T#`*R#`#a+t#a#o*R~+yVX~}!O*R!Q![*R!c!}*R#R#S*R#T#g*R#g#h,`#h#o*R~,eVX~}!O*R!Q![*R!c!}*R#R#S*R#T#X*R#X#Y,z#Y#o*R~-RTV~X~}!O*R!Q![*R!c!}*R#R#S*R#T#o*R~-gVX~}!O*R!Q![*R!c!}*R#R#S*R#T#b*R#b#c-|#c#o*R~.TTc~X~}!O*R!Q![*R!c!}*R#R#S*R#T#o*R~.iVX~}!O*R!Q![*R!c!}*R#R#S*R#T#f*R#f#g/O#g#o*R~/TVX~}!O*R!Q![*R!c!}*R#R#S*R#T#i*R#i#j,`#j#o*R", - tokenizers: [0, 1], - topRules: {"Program":[0,1]}, - specialized: [{term: 9, get: (value) => spec_Identifier[value] || -1}], - tokenPrec: 431 + states: ",fOYQPOOOOQO'#Cc'#CcO!WQPO'#ChOOQO'#C`'#C`Q!_QQOOOOQO'#Ck'#CkOYQPO'#ClO#iQSO'#D]OYQPO'#DVO%UQQO'#C_OOQO'#C_'#C_O&OQQO'#DhO&]QPO,59SOOQO,59S,59SO&bQPO,59UOYQPO,59sOYQPO,59sOYQPO,59sOYQPO,59uOYQPO,59XOYQPO,59XO&gQQO,59WO'PQPO'#CqO'WQPO,59[O']QPO'#D^O'bQPO'#ESO'jQPO,59wOOQO,59w,59wO'oQQO,59qO)rQPO,59vOYQPO'#D_O)yQPO,5:SOOQO1G.n1G.nOOQO1G.p1G.pO,RQQO1G/_O.VQQO1G/_O.dQQO1G/_O.nQQO1G/aO0uQQO1G.sO0|QQO1G.sOOQO1G.r1G.rOYQPO'#CwOOQO'#D`'#D`O1TQPO,59]OYQPO'#CsOYQPO'#CuO1[QPO'#C|O2YQPO'#DQOOQO1G.v1G.vOYQPO,59xO2eQPO'#DcO2jQPO,5:nOOQO1G/c1G/cO2rQPO1G/bOOQO1G/b1G/bO2wQQO,59yOOQO-E7]-E7]OYQPO7+${O3UQQO'#CyO3rQPO'#DoOOQO,59c,59cOOQO-E7^-E7^O4ZQQO,59_O4qQQO,59aO5XQQO'#DOO5`QQO'#DOO5yQPO'#DpOOQO,59h,59hOOQO,59l,59lO6bQPO,59lO6gQQO1G/dOOQO,59},59}OOQO-E7a-E7aOOQO7+$|7+$|O6qQQO< spec_Identifier[value] || -1}], + tokenPrec: 1145 }) diff --git a/common/markdown_parser/parse-expression.terms.js b/common/markdown_parser/parse-expression.terms.js index d321a969..fc90b240 100644 --- a/common/markdown_parser/parse-expression.terms.js +++ b/common/markdown_parser/parse-expression.terms.js @@ -1,19 +1,40 @@ // This file was generated by lezer-generator. You probably shouldn't edit it. export const - Program = 1, - Expression = 2, - Value = 3, - Number = 4, - String = 5, - Bool = 6, - BooleanKW = 7, - Regex = 8, - Identifier = 9, - List = 11, - LVal = 12, - Attribute = 13, - ParenthesizedExpression = 14, - LogicalExpression = 15, - BinExpression = 18, - InKW = 19, - Call = 20 + LineComment = 1, + Program = 2, + Expression = 3, + Value = 4, + Number = 5, + String = 6, + Bool = 7, + BooleanKW = 8, + Regex = 9, + Identifier = 10, + List = 12, + GlobalIdentifier = 13, + Attribute = 14, + TopLevelVal = 15, + ParenthesizedExpression = 16, + LogicalExpression = 17, + QueryExpression = 20, + Query = 21, + TagIdentifier = 22, + WhereClause = 23, + LimitClause = 25, + OrderClause = 27, + Order = 28, + OrderBy = 29, + OrderDirection = 30, + OrderKW = 31, + SelectClause = 32, + Select = 34, + RenderClause = 36, + PageRef = 40, + UnaryExpression = 41, + NotKW = 42, + BinExpression = 43, + InKW = 44, + TernaryExpression = 45, + Call = 46, + Object = 47, + KeyVal = 48 diff --git a/common/markdown_parser/parse-query.js b/common/markdown_parser/parse-query.js index c580490a..65387172 100644 --- a/common/markdown_parser/parse-query.js +++ b/common/markdown_parser/parse-query.js @@ -1,18 +1,18 @@ // This file was generated by lezer-generator. You probably shouldn't edit it. import {LRParser} from "@lezer/lr" -const spec_Identifier = {__proto__:null,where:12, null:28, and:40, or:42, limit:52, select:66, as:70, render:74, each:76, all:78} +const spec_Identifier = {__proto__:null,where:14, null:30, and:44, or:46, limit:70, select:84, as:88, render:92, each:94, all:96} export const parser = LRParser.deserialize({ version: 14, - states: ")xOVQPOOOmQQO'#C^QOQPOOOtQQO'#CwOOQO'#DV'#DVO!`QQO,58xOtQQO'#C`OtQQO'#CuO!gQQO'#C|O#RQSO'#DQOOQO'#Cg'#CgOtQQO'#CkOOQO'#Cd'#CdO#^QSO'#CcO&eQSO'#ClOOQO'#Cl'#ClOtQQO'#CnO(]QSO'#CyOOQO'#Cc'#CcO(yQQO'#DrOOQO,59c,59cOOQO-E7T-E7TO)bQSO,58zO)xQSO,59aO*`QSO'#ClO*yQSO'#DOO+QQQO'#DsOOQO,59h,59hOOQO,59l,59lO+iQPO,59lO+nQSO'#D_O+{QPO,59VO,QQQO,59XO,VQQO,59`O,^QSO,59YOtQQO,59^OtQQO,59^OtQQO,59^OOQO'#Cz'#CzOOQO,59e,59eOtQQO,59ZOtQQO,59ZOtQQO'#DXO,eQQO,5:^O,|QQO,59jO!gQQO'#DYO-RQQO,5:_OOQO1G/W1G/WOtQQO'#DWO-jQPO,59yOOQO1G.q1G.qOOQO1G.s1G.sO-uQPO1G.zOOQO1G.z1G.zOOQO1G.t1G.tO/kQSO1G.xOOQO1G.x1G.xO/uQSO1G.xO1sQSO1G.uO1zQSO1G.uOOQO,59s,59sOOQO-E7V-E7VOOQO1G/U1G/UOOQO,59t,59tOOQO-E7W-E7WO2RQSO,59rOOQO-E7U-E7UOOQO7+$f7+$f", - stateData: "2h~O!POS~ORPO~OUUOjVOlROqWOuXO~O}QX~P[OT^OX[OY[O[YO][O^[O!QZO!V`O~O}Qa~P[OThOX[OY[O[YO][O^[O!QZO!V`O~OvmOwmOxlO~O!UpOUVXdVXeVXgVXjVXlVXoVXqVXuVX}VX!SVX!XVX!YVX!ZVX![VX!]VX!^VX!_VX!`VX!aVX!bVX!cVX!dVX!eVXsVX!TVX!WVX~O!VqOd`Xe`Xg`X!U`X!X`X!Y`X!Z`X![`X!]`X!^`X!_`X!``X!a`X!b`X!c`X!d`X!e`Xs`X~OU`Xj`Xl`Xo`Xq`Xu`X}`X!S`X!T`X!W`X~P%WOdxOeyOgsO!XsO!YsO!ZsO![sO!]sO!^sO!_sO!`sO!atO!btO!ctO!duO!euO~OovOUmXjmXlmXqmXumX}mX!SmX~P'XO!SzOU!fXj!fXl!fXq!fXu!fX}!fX~OUSajSalSaqSauSa}Sa~P'XOUiajialiaqiauia}ia~P'XOUrXjrXlrXqrXurX}rX!SrX~P%WOs|O~P'XO!S}OU!gXj!gXl!gXq!gXu!gX}!gX~Ox!PO~O!S!QO!T!RX!W!RX~P'XO!T!SO~OT!TO~O!W!VO~PtO!W!WO~P'XO!SzOU!faj!fal!faq!fau!fa}!fa~OT!`O~O!S}OU!gaj!gal!gaq!gau!ga}!ga~O!S!QO!T!Ra!W!Ra~O!W!eO~O!atO!btO!ctOUfidfiefigfijfilfiofiqfiufi}fi!Sfi!Xfi!Yfi!Zfi![fi!]fi!^fi!_fi!`fisfi!Tfi!Wfi~O!duO!euO~P-zO!dfi!efi~P-zOgsO!XsO!YsO!ZsO![sO!]sO!^sO!_sO!`sO!atO!btO!ctO!duO!euOUciecijcilciociqciuci}ci!Scisci!Tci!Wci~Odci~P0POdxO~P0PO!Sza!Tza!Wza~P'XOl[goTXo~", - goto: "&U!hPP!iP!lPP!p#iPP#zPPP#z$]$n#i#iPP#iP#i!lP!lP%P%VP!lP%YP!lPPPP%`%f%l%rPPPP%xPPPPPPPPPPPPPPPPPP&O&RRQOTSPTSaRzQfUQgVSiW}SnZqQr`Q!XsQ!YtQ!ZuQ![xQ!]yR!c!QobRUVWZ`qstuxyz}!Qo[RUVWZ`qstuxyz}!Qo]RUVWZ`qstuxyz}!Qo_RUVWZ`qstuxyz}!QQcRR!^zRwaQjWR!a}QTPReTQ!RnR!d!RQ{cR!_{Q!OjR!b!OQoZR!UqRdRRkW", - nodeNames: "⚠ Program Query TagIdentifier WhereClause Identifier where Expression Value Number String Bool BooleanKW Regex null List LVal Attribute ParenthesizedExpression LogicalExpression and or BinExpression InKW Call LimitClause limit OrderClause Order OrderBy OrderDirection OrderKW SelectClause select Select as RenderClause render each all PageRef", - maxTerm: 69, - skippedNodes: [0], - repeatNodeCount: 4, - tokenData: "8k~RzX^#upq#uqr$jrs$}uv%uxy%zyz&Pz{&U{|&Z|}&`}!O&e!O!P&j!P!Q&o!Q![)a!^!_)i!_!`)v!`!a*T!c!}*b!}#O+d#P#Q,a#T#U,f#U#W*b#W#X.c#X#Y*b#Y#Z/S#Z#]*b#]#^2^#^#c*b#c#d3j#d#h*b#h#i7Z#i#o*b#y#z#u$f$g#u#BY#BZ#u$IS$I_#u$Ip$Iq$}$Iq$Ir$}$I|$JO#u$JT$JU#u$KV$KW#u&FU&FV#u~#zY!P~X^#upq#u#y#z#u$f$g#u#BY#BZ#u$IS$I_#u$I|$JO#u$JT$JU#u$KV$KW#u&FU&FV#u~$mP!_!`$p~$uP![~#r#s$x~$}O!`~~%QWOr$}rs%js$Ip$}$Ip$Iq%j$Iq$Ir%j$Ir;'S$};'S;=`%o<%lO$}~%oOY~~%rP;=`<%l$}~%zO!c~~&PO!V~~&UO!W~~&ZO!a~~&`O!d~~&eO!S~~&jO!e~~&oO!U~U&vX!bS]QOY'cZ]'c^!P'c!P!Q(T!Q#O'c#O#P(Y#P;'S'c;'S;=`)Z<%lO'cQ'hX]QOY'cZ]'c^!P'c!P!Q(T!Q#O'c#O#P(Y#P;'S'c;'S;=`)Z<%lO'cQ(YO]QQ(]RO;'S'c;'S;=`(f;=`O'cQ(kY]QOY'cZ]'c^!P'c!P!Q(T!Q#O'c#O#P(Y#P;'S'c;'S;=`)Z;=`<%l'c<%lO'cQ)^P;=`<%l'c~)fPX~!Q![)a~)nP!X~!_!`)q~)vO!Y~~){P!Z~#r#s*O~*TO!_~~*YP!^~!_!`*]~*bO!]~V*iURPTU}!O*b!P!Q*{!Q![*b!c!}*b#R#S*b#T#o*bP+QURP}!O*{!P!Q*{!Q![*{!c!}*{#R#S*{#T#o*{V+iP!QQ!}#O+lT+oTO#P+l#P#Q,O#Q;'S+l;'S;=`,Z<%lO+lT,RP#P#Q,UT,ZOxTT,^P;=`<%l+l~,fO!T~V,mWRPTU}!O*b!P!Q*{!Q![*b!c!}*b#R#S*b#T#g*b#g#h-V#h#o*bV-^WRPTU}!O*b!P!Q*{!Q![*b!c!}*b#R#S*b#T#V*b#V#W-v#W#o*bV.PURPoSTU}!O*b!P!Q*{!Q![*b!c!}*b#R#S*b#T#o*bV.jWRPTU}!O*b!P!Q*{!Q![*b!c!}*b#R#S*b#T#X*b#X#Y,f#Y#o*bV/ZVRPTU}!O*b!P!Q*{!Q![*b!c!}*b#R#S*b#T#U/p#U#o*bV/wWRPTU}!O*b!P!Q*{!Q![*b!c!}*b#R#S*b#T#`*b#`#a0a#a#o*bV0hWRPTU}!O*b!P!Q*{!Q![*b!c!}*b#R#S*b#T#g*b#g#h1Q#h#o*bV1XWRPTU}!O*b!P!Q*{!Q![*b!c!}*b#R#S*b#T#X*b#X#Y1q#Y#o*bV1zURP[QTU}!O*b!P!Q*{!Q![*b!c!}*b#R#S*b#T#o*bV2eWRPTU}!O*b!P!Q*{!Q![*b!c!}*b#R#S*b#T#b*b#b#c2}#c#o*bV3WURPgSTU}!O*b!P!Q*{!Q![*b!c!}*b#R#S*b#T#o*bV3qWRPTU}!O*b!P!Q*{!Q![*b!c!}*b#R#S*b#T#f*b#f#g4Z#g#o*bV4bWRPTU}!O*b!P!Q*{!Q![*b!c!}*b#R#S*b#T#W*b#W#X4z#X#o*bV5RWRPTU}!O*b!P!Q*{!Q![*b!c!}*b#R#S*b#T#X*b#X#Y5k#Y#o*bV5rWRPTU}!O*b!P!Q*{!Q![*b!c!}*b#R#S*b#T#f*b#f#g6[#g#o*bV6cVRPTUpq6x}!O*b!P!Q*{!Q![*b!c!}*b#R#S*b#T#o*bU6{P#U#V7OU7RP#m#n7UU7ZOlUV7bWRPTU}!O*b!P!Q*{!Q![*b!c!}*b#R#S*b#T#f*b#f#g7z#g#o*bV8RWRPTU}!O*b!P!Q*{!Q![*b!c!}*b#R#S*b#T#i*b#i#j1Q#j#o*b", + states: ",fOYQPOOOpQQO'#C_QOQPOOOzQQO'#DQOOQO'#D_'#D_O!xQQO,58yOzQQO'#CaOzQQO'#DOO#SQQO'#DVO$QQQO'#DZOOQO'#Ch'#ChO$]QQO'#ClOOQO'#Ce'#CeO%nQSO'#DSOOQO'#Co'#CoOzQQO'#CpO&_QPO'#C|OzQQO'#CuO'zQSO'#CdOOQO'#Cd'#CdO(tQQO'#EROOQO,59l,59lOOQO-E7]-E7]O)`QSO,58{O)yQSO,59jO*dQSO'#DXO*kQSO'#CdO+XQQO'#ESOOQO,59q,59qOOQO,59u,59uO+sQPO,59uO+xQSO'#DhO,VQPO,59WOOQO,59W,59WO,[QQO,59YOzQQO,59cOzQQO,59cOzQQO,59cOzQQO,59eOOQO'#DT'#DTOOQO,59n,59nOzQQO,59]OzQQO,59]O,aQSO,59[O,hQPO,59`O,mQPO'#C}O,rQPO'#EQO,zQPO,59hOOQO,59h,59hO-PQSO,59aO/SQQO,59fOzQQO'#DbO/ZQQO,5:mO/uQQO,59sO#SQQO'#DcO/zQQO,5:nOOQO1G/a1G/aOzQQO'#D`O0fQPO,5:SOOQO1G.r1G.rOOQO1G.t1G.tO2nQSO1G.}O4rQSO1G.}O5PQSO1G.}O5ZQSO1G/PO7bQSO1G.wO7iQSO1G.wOOQO1G.v1G.vOOQO1G.z1G.zOzQQO,59iO7pQPO'#DaO7uQPO,5:lOOQO1G/S1G/SO7}QPO1G/QOOQO1G/Q1G/QOOQO,59|,59|OOQO-E7`-E7`OOQO1G/_1G/_OOQO,59},59}OOQO-E7a-E7aO8SQSO,59zOOQO-E7^-E7^OzQQO7+$kO8aQSO1G/TOOQO,59{,59{OOQO-E7_-E7_OOQO7+$l7+$lO8kQSO< spec_Identifier[value] || -1}], - tokenPrec: 798 + topRules: {"Program":[0,2]}, + specialized: [{term: 6, get: (value) => spec_Identifier[value] || -1}], + tokenPrec: 1180 }) diff --git a/common/markdown_parser/parse-query.terms.js b/common/markdown_parser/parse-query.terms.js index 42e872de..21173264 100644 --- a/common/markdown_parser/parse-query.terms.js +++ b/common/markdown_parser/parse-query.terms.js @@ -1,32 +1,40 @@ // This file was generated by lezer-generator. You probably shouldn't edit it. export const - Program = 1, - Query = 2, - TagIdentifier = 3, - WhereClause = 4, - Identifier = 5, - Expression = 7, - Value = 8, - Number = 9, - String = 10, - Bool = 11, - BooleanKW = 12, - Regex = 13, - List = 15, - LVal = 16, - Attribute = 17, - ParenthesizedExpression = 18, - LogicalExpression = 19, - BinExpression = 22, - InKW = 23, - Call = 24, - LimitClause = 25, - OrderClause = 27, - Order = 28, - OrderBy = 29, - OrderDirection = 30, - OrderKW = 31, - SelectClause = 32, - Select = 34, - RenderClause = 36, - PageRef = 40 + LineComment = 1, + Program = 2, + Query = 3, + TagIdentifier = 4, + WhereClause = 5, + Identifier = 6, + Expression = 8, + Value = 9, + Number = 10, + String = 11, + Bool = 12, + BooleanKW = 13, + Regex = 14, + List = 16, + GlobalIdentifier = 17, + Attribute = 18, + TopLevelVal = 19, + ParenthesizedExpression = 20, + LogicalExpression = 21, + QueryExpression = 24, + UnaryExpression = 25, + NotKW = 26, + BinExpression = 27, + InKW = 28, + TernaryExpression = 29, + Call = 30, + PageRef = 31, + Object = 32, + KeyVal = 33, + LimitClause = 34, + OrderClause = 36, + Order = 37, + OrderBy = 38, + OrderDirection = 39, + OrderKW = 40, + SelectClause = 41, + Select = 43, + RenderClause = 45 diff --git a/common/markdown_parser/parser.test.ts b/common/markdown_parser/parser.test.ts index d75c7042..99c37bac 100644 --- a/common/markdown_parser/parser.test.ts +++ b/common/markdown_parser/parser.test.ts @@ -80,6 +80,14 @@ Deno.test("Test inline attribute syntax", () => { assertEquals(valueNode?.children![0].text, "[1, 2, 3]"); }); +Deno.test("Test template directive parsing", () => { + const tree = parse( + extendedMarkdownLanguage, + "Simple {{name}} and {{count({ page })}}", + ); + console.log("Template directive", JSON.stringify(tree, null, 2)); +}); + const multiStatusTaskExample = ` * [ ] Task 1 - [x] Task 2 @@ -106,7 +114,7 @@ const commandLinkSample = ` Deno.test("Test command links", () => { const tree = parse(extendedMarkdownLanguage, commandLinkSample); const commands = collectNodesOfType(tree, "CommandLink"); - console.log("Command links parsed", JSON.stringify(commands, null, 2)); + // console.log("Command links parsed", JSON.stringify(commands, null, 2)); assertEquals(commands.length, 3); assertEquals(commands[0].children![1].children![0].text, "Some: Command"); assertEquals(commands[1].children![1].children![0].text, "Other: Command"); @@ -132,7 +140,24 @@ Deno.test("Test command link arguments", () => { assertEquals(args2!.children![0].text, '"other", "args", 123'); }); -Deno.test("Test template directives", () => { - const tree = parse(extendedMarkdownLanguage, `Hello there {{name}}!`); - console.log("Template directive", JSON.stringify(tree, null, 2)); +Deno.test("Test directive parser", () => { + const simpleExample = `Simple {{.}}`; + let tree = parse(extendedMarkdownLanguage, simpleExample); + assertEquals(renderToText(tree), simpleExample); + + const eachExample = `{{#each .}}Sup{{/each}}`; + tree = parse(extendedMarkdownLanguage, eachExample); + + const ifExample = `{{#if true}}Sup{{/if}}`; + tree = parse(extendedMarkdownLanguage, ifExample); + assertEquals(renderToText(tree), ifExample); + + const ifElseExample = `{{#if true}}Sup{{else}}Sup2{{/if}}`; + tree = parse(extendedMarkdownLanguage, ifElseExample); + assertEquals(renderToText(tree), ifElseExample); + console.log("Final tree", JSON.stringify(tree, null, 2)); + + const letExample = `{{#let @p = true}}{{/let}}`; + tree = parse(extendedMarkdownLanguage, letExample); + assertEquals(renderToText(tree), letExample); }); diff --git a/common/markdown_parser/parser.ts b/common/markdown_parser/parser.ts index f57b6826..3c141d87 100644 --- a/common/markdown_parser/parser.ts +++ b/common/markdown_parser/parser.ts @@ -129,29 +129,137 @@ const CommandLink: MarkdownConfig = { ], }; -export const templateDirectiveRegex = /^\{\{([^\}]+)\}\}/; - const TemplateDirective: MarkdownConfig = { defineNodes: [ - { name: "TemplateDirective", style: t.monospace }, - { name: "TemplateDirectiveMark", style: t.monospace }, + { name: "TemplateDirective" }, + { name: "TemplateExpressionDirective" }, + { name: "TemplateIfStartDirective", style: ct.DirectiveTag }, + { name: "TemplateEachStartDirective", style: ct.DirectiveTag }, + { name: "TemplateLetStartDirective", style: ct.DirectiveTag }, + { name: "TemplateIfEndDirective", style: ct.DirectiveTag }, + { name: "TemplateEachEndDirective", style: ct.DirectiveTag }, + { name: "TemplateLetEndDirective", style: ct.DirectiveTag }, + { name: "TemplateLetVar", style: t.variableName }, + { name: "TemplateDirectiveMark", style: ct.DirectiveMarkTag }, ], parseInline: [ { name: "TemplateDirective", parse(cx, next, pos) { - let match: RegExpMatchArray | null; + const textFromPos = cx.slice(pos, cx.end); if ( next != 123 /* '{' */ || - !(match = templateDirectiveRegex.exec(cx.slice(pos, cx.end))) + cx.slice(pos, pos + 2) !== "{{" ) { return -1; } - const fullMatch = match[0]; - const endPos = pos + fullMatch.length; + + let bracketNestingDepth = 0; + let valueLength = 0; + // We need to ensure balanced { and } pairs + loopLabel: + for (; valueLength < textFromPos.length; valueLength++) { + switch (textFromPos[valueLength]) { + case "{": + bracketNestingDepth++; + break; + case "}": + bracketNestingDepth--; + if (bracketNestingDepth === 0) { + // Done! + break loopLabel; + } + break; + } + } + if (bracketNestingDepth !== 0) { + return -1; + } + + const bodyText = textFromPos.slice(2, valueLength - 1); + // console.log("Body text", bodyText); + + const endPos = pos + valueLength + 1; + let bodyEl: any; + + // Is this an open block directive? + const openBlockMatch = /^(\s*#(if|each)\s*)(.+)$/s.exec(bodyText); + if (openBlockMatch) { + const [_, directiveStart, directiveType, directiveBody] = + openBlockMatch; + const parsedExpression = highlightingExpressionParser.parse( + directiveBody, + ); + bodyEl = cx.elt( + directiveType === "if" + ? "TemplateIfStartDirective" + : "TemplateEachStartDirective", + pos + 2, + endPos - 2, + [cx.elt(parsedExpression, pos + 2 + directiveStart.length)], + ); + } + + if (!bodyEl) { + // Is this an open block directive? + const openLetBlockMatch = /^(\s*#let\s*)(@\w+)(\s*=\s*)(.+)$/s.exec( + bodyText, + ); + if (openLetBlockMatch) { + const [_, directiveStart, varName, eq, expr] = openLetBlockMatch; + const parsedExpression = highlightingExpressionParser.parse( + expr, + ); + bodyEl = cx.elt( + "TemplateLetStartDirective", + pos + 2, + endPos - 2, + [ + cx.elt( + "TemplateLetVar", + pos + 2 + directiveStart.length, + pos + 2 + directiveStart.length + varName.length, + ), + cx.elt( + parsedExpression, + pos + 2 + directiveStart.length + varName.length + eq.length, + ), + ], + ); + } + } + + if (!bodyEl) { + // Is this a directive close? + const closeBlockMatch = /^\s*\/(if|each|let)/.exec(bodyText); + + if (closeBlockMatch) { + const [_, directiveType] = closeBlockMatch; + const upCaseDirectiveType = directiveType[0].toUpperCase() + + directiveType.slice(1); + bodyEl = cx.elt( + `Template${upCaseDirectiveType}EndDirective`, + pos + 2, + endPos - 2, + ); + } + } + + if (!bodyEl) { + // Let's parse as an expression + const parsedExpression = highlightingExpressionParser.parse(bodyText); + bodyEl = cx.elt( + "TemplateExpressionDirective", + pos + 2, + endPos - 2, + [cx.elt(parsedExpression, pos + 2)], + ); + } + return cx.addElement( cx.elt("TemplateDirective", pos, endPos, [ cx.elt("TemplateDirectiveMark", pos, pos + 2), + bodyEl!, cx.elt("TemplateDirectiveMark", endPos - 2, endPos), ]), ); @@ -188,20 +296,31 @@ export const Highlight: MarkdownConfig = { import { parser as queryParser } from "./parse-query.js"; +const expressionStyleTags = styleTags({ + Identifier: t.variableName, + TagIdentifier: t.variableName, + GlobalIdentifier: t.variableName, + String: t.string, + Number: t.number, + PageRef: ct.WikiLinkTag, + BinExpression: t.operator, + TernaryExpression: t.operator, + Regex: t.regexp, + "where limit select render Order OrderKW and or null as InKW NotKW BooleanKW each all": + t.keyword, +}); + export const highlightingQueryParser = queryParser.configure({ props: [ - styleTags({ - "Name": t.variableName, - "String": t.string, - "Number": t.number, - "PageRef": ct.WikiLinkTag, - "where limit select render Order OrderKW and or as InKW each all": - t.keyword, - }), + expressionStyleTags, ], }); -export { parser as expressionParser } from "./parse-expression.js"; +import { parser as expressionParser } from "./parse-expression.js"; + +export const highlightingExpressionParser = expressionParser.configure({ + props: [expressionStyleTags], +}); export const attributeStartRegex = /^\[([\w\$]+)(::?\s*)/; @@ -250,7 +369,6 @@ export const Attribute: MarkdownConfig = { } if (textFromPos[valueLength + 1] === "(") { - console.log("Link", fullMatch, textFromPos); // This turns out to be a link, back out! return -1; } @@ -380,6 +498,7 @@ const NamedAnchor = regexParser({ import { Table } from "./table_parser.ts"; import { foldNodeProp } from "@codemirror/language"; +import { c } from "https://esm.sh/@codemirror/legacy-modes@6.3.3/mode/clike?external=@codemirror/language&target=es2022"; // FrontMatter parser diff --git a/common/markdown_parser/query.grammar b/common/markdown_parser/query.grammar index d4419e68..b4e0868b 100644 --- a/common/markdown_parser/query.grammar +++ b/common/markdown_parser/query.grammar @@ -1,22 +1,24 @@ @top Program { Query } @precedence { + attr @left + unaryop mulop @left addop @left binop @left + terop @left and @left or @left } @skip { - space + space | LineComment } commaSep { content ("," content)* } kw { @specialize[@name={term}] } - Query { TagIdentifier ( WhereClause | LimitClause | OrderClause | SelectClause | RenderClause )* } @@ -37,17 +39,15 @@ OrderDirection { Value { Number | String | Bool | Regex | kw<"null"> | List } Attribute { - LVal "." Identifier + Expression !attr "." Identifier } Call { - Identifier "(" commaSep ")" | Identifier "(" ")" + Identifier "(" commaSep ")" + | Identifier "(" ")" } -LVal { - Identifier -| Attribute -} +TopLevelVal { "." } ParenthesizedExpression { "(" Expression ")" } @@ -58,11 +58,28 @@ Expression !and kw<"and"> Expression Expression { Value -| LVal +| Identifier +| GlobalIdentifier +| Attribute +| TopLevelVal | ParenthesizedExpression | LogicalExpression +| QueryExpression +| UnaryExpression | BinExpression +| TernaryExpression | Call +| PageRef +| Object +} + +QueryExpression { + "{" Query "}" +} + +UnaryExpression { + !unaryop NotKW Expression + | !unaryop "!" Expression } BinExpression { @@ -83,8 +100,17 @@ BinExpression { | Expression !addop "-" Expression } -List { "[" commaSep "]" } +TernaryExpression { + Expression !terop "?" Expression !terop ":" Expression +} +List { "[" commaSep "]" | "[" "]" } + +KeyVal { + String ":" Expression +} + +Object { "{" commaSep "}" | "{" "}" } Bool { BooleanKW @@ -94,18 +120,24 @@ Bool { @tokens { space { std.whitespace+ } + LineComment { "#" ![\n]* } + TagIdentifier { @asciiLetter (@asciiLetter | @digit | "-" | "_" | "/" )* } Identifier { @asciiLetter (@asciiLetter | @digit | "-" | "_")* } + GlobalIdentifier { "@" @asciiLetter (@asciiLetter | @digit | "-" | "_")* } + String { - ("\"" | "“" | "”") ![\"”“]* ("\"" | "“" | "”") + "\"" ![\"]* "\"" + | "'" ![']* "'" + | "`" ![`]* "`" } PageRef { - "[" "[" ![\]]* "]" "]" + "[[" ![\]]* "]]" } Order { "order by" } - Regex { "/" ( ![/\\\n\r] | "\\" _ )* "/"? } + Regex { "/" ( ![/\\\n\r] | "\\" _ )+ "/" } Number { std.digit+ } @@ -113,7 +145,11 @@ Bool { InKW { "in" } + NotKW { "not" } + OrderKW { "asc" | "desc" } - @precedence { Order, BooleanKW, InKW, OrderKW, Identifier, Number } + @precedence { Order, BooleanKW, InKW, OrderKW, NotKW, Identifier, Number } + + } diff --git a/common/query_functions.ts b/common/query_functions.ts index 640c334b..c4e7bf32 100644 --- a/common/query_functions.ts +++ b/common/query_functions.ts @@ -1,15 +1,53 @@ import { FunctionMap } from "$sb/types.ts"; import { builtinFunctions } from "$sb/lib/builtin_query_functions.ts"; +import { System } from "../plugos/system.ts"; +import { Query } from "$sb/types.ts"; +import { LimitedMap } from "$sb/lib/limited_map.ts"; -export function buildQueryFunctions(allKnownPages: Set): FunctionMap { +const pageCacheTtl = 10 * 1000; // 10s + +export function buildQueryFunctions( + allKnownPages: Set, + system: System, +): FunctionMap { + const pageCache = new LimitedMap(10); return { ...builtinFunctions, - pageExists: (name: string) => { + pageExists(name: string) { if (name.startsWith("!") || name.startsWith("{{")) { // Let's assume federated pages exist, and ignore template variable ones return true; } return allKnownPages.has(name); }, + async template(template: string, obj: any) { + return (await system.invokeFunction("template.renderTemplate", [ + template, + obj, + ])).text; + }, + // INTERNAL: Used for implementing the { query } syntax in expressions + $query(query: Query, variables: Record) { + return system.invokeFunction("query.renderQuery", [ + query, + variables, + ]); + }, + // INTERNAL: Used to implement resolving [[links]] in expressions + $readPage(name: string): Promise | string { + const cachedPage = pageCache.get(name); + if (cachedPage) { + return cachedPage; + } else { + return system.syscall({}, "space.readPage", [name]).then((page) => { + pageCache.set(name, page, pageCacheTtl); + return page; + }).catch((e: any) => { + if (e.message === "Not found") { + throw new Error(`Page not found: ${name}`); + } + }); + } + }, }; } diff --git a/common/syscalls/handlebar_helpers.ts b/common/syscalls/handlebar_helpers.ts deleted file mode 100644 index ffc0af0a..00000000 --- a/common/syscalls/handlebar_helpers.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { niceDate, niceTime } from "$sb/lib/dates.ts"; - -export function handlebarHelpers() { - return { - json: (v: any) => JSON.stringify(v), - niceDate: (ts: any) => niceDate(new Date(ts)), - escapeRegexp: (ts: any) => { - return ts.replace(/[-\/\\^$*+?.()|[\]{}]/g, "\\$&"); - }, - escape: (handlebarsExpr: string) => { - return `{{${handlebarsExpr}}}`; - }, - replaceRegexp: (s: string, regexp: string, replacement: string) => { - return s.replace(new RegExp(regexp, "g"), replacement); - }, - prefixLines: (v: string, prefix: string) => - v.split("\n").map((l) => prefix + l).join("\n"), - substring: (s: string, from: number, to: number, elipsis = "") => - s.length > to - from ? s.substring(from, to) + elipsis : s, - - time: () => niceTime(new Date()), - today: () => niceDate(new Date()), - tomorrow: () => { - const tomorrow = new Date(); - tomorrow.setDate(tomorrow.getDate() + 1); - return niceDate(tomorrow); - }, - yesterday: () => { - const yesterday = new Date(); - yesterday.setDate(yesterday.getDate() - 1); - return niceDate(yesterday); - }, - lastWeek: () => { - const lastWeek = new Date(); - lastWeek.setDate(lastWeek.getDate() - 7); - return niceDate(lastWeek); - }, - nextWeek: () => { - const nextWeek = new Date(); - nextWeek.setDate(nextWeek.getDate() + 7); - return niceDate(nextWeek); - }, - weekStart: (startOnMonday = true) => { - const d = new Date(); - const day = d.getDay(); - let diff = d.getDate() - day; - if (startOnMonday) { - diff += day == 0 ? -6 : 1; - } - return niceDate(new Date(d.setDate(diff))); - }, - ifEq: function (v1: any, v2: any, options: any) { - if (v1 === v2) { - return options.fn(this); - } - return options.inverse(this); - }, - ifNeq: function (v1: any, v2: any, options: any) { - if (v1 !== v2) { - return options.fn(this); - } - return options.inverse(this); - }, - }; -} diff --git a/common/syscalls/handlebars.ts b/common/syscalls/handlebars.ts deleted file mode 100644 index 6c168e1d..00000000 --- a/common/syscalls/handlebars.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { SysCallMapping } from "../../plugos/system.ts"; -import { handlebarHelpers } from "./handlebar_helpers.ts"; -import Handlebars from "handlebars"; - -export function handlebarsSyscalls(): SysCallMapping { - return { - "handlebars.renderTemplate": ( - _ctx, - template: string, - obj: any, - globals: Record = {}, - ): string => { - return renderHandlebarsTemplate(template, obj, globals); - }, - }; -} - -export function renderHandlebarsTemplate( - template: string, - obj: any, - globals: Record, -) { - const templateFn = Handlebars.compile( - template, - { noEscape: true }, - ); - return templateFn(obj, { - helpers: handlebarHelpers(), - data: globals, - }); -} diff --git a/common/syscalls/template.ts b/common/syscalls/template.ts new file mode 100644 index 00000000..cfd33a99 --- /dev/null +++ b/common/syscalls/template.ts @@ -0,0 +1,34 @@ +import { FunctionMap } from "$sb/types.ts"; +import { AST } from "$sb/lib/tree.ts"; +import { SysCallMapping } from "../../plugos/system.ts"; +import { renderTemplate } from "../template/render.ts"; +import { parseTemplate } from "../template/template_parser.ts"; + +export function templateSyscalls(functionMap: FunctionMap): SysCallMapping { + return { + "template.renderTemplate": ( + _ctx, + template: string, + obj: any, + globals: Record = {}, + ): Promise => { + return renderTheTemplate(template, obj, globals, functionMap); + }, + "template.parseTemplate": ( + _ctx, + template: string, + ): AST => { + return parseTemplate(template); + }, + }; +} + +export function renderTheTemplate( + template: string, + obj: any, + globals: Record, + functionMap: FunctionMap, +): Promise { + const parsedTemplate = parseTemplate(template); + return renderTemplate(parsedTemplate, obj, globals, functionMap); +} diff --git a/common/template/parse-template.js b/common/template/parse-template.js new file mode 100644 index 00000000..50e21d36 --- /dev/null +++ b/common/template/parse-template.js @@ -0,0 +1,16 @@ +// This file was generated by lezer-generator. You probably shouldn't edit it. +import {LRParser} from "@lezer/lr" +export const parser = LRParser.deserialize({ + version: 14, + states: "#fQQOPOOO]OQO'#C`ObOSO'#CdOOOO'#C^'#C^OOOO'#Ch'#ChQQOPOOOgOQO,58zOlOQO,59OOOOO-E6f-E6fOOOO1G.f1G.fOqOQO1G.jOvOPO7+$UO!UOSO< { + const functionMap = builtinFunctions; + + const variables = { + page: { name: "this page" }, + }; + + // Base case + assertEquals(await parseAndRender(`Hello World`, {}), `Hello World`); + + // Variable + assertEquals( + await parseAndRender(`Hello {{name}}`, { name: "World" }), + `Hello World`, + ); + + // Global variable + assertEquals( + await parseAndRender(`Hello {{@page.name}}`, {}), + `Hello this page`, + ); + + // Function invocation + assertEquals( + await parseAndRender(`Hello {{replace(name, "o", "e")}}`, { + name: "World", + }), + `Hello Werld`, + ); + + // Two variables + assertEquals( + await parseAndRender(`Hello {{firstName}} {{lastName}}`, { + firstName: "Pete", + lastName: "Smith", + }), + `Hello Pete Smith`, + ); + + // Each directive + assertEquals( + await parseAndRender( + `a{{#each .}}* {{name}}\n{{/each}}b`, + [ + { name: "Pete" }, + { name: "John" }, + ], + ), + `a* Pete\n* John\nb`, + ); + + // If directive + assertEquals( + await parseAndRender( + `{{#if .}}Hello{{/if}}`, + true, + ), + `Hello`, + ); + assertEquals( + await parseAndRender( + `{{#if .}}Hello{{/if}}`, + false, + ), + ``, + ); + + assertEquals( + await parseAndRender( + `{{#if .}}Hello{{else}}Bye{{/if}}`, + false, + ), + `Bye`, + ); + + assertEquals( + await parseAndRender( + `{{#if .}}Hello{{else}}Bye{{/if}}`, + true, + ), + `Hello`, + ); + + assertEquals( + await parseAndRender( + `{{#if true}}{{#each people}}* {{name}}\n{{/each}}{{/if}}`, + { + people: [ + { name: "Pete" }, + { name: "John" }, + ], + }, + ), + `* Pete\n* John\n`, + ); + + // Let directive + assertEquals( + await parseAndRender( + `{{#let @name = "Pete"}}Hello {{@name}}{{/let}} and now {{@name}}`, + {}, + ), + `Hello Pete and now undefined`, + ); + + // Test line removal + console.log( + "Got this", + JSON.stringify( + parseTemplate( + "{{#if .}}\nHello\n{{/if}}", + ), + null, + 2, + ), + ); + assertEquals( + await parseAndRender( + "{{#if .}}\nHello\n{{/if}}", + true, + ), + "Hello\n", + ); + + assertEquals( + await parseAndRender( + "{{#each [1, 2, 3]}}\n{{.}}\n{{/each}}", + true, + ), + "1\n2\n3\n", + ); + + function parseAndRender(template: string, value: any): Promise { + const parsedTemplate = parseTemplate(template); + return renderTemplate(parsedTemplate, value, variables, functionMap); + } +}); diff --git a/common/template/render.ts b/common/template/render.ts new file mode 100644 index 00000000..81a9aa2f --- /dev/null +++ b/common/template/render.ts @@ -0,0 +1,192 @@ +import { AST } from "$sb/lib/tree.ts"; +import { evalQueryExpression } from "$sb/lib/query_expression.ts"; +import { expressionToKvQueryExpression } from "$sb/lib/parse-query.ts"; +import { FunctionMap } from "$sb/types.ts"; +import { + jsonObjectToMDTable, + jsonToMDTable, +} from "../../plugs/template/util.ts"; + +export async function renderTemplate( + ast: AST, + value: any, + variables: Record, + functionMap: FunctionMap, +): Promise { + const [_, ...elements] = ast; + const renderedElements = await Promise.all( + elements.map((e) => + renderTemplateElement(e, value, variables, functionMap) + ), + ); + return renderedElements.join(""); +} + +async function renderTemplateElement( + ast: AST, + value: any, + variables: Record, + functionMap: FunctionMap, +): Promise { + const [type, ...children] = ast; + switch (type) { + case "TemplateElement": + return (await Promise.all( + children.map((c) => + renderTemplateElement(c, value, variables, functionMap) + ), + )).join(""); + case "ExpressionDirective": + return await renderExpressionDirective( + ast, + value, + variables, + functionMap, + ); + case "EachDirective": + return await renderEachDirective( + ast, + value, + variables, + functionMap, + ); + case "IfDirective": + return await renderIfDirective(ast, value, variables, functionMap); + case "LetDirective": + return await renderLetDirective(ast, value, variables, functionMap); + case "Text": + return children[0] as string; + default: + throw new Error(`Unknown template element type ${type}`); + } +} + +async function renderExpressionDirective( + ast: AST, + value: any, + variables: Record, + functionMap: FunctionMap, +): Promise { + const [_, expression] = ast; + const expr = expressionToKvQueryExpression(expression); + const result = await evalQueryExpression( + expr, + value, + variables, + functionMap, + ); + + if ( + Array.isArray(result) && result.length > 0 && typeof result[0] === "object" + ) { + // If result is an array of objects, render as a markdown table + try { + return jsonToMDTable(result); + } catch (e: any) { + console.error( + `Error rendering expression directive: ${e.message} for value ${ + JSON.stringify(result) + }`, + ); + return JSON.stringify(result); + } + } else if (typeof result === "object" && result.constructor === Object) { + // if result is a plain object, render as a markdown table + return jsonToMDTable([result]); + } else if (Array.isArray(result)) { + // Not-object array + return JSON.stringify(result); + } else { + return "" + result; + } +} + +async function renderEachDirective( + ast: AST, + value: any[], + variables: Record, + functionMap: FunctionMap, +): Promise { + const [_eachDirective, expression, template] = ast; + const expr = expressionToKvQueryExpression(expression); + const values = await evalQueryExpression( + expr, + value, + variables, + functionMap, + ); + if (!Array.isArray(values)) { + throw new Error( + `Expecting a list expression for #each directive, instead got ${values}`, + ); + } + const resultPieces: string[] = []; + for (const itemValue of values) { + try { + resultPieces.push( + await renderTemplate( + template, + itemValue, + variables, + functionMap, + ), + ); + } catch (e: any) { + throw new Error( + `Error rendering #each directive: ${e.message} for item ${ + JSON.stringify(itemValue) + }`, + ); + } + } + return resultPieces.join(""); +} + +async function renderIfDirective( + ast: AST, + value: any, + variables: Record, + functionMap: FunctionMap, +) { + const [_, expression, trueTemplate, falseTemplate] = ast; + const expr = expressionToKvQueryExpression(expression); + const condVal = await evalQueryExpression( + expr, + value, + variables, + functionMap, + ); + if ( + !Array.isArray(condVal) && condVal || + (Array.isArray(condVal) && condVal.length > 0) + ) { + return renderTemplate(trueTemplate, value, variables, functionMap); + } else { + return falseTemplate + ? renderTemplate(falseTemplate, value, variables, functionMap) + : ""; + } +} + +async function renderLetDirective( + ast: AST, + value: any, + variables: Record, + functionMap: FunctionMap, +) { + const [_letDirective, name, expression, template] = ast; + const expr = expressionToKvQueryExpression(expression); + const val = await evalQueryExpression( + expr, + value, + variables, + functionMap, + ); + const newGlobalVariables = { ...variables, [name as any]: val }; + return await renderTemplate( + template, + value, + newGlobalVariables, + functionMap, + ); +} diff --git a/common/template/template.grammar b/common/template/template.grammar new file mode 100644 index 00000000..cb08f0dc --- /dev/null +++ b/common/template/template.grammar @@ -0,0 +1,36 @@ +@top Template { TemplateElement* } + +TemplateElement { + Text + | ExpressionDirective + | BlockDirective +} + + +BlockDirective { + BlockDirectiveOpenTagStart BlockName BlockDirectiveContent DirectiveTagEnd + TemplateElement* + BlockDirectiveCloseTagStart BlockName DirectiveTagEnd +} + +ExpressionDirective { + ExpressionDirectiveOpenTag BlockDirectiveContent DirectiveTagEnd +} + +@tokens { + @precedence { BlockDirectiveOpenTagStart, ExpressionDirectiveOpenTag, Text } + + Text { (![{] | "{" ![{])+ } + + BlockName { @asciiLetter+ } + + BlockDirectiveContent { ( ![}] | "{" ![}]* "}" )+ } + + ExpressionDirectiveOpenTag { "{{" } + + BlockDirectiveOpenTagStart { "{{#" } + BlockDirectiveCloseTagStart { "{{/" } + + DirectiveTagEnd { "}}" } + +} \ No newline at end of file diff --git a/common/template/template_parser.test.ts b/common/template/template_parser.test.ts new file mode 100644 index 00000000..f0fec182 --- /dev/null +++ b/common/template/template_parser.test.ts @@ -0,0 +1,56 @@ +import { assertEquals } from "../../test_deps.ts"; +import { parseTemplate } from "./template_parser.ts"; + +Deno.test("Test template", () => { + assertEquals(parseTemplate(`{{.}}`), ["Template", ["TemplateElement", [ + "ExpressionDirective", + [ + "Expression", + ["TopLevelVal", "."], + ], + ]]]); + + assertEquals(parseTemplate(`{{{page}}}`), ["Template", ["TemplateElement", [ + "ExpressionDirective", + ["Expression", ["QueryExpression", "{", ["Query", [ + "TagIdentifier", + "page", + ]], "}"]], + ]]]); + + assertEquals(parseTemplate(`{{escapeRegexp @page.name}}`), ["Template", [ + "TemplateElement", + ["ExpressionDirective", ["Expression", [ + "Call", + ["Identifier", "escapeRegexp"], + "(", + ["Expression", [ + "Attribute", + [ + "Expression", + ["GlobalIdentifier", "@page"], + ], + ".", + [ + "Identifier", + "name", + ], + ]], + ")", + ]]], + ]]); + + const tree = parseTemplate(` + # My template + {{#each .}} + * Page: {{name}} + {{#if somethingComplicated(something)}} + Sup + {{else}} + Not sup + {{/if}} + {{/each}} + `); + + console.log(JSON.stringify(tree, null, 2)); +}); diff --git a/common/template/template_parser.ts b/common/template/template_parser.ts new file mode 100644 index 00000000..70f9f0bd --- /dev/null +++ b/common/template/template_parser.ts @@ -0,0 +1,175 @@ +import { LRLanguage } from "@codemirror/language"; +import { parse } from "../markdown_parser/parse_tree.ts"; + +import { parser as templateParser } from "./parse-template.js"; +import { parser as expressionParser } from "../markdown_parser/parse-expression.js"; +import { parser as queryParser } from "../markdown_parser/parse-query.js"; +import { AST } from "$sb/lib/tree.ts"; +import { parseTreeToAST } from "$sb/lib/tree.ts"; +import { deepEqual } from "$sb/lib/json.ts"; + +export const templateLanguage = LRLanguage.define({ + name: "template", + parser: templateParser, +}); + +export const expressionLanguage = LRLanguage.define({ + name: "expression", + parser: expressionParser, +}); + +export const queryLanguage = LRLanguage.define({ + name: "query", + parser: queryParser, +}); + +export function parseTemplate(text: string) { + // Remove a newline after a singleton (only thing on the line) block open or close tag + // text = text.replaceAll(/(^|\n)(\{\{[#\/][^}]+\}\})(\n)/g, "$1$2"); + const tree = parse(templateLanguage, text); + const ast = processTree(parseTreeToAST(tree, false)); + // console.log("AST", JSON.stringify(ast, null, 2)); + return ast; +} + +function processTree(tree: AST): AST { + switch (tree[0]) { + case "Template": + return [ + "Template", + ...stripInitialNewline((tree.slice(1) as AST[]).map(processTree)), + ]; + case "TemplateElement": + return ["TemplateElement", ...(tree.slice(1) as AST[]).map(processTree)]; + case "ExpressionDirective": { + let exprString = tree[2][1] as string; + const legacyCallSyntax = /^([A-Za-z]+)\s+([^(]+$)/.exec(exprString); + if (legacyCallSyntax) { + // Translates "escapeRegex @page.name" -> "escapeRegex(@page.name)" + const [_, fn, args] = legacyCallSyntax; + exprString = `${fn}(${args})`; + console.warn( + "Translated legacy function call to new syntax", + exprString, + ); + } + const expressionTree = parseTreeToAST(parse( + expressionLanguage, + exprString, + )); + return ["ExpressionDirective", expressionTree[1]]; + } + case "BlockDirective": { + const blockType = tree[2][1] as string; + const blockTextContent = tree[3][1] as string; + const bodyElements = (tree as any[]).filter((n) => + n[0] === "TemplateElement" + ); + const closingBlockName = tree[tree.length - 2][1]; + if (closingBlockName !== blockType) { + throw new Error( + `Block #${blockType} is not properly closed, saw /${closingBlockName} instead`, + ); + } + // const body = stripInitialNewline(bodyElements.map(processTree)); + const body = bodyElements.map(processTree); + switch (blockType) { + case "each": { + const expressionTree = parseTreeToAST(parse( + expressionLanguage, + blockTextContent.trim(), + )); + // console.log("Each body", bodyElements); + return ["EachDirective", expressionTree[1], [ + "Template", + ...stripInitialNewline(body), + ]]; + } + case "let": { + const letExpr = blockTextContent.trim(); + const letMatch = letExpr.match(/@(\w+)\s*=\s*(.+)$/s); + if (!letMatch) { + throw new Error( + `A #let directive should be of the shape {{#let @var = expression}}, got instead: ${blockTextContent}`, + ); + } + const expressionTree = parseTreeToAST(parse( + expressionLanguage, + letMatch[2], + )); + return [ + "LetDirective", + letMatch[1], + expressionTree[1], + ["Template", ...stripInitialNewline(body)], + ]; + } + case "if": { + const expressionTree = parseTreeToAST(parse( + expressionLanguage, + blockTextContent.trim(), + )); + const elseIndex = body.findIndex((n) => + deepEqual(n, ["TemplateElement", ["ExpressionDirective", [ + "Expression", + ["Identifier", "else"], + ]]]) + ); + if (elseIndex !== -1) { + return [ + "IfDirective", + expressionTree[1], + ["Template", ...stripInitialNewline(body.slice(0, elseIndex))], + ["Template", ...stripInitialNewline(body.slice(elseIndex + 1))], + ]; + } else { + return ["IfDirective", expressionTree[1], [ + "Template", + ...stripInitialNewline(body), + ]]; + } + } + default: { + throw new Error(`Unknown block type: ${blockType}`); + } + } + } + case "Text": + return tree; + default: + console.log("tree", tree); + throw new Error(`Unknown node type: ${tree[0]}`); + } +} + +function stripInitialNewline(body: any[]) { + // body = [["TemplateElement", ["Text", "\n..."], ...]] + let first = true; + let stripNext = false; + for (const el of body) { + // Strip initial newline + if (first && el[1][0] === "Text" && el[1][1].startsWith("\n")) { + // Remove initial newline + el[1][1] = el[1][1].slice(1); + } + first = false; + + // After each block directive, strip the next newline + if ( + ["IfDirective", "EachDirective", "LetDirective"].includes(el[1][0]) + ) { + // console.log("Got a block directive, consider stripping the next one", el); + stripNext = true; + continue; + } + if ( + el[1][0] === "Text" && + el[1][1].startsWith("\n") && stripNext + ) { + // console.log("Stripping initial newline from", el); + el[1][1] = el[1][1].slice(1); + } + stripNext = false; + } + return body; +} diff --git a/deno.jsonc b/deno.jsonc index ac348d2e..3be35ec1 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -2,12 +2,12 @@ "tasks": { "clean": "rm -rf dist dist_client_bundle dist_plug_bundle website_build", "deep-clean-mac": "rm -f deno.lock && rm -rf $HOME/Library/Caches/deno && deno task clean", - "install": "deno install -f --unstable -A --importmap import_map.json silverbullet.ts", + "install": "deno install -f --unstable-kv --unstable-worker-options -A --importmap import_map.json silverbullet.ts", "check": "find . -name '*.ts*' | xargs deno check", "test": "deno test -A --unstable-kv --unstable-worker-options", "build": "deno run -A build_plugs.ts && deno run -A build_web.ts", "plugs": "deno run -A build_plugs.ts", - "server": "deno run -A --unstable-kv --unstable-worker-options --check silverbullet.ts", + "server": "deno run -A --unstable-kv --unstable-worker-options --check silverbullet.ts", "watch-web": "deno run -A --check build_web.ts --watch", "watch-server": "deno run -A --unstable-kv --unstable-worker-options --check --watch silverbullet.ts", diff --git a/import_map.json b/import_map.json index 0cce8e6e..a10f34bc 100644 --- a/import_map.json +++ b/import_map.json @@ -19,7 +19,6 @@ "preact": "https://esm.sh/preact@10.11.1", "$sb/": "./plug-api/", - "handlebars": "https://esm.sh/handlebars@4.7.7?target=es2022", "zod": "https://deno.land/x/zod@v3.22.4/mod.ts" } } diff --git a/plug-api/app_event.ts b/plug-api/app_event.ts index 5ed624f2..5ac6e50d 100644 --- a/plug-api/app_event.ts +++ b/plug-api/app_event.ts @@ -19,7 +19,7 @@ export type AppEvent = export type QueryProviderEvent = { query: Query; - pageName: string; + variables?: Record; }; export type ClickEvent = { @@ -62,6 +62,7 @@ export type SlashCompletion = { label: string; detail?: string; invoke: string; + order?: number; } & Record; export type WidgetContent = { diff --git a/plug-api/lib/builtin_query_functions.ts b/plug-api/lib/builtin_query_functions.ts index 0fe8a97f..d14d7e4b 100644 --- a/plug-api/lib/builtin_query_functions.ts +++ b/plug-api/lib/builtin_query_functions.ts @@ -1,51 +1,70 @@ import type { FunctionMap } from "$sb/types.ts"; -import { niceDate } from "$sb/lib/dates.ts"; +import { niceDate, niceTime } from "$sb/lib/dates.ts"; export const builtinFunctions: FunctionMap = { today() { return niceDate(new Date()); }, - max(...args: number[]) { - return Math.max(...args); - }, - min(...args: number[]) { - return Math.min(...args); - }, - replace(str: string, match: [string, string] | string, replace: string) { + replace( + str: string, + match: [string, string] | string, + replace: string, + ) { const matcher = Array.isArray(match) ? new RegExp(match[0], match[1] + "g") : match; return str.replaceAll(matcher, replace); }, - toJSON(obj: any) { - return JSON.stringify(obj); + json: (v: any) => { + return JSON.stringify(v); }, - startsWith(str: string, prefix: string) { - return str.startsWith(prefix); + niceDate: (ts: any) => niceDate(new Date(ts)), + escapeRegexp: (ts: any) => { + return ts.replace(/[-\/\\^$*+?.()|[\]{}]/g, "\\$&"); }, - endsWith(str: string, suffix: string) { - return str.endsWith(suffix); + // Legacy name: don't use + escape: (handlebarsExpr: string) => { + return `{{${handlebarsExpr}}}`; }, - // Note: these assume Monday as the first day of the week - firstDayOfWeek(dateString: string): string { - const date = new Date(dateString); - const dayOfWeek = date.getDay(); - const daysToSubtract = (dayOfWeek + 7 - 1) % 7; - const firstDayOfWeek = new Date(date); - firstDayOfWeek.setDate(date.getDate() - daysToSubtract); - return niceDate(firstDayOfWeek); + escapeDirective: (directiveText: string) => { + return `{{${directiveText}}}`; }, - lastDayOfWeek(dateString: string): string { - const date = new Date(dateString); - const dayOfWeek = date.getDay(); - const daysToAdd = (7 - dayOfWeek) % 7; - const lastDayOfWeek = new Date(date); - lastDayOfWeek.setDate(date.getDate() + daysToAdd); - return niceDate(lastDayOfWeek); + time: () => niceTime(new Date()), + tomorrow: () => { + const tomorrow = new Date(); + tomorrow.setDate(tomorrow.getDate() + 1); + return niceDate(tomorrow); }, - addDays(dateString: string, daysToAdd: number): string { - const date = new Date(dateString); - date.setDate(date.getDate() + daysToAdd); - return niceDate(date); + yesterday: () => { + const yesterday = new Date(); + yesterday.setDate(yesterday.getDate() - 1); + return niceDate(yesterday); + }, + lastWeek: () => { + const lastWeek = new Date(); + lastWeek.setDate(lastWeek.getDate() - 7); + return niceDate(lastWeek); + }, + nextWeek: () => { + const nextWeek = new Date(); + nextWeek.setDate(nextWeek.getDate() + 7); + return niceDate(nextWeek); + }, + weekStart: (startOnMonday = true) => { + const d = new Date(); + const day = d.getDay(); + let diff = d.getDate() - day; + if (startOnMonday) { + diff += day == 0 ? -6 : 1; + } + return niceDate(new Date(d.setDate(diff))); + }, + + // List functions + count: (list: any[]) => { + return list.length; + }, + at: (list: any[], index: number) => { + return list[index]; }, }; diff --git a/plug-api/lib/parse-query.ts b/plug-api/lib/parse-query.ts index 48b82f54..33c09d05 100644 --- a/plug-api/lib/parse-query.ts +++ b/plug-api/lib/parse-query.ts @@ -122,6 +122,22 @@ export function expressionToKvQueryExpression(node: AST): QueryExpression { } return ["array", exprs.map(expressionToKvQueryExpression)]; } + case "Object": { + const objAttrs: [string, QueryExpression][] = []; + for (const kv of node.slice(2)) { + if (typeof kv === "string") { + continue; + } + const [_, key, _colon, expr] = kv; + objAttrs.push([ + key[1].slice(1, -1) as string, + expressionToKvQueryExpression( + expr, + ), + ]); + } + return ["object", objAttrs]; + } case "BinExpression": { const lval = expressionToKvQueryExpression(node[1]); const binOp = node[2][0] === "InKW" ? "in" : (node[2] as string).trim(); @@ -148,6 +164,34 @@ export function expressionToKvQueryExpression(node: AST): QueryExpression { } return ["call", fn, args.map(expressionToKvQueryExpression)]; } + case "UnaryExpression": { + // console.log("UnaryExpression", node); + if (node[1][0] === "NotKW" || node[1][0] === "!") { + return ["not", expressionToKvQueryExpression(node[2])]; + } + throw new Error(`Unknown unary expression: ${node[1][0]}`); + } + case "TopLevelVal": { + return ["attr"]; + } + case "GlobalIdentifier": { + return ["global", (node[1] as string).substring(1)]; + } + case "TernaryExpression": { + const [_, condition, _space, ifTrue, _space2, ifFalse] = node; + return [ + "?", + expressionToKvQueryExpression(condition), + expressionToKvQueryExpression(ifTrue), + expressionToKvQueryExpression(ifFalse), + ]; + } + case "QueryExpression": { + return ["query", astToKvQuery(node[2])]; + } + case "PageRef": { + return ["pageref", (node[1] as string).slice(2, -2)]; + } default: throw new Error(`Not supported: ${node[0]}`); } diff --git a/plug-api/lib/parser-query.test.ts b/plug-api/lib/parser-query.test.ts index 8b93e7a7..72ade1d8 100644 --- a/plug-api/lib/parser-query.test.ts +++ b/plug-api/lib/parser-query.test.ts @@ -1,12 +1,18 @@ import { parse } from "../../common/markdown_parser/parse_tree.ts"; -import { AST, parseTreeToAST } from "$sb/lib/tree.ts"; -import { assertEquals } from "../../test_deps.ts"; +import { AST, collectNodesOfType, parseTreeToAST } from "$sb/lib/tree.ts"; +import { assert, assertEquals } from "../../test_deps.ts"; import { astToKvQuery } from "$sb/lib/parse-query.ts"; import { languageFor } from "../../common/languages.ts"; function wrapQueryParse(query: string): AST | null { const tree = parse(languageFor("query")!, query); // console.log("tree", tree); + // Check for no ambiguitiies + if (collectNodesOfType(tree, "⚠").length > 0) { + console.error("Parse error:", JSON.stringify(tree, null, 2)); + assert(false); + } + return parseTreeToAST(tree.children![0]); } @@ -29,6 +35,63 @@ Deno.test("Test query parser", () => { }, ); + assertEquals( + astToKvQuery(wrapQueryParse(`page where .`)!), + { + querySource: "page", + filter: ["attr"], + }, + ); + + assertEquals( + astToKvQuery(wrapQueryParse(`page where @page`)!), + { + querySource: "page", + filter: ["global", "page"], + }, + ); + + // Comment check + assertEquals( + astToKvQuery( + wrapQueryParse(`page # comment`)!, + ), + { + querySource: "page", + }, + ); + + // Nested query check + + assertEquals( + astToKvQuery( + wrapQueryParse(`page select {page} as p`)!, + ), + { + querySource: "page", + select: [{ + expr: ["query", { querySource: "page" }], + name: "p", + }], + }, + ); + + assertEquals( + astToKvQuery(wrapQueryParse(`page where not true`)!), + { + querySource: "page", + filter: ["not", ["boolean", true]], + }, + ); + + assertEquals( + astToKvQuery(wrapQueryParse(`page where !isSomething`)!), + { + querySource: "page", + filter: ["not", ["attr", "isSomething"]], + }, + ); + assertEquals( astToKvQuery(wrapQueryParse(`page where name =~ /test/`)!), { @@ -45,6 +108,65 @@ Deno.test("Test query parser", () => { }, ); + assertEquals( + astToKvQuery( + wrapQueryParse(`page where name = [[my page]]`)!, + ), + { + querySource: "page", + filter: ["=", ["attr", "name"], [ + "pageref", + "my page", + ]], + }, + ); + + assertEquals( + astToKvQuery( + wrapQueryParse(`page where name = {"name": "Pete", "age": 27}`)!, + ), + { + querySource: "page", + filter: ["=", ["attr", "name"], ["object", [ + ["name", ["string", "Pete"]], + ["age", ["number", 27]], + ]]], + }, + ); + + assertEquals( + astToKvQuery( + wrapQueryParse(`page where json({})`)!, + ), + { + querySource: "page", + filter: ["call", "json", [["object", []]]], + }, + ); + + assertEquals( + astToKvQuery( + wrapQueryParse(`page where name = [1, 2, 3]`)!, + ), + { + querySource: "page", + filter: ["=", ["attr", "name"], ["array", [["number", 1], ["number", 2], [ + "number", + 3, + ]]]], + }, + ); + + assertEquals( + astToKvQuery( + wrapQueryParse(`page where name = []`)!, + ), + { + querySource: "page", + filter: ["=", ["attr", "name"], ["array", []]], + }, + ); + assertEquals( astToKvQuery( wrapQueryParse(`page where name = "test" and age > 20`)!, @@ -195,6 +317,16 @@ Deno.test("Test query parser", () => { }, ); + assertEquals( + astToKvQuery( + wrapQueryParse(`task where myCall().thing`)!, + ), + { + querySource: "task", + filter: ["attr", ["call", "myCall", []], "thing"], + }, + ); + assertEquals( astToKvQuery( wrapQueryParse(`task select today() as today2`)!, @@ -220,4 +352,20 @@ Deno.test("Test query parser", () => { }], }, ); + + assertEquals( + astToKvQuery( + wrapQueryParse(`page select 8 > 3 ? "yes" : "no" as truth`)!, + ), + { + querySource: "page", + select: [{ + name: "truth", + expr: ["?", [">", ["number", 8], ["number", 3]], ["string", "yes"], [ + "string", + "no", + ]], + }], + }, + ); }); diff --git a/plug-api/lib/query.ts b/plug-api/lib/query.ts index e20fcab4..c231370b 100644 --- a/plug-api/lib/query.ts +++ b/plug-api/lib/query.ts @@ -1,128 +1,5 @@ import { FunctionMap, KV, Query, QueryExpression } from "$sb/types.ts"; - -export function evalQueryExpression( - val: QueryExpression, - obj: any, - functionMap: FunctionMap = {}, -): any { - const [type, op1] = val; - - switch (type) { - // Logical operators - case "and": - return evalQueryExpression(op1, obj, functionMap) && - evalQueryExpression(val[2], obj, functionMap); - case "or": - return evalQueryExpression(op1, obj, functionMap) || - evalQueryExpression(val[2], obj, functionMap); - // Value types - case "null": - return null; - // TODO: Add this to the actualy query syntax - case "not": - return !evalQueryExpression(op1, obj, functionMap); - case "number": - case "string": - case "boolean": - return op1; - case "regexp": - return [op1, val[2]]; - case "attr": { - let attributeVal = obj; - if (val.length === 3) { - attributeVal = evalQueryExpression(val[1], obj, functionMap); - if (attributeVal) { - return attributeVal[val[2]]; - } else { - return null; - } - } else if (!val[1]) { - return obj; - } else { - return attributeVal[val[1]]; - } - } - case "array": { - return op1.map((v) => evalQueryExpression(v, obj, functionMap)); - } - case "object": - return obj; - case "call": { - const fn = functionMap[op1]; - if (!fn) { - throw new Error(`Unknown function: ${op1}`); - } - return fn( - ...val[2].map((v) => evalQueryExpression(v, obj, functionMap)), - ); - } - } - - // Binary operators, here we can pre-calculate the two operand values - const val1 = evalQueryExpression(op1, obj, functionMap); - const val2 = evalQueryExpression(val[2], obj, functionMap); - - switch (type) { - case "+": - return val1 + val2; - case "-": - return val1 - val2; - case "*": - return val1 * val2; - case "/": - return val1 / val2; - case "%": - return val1 % val2; - case "=": { - if (Array.isArray(val1) && !Array.isArray(val2)) { - // Record property is an array, and value is a scalar: find the value in the array - return val1.includes(val2); - } else if (Array.isArray(val1) && Array.isArray(val2)) { - // Record property is an array, and value is an array: compare the arrays - return val1.length === val2.length && - val1.every((v) => val2.includes(v)); - } - return val1 == val2; - } - case "!=": { - if (Array.isArray(val1) && !Array.isArray(val2)) { - // Record property is an array, and value is a scalar: find the value in the array - return !val1.includes(val2); - } else if (Array.isArray(val1) && Array.isArray(val2)) { - // Record property is an array, and value is an array: compare the arrays - return !(val1.length === val2.length && - val1.every((v) => val2.includes(v))); - } - return val1 != val2; - } - case "=~": { - if (!Array.isArray(val2)) { - throw new Error(`Invalid regexp: ${val2}`); - } - const r = new RegExp(val2[0], val2[1]); - return r.test(val1); - } - case "!=~": { - if (!Array.isArray(val2)) { - throw new Error(`Invalid regexp: ${val2}`); - } - const r = new RegExp(val2[0], val2[1]); - return !r.test(val1); - } - case "<": - return val1 < val2; - case "<=": - return val1 <= val2; - case ">": - return val1 > val2; - case ">=": - return val1 >= val2; - case "in": - return val2.includes(val1); - default: - throw new Error(`Unupported operator: ${type}`); - } -} +import { evalQueryExpression } from "$sb/lib/query_expression.ts"; /** * Looks for an attribute assignment in the expression, and returns the expression assigned to the attribute or throws an error when not found @@ -162,33 +39,68 @@ export function liftAttributeFilter( throw new Error(`Cannot find attribute assignment for ${attributeName}`); } -export function applyQuery(query: Query, allItems: T[]): T[] { +export async function applyQuery( + query: Query, + allItems: T[], + variables: Record, + functionMap: FunctionMap, +): Promise { // Filter if (query.filter) { - allItems = allItems.filter((item) => - evalQueryExpression(query.filter!, item) - ); + const filteredItems: T[] = []; + for (const item of allItems) { + if ( + await evalQueryExpression( + query.filter, + item, + variables, + functionMap, + ) + ) { + filteredItems.push(item); + } + } + allItems = filteredItems; } // Add dummy keys, then remove them - return applyQueryNoFilterKV( + return (await applyQueryNoFilterKV( query, allItems.map((v) => ({ key: [], value: v })), - ).map((v) => v.value); + variables, + functionMap, + )).map((v) => v.value); } -export function applyQueryNoFilterKV( +export async function applyQueryNoFilterKV( query: Query, allItems: KV[], - functionMap: FunctionMap = {}, // TODO: Figure this out later -): KV[] { + variables: Record, + functionMap: FunctionMap, +): Promise { // Order by if (query.orderBy) { allItems = allItems.sort((a, b) => { const aVal = a.value; const bVal = b.value; for (const { expr, desc } of query.orderBy!) { - const evalA = evalQueryExpression(expr, aVal, functionMap); - const evalB = evalQueryExpression(expr, bVal, functionMap); + const evalA = evalQueryExpression( + expr, + aVal, + variables, + functionMap, + ); + if (evalA instanceof Promise) { + throw new Error("Cannot order by a promise"); + } + const evalB = evalQueryExpression( + expr, + bVal, + variables, + functionMap, + ); + if (evalB instanceof Promise) { + throw new Error("Cannot order by a promise"); + } if ( evalA < evalB || evalA === undefined ) { @@ -211,7 +123,7 @@ export function applyQueryNoFilterKV( const newRec: any = {}; for (const { name, expr } of query.select) { newRec[name] = expr - ? evalQueryExpression(expr, rec, functionMap) + ? await evalQueryExpression(expr, rec, variables, functionMap) : rec[name]; } allItems[i].value = newRec; @@ -232,7 +144,12 @@ export function applyQueryNoFilterKV( } if (query.limit) { - const limit = evalQueryExpression(query.limit, {}, functionMap); + const limit = await evalQueryExpression( + query.limit, + {}, + variables, + functionMap, + ); if (allItems.length > limit) { allItems = allItems.slice(0, limit); } diff --git a/plug-api/lib/query_expression.test.ts b/plug-api/lib/query_expression.test.ts new file mode 100644 index 00000000..c482c6f2 --- /dev/null +++ b/plug-api/lib/query_expression.test.ts @@ -0,0 +1,45 @@ +import { evalQueryExpression } from "$sb/lib/query_expression.ts"; +import { assert, assertEquals } from "../../test_deps.ts"; + +Deno.test("Test query expression evaluation", async () => { + assertEquals( + evalQueryExpression( + ["+", ["number", 1], ["number", 2]], + {}, + {}, + {}, + ), + 3, + ); + + assertEquals( + evalQueryExpression( + ["?", [">", ["number", 8], ["number", 3]], ["string", "yes"], [ + "string", + "no", + ]], + {}, + {}, + {}, + ), + "yes", + ); + + // Only when an expression calls an async function should the result be a + const prom = evalQueryExpression( + ["+", ["call", "asyncCall", []], ["number", 2]], + {}, + {}, + { asyncCall: () => Promise.resolve(1) }, + ); + assert(prom instanceof Promise); + assertEquals(await prom, 3); + const nonProm = evalQueryExpression( + ["+", ["call", "asyncCall", []], ["number", 2]], + {}, + {}, + { asyncCall: () => 1 }, + ); + assert(!(nonProm instanceof Promise)); + assertEquals(nonProm, 3); +}); diff --git a/plug-api/lib/query_expression.ts b/plug-api/lib/query_expression.ts new file mode 100644 index 00000000..447a8f4e --- /dev/null +++ b/plug-api/lib/query_expression.ts @@ -0,0 +1,274 @@ +import { FunctionMap, QueryExpression } from "$sb/types.ts"; + +export function evalQueryExpression( + val: QueryExpression, + obj: any, + variables: Record, + functionMap: FunctionMap, +): Promise | any { + const [type, op1] = val; + + switch (type) { + // Logical operators + case "and": { + const op1Val = evalQueryExpression( + op1, + obj, + variables, + functionMap, + ); + if (op1Val instanceof Promise) { + return op1Val.then((v) => + v && + evalQueryExpression(val[2], obj, variables, functionMap) + ); + } else if (op1Val) { + return evalQueryExpression(val[2], obj, variables, functionMap); + } else { + return false; + } + } + case "or": { + const op1Val = evalQueryExpression( + op1, + obj, + variables, + functionMap, + ); + if (op1Val instanceof Promise) { + return op1Val.then((v) => + v || + evalQueryExpression(val[2], obj, variables, functionMap) + ); + } else if (op1Val) { + return true; + } else { + return evalQueryExpression(val[2], obj, variables, functionMap); + } + } + // Value types + case "null": + return null; + // TODO: Add this to the actualy query syntax + case "not": { + const val = evalQueryExpression(op1, obj, variables, functionMap); + if (val instanceof Promise) { + return val.then((v) => !v); + } else { + return !val; + } + } + case "number": + case "string": + case "boolean": + return op1; + case "regexp": + return [op1, val[2]]; + case "attr": { + const attributeVal = obj; + if (val.length === 3) { + const attributeVal = evalQueryExpression( + val[1], + obj, + variables, + functionMap, + ); + if (attributeVal instanceof Promise) { + return attributeVal.then((v) => v[val[2]]); + } else if (attributeVal) { + return attributeVal[val[2]]; + } else { + return null; + } + } else if (!val[1]) { + return obj; + } else { + let attrVal = attributeVal[val[1]]; + const func = functionMap[val[1]]; + if (attrVal === undefined && func !== undefined) { + // Fallback to function call, if one is defined with this name + attrVal = func(variables); + } + return attrVal; + } + } + case "global": { + return variables[op1]; + } + case "array": { + const argValues = op1.map((v) => + evalQueryExpression(v, obj, variables, functionMap) + ); + // Check if any arg value is a promise, and if so wait for it + const waitForPromises: Promise[] = []; + for (let i = 0; i < argValues.length; i++) { + if (argValues[i] instanceof Promise) { + // Wait and replace in-place + waitForPromises.push(argValues[i].then((v: any) => argValues[i] = v)); + } + } + if (waitForPromises.length > 0) { + return Promise.all(waitForPromises).then(() => argValues); + } else { + return argValues; + } + } + case "object": { + const objEntries: [string, any][] = op1.map(( + [key, expr], + ) => [key, evalQueryExpression(expr, obj, variables, functionMap)]); + // Check if any arg value is a promise, and if so wait for it + const waitForPromises: Promise[] = []; + for (let i = 0; i < objEntries.length; i++) { + if (objEntries[i][1] instanceof Promise) { + // Wait and replace in-place + waitForPromises.push( + objEntries[i][1].then((v: any) => objEntries[i] = v), + ); + } + } + if (waitForPromises.length > 0) { + return Promise.all(waitForPromises).then(() => + Object.fromEntries(objEntries) + ); + } else { + return Object.fromEntries(objEntries); + } + } + case "pageref": { + const readPageFunction = functionMap.$readPage; + if (!readPageFunction) { + throw new Error(`No $readPage function defined`); + } + return readPageFunction(op1); + } + case "query": { + const parsedQuery = val[1]; + const queryFunction = functionMap.$query; + if (!queryFunction) { + throw new Error(`No $query function defined`); + } + return queryFunction(parsedQuery, variables); + } + case "call": { + const fn = functionMap[op1]; + if (!fn) { + throw new Error(`Unknown function: ${op1}`); + } + const argValues = val[2].map((v) => + evalQueryExpression(v, obj, variables, functionMap) + ); + // Check if any arg value is a promise, and if so wait for it + const waitForPromises: Promise[] = []; + for (let i = 0; i < argValues.length; i++) { + if (argValues[i] instanceof Promise) { + // Wait and replace in-place + waitForPromises.push(argValues[i].then((v: any) => argValues[i] = v)); + } + } + if (waitForPromises.length > 0) { + return Promise.all(waitForPromises).then(() => fn(...argValues)); + } else { + return fn(...argValues); + } + } + } + + // Binary operators, here we can pre-calculate the two operand values + const val1 = evalQueryExpression( + op1, + obj, + variables, + functionMap, + ); + const val2 = evalQueryExpression( + val[2], + obj, + variables, + functionMap, + ); + + const val3 = val[3] + ? evalQueryExpression( + val[3], + obj, + variables, + functionMap, + ) + : undefined; + + if ( + val1 instanceof Promise || val2 instanceof Promise || + val3 instanceof Promise + ) { + return Promise.all([val1, val2, val3]).then(([v1, v2, v3]) => + evalSimpleExpression(type, v1, v2, v3) + ); + } else { + return evalSimpleExpression(type, val1, val2, val3); + } +} +function evalSimpleExpression(type: string, val1: any, val2: any, val3: any) { + switch (type) { + case "+": + return val1 + val2; + case "-": + return val1 - val2; + case "*": + return val1 * val2; + case "/": + return val1 / val2; + case "%": + return val1 % val2; + case "=": { + if (Array.isArray(val1) && !Array.isArray(val2)) { + // Record property is an array, and value is a scalar: find the value in the array + return val1.includes(val2); + } else if (Array.isArray(val1) && Array.isArray(val2)) { + // Record property is an array, and value is an array: compare the arrays + return val1.length === val2.length && + val1.every((v) => val2.includes(v)); + } + return val1 == val2; + } + case "!=": { + if (Array.isArray(val1) && !Array.isArray(val2)) { + // Record property is an array, and value is a scalar: find the value in the array + return !val1.includes(val2); + } else if (Array.isArray(val1) && Array.isArray(val2)) { + // Record property is an array, and value is an array: compare the arrays + return !(val1.length === val2.length && + val1.every((v) => val2.includes(v))); + } + return val1 != val2; + } + case "=~": { + if (!Array.isArray(val2)) { + throw new Error(`Invalid regexp: ${val2}`); + } + const r = new RegExp(val2[0], val2[1]); + return r.test(val1); + } + case "!=~": { + if (!Array.isArray(val2)) { + throw new Error(`Invalid regexp: ${val2}`); + } + const r = new RegExp(val2[0], val2[1]); + return !r.test(val1); + } + case "<": + return val1 < val2; + case "<=": + return val1 <= val2; + case ">": + return val1 > val2; + case ">=": + return val1 >= val2; + case "in": + return val2.includes(val1); + case "?": + return val1 ? val2 : val3; + default: + throw new Error(`Unupported operator: ${type}`); + } +} diff --git a/plug-api/lib/tree.ts b/plug-api/lib/tree.ts index 2af75b6f..5228d177 100644 --- a/plug-api/lib/tree.ts +++ b/plug-api/lib/tree.ts @@ -214,16 +214,22 @@ export function cloneTree(tree: ParseTree): ParseTree { return newTree; } -export function parseTreeToAST(tree: ParseTree): AST { +export function parseTreeToAST(tree: ParseTree, omitTrimmable = true): AST { + const parseErrorNodes = collectNodesOfType(tree, "⚠"); + if (parseErrorNodes.length > 0) { + throw new Error( + `Parse error in: ${renderToText(tree)}`, + ); + } if (tree.text !== undefined) { return tree.text; } const ast: AST = [tree.type!]; for (const node of tree.children!) { if (node.type && !node.type.endsWith("Mark")) { - ast.push(parseTreeToAST(node)); + ast.push(parseTreeToAST(node, omitTrimmable)); } - if (node.text && node.text.trim()) { + if (node.text && (omitTrimmable && node.text.trim() || !omitTrimmable)) { ast.push(node.text); } } diff --git a/plug-api/plugos-syscall/datastore.ts b/plug-api/plugos-syscall/datastore.ts index 6fd3f2f7..efd94b29 100644 --- a/plug-api/plugos-syscall/datastore.ts +++ b/plug-api/plugos-syscall/datastore.ts @@ -27,12 +27,18 @@ export function batchDel(keys: KvKey[]): Promise { export function query( query: KvQuery, + variables: Record = {}, ): Promise { - return syscall("datastore.query", query); + return syscall("datastore.query", query, variables); } export function queryDelete( query: KvQuery, + variables?: Record, ): Promise { - return syscall("datastore.queryDelete", query); + return syscall("datastore.queryDelete", query, variables); +} + +export function listFunctions(): Promise { + return syscall("datastore.listFunctions"); } diff --git a/plug-api/silverbullet-syscall/mod.ts b/plug-api/silverbullet-syscall/mod.ts index f1ab19df..7b87093a 100644 --- a/plug-api/silverbullet-syscall/mod.ts +++ b/plug-api/silverbullet-syscall/mod.ts @@ -6,5 +6,5 @@ export * as clientStore from "./clientStore.ts"; export * as sync from "./sync.ts"; export * as debug from "./debug.ts"; export * as language from "./language.ts"; -export * as handlebars from "./handlebars.ts"; +export * as template from "./template.ts"; export * as codeWidget from "./code_widget.ts"; diff --git a/plug-api/silverbullet-syscall/handlebars.ts b/plug-api/silverbullet-syscall/template.ts similarity index 57% rename from plug-api/silverbullet-syscall/handlebars.ts rename to plug-api/silverbullet-syscall/template.ts index 9b5dbeea..c6c428f6 100644 --- a/plug-api/silverbullet-syscall/handlebars.ts +++ b/plug-api/silverbullet-syscall/template.ts @@ -12,5 +12,11 @@ export function renderTemplate( obj: any, globals: Record = {}, ): Promise { - return syscall("handlebars.renderTemplate", template, obj, globals); + return syscall("template.renderTemplate", template, obj, globals); +} + +export function parseTemplate( + template: string, +): Promise { + return syscall("template.parseTemplate", template); } diff --git a/plug-api/types.ts b/plug-api/types.ts index 0786cde9..c8b7b96c 100644 --- a/plug-api/types.ts +++ b/plug-api/types.ts @@ -91,24 +91,32 @@ export type QueryExpression = | [">", QueryExpression, QueryExpression] | [">=", QueryExpression, QueryExpression] | ["in", QueryExpression, QueryExpression] - | ["attr", QueryExpression, string] - | ["attr", string] + | ["attr"] // . + | ["attr", string] // name + | ["attr", QueryExpression, string] // something.name + | ["global", string] // @name | ["number", number] | ["string", string] | ["boolean", boolean] | ["null"] | ["not", QueryExpression] | ["array", QueryExpression[]] - | ["object", Record] + | ["object", [string, QueryExpression][]] | ["regexp", string, string] // regex, modifier + | ["pageref", string] | ["+", QueryExpression, QueryExpression] | ["-", QueryExpression, QueryExpression] | ["*", QueryExpression, QueryExpression] | ["%", QueryExpression, QueryExpression] | ["/", QueryExpression, QueryExpression] + | ["?", QueryExpression, QueryExpression, QueryExpression] + | ["query", Query] | ["call", string, QueryExpression[]]; -export type FunctionMap = Record any>; +export type FunctionMap = Record< + string, + (...args: any[]) => any +>; /** * An ObjectValue that can be indexed by the `index` plug, needs to have a minimum of diff --git a/plugos/lib/datastore.test.ts b/plugos/lib/datastore.test.ts index 15ca0e75..0154c2cb 100644 --- a/plugos/lib/datastore.test.ts +++ b/plugos/lib/datastore.test.ts @@ -5,10 +5,14 @@ import { DenoKvPrimitives } from "./deno_kv_primitives.ts"; import { KvPrimitives } from "./kv_primitives.ts"; import { assertEquals } from "https://deno.land/std@0.165.0/testing/asserts.ts"; import { PrefixedKvPrimitives } from "./prefixed_kv_primitives.ts"; +import { Query } from "$sb/types.ts"; async function test(db: KvPrimitives) { const datastore = new DataStore(new PrefixedKvPrimitives(db, ["ds"]), { count: (arr: any[]) => arr.length, + $query: (_query: Query) => { + return Promise.resolve([]); + }, }); await datastore.set(["user", "peter"], { name: "Peter" }); await datastore.set(["user", "hank"], { name: "Hank" }); @@ -85,6 +89,20 @@ async function test(db: KvPrimitives) { key: ["kv", "complicated"], value: { name: "Frank!", parentCount: 2, parents: ["John", "Jane"] }, }); + + assertEquals( + await datastore.query({ + prefix: ["kv"], + limit: ["number", 1], + select: [ + { + name: "random", + expr: ["query", { querySource: "bla" }], + }, + ], + }), + [{ key: ["kv", "complicated"], value: { random: [] } }], + ); } Deno.test("Test Deno KV DataStore", async () => { diff --git a/plugos/lib/datastore.ts b/plugos/lib/datastore.ts index 97532164..9c872775 100644 --- a/plugos/lib/datastore.ts +++ b/plugos/lib/datastore.ts @@ -1,7 +1,8 @@ -import { applyQueryNoFilterKV, evalQueryExpression } from "$sb/lib/query.ts"; +import { applyQueryNoFilterKV } from "$sb/lib/query.ts"; import { FunctionMap, KV, KvKey, KvQuery } from "$sb/types.ts"; import { builtinFunctions } from "$sb/lib/builtin_query_functions.ts"; import { KvPrimitives } from "./kv_primitives.ts"; +import { evalQueryExpression } from "$sb/lib/query_expression.ts"; /** * This is the data store class you'll actually want to use, wrapping the primitives * in a more user-friendly way @@ -9,7 +10,7 @@ import { KvPrimitives } from "./kv_primitives.ts"; export class DataStore { constructor( readonly kv: KvPrimitives, - private functionMap: FunctionMap = builtinFunctions, + public functionMap: FunctionMap = builtinFunctions, ) { } @@ -57,13 +58,21 @@ export class DataStore { return this.kv.batchDelete(keys); } - async query(query: KvQuery): Promise[]> { + async query( + query: KvQuery, + variables: Record = {}, + ): Promise[]> { const results: KV[] = []; let itemCount = 0; // Accumulate results let limit = Infinity; if (query.limit) { - limit = evalQueryExpression(query.limit, {}, this.functionMap); + limit = await evalQueryExpression( + query.limit, + {}, + variables, + this.functionMap, + ); } for await ( const entry of this.kv.query(query) @@ -71,7 +80,12 @@ export class DataStore { // Filter if ( query.filter && - !evalQueryExpression(query.filter, entry.value, this.functionMap) + !await evalQueryExpression( + query.filter, + entry.value, + variables, + this.functionMap, + ) ) { continue; } @@ -84,13 +98,21 @@ export class DataStore { } } // Apply order by, limit, and select - return applyQueryNoFilterKV(query, results, this.functionMap); + return applyQueryNoFilterKV( + query, + results, + variables, + this.functionMap, + ); } - async queryDelete(query: KvQuery): Promise { + async queryDelete( + query: KvQuery, + variables: Record = {}, + ): Promise { const keys: KvKey[] = []; for ( - const { key } of await this.query(query) + const { key } of await this.query(query, variables) ) { keys.push(key); } diff --git a/plugos/syscalls/datastore.ts b/plugos/syscalls/datastore.ts index 84ec7cd0..f9bd528d 100644 --- a/plugos/syscalls/datastore.ts +++ b/plugos/syscalls/datastore.ts @@ -20,8 +20,16 @@ export function dataStoreReadSyscalls(ds: DataStore): SysCallMapping { return ds.get(key); }, - "datastore.query": async (_ctx, query: KvQuery): Promise => { - return (await ds.query(query)); + "datastore.query": ( + _ctx, + query: KvQuery, + variables?: Record, + ): Promise => { + return ds.query(query, variables); + }, + + "datastore.listFunctions": (): string[] => { + return Object.keys(ds.functionMap); }, }; } @@ -44,8 +52,12 @@ export function dataStoreWriteSyscalls(ds: DataStore): SysCallMapping { return ds.batchDelete(keys); }, - "datastore.queryDelete": (_ctx, query: KvQuery): Promise => { - return ds.queryDelete(query); + "datastore.queryDelete": ( + _ctx, + query: KvQuery, + variables?: Record, + ): Promise => { + return ds.queryDelete(query, variables); }, }; } diff --git a/plugs/editor/complete.ts b/plugs/editor/complete.ts index 5b8a74c2..6ead7222 100644 --- a/plugs/editor/complete.ts +++ b/plugs/editor/complete.ts @@ -9,22 +9,33 @@ export async function pageComplete(completeEvent: CompleteEvent) { if (!match) { return null; } - // When we're in fenced code block, we likely want to complete a page name without an alias, and only complete template pages - // so let's check if we're in a template context - const isInTemplateContext = - completeEvent.parentNodes.find((node) => node.startsWith("FencedCode")) && - // either a render [[bla]] clause or page: "[[bla]]" template block - /render\s+\[\[|page:\s*["']\[\[/.test( - completeEvent.linePrefix, - ); - // When in a template context, we only want to complete template pages - // When outside of a template context, we want to complete all pages except template pages - let allPages: PageMeta[] = isInTemplateContext - ? await queryObjects("template", {}, 5) - : await queryObjects("page", { + let allPages: PageMeta[] = []; + + if ( + completeEvent.parentNodes.find((node) => node.startsWith("FencedCode")) && + // either a render [[bla]] clause + /render\s+\[\[/.test( + completeEvent.linePrefix, + ) + ) { + // We're in a template context, let's only complete templates + allPages = await queryObjects("template", {}, 5); + } else if ( + completeEvent.parentNodes.find((node) => + node.startsWith("FencedCode:include") || + node.startsWith("FencedCode:template") + ) + ) { + // Include both pages and templates in page completion in ```include and ```template blocks + allPages = await queryObjects("page", {}, 5); + } else { + // Otherwise, just complete non-template pages + allPages = await queryObjects("page", { filter: ["!=", ["attr", "tags"], ["string", "template"]], }, 5); + } + const prefix = match[1]; if (prefix.startsWith("!")) { // Federation prefix, let's first see if we're matching anything from federation that is locally synced @@ -54,7 +65,7 @@ export async function pageComplete(completeEvent: CompleteEvent) { completions.push({ label: `${pageMeta.displayName}`, boost: new Date(pageMeta.lastModified).getTime(), - apply: isInTemplateContext + apply: pageMeta.tag === "template" ? pageMeta.name : `${pageMeta.name}|${pageMeta.displayName}`, detail: `displayName for: ${pageMeta.name}`, @@ -66,7 +77,7 @@ export async function pageComplete(completeEvent: CompleteEvent) { completions.push({ label: `${alias}`, boost: new Date(pageMeta.lastModified).getTime(), - apply: isInTemplateContext + apply: pageMeta.tag === "template" ? pageMeta.name : `${pageMeta.name}|${alias}`, detail: `alias to: ${pageMeta.name}`, diff --git a/plugs/editor/editor.plug.yaml b/plugs/editor/editor.plug.yaml index a88d0895..69aa02c6 100644 --- a/plugs/editor/editor.plug.yaml +++ b/plugs/editor/editor.plug.yaml @@ -84,6 +84,11 @@ functions: command: name: "Navigate: To Page" hide: true + navigateToURL: + path: "./navigate.ts:navigateToURL" + command: + name: "Navigate: To URL" + hide: true # Text editing commands quoteSelectionCommand: diff --git a/plugs/editor/navigate.ts b/plugs/editor/navigate.ts index d9777d50..c03ccce1 100644 --- a/plugs/editor/navigate.ts +++ b/plugs/editor/navigate.ts @@ -18,6 +18,7 @@ async function actionClickOrActionEnter( if (!mdTree) { return; } + // console.log("Got a click on", mdTree); const navigationNodeFinder = (t: ParseTree) => [ "WikiLink", @@ -99,7 +100,6 @@ async function actionClickOrActionEnter( break; } case "Hashtag": { - console.log("Got myself a hash tag", mdTree); const hashtag = mdTree.children![0].text!.slice(1); await editor.navigate( { page: `${tagPrefix}${hashtag}`, pos: 0 }, @@ -136,3 +136,7 @@ export async function navigateCommand(cmdDef: any) { export async function navigateToPage(_cmdDef: any, pageName: string) { await editor.navigate({ page: pageName, pos: 0 }); } + +export async function navigateToURL(_cmdDef: any, url: string) { + await editor.openUrl(url, false); +} diff --git a/plugs/index/api.ts b/plugs/index/api.ts index 377d9132..2f359d2f 100644 --- a/plugs/index/api.ts +++ b/plugs/index/api.ts @@ -181,11 +181,12 @@ export function queryObjects( export async function query( query: KvQuery, + variables?: Record, ): Promise { return (await datastore.query({ ...query, prefix: [indexKey, ...query.prefix ? query.prefix : []], - })).map(({ key, value }) => ({ key: key.slice(1), value })); + }, variables)).map(({ key, value }) => ({ key: key.slice(1), value })); } export function getObjectByRef( @@ -198,13 +199,14 @@ export function getObjectByRef( export async function objectSourceProvider({ query, + variables, }: QueryProviderEvent): Promise { const tag = query.querySource!; const results = await datastore.query({ ...query, prefix: [indexKey, tag], distinct: true, - }); + }, variables); return results.map((r) => r.value); } diff --git a/plugs/index/attachment.ts b/plugs/index/attachment.ts deleted file mode 100644 index 25686722..00000000 --- a/plugs/index/attachment.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { QueryProviderEvent } from "$sb/app_event.ts"; -import { applyQuery } from "$sb/lib/query.ts"; -import { space } from "$sb/syscalls.ts"; - -export async function attachmentQueryProvider({ query }: QueryProviderEvent) { - return applyQuery(query, await space.listAttachments()); -} diff --git a/plugs/index/index.plug.yaml b/plugs/index/index.plug.yaml index 1165b36d..b142c3b5 100644 --- a/plugs/index/index.plug.yaml +++ b/plugs/index/index.plug.yaml @@ -62,12 +62,6 @@ functions: path: "./command.ts:reindexSpace" env: server - # Attachments - attachmentQueryProvider: - path: ./attachment.ts:attachmentQueryProvider - events: - - query:attachment - indexPage: path: page.ts:indexPage events: diff --git a/plugs/index/lint.ts b/plugs/index/lint.ts index 751848fe..c5fccf80 100644 --- a/plugs/index/lint.ts +++ b/plugs/index/lint.ts @@ -50,7 +50,7 @@ export async function lintYAML({ tree }: LintEvent): Promise { const codeLang = codeInfo.children![0].text!; // All known YAML formats if ( - codeLang === "template" || codeLang === "yaml" || + ["include", "embed", "yaml"].includes(codeLang) || codeLang.startsWith("#") ) { const codeText = findNodeOfType(node, "CodeText"); diff --git a/plugs/index/widget.ts b/plugs/index/widget.ts index addbb486..72f5b487 100644 --- a/plugs/index/widget.ts +++ b/plugs/index/widget.ts @@ -11,7 +11,7 @@ import { loadPageObject } from "../template/page.ts"; import { queryObjects } from "./api.ts"; import { TemplateObject, WidgetConfig } from "../template/types.ts"; import { expressionToKvQueryExpression } from "$sb/lib/parse-query.ts"; -import { evalQueryExpression } from "$sb/lib/query.ts"; +import { evalQueryExpression } from "$sb/lib/query_expression.ts"; import { renderTemplate } from "../template/plug_api.ts"; import { extractFrontmatter } from "$sb/lib/frontmatter.ts"; import { rewritePageRefs } from "$sb/lib/resolve.ts"; @@ -59,13 +59,13 @@ export async function renderTemplateWidgets(side: "top" | "bottom"): Promise< await language.parseLanguage("expression", blockDef.where!), ); const parsedExpression = expressionToKvQueryExpression(exprAST[1]); - if (evalQueryExpression(parsedExpression, pageMeta)) { + if (await evalQueryExpression(parsedExpression, pageMeta, {}, {})) { // Match! We're happy const templateText = await space.readPage(template.ref); let renderedTemplate = (await renderTemplate( templateText, - pageMeta, frontmatter, + { page: pageMeta }, )).text; const parsedMarkdown = await markdown.parseMarkdown(renderedTemplate); diff --git a/plugs/query/api.ts b/plugs/query/api.ts new file mode 100644 index 00000000..59b20358 --- /dev/null +++ b/plugs/query/api.ts @@ -0,0 +1,70 @@ +import { parseQuery } from "$sb/lib/parse-query.ts"; +import { Query } from "$sb/types.ts"; +import { events } from "$sb/syscalls.ts"; +import { QueryProviderEvent } from "$sb/app_event.ts"; +import { resolvePath } from "$sb/lib/resolve.ts"; +import { renderQueryTemplate } from "../template/util.ts"; + +export async function query( + query: string, + variables?: Record, +): Promise { + const parsedQuery = await parseQuery(query); + + return queryParsed(parsedQuery, variables); +} + +/** + * Renders a query either as a result array, or as a rendered template when the `render` clause is defined + * @param parsedQuery + * @param variables + * @returns Promise: a string if the query has a `render` clause, or an array of results + */ +export async function renderQuery( + parsedQuery: Query, + variables?: Record, +): Promise { + const results = await queryParsed(parsedQuery, variables); + if (results.length === 0 && !parsedQuery.renderAll) { + return results; + } else { + if (parsedQuery.render) { + // Configured a custom rendering template, let's use it! + const templatePage = resolvePath( + variables?.page?.name, + parsedQuery.render, + ); + const rendered = await renderQueryTemplate( + variables?.page, + templatePage, + results, + parsedQuery.renderAll!, + ); + return rendered.trim(); + } + + return results; + } +} + +export async function queryParsed( + parsedQuery: Query, + variables?: Record, +) { + if (!parsedQuery.limit) { + parsedQuery.limit = ["number", 1000]; + } + + const eventName = `query:${parsedQuery.querySource}`; + // console.log("Parsed query", parsedQuery); + // Let's dispatch an event and see what happens + const event: QueryProviderEvent = { query: parsedQuery }; + if (variables) { + event.variables = variables; + } + const results = await events.dispatchEvent(eventName, event, 30 * 1000); + if (results.length === 0) { + throw new Error(`Unsupported query source '${parsedQuery.querySource}'`); + } + return results.flat(); +} diff --git a/plugs/query/complete.ts b/plugs/query/complete.ts index 2d634d76..e192b07b 100644 --- a/plugs/query/complete.ts +++ b/plugs/query/complete.ts @@ -6,16 +6,32 @@ import { } from "../index/attributes.ts"; export async function queryComplete(completeEvent: CompleteEvent) { - const fencedParent = completeEvent.parentNodes.find((node) => + let querySourceMatch: RegExpExecArray | null = null; + + // Let's check if this is a query block + let fencedParent = completeEvent.parentNodes.find((node) => node.startsWith("FencedCode:query") ); - if (!fencedParent) { - return null; + if (fencedParent) { + // Yep, let's see if we can do source completion + querySourceMatch = /^\s*([\w\-_]*)$/.exec( + completeEvent.linePrefix, + ); + } else { + // Not a query, perhaps a template then? + fencedParent = completeEvent.parentNodes.find((node) => + node.startsWith("FencedCode:template") + ); + + if (fencedParent) { + querySourceMatch = /\{(\s*[\w\-_]*)$/.exec( + completeEvent.linePrefix, + ); + } else { + // No? Then we're out, sorry. + return null; + } } - // First let's try to match the query source - let querySourceMatch = /^\s*([\w\-_]*)$/.exec( - completeEvent.linePrefix, - ); if (querySourceMatch) { const allEvents = await events.listEvents(); @@ -44,7 +60,7 @@ export async function queryComplete(completeEvent: CompleteEvent) { // If that doesn't work, let's try to match other bits of the query // For this we do need to find the query source, though, so let's look for it in fencedParent - querySourceMatch = /^[\n\r\s]*([\w\-_]+)/.exec( + querySourceMatch = /(^[\n\r\s]*|\{\s*)([\w\-_]+)/.exec( fencedParent.slice("FencedCode:query".length), ); const whereMatch = @@ -53,7 +69,7 @@ export async function queryComplete(completeEvent: CompleteEvent) { completeEvent.linePrefix, ); if (querySourceMatch && whereMatch) { - const type = querySourceMatch[1]; + const type = querySourceMatch[2]; const attributePrefix = whereMatch[3]; const completions = (await events.dispatchEvent( `attribute:complete:${type}`, diff --git a/plugs/query/lint.ts b/plugs/query/lint.ts index 450c05e7..3803aa6a 100644 --- a/plugs/query/lint.ts +++ b/plugs/query/lint.ts @@ -32,7 +32,6 @@ export async function lintQuery( const parsedQuery = await parseQuery( await replaceTemplateVars(bodyText, pageObject), ); - const allSources = await allQuerySources(); if ( parsedQuery.querySource && diff --git a/plugs/query/query.plug.yaml b/plugs/query/query.plug.yaml index aa9030ef..930fedaa 100644 --- a/plugs/query/query.plug.yaml +++ b/plugs/query/query.plug.yaml @@ -1,5 +1,13 @@ name: query functions: + # API + queryParsed: + path: api.ts:queryParsed + query: + path: api.ts:query + renderQuery: + path: api.ts:renderQuery + queryWidget: path: widget.ts:widget codeWidget: query diff --git a/plugs/query/widget.ts b/plugs/query/widget.ts index a3b496d2..3caf9863 100644 --- a/plugs/query/widget.ts +++ b/plugs/query/widget.ts @@ -2,8 +2,9 @@ import { codeWidget, editor, events } from "$sb/syscalls.ts"; import { parseQuery } from "$sb/lib/parse-query.ts"; import { loadPageObject, replaceTemplateVars } from "../template/page.ts"; import { resolvePath } from "$sb/lib/resolve.ts"; -import { CodeWidgetContent, PageMeta, Query } from "$sb/types.ts"; +import { CodeWidgetContent } from "$sb/types.ts"; import { jsonToMDTable, renderQueryTemplate } from "../template/util.ts"; +import { renderQuery } from "./api.ts"; export async function widget( bodyText: string, @@ -16,28 +17,11 @@ export async function widget( await replaceTemplateVars(bodyText, pageObject), ); - const results = await performQuery( - parsedQuery, - pageObject, - ); - if (results.length === 0 && !parsedQuery.renderAll) { - resultMarkdown = "No results"; + const results = await renderQuery(parsedQuery, { page: pageObject }); + if (Array.isArray(results)) { + resultMarkdown = jsonToMDTable(results); } else { - if (parsedQuery.render) { - // Configured a custom rendering template, let's use it! - const templatePage = resolvePath(pageName, parsedQuery.render); - const rendered = await renderQueryTemplate( - pageObject, - templatePage, - results, - parsedQuery.renderAll!, - ); - resultMarkdown = rendered.trim(); - } else { - // TODO: At this point it's a bit pointless to first render a markdown table, and then convert that to HTML - // We should just render the HTML table directly - resultMarkdown = jsonToMDTable(results); - } + resultMarkdown = results; } return { @@ -62,25 +46,6 @@ export async function widget( } } -export async function performQuery(parsedQuery: Query, pageObject: PageMeta) { - if (!parsedQuery.limit) { - parsedQuery.limit = ["number", 1000]; - } - - const eventName = `query:${parsedQuery.querySource}`; - // console.log("Parsed query", parsedQuery); - // Let's dispatch an event and see what happens - const results = await events.dispatchEvent( - eventName, - { query: parsedQuery, pageName: pageObject.name }, - 30 * 1000, - ); - if (results.length === 0) { - throw new Error(`Unsupported query source '${parsedQuery.querySource}'`); - } - return results.flat(); -} - export function refreshAllWidgets() { codeWidget.refreshAll(); } diff --git a/plugs/search/search.ts b/plugs/search/search.ts index ecc3f195..e207d487 100644 --- a/plugs/search/search.ts +++ b/plugs/search/search.ts @@ -1,14 +1,11 @@ import { IndexTreeEvent, QueryProviderEvent } from "$sb/app_event.ts"; import { renderToText } from "$sb/lib/tree.ts"; -import { - applyQuery, - evalQueryExpression, - liftAttributeFilter, -} from "$sb/lib/query.ts"; +import { applyQuery, liftAttributeFilter } from "$sb/lib/query.ts"; import { editor } from "$sb/syscalls.ts"; import { FileMeta } from "$sb/types.ts"; import { PromiseQueue } from "$sb/lib/async.ts"; import { ftsIndexPage, ftsSearch } from "./engine.ts"; +import { evalQueryExpression } from "$sb/lib/query_expression.ts"; const searchPrefix = "🔍 "; @@ -31,7 +28,7 @@ export async function queryProvider({ if (!phraseFilter) { throw Error("No 'phrase' filter specified, this is mandatory"); } - const phrase = evalQueryExpression(phraseFilter, {}); + const phrase = await evalQueryExpression(phraseFilter, {}, {}, {}); // console.log("Phrase", phrase); let results: any[] = await ftsSearch(phrase); @@ -41,7 +38,7 @@ export async function queryProvider({ delete r.id; } - results = applyQuery(query, results); + results = await applyQuery(query, results, {}, {}); return results; } diff --git a/plugs/template/api.ts b/plugs/template/api.ts index 169268ed..f82311a8 100644 --- a/plugs/template/api.ts +++ b/plugs/template/api.ts @@ -1,5 +1,4 @@ -import { handlebars, markdown, YAML } from "$sb/syscalls.ts"; -import type { PageMeta } from "$sb/types.ts"; +import { markdown, template, YAML } from "$sb/syscalls.ts"; import { extractFrontmatter } from "$sb/lib/frontmatter.ts"; import { TemplateObject } from "./types.ts"; import { renderToText } from "$sb/lib/tree.ts"; @@ -14,34 +13,42 @@ import { renderToText } from "$sb/lib/tree.ts"; */ export async function renderTemplate( templateText: string, - pageMeta: PageMeta, data: any = {}, + variables: Record = {}, ): Promise<{ renderedFrontmatter?: string; frontmatter: any; text: string }> { - const tree = await markdown.parseMarkdown(templateText); - const frontmatter: Partial = await extractFrontmatter(tree, { - removeFrontmatterSection: true, - removeTags: ["template"], - }); - templateText = renderToText(tree).trimStart(); - // If a 'frontmatter' key was specified in the frontmatter, use that as the frontmatter - let frontmatterText: string | undefined; - if (frontmatter.frontmatter) { - if (typeof frontmatter.frontmatter === "string") { - frontmatterText = frontmatter.frontmatter; - } else { - frontmatterText = await YAML.stringify(frontmatter.frontmatter); + try { + const tree = await markdown.parseMarkdown(templateText); + const frontmatter: Partial = await extractFrontmatter( + tree, + { + removeFrontmatterSection: true, + removeTags: ["template"], + }, + ); + templateText = renderToText(tree).trimStart(); + // If a 'frontmatter' key was specified in the frontmatter, use that as the frontmatter + let frontmatterText: string | undefined; + if (frontmatter.frontmatter) { + if (typeof frontmatter.frontmatter === "string") { + frontmatterText = frontmatter.frontmatter; + } else { + frontmatterText = await YAML.stringify(frontmatter.frontmatter); + } + frontmatterText = await template.renderTemplate( + frontmatterText, + data, + variables, + ); } - frontmatterText = await handlebars.renderTemplate(frontmatterText, data, { - page: pageMeta, - }); + return { + frontmatter, + renderedFrontmatter: frontmatterText, + text: await template.renderTemplate(templateText, data, variables), + }; + } catch (e) { + console.error("Error rendering template", e); + throw e; } - return { - frontmatter, - renderedFrontmatter: frontmatterText, - text: await handlebars.renderTemplate(templateText, data, { - page: pageMeta, - }), - }; } /** diff --git a/plugs/template/lint.ts b/plugs/template/lint.ts index e39fb991..3ea85203 100644 --- a/plugs/template/lint.ts +++ b/plugs/template/lint.ts @@ -1,8 +1,13 @@ import { LintEvent } from "$sb/app_event.ts"; import { LintDiagnostic } from "$sb/types.ts"; -import { findNodeOfType } from "$sb/lib/tree.ts"; +import { + findNodeOfType, + renderToText, + traverseTreeAsync, +} from "$sb/lib/tree.ts"; import { FrontmatterConfig } from "./types.ts"; import { extractFrontmatter } from "$sb/lib/frontmatter.ts"; +import { template, YAML } from "$sb/syscalls.ts"; export async function lintTemplateFrontmatter( { tree }: LintEvent, @@ -41,3 +46,77 @@ export async function lintTemplateFrontmatter( } return diagnostics; } + +export async function lintTemplateBlocks( + { tree }: LintEvent, +): Promise { + const frontmatter = await extractFrontmatter(tree); + const diagnostics: LintDiagnostic[] = []; + + if (frontmatter.tags?.includes("template")) { + // Parse the whole page as a template to check for errors if this is a template + try { + await template.parseTemplate(renderToText(tree)); + } catch (e: any) { + diagnostics.push({ + from: tree.from!, + to: tree.to!, + message: e.message, + severity: "error", + }); + } + } + + await traverseTreeAsync(tree, async (node) => { + if (node.type === "FencedCode") { + const codeInfo = findNodeOfType(node, "CodeInfo")!; + if (!codeInfo) { + return true; + } + const codeLang = codeInfo.children![0].text!; + if (codeLang !== "template") { + return true; + } + + const codeText = findNodeOfType(node, "CodeText"); + if (!codeText) { + return true; + } + const bodyText = renderToText(codeText); + // See if it parses as YAML, then issue a warning + try { + const parsedYaml = await YAML.parse(bodyText); + if ( + typeof parsedYaml === "object" && + (parsedYaml.template || parsedYaml.page || parsedYaml.raw) + ) { + diagnostics.push({ + from: codeText.from!, + to: codeText.to!, + message: + "Legacy template syntax detected, please replace ```template with ```include to fix.", + severity: "warning", + }); + } + } catch { + // Ignore + } + + // Ok, now parse it as a template and report any parse errors + try { + await template.parseTemplate(bodyText); + } catch (e: any) { + diagnostics.push({ + from: codeText.from!, + to: codeText.to!, + message: e.message, + severity: "error", + }); + } + } + + return false; + }); + + return diagnostics; +} diff --git a/plugs/template/page.ts b/plugs/template/page.ts index d1da159c..b7c623eb 100644 --- a/plugs/template/page.ts +++ b/plugs/template/page.ts @@ -1,4 +1,4 @@ -import { editor, handlebars, space } from "$sb/syscalls.ts"; +import { editor, space, template } from "$sb/syscalls.ts"; import { PageMeta } from "$sb/types.ts"; import { getObjectByRef, queryObjects } from "../index/plug_api.ts"; import { FrontmatterConfig, TemplateObject } from "./types.ts"; @@ -225,5 +225,5 @@ export function replaceTemplateVars( s: string, pageMeta: PageMeta, ): Promise { - return handlebars.renderTemplate(s, {}, { page: pageMeta }); + return template.renderTemplate(s, {}, { page: pageMeta }); } diff --git a/plugs/template/plug_api.ts b/plugs/template/plug_api.ts index 4087e02b..c88651d8 100644 --- a/plugs/template/plug_api.ts +++ b/plugs/template/plug_api.ts @@ -1,16 +1,15 @@ -import type { PageMeta } from "$sb/types.ts"; import { system } from "../../plug-api/syscalls.ts"; export function renderTemplate( templateText: string, - pageMeta: PageMeta, data: any = {}, + variables: Record = {}, ): Promise<{ frontmatter?: string; text: string }> { return system.invokeFunction( "template.renderTemplate", templateText, - pageMeta, data, + variables, ); } diff --git a/plugs/template/snippet.ts b/plugs/template/snippet.ts index 08562657..37fcec9c 100644 --- a/plugs/template/snippet.ts +++ b/plugs/template/snippet.ts @@ -7,7 +7,6 @@ import { loadPageObject } from "./page.ts"; import { renderTemplate } from "./api.ts"; import { prepareFrontmatterDispatch } from "$sb/lib/frontmatter.ts"; import { SnippetConfig } from "./types.ts"; -import { snippet } from "@codemirror/autocomplete"; export async function snippetSlashComplete( completeEvent: CompleteEvent, @@ -22,6 +21,7 @@ export async function snippetSlashComplete( return { label: snippetTemplate.slashCommand, detail: template.description, + order: snippetTemplate.order || 0, templatePage: template.ref, pageName: completeEvent.pageName, invoke: "template.insertSnippetTemplate", diff --git a/plugs/template/template.plug.yaml b/plugs/template/template.plug.yaml index 4502b650..74ad7a2b 100644 --- a/plugs/template/template.plug.yaml +++ b/plugs/template/template.plug.yaml @@ -29,8 +29,13 @@ functions: - editor:complete # Widget - templateWidget: # Legacy - path: template_block.ts:widget + templateWidget: + path: widget.ts:includeWidget + codeWidget: include + renderMode: markdown + + blockWidget: + path: widget.ts:templateWidget codeWidget: template renderMode: markdown @@ -52,3 +57,8 @@ functions: path: lint.ts:lintTemplateFrontmatter events: - editor:lint + + lintTemplateBlocks: + path: lint.ts:lintTemplateBlocks + events: + - editor:lint diff --git a/plugs/template/types.ts b/plugs/template/types.ts index 85a63cad..45b0925c 100644 --- a/plugs/template/types.ts +++ b/plugs/template/types.ts @@ -34,6 +34,7 @@ export type NewPageConfig = z.infer; export const SnippetConfig = refineCommand( z.object({ slashCommand: z.string(), // trigger + order: z.number().optional(), // order in the list // Regex match to apply (implicitly makes the body the regex replacement) matchRegex: z.string().optional(), // Deprecated: use matchRegex instead (for backwards compatibility) diff --git a/plugs/template/util.ts b/plugs/template/util.ts index ee287c45..9a26d6bf 100644 --- a/plugs/template/util.ts +++ b/plugs/template/util.ts @@ -1,15 +1,7 @@ -import { handlebarHelpers } from "../../common/syscalls/handlebar_helpers.ts"; import { PageMeta } from "$sb/types.ts"; -import { handlebars, space } from "$sb/syscalls.ts"; +import { space, template } from "$sb/syscalls.ts"; import { cleanTemplate } from "./plug_api.ts"; -export function buildHandebarOptions(pageMeta: PageMeta) { - return { - helpers: handlebarHelpers(), - data: { page: pageMeta }, - }; -} - export function defaultJsonTransformer(v: any): string { if (v === undefined) { return ""; @@ -27,6 +19,19 @@ export function defaultJsonTransformer(v: any): string { return "" + v; } +export function jsonObjectToMDTable( + obj: Record, + valueTransformer: (v: any) => string = defaultJsonTransformer, +): string { + const lines = []; + lines.push("| Key | Value |"); + lines.push("| --- | --- |"); + for (const [k, v] of Object.entries(obj)) { + lines.push(`| ${k} | ${valueTransformer(v)} |`); + } + return lines.join("\n"); +} + // Nicely format an array of JSON objects as a Markdown table export function jsonToMDTable( jsonArray: any[], @@ -80,5 +85,5 @@ export async function renderQueryTemplate( if (!renderAll) { templateText = `{{#each .}}\n${templateText}\n{{/each}}`; } - return handlebars.renderTemplate(templateText, data, { page: pageMeta }); + return template.renderTemplate(templateText, data, { page: pageMeta }); } diff --git a/plugs/template/var.ts b/plugs/template/var.ts index 35939dbc..f8748b7a 100644 --- a/plugs/template/var.ts +++ b/plugs/template/var.ts @@ -1,7 +1,6 @@ import { CompleteEvent } from "$sb/app_event.ts"; -import { PageMeta } from "$sb/types.ts"; -import { events } from "$sb/syscalls.ts"; -import { buildHandebarOptions } from "./util.ts"; +import { datastore, events } from "$sb/syscalls.ts"; + import { AttributeCompleteEvent, AttributeCompletion, @@ -9,36 +8,58 @@ import { import { attributeCompletionsToCMCompletion } from "./snippet.ts"; export async function templateVariableComplete(completeEvent: CompleteEvent) { - const match = /\{\{([\w@]*)$/.exec(completeEvent.linePrefix); + // Check if we're in a query, block or template context + const fencedParent = completeEvent.parentNodes.find((node) => + node.startsWith("FencedCode:query") || + node.startsWith("FencedCode:template") + ); + + if (!fencedParent) { + return null; + } + + const match = /(@|[^\{]\{\{)(\w*)$/.exec(completeEvent.linePrefix); if (!match) { return null; } - const handlebarOptions = buildHandebarOptions({ name: "" } as PageMeta); - let allCompletions: any[] = Object.keys(handlebarOptions.helpers).map( - (name) => ({ label: name, detail: "helper" }), - ); + let allCompletions: any[] = []; + + if (match[1] !== "@") { + // Not a variable + // Function completions + const builtinFunctions = await datastore.listFunctions(); + allCompletions = builtinFunctions.map((name) => ({ + label: name, + detail: "function", + })); + + // Attribute completions + const completions = (await events.dispatchEvent( + `attribute:complete:_`, + { + source: "", + prefix: match[2], + } as AttributeCompleteEvent, + )).flat() as AttributeCompletion[]; + + allCompletions = allCompletions.concat( + attributeCompletionsToCMCompletion(completions), + ); + } + + const allVariables = [...fencedParent.match(/@(\w+)/g) || []]; + allVariables.push("@page"); allCompletions = allCompletions.concat( - Object.keys(handlebarOptions.data).map((key) => ({ - label: `@${key}`, - detail: "global variable", + allVariables.filter((v) => v !== match[0]).map((key) => ({ + label: key, + apply: key.substring(1), + detail: "variable", })), ); - const completions = (await events.dispatchEvent( - `attribute:complete:_`, - { - source: "", - prefix: match[1], - } as AttributeCompleteEvent, - )).flat() as AttributeCompletion[]; - - allCompletions = allCompletions.concat( - attributeCompletionsToCMCompletion(completions), - ); - return { - from: completeEvent.pos - match[1].length, + from: completeEvent.pos - match[2].length, options: allCompletions, }; } diff --git a/plugs/template/template_block.ts b/plugs/template/widget.ts similarity index 61% rename from plugs/template/template_block.ts rename to plugs/template/widget.ts index 12849d7e..cb200578 100644 --- a/plugs/template/template_block.ts +++ b/plugs/template/widget.ts @@ -4,7 +4,7 @@ import { CodeWidgetContent, PageMeta } from "$sb/types.ts"; import { renderTemplate } from "./plug_api.ts"; import { renderToText } from "$sb/lib/tree.ts"; import { rewritePageRefs, rewritePageRefsInString } from "$sb/lib/resolve.ts"; -import { performQuery } from "../query/widget.ts"; +import { queryParsed } from "../query/api.ts"; import { parseQuery } from "$sb/lib/parse-query.ts"; type TemplateWidgetConfig = { @@ -21,7 +21,7 @@ type TemplateWidgetConfig = { query?: string; }; -export async function widget( +export async function includeWidget( bodyText: string, pageName: string, ): Promise { @@ -62,15 +62,15 @@ export async function widget( const parsedQuery = await parseQuery( await replaceTemplateVars(config.query, pageMeta), ); - value = await performQuery(parsedQuery, pageMeta); + value = await queryParsed(parsedQuery); } let { text: rendered } = config.raw ? { text: templateText } : await renderTemplate( templateText, - pageMeta, value, + { page: pageMeta }, ); if (templatePage) { @@ -99,3 +99,54 @@ export async function widget( }; } } + +export async function templateWidget( + bodyText: string, + pageName: string, +): Promise { + // Check if this is a legacy syntax template widget (with YAML body) + try { + const parsedYaml = await YAML.parse(bodyText); + if ( + typeof parsedYaml === "object" && + (parsedYaml.template || parsedYaml.page || parsedYaml.raw) + ) { + // Yeah... this looks like a legacy widget + console.warn( + "Found a template widget with legacy syntax, implicitly rewriting it to 'include'", + ); + return includeWidget(bodyText, pageName); + } + } catch { + // Not a legacy widget, good! + } + + const pageMeta: PageMeta = await loadPageObject(pageName); + + try { + const { text: rendered } = await renderTemplate( + bodyText, + pageMeta, + { page: pageMeta }, + ); + + return { + markdown: rendered, + buttons: [{ + description: "Edit", + svg: + ``, + invokeFunction: "query.editButton", + }, { + description: "Reload", + svg: + ``, + invokeFunction: "query.refreshAllWidgets", + }], + }; + } catch (e: any) { + return { + markdown: `**Error:** ${e.message}`, + }; + } +} diff --git a/scripts/generate.sh b/scripts/generate.sh index 2bafc3c0..c6bf645b 100755 --- a/scripts/generate.sh +++ b/scripts/generate.sh @@ -2,6 +2,7 @@ QUERY_GRAMMAR=common/markdown_parser/query.grammar EXPRESSION_GRAMMAR=common/markdown_parser/expression.grammar.generated +TEMPLATE_GRAMMAR=common/template/template.grammar LEZER_GENERATOR_VERSION=1.5.1 # Generate a patched grammer for just expressions @@ -9,4 +10,5 @@ echo "@top Program { Expression }" > $EXPRESSION_GRAMMAR tail -n +2 $QUERY_GRAMMAR >> $EXPRESSION_GRAMMAR deno run -A npm:@lezer/generator@$LEZER_GENERATOR_VERSION $QUERY_GRAMMAR -o common/markdown_parser/parse-query.js -deno run -A npm:@lezer/generator@$LEZER_GENERATOR_VERSION $EXPRESSION_GRAMMAR -o common/markdown_parser/parse-expression.js \ No newline at end of file +deno run -A npm:@lezer/generator@$LEZER_GENERATOR_VERSION $EXPRESSION_GRAMMAR -o common/markdown_parser/parse-expression.js +deno run -A npm:@lezer/generator@$LEZER_GENERATOR_VERSION $TEMPLATE_GRAMMAR -o common/template/parse-template.js \ No newline at end of file diff --git a/server/server_system.ts b/server/server_system.ts index 2c43b204..4020de76 100644 --- a/server/server_system.ts +++ b/server/server_system.ts @@ -27,7 +27,7 @@ import { } from "../plugos/syscalls/datastore.ts"; import { DataStoreMQ } from "../plugos/lib/mq.datastore.ts"; import { languageSyscalls } from "../common/syscalls/language.ts"; -import { handlebarsSyscalls } from "../common/syscalls/handlebars.ts"; +import { templateSyscalls } from "../common/syscalls/template.ts"; import { codeWidgetSyscalls } from "../web/syscalls/code_widget.ts"; import { CodeWidgetHook } from "../web/hooks/code_widget.ts"; import { KVPrimitivesManifestCache } from "../plugos/manifest_cache.ts"; @@ -75,7 +75,7 @@ export class ServerSystem { async init(awaitIndex = false) { this.ds = new DataStore( this.kvPrimitives, - buildQueryFunctions(this.allKnownPages), + buildQueryFunctions(this.allKnownPages, this.system), ); this.system = new System( @@ -132,7 +132,7 @@ export class ServerSystem { systemSyscalls(this.system, this.readOnlyMode), mqSyscalls(mq), languageSyscalls(), - handlebarsSyscalls(), + templateSyscalls(this.ds.functionMap), dataStoreReadSyscalls(this.ds), codeWidgetSyscalls(codeWidgetHook), markdownSyscalls(), diff --git a/web/client.ts b/web/client.ts index 14d461e0..283aac02 100644 --- a/web/client.ts +++ b/web/client.ts @@ -56,11 +56,12 @@ import { markFullSpaceIndexComplete, } from "../common/space_index.ts"; import { LimitedMap } from "$sb/lib/limited_map.ts"; -import { renderHandlebarsTemplate } from "../common/syscalls/handlebars.ts"; +import { renderTheTemplate } from "../common/syscalls/template.ts"; import { buildQueryFunctions } from "../common/query_functions.ts"; import { PageRef } from "$sb/lib/page.ts"; import { ReadOnlySpacePrimitives } from "../common/spaces/ro_space_primitives.ts"; import { KvPrimitives } from "../plugos/lib/kv_primitives.ts"; +import { builtinFunctions } from "$sb/lib/builtin_query_functions.ts"; const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/; const autoSaveInterval = 1000; @@ -140,10 +141,8 @@ export class Client { `${this.dbPrefix}_state`, ); await stateKvPrimitives.init(); - this.stateDataStore = new DataStore( - stateKvPrimitives, - buildQueryFunctions(this.allKnownPages), - ); + + this.stateDataStore = new DataStore(stateKvPrimitives); // Setup message queue this.mq = new DataStoreMQ(this.stateDataStore); @@ -152,7 +151,6 @@ export class Client { // Timeout after 5s, retries 3 times, otherwise drops the message (no DLQ) this.mq.requeueTimeouts(5000, 3, true).catch(console.error); }, 20000); // Look to requeue every 20s - // Event hook this.eventHook = new EventHook(); @@ -165,6 +163,12 @@ export class Client { window.silverBulletConfig.readOnly, ); + // Swap in the expanded function map + this.stateDataStore.functionMap = buildQueryFunctions( + this.allKnownPages, + this.system.system, + ); + const localSpacePrimitives = await this.initSpace(); this.syncService = this.syncMode @@ -225,7 +229,7 @@ export class Client { } await this.loadPlugs(); - this.initNavigator(); + await this.initNavigator(); await this.initSync(); this.loadCustomStyles().catch(console.error); @@ -415,9 +419,11 @@ export class Client { } } - private initNavigator() { + private async initNavigator() { this.pageNavigator = new PathPageNavigator(this); + await this.pageNavigator.init(); + this.pageNavigator.subscribe(async (pageState) => { console.log("Now navigating to", pageState); @@ -554,7 +560,7 @@ export class Client { new DataStoreSpacePrimitives( new DataStore( spaceKvPrimitives, - buildQueryFunctions(this.allKnownPages), + buildQueryFunctions(this.allKnownPages, this.system.system), ), ), this.plugSpaceRemotePrimitives, @@ -927,7 +933,12 @@ export class Client { ) { if (!pageRef.page) { pageRef.page = cleanPageRef( - renderHandlebarsTemplate(this.settings.indexPage, {}, {}), + await renderTheTemplate( + this.settings.indexPage, + {}, + {}, + builtinFunctions, + ), ); } diff --git a/web/client_system.ts b/web/client_system.ts index 91ed96e1..977388f5 100644 --- a/web/client_system.ts +++ b/web/client_system.ts @@ -33,7 +33,7 @@ import { import { DataStore } from "../plugos/lib/datastore.ts"; import { MessageQueue } from "../plugos/lib/mq.ts"; import { languageSyscalls } from "../common/syscalls/language.ts"; -import { handlebarsSyscalls } from "../common/syscalls/handlebars.ts"; +import { templateSyscalls } from "../common/syscalls/template.ts"; import { codeWidgetSyscalls } from "./syscalls/code_widget.ts"; import { clientCodeWidgetSyscalls } from "./syscalls/client_code_widget.ts"; import { KVPrimitivesManifestCache } from "../plugos/manifest_cache.ts"; @@ -162,7 +162,7 @@ export class ClientSystem { markdownSyscalls(), assetSyscalls(this.system), yamlSyscalls(), - handlebarsSyscalls(), + templateSyscalls(this.ds.functionMap), codeWidgetSyscalls(this.codeWidgetHook), clientCodeWidgetSyscalls(), languageSyscalls(), diff --git a/web/components/fuse_search.ts b/web/components/fuse_search.ts index 6a796277..3dfa080c 100644 --- a/web/components/fuse_search.ts +++ b/web/components/fuse_search.ts @@ -35,6 +35,9 @@ export const fuzzySearchAndSort = ( }, { name: "aliases", weight: 0.5, + }, { + name: "description", + weight: 0.3, }], includeScore: true, shouldSort: true, diff --git a/web/editor_state.ts b/web/editor_state.ts index 1e2bfc46..9de5edee 100644 --- a/web/editor_state.ts +++ b/web/editor_state.ts @@ -20,7 +20,6 @@ import { LanguageDescription, LanguageSupport, markdown, - searchKeymap, standardKeymap, syntaxHighlighting, ViewPlugin, diff --git a/web/hooks/slash_command.ts b/web/hooks/slash_command.ts index c85798c2..90f50be8 100644 --- a/web/hooks/slash_command.ts +++ b/web/hooks/slash_command.ts @@ -21,7 +21,7 @@ export type SlashCommandHookT = { slashCommand?: SlashCommandDef; }; -const slashCommandRegexp = /([^\w:]|^)\/[\w\-]*/; +const slashCommandRegexp = /([^\w:]|^)\/[\w#\-]*/; export class SlashCommandHook implements Hook { slashCommands = new Map(); @@ -104,6 +104,7 @@ export class SlashCommandHook implements Hook { options.push({ label: slashCompletion.label, detail: slashCompletion.detail, + boost: slashCompletion.order && -slashCompletion.order, apply: () => { // Delete slash command part this.editor.editorView.dispatch({ diff --git a/web/navigator.ts b/web/navigator.ts index f53a2956..848f36cb 100644 --- a/web/navigator.ts +++ b/web/navigator.ts @@ -2,7 +2,8 @@ import { safeRun } from "../common/util.ts"; import { PageRef, parsePageRef } from "$sb/lib/page.ts"; import { Client } from "./client.ts"; import { cleanPageRef } from "$sb/lib/resolve.ts"; -import { renderHandlebarsTemplate } from "../common/syscalls/handlebars.ts"; +import { renderTheTemplate } from "../common/syscalls/template.ts"; +import { builtinFunctions } from "$sb/lib/builtin_query_functions.ts"; export type PageState = PageRef & { scrollTop?: number; @@ -14,15 +15,23 @@ export type PageState = PageRef & { export class PathPageNavigator { navigationResolve?: () => void; - indexPage: string; + indexPage!: string; openPages = new Map(); constructor( private client: Client, ) { + } + + async init() { this.indexPage = cleanPageRef( - renderHandlebarsTemplate(client.settings.indexPage, {}, {}), + await renderTheTemplate( + this.client.settings.indexPage, + {}, + {}, + builtinFunctions, + ), ); } diff --git a/web/style.ts b/web/style.ts index 73ef9a77..7ec03511 100644 --- a/web/style.ts +++ b/web/style.ts @@ -36,7 +36,8 @@ export default function highlightStyles() { { tag: t.literal, class: "sb-literal" }, { tag: t.keyword, class: "sb-keyword" }, { tag: t.list, class: "sb-list" }, - // { tag: t.def, class: "sb-li" }, + { tag: t.operator, class: "sb-operator" }, + { tag: t.regexp, class: "sb-string" }, { tag: t.string, class: "sb-string" }, { tag: t.number, class: "sb-number" }, { tag: [t.regexp, t.escape, t.special(t.string)], class: "sb-string2" }, @@ -52,5 +53,8 @@ export default function highlightStyles() { { tag: ct.NakedURLTag, class: "sb-naked-url" }, { tag: ct.TaskDeadlineTag, class: "sb-task-deadline" }, { tag: ct.NamedAnchorTag, class: "sb-named-anchor" }, + + { tag: ct.DirectiveMarkTag, class: "sb-directive-mark" }, + { tag: ct.DirectiveTag, class: "sb-directive" }, ]); } diff --git a/web/styles/colors.scss b/web/styles/colors.scss index e8e66f98..e4ea13fb 100644 --- a/web/styles/colors.scss +++ b/web/styles/colors.scss @@ -251,6 +251,20 @@ background-color: var(--editor-code-background-color); } + .sb-line-fenced-code .sb-code { + color: var(--editor-code-color); + } + + .sb-directive-mark { + color: var(--editor-directive-color); + font-weight: bold; + } + + .sb-directive { + color: var(--editor-directive-color); + font-weight: bold; + } + .sb-highlight { background-color: var(--editor-highlight-background-color); } @@ -280,6 +294,10 @@ color: var(--editor-code-number-color); } + .sb-operator { + color: var(--editor-code-operator-color); + } + .sb-meta { color: var(--editor-meta-color); } diff --git a/web/styles/editor.scss b/web/styles/editor.scss index 6814141a..abe4727b 100644 --- a/web/styles/editor.scss +++ b/web/styles/editor.scss @@ -26,6 +26,10 @@ } } + .cm-line:has(.sb-directive) { + background-color: var(--editor-directive-background-color); + } + .sb-line-h1 { font-size: 1.5em; } @@ -306,7 +310,7 @@ } .sb-wiki-link { - cursor: pointer; + // cursor: pointer; } .sb-task-mark { diff --git a/web/styles/theme.scss b/web/styles/theme.scss index 8462ac7a..2e94fff6 100644 --- a/web/styles/theme.scss +++ b/web/styles/theme.scss @@ -63,6 +63,7 @@ html { --editor-hashtag-border-color: #0120416b; --editor-ruler-color: rgb(76, 75, 75); --editor-naked-url-color: var(--link-color); + --editor-code-color: #7c828e; --editor-link-color: var(--link-color); --editor-link-url-color: var(--link-color); --editor-link-meta-color: var(--meta-subtle-color); @@ -90,8 +91,9 @@ html { --editor-code-comment-color: var(--meta-subtle-color); --editor-code-variable-color: #024866; --editor-code-typename-color: #038138; - --editor-code-string-color: #440377; - --editor-code-number-color: #440377; + --editor-code-string-color: #540293; + --editor-code-number-color: #01833c; + --editor-code-operator-color: #808080; --editor-code-info-color: var(--subtle-color); --editor-code-atom-color: #5a0000; --editor-admonition-note-border-color: rgb(0, 184, 212); @@ -105,6 +107,10 @@ html { --editor-task-marker-color: var(--subtle-color); --editor-task-state-color: var(--subtle-color); + --editor-directive-mark-color: #6c0101; + --editor-directive-color: #696969; + --editor-directive-background-color: #ebebeb7d; + --ui-font: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", diff --git a/web/syscalls/datastore.proxy.ts b/web/syscalls/datastore.proxy.ts index a11acd7e..594a2888 100644 --- a/web/syscalls/datastore.proxy.ts +++ b/web/syscalls/datastore.proxy.ts @@ -11,5 +11,6 @@ export function dataStoreProxySyscalls(client: Client): SysCallMapping { "datastore.batchGet", "datastore.query", "datastore.get", + "datastore.listFunctions", ]); } diff --git a/website/Attributes.md b/website/Attributes.md index 47f9d1e6..6508f70c 100644 --- a/website/Attributes.md +++ b/website/Attributes.md @@ -33,7 +33,7 @@ However, usually, [[Frontmatter]] is used for this purpose instead. Example query: ```query -page where name = "{{@page.name}}" select name, pageAttribute +page where name = @page.name select name, pageAttribute ``` This attaches an attribute to an item: diff --git a/website/Blocks.md b/website/Blocks.md index f2a5d5ce..0aacbbd1 100644 --- a/website/Blocks.md +++ b/website/Blocks.md @@ -2,7 +2,7 @@ Blocks use the fenced code block notation of [[Markdown]], and assign special be The general syntax is: - ```block-type + ```template-type block configuration ``` diff --git a/website/CHANGELOG.md b/website/CHANGELOG.md index 5e4a2f76..f7e17360 100644 --- a/website/CHANGELOG.md +++ b/website/CHANGELOG.md @@ -3,12 +3,37 @@ release. --- -## Edge -$edge +## 0.7.0 +I know what you’re thinking: another “major” minor release, so quickly? -_The changes below are not yet released “properly.” To try them out early, check out [the docs on edge](https://community.silverbullet.md/t/living-on-the-edge-builds/27)._ +Yeah... we’re swapping out some of the guts of SilverBullet here, so this warrants the version bump. Honestly, this should have been 100.0, but you know... -* No edgy updates since 0.6.1. Hang tight! +This is the one where the template engine SilverBullet is swapped out and no longer using [Handlebars.js](https://handlebarsjs.com/) under the hood. We’d like to thank handlebars for its effort, but everything has to end some day. It’s time to take our destiny into our own hands. + +SilverBullet now uses its own custom [[Template Language]] that is _way, way, way_ dare I say _way_ more powerful than Handlebars. I put in a lot of effort to ensure backwards compatibility, so unless you did some obscure things in your templates, everything should keep working as is. If not, please reach out on our [community](https://community.silverbullet.md) and I’ll do my best to support you in this journey that for sure will be worth it. + +Trust me, I’m an engineer. + +> **note** **Upgrade note** +> While the [[Library/Core]] library that you (probably) imported previously will keep working (I hope), I would recommend you do another import overwriting the old files. Mainly because many templates have been rebuilt to leverage the new features of our new [[Template Language]] and therefore much cleaner. So please follow the [[Library/Core#Installation]] instructions, and hit “Ok” about 30 times. + +So, what’s the fuss all about? + +* **Soft breaking change**: `include` is the new name of the old `template` [[Blocks]]. Previously, a template block’s primary function was to include an external template or page — that role has now been replaced with [[Live Templates#Include]]. If you used `template` blocks in the past, you will see a warning squiggly warning appear suggesting you turn it into an `include` block, which is functionally equivalent to the old `template.` To fix this across your space, simply replace triple-backtick `template` with triple-backtick `include` across all your files and you should be done. That said, SilverBullet will detect if you’re using a legacy syntax `template` blocks and automatically interpret them as `include` blocks instead so _nothing should break_. Famous last words. +* So, funny story, we’re also **instantly deprecating** these new `include` blocks and suggest you `template` all the things, see [[Live Templates#Recommended alternative]] for the rationale behind this insanity. You will ask: “So what is he going to deprecate next? [[Live Queries]]? _Hahaha!_” Honestly, probably yes, because everything those can do, templates do better — basically. But let’s take it slow for now. One step at a time, as my mom says. +* **`template`s are the new hotness**. The body of a `template` block is now interpreted as the new and shiny [[Template Language]]! This means you get syntax highlighting and (best effort) code completion for all your markdown and template directives endeavors. +* The [[Template Language]] and [[Query Language]] have been unified. It may not have been visible to everybody, but the handlebars syntax and SB’s own query language were different before. No longer. +* The [[Expression Language]] used as part of the new [[Template Language]] has been significantly expanded. Some notable new features: + * It now supports some basics like a `not` operator, and ternary operator (`bla ? true : false`). I know, the level of innovation here is crazy. + * It now supports (sub) queries as values, simply wrap any query in `{` and `}` and you’re good to go: [[Expression Language#Queries]] + * It now supports _page references_ as values: [[Expression Language#Page references]] + * It now (officially) supports function calls, and while I’ve kept the set of built-in [[Functions]] deliberately small for now — this is where the big wins are going to be gained in future releases: [[Expression Language#Function calls]]. Let me know in the [community](https://community.silverbullet.md) what functions you’d like to see. _Even the ability to expand the set of functions using JavaScript from within your space_ is now on the table for future iterations. +* A [[CHANGELOG]] entry is never not going to do this justice. I recommend you simply (re)read the following pages and check out the examples to have your 🤯 and start to think what you can do with this: + * [[Templates]] + * [[Template Language]] + * [[Query Language]] + * [[Expression Language]] +* Ah yes, you can also still take notes with SilverBullet. That still works. I think. ## 0.6.1 * Tag pages: when you click on a #tag you will now be directed to a page that shows all pages, tasks, items and paragraphs tagged with that tag. diff --git a/website/Expression Language.md b/website/Expression Language.md new file mode 100644 index 00000000..734208ca --- /dev/null +++ b/website/Expression Language.md @@ -0,0 +1,111 @@ +SilverBullet has a simple expression language that is used by the [[Query Language]], [[Template Language]] and `where` condition in [[Live Template Widgets]]. + +Examples in this page will be demonstrated by embedding expressions inside of a [[Templates|template]]. + +While a custom language, it takes a lot of inspiration from JavaScript and SQL, but includes some features very specific to SilverBullet, including syntax for embedded queries and page references. + +# Primitive values +* strings: `"a string"`, `'a string'` or `a string` (with backticks). Escaping is not currently supported, so use your quotes wisely. +* numbers: `10` +* booleans: `true` or `false` +* regular expressions: `/[a-z]+/` +* null: `null` +* lists: `["value 1", 10, false]` +* objects: `{"name": "Jack", "age": 1232}` +* attributes: + * `.` for the current object + * `attr` for the current object’s attribute with the name `attr` + * `.attr` to access an attribute from the object that `` evaluates to +* variables: `@page` + +## Examples +```template +String expression: {{"This is a string"}} +List expression: {{[1, 2, 3]}} +Attribute of variable: {{@page.name}} +``` + +# Function calls +* `functionName(argument1, argument2, ...)`: call function `functionName` +* `functionName`: call `functionName` without any arguments + +## Examples +```template +Today with argument list: {{today()}} +Today without an argument list: {{today}} +``` + +## Supported functions +[[Functions]] + +# Queries +Expression can include [[Query Language|queries]] using the `{ page limit 1 }` syntax. Queries evaluate to one of two things: + +* Queries with a `render` or `render all` clause will evaluate a string, where the referenced template is applied (similar to [[Live Queries]]). +* Queries _without_ a `clause` will evaluate to a list of results. + +Note that you can reference variables inside your query as well. + +## Example +Incomplete task: +* [ ] This task is not yet complete + +```template +Number of incomplete tasks on this page: {{count({task where not done and page = @page.name})}} + +A rendered query: +{{{page limit 3 render [[Library/Core/Query/Page]]}}} +``` + +# Page references +Page references use the `[[page name]]` syntax and evaluate to the content of the referenced page (as a string), this makes them a good candidate to be used in conjunction with [[Functions#template(text, value)]] or to simply inline another page: + +```template +Including another page directly, without template rendering: {{[[internal/test page]]}} + +And rendered as a template: {{template([[internal/test page]], "actual value")}} +``` + +# Logical expressions +* and: `name = "this" and age > 10` +* or: `name = "this" or age > 10` + +## Examples +```template +Simple boolean expression: {{"pete" = "pete" or "hank" = "pete"}} +``` + +# Operators +* `=` equals + * For scalar values this performs an equivalence test (e.g. `10 = 10`) + * If the left operand is an array and the right operand is _not_, this will check if the right operand is _included_ in the left operand’s value, e.g. `[1, 2, 3] = 2` will be true. + * If both operands are arrays, they will be compared for equivalence ignoring order, so this will be true: `[1, 2, 3] = [3, 2, 1]` +* `!=` the exact inverse of the meaning of `=`, e.g. `name != "Pete"` +* `<` less than, e.g. `age < 10` +* `<=` less than or equals, e.g. `age <= 10` +* `>` greater than, e.g. `age > 10` +* `>=` greater than or equals, e.g. `age >= 10` +* `=~` to match against a regular expression, e.g. `name =~ /^template\//` +* `!=~` to not match a regular expression, e.g. `name !=~ /^template\//` +* `in` member of a list (e.g. `prop in ["foo", "bar"]`) +* `+` addition (can also concatenate strings), e.g. `10 + 12` or `name + "!!!"` +* `-` subtraction, e.g. `10 - 12` +* `/` addition, e.g. `10 / 12` +* `*` multiplication, e.g. `10 * 12` +* `%` modulo, e.g. `10 % 12` +* `not ` or `!` to negate the truthiness value +* ` ? : ` + +## Operator precedence +Operator precedence follows standard rules, use parentheses when in doubt, e.g. `(age > 10) or (name = "Pete")` + +## Examples +```template +Some arithmatic: {{10 * 100 / 1000}} +Some regexp match: {{"hello" =~ /ell/}} +Some list check: {{["template", "other"] = "template"}} +{{8 > 3 ? "8 is larger than 3" : "8 is not larger than 3"}} +``` + +# Comments +Line comments are supported via `# this is a line comment` diff --git a/website/Functions.md b/website/Functions.md new file mode 100644 index 00000000..08c421b2 --- /dev/null +++ b/website/Functions.md @@ -0,0 +1,62 @@ +Here is a list of all functions you can use in SilverBullet’s [[Expression Language]]: + +# Lists +## count(list) +Count the number of elements in a list: + +```template +There are **{{count({page})}}** pages in this space. +``` + +## at(list, index) +Returns the `index`th element of `list` (starting from 0 of course). + +```template +Index 1 of our count to three is {{at([1, 2, 3], 1)}} +``` + +# Templates +## template(text, value) +Renders a template with an optional value: + +```template +{{template([[Library/Core/Query/Page]], {"name": "Some page"})}} +``` + + +# Date and time +## today() +Today’s date in a `YYYY-MM-dd` format. + +## tomorrow() +Tomorrow’s date in a `YYYY-MM-dd` format. + +## yesterday() +Yesterday’s date in `YYYY-MM-dd` format. + +## niceDate(timestamp) +Convert a unix timestamp to a `YYYY-MM-dd` format. + +## time() +Current time. + +# String manipulation +## replace(str, match, replacement) +Replace text in a string. `match` can either be a literal string or a regular expression: `replace("hello", "ell", "all")` would produce “hallo”, and `replace("hello", /l/, "b")` would produce “hebbo”. + +## json(obj) +Convert the argument to a JSON string (for debugging purposes). + +```template +The current page object: {{json(@page)}} +``` + +# Space related +## pageExists(name) +Checks if the page `name` exists: + +```template +This very page exists: {{pageExists(@page.name)}} + +And this one: {{pageExists("non-existing")}} +``` diff --git a/website/Install/Configuration.md b/website/Install/Configuration.md index 1f1da002..163e9c31 100644 --- a/website/Install/Configuration.md +++ b/website/Install/Configuration.md @@ -23,8 +23,7 @@ This is the default and simplest backend to use: a folder on disk. It is configu ## AWS S3 bucket storage It is also possible to use an S3 bucket as storage. For this, you need to create a bucket, create an IAM user and configure access to it appropriately. -Since S3 doesn’t support an efficient way to store custom metadata, this mode does require a [[ -]] configuration (see below) to keep all file metadata. +Since S3 doesn’t support an efficient way to store custom metadata, this mode does require a [[#Database]] configuration (see below) to keep all file metadata. S3 is configured as follows: diff --git a/website/Install/Deno.md b/website/Install/Deno.md index 8c67aedc..15b44dd2 100644 --- a/website/Install/Deno.md +++ b/website/Install/Deno.md @@ -9,7 +9,7 @@ $ curl -fsSL https://deno.land/install.sh | sh After having installed Deno, run: ```shell -$ deno install -f --name silverbullet --unstable -A https://get.silverbullet.md +$ deno install -f --name silverbullet --unstable-kv --unstable-worker-options -A https://get.silverbullet.md ``` You only have to do this once. This will download the currently _released_ version of SilverBullet onto your machine. diff --git a/website/Library/Core/Page/Maintenance.md b/website/Library/Core/Page/Maintenance.md index cde943dc..ea847955 100644 --- a/website/Library/Core/Page/Maintenance.md +++ b/website/Library/Core/Page/Maintenance.md @@ -4,33 +4,34 @@ We would like to keep our space clean, these are some tools that help you do tha This shows all internal links that are broken. ```template -query: | - link where toPage and pageExists(toPage) = false -template: | - {{#if .}} - {{#each .}} - * [[{{ref}}]]: broken link to [[{{toPage}}]] - {{/each}} - {{else}} - No broken links, all good! - {{/if}} +{{#let @brokenLinks = { + link where toPage and not pageExists(toPage) +}}} +{{#if @brokenLinks}} +{{#each @brokenLinks}} +* [[{{ref}}]]: broken link to [[{{toPage}}]] +{{/each}} +{{else}} +No broken links, all good! +{{/if}} +{{/let}} ``` # Conflict copies These are pages that have conflicted copies (as a result of sync), have a look at them as well as their original (non-conflicted versions) and decide which one to keep. ```template -query: | - page where name =~ /\.conflicted\.\d+$/ - select - name as conflictedName, - replace(name, /\.conflicted\.\d+$/, "") as cleanName -template: | - {{#if .}} - {{#each .}} - * [[{{cleanName}}]]: confict copy [[{{conflictedName}}]] - {{/each}} - {{else}} - No conflicting pages! - {{/if}} +{{#let @conflictPages = { + page where name =~ /\.conflicted\.\d+$/ + select name as conflictedName, + replace(name, /\.conflicted\.\d+$/, "") as cleanName +}}} +{{#if @conflictPages}} +{{#each @conflictPages}} +* [[{{cleanName}}]]: confict copy [[{{conflictedName}}]] +{{/each}} +{{else}} +No conflicting pages! +{{/if}} +{{/let}} ``` diff --git a/website/Library/Core/Page/Template Index.md b/website/Library/Core/Page/Template Index.md index 4758c023..320020db 100644 --- a/website/Library/Core/Page/Template Index.md +++ b/website/Library/Core/Page/Template Index.md @@ -1,23 +1,26 @@ This page lists all templates currently available in your space. # New Page -$newPage -These [[!silverbullet.md/Page Templates]] are available through the {[Page: From Template]} command. +These [[!silverbullet.md/Page Templates]] are available through the {[Page: From Template]} command: ```query template where hooks.newPage render [[Library/Core/Query/Template]] ``` # Snippets -$snippets These can be used as [[!silverbullet.md/Snippets]] via [[!silverbullet.md/Slash Commands]]: -```query -template where hooks.snippet render [[Library/Core/Query/Template]] +```template +{{#each { + template + where hooks.snippet + order by hooks.snippet.slashCommand +}}} +* [[{{ref}}|/{{hooks.snippet.slashCommand}}]] {{description}} +{{/each}} ``` # Widgets -$widgets Widgets can either be automatically attached to the top or bottom of pages (matching certain criteria) or used inline via [[!silverbullet.md/Live Templates]]. ## Top @@ -37,11 +40,11 @@ render [[Library/Core/Query/Template]] ``` ## Inline -Use these as `page` in [[!silverbullet.md/Live Templates]] to render useful things in your pages: +Use with [[!silverbullet.md/Live Templates#Include]] to render useful things in your pages: ```query template -where name =~ /\/Widget\// and hooks.top = null and hooks.bottom = null +where name =~ /\/Widget\// and not hooks.top and not hooks.bottom order by order render [[Library/Core/Query/Template]] ``` diff --git a/website/Library/Core/Query/Template.md b/website/Library/Core/Query/Template.md index 0bf123ab..51dee549 100644 --- a/website/Library/Core/Query/Template.md +++ b/website/Library/Core/Query/Template.md @@ -1,4 +1,7 @@ -#template +--- +tags: template +--- + * [[{{ref}}]] {{description}} {{#if usage}} * **Usage:** {{usage}} diff --git a/website/Library/Core/Snippet/each-directive.md b/website/Library/Core/Snippet/each-directive.md new file mode 100644 index 00000000..a10fb5e6 --- /dev/null +++ b/website/Library/Core/Snippet/each-directive.md @@ -0,0 +1,10 @@ +--- +description: (Template) Insert an `#each` directive +tags: template +hooks.snippet: + slashCommand: "#each" + order: 10 +--- +{{escapeDirective("#each |^|")}} + +{{escapeDirective("/each")}} \ No newline at end of file diff --git a/website/Library/Core/Snippet/if-directive.md b/website/Library/Core/Snippet/if-directive.md new file mode 100644 index 00000000..003990a5 --- /dev/null +++ b/website/Library/Core/Snippet/if-directive.md @@ -0,0 +1,10 @@ +--- +description: (Template) Insert an `#if` directive +tags: template +hooks.snippet: + slashCommand: "#if" + order: 10 +--- +{{escapeDirective("#if |^|")}} + +{{escapeDirective("/if")}} \ No newline at end of file diff --git a/website/Library/Core/Snippet/if-else-directive.md b/website/Library/Core/Snippet/if-else-directive.md new file mode 100644 index 00000000..638a231a --- /dev/null +++ b/website/Library/Core/Snippet/if-else-directive.md @@ -0,0 +1,12 @@ +--- +description: (Template) Insert `#if` directive with 'else' +tags: template +hooks.snippet: + slashCommand: "#if-else" + order: 10 +--- +{{escapeDirective("#if |^|")}} + +{{escapeDirective("else")}} + +{{escapeDirective("/if")}} \ No newline at end of file diff --git a/website/Library/Core/Snippet/let-directive.md b/website/Library/Core/Snippet/let-directive.md new file mode 100644 index 00000000..d0c0d807 --- /dev/null +++ b/website/Library/Core/Snippet/let-directive.md @@ -0,0 +1,10 @@ +--- +description: (Template) Insert a `#let` directive +tags: template +hooks.snippet: + slashCommand: "#let" + order: 10 +--- +{{escapeDirective("#let @|^| = ")}} + +{{escapeDirective("/let")}} \ No newline at end of file diff --git a/website/Library/Core/Snippet/task.md b/website/Library/Core/Snippet/task.md index 07b902b3..477d24de 100644 --- a/website/Library/Core/Snippet/task.md +++ b/website/Library/Core/Snippet/task.md @@ -1,6 +1,6 @@ --- tags: template -description: Turn the current line into a task +description: Make this a task hooks.snippet: slashCommand: task matchRegex: "^(\\s*)[\\-\\*]?\\s*(\\[[ xX]\\])?\\s*" diff --git a/website/Library/Core/Snippet/template.md b/website/Library/Core/Snippet/template.md index 24efd957..90ba62b0 100644 --- a/website/Library/Core/Snippet/template.md +++ b/website/Library/Core/Snippet/template.md @@ -1,8 +1,8 @@ --- -description: Insert an inline template +description: Insert a template tags: template hooks.snippet.slashCommand: "template" --- ```template -page: "[[|^|]]" +|^| ``` diff --git a/website/Library/Core/Widget/Linked Mentions.md b/website/Library/Core/Widget/Linked Mentions.md index a2c9554e..f518ae2e 100644 --- a/website/Library/Core/Widget/Linked Mentions.md +++ b/website/Library/Core/Widget/Linked Mentions.md @@ -3,16 +3,11 @@ description: Adds Linked Mentions to pages tags: template hooks.bottom.where: 'true' --- -```template -# We need to escape handlebars directives here, since we're embedding -# this template into a template (INCEPTION) -template: | - {{escape "#if ."}} - # Linked Mentions - {{escape "#each ."}} - * [[{{escape "ref"}}]]: `{{escape "snippet"}}` - {{escape "/each"}} - {{escape "/if"}} -query: | - link where toPage = "{{@page.name}}" and page != "{{@page.name}}" -``` +{{#let @linkedMentions = {link where toPage = @page.name and page != @page.name order by page}}} +{{#if @linkedMentions}} +# Linked Mentions +{{#each @linkedMentions}} +* [[{{ref}}]]: `{{snippet}}` +{{/each}} +{{/if}} +{{/let}} diff --git a/website/Library/Core/Widget/Linked Tasks.md b/website/Library/Core/Widget/Linked Tasks.md index 57412e2e..25f28627 100644 --- a/website/Library/Core/Widget/Linked Tasks.md +++ b/website/Library/Core/Widget/Linked Tasks.md @@ -5,5 +5,5 @@ description: | --- ```query -task where name =~ /\[\[{{escapeRegexp @page.name}}\]\]/ where done = false render [[Library/Core/Query/Task]] +task where name =~ /\[\[{{escapeRegexp(@page.name)}}\]\]/ where done = false render [[Library/Core/Query/Task]] ``` \ No newline at end of file diff --git a/website/Library/Core/Widget/Tagged Tasks.md b/website/Library/Core/Widget/Tagged Tasks.md index f9d4bd1c..f6966073 100644 --- a/website/Library/Core/Widget/Tagged Tasks.md +++ b/website/Library/Core/Widget/Tagged Tasks.md @@ -5,5 +5,5 @@ usage: Pass in the tag to filter on as the `value` of this template --- ```query -task where tags = "{{.}}" and done = false render [[Library/Core/Query/Task]] -``` +task where tags = "{{.}}" and not done render [[Library/Core/Query/Task]] +``` \ No newline at end of file diff --git a/website/Live Queries.md b/website/Live Queries.md index 61112122..0f2d4056 100644 --- a/website/Live Queries.md +++ b/website/Live Queries.md @@ -1,114 +1,9 @@ -Live Queries enable a (quasi) live view on various data sources, usually [[Objects]], and renders their results inline via [[Live Preview]] either as a template or using [[Templates]]. +Live Queries are a [[Blocks|block]] that generates a (quasi) live view on various data sources, usually [[Objects]], and renders their results inline via [[Live Preview]] either as a table or using [[Templates]]. -# Syntax -The syntax of live queries is inspired by [SQL](https://en.wikipedia.org/wiki/SQL). Below is a query that demonstrates some of the supported clauses. Hover over the result and click the edit icon to show the code that generates the view: -```query -page -order by lastModified desc -where size > 100 -select name -limit 10 -render [[Library/Core/Query/Page]] -``` -It’s most convenient to use the `/query` [[Snippets|snippet]] to insert a query in a page. +The syntax used is: -For those comfortable reading such things, [here you can find the full query grammar](https://github.com/silverbulletmd/silverbullet/blob/main/common/markdown_parser/query.grammar). + ```query + page limit 3 + ``` -The general syntax is to specify a `querySource` followed by a number of clauses that modify or restrict. If you haven’t already, check out how [[Objects]] work in SilverBullet. - -# Clauses -## where [[@expression]] -A `where` clause filters out all objects that do not match a certain condition. You can have multiple `where` clauses if you like, which will have the same effect as combining them with the `and` keyword. - -Here is a simple example based on a custom tag `#person` (see [[Objects]] on how this works): - -```#person -name: John -age: 7 ---- -name: Pete -age: 25 -``` - -To query all `person`s that are above age 21, we can use the following `where` clause: - -```query -person where page = "{{@page.name}}" and age > 21 -``` -## order by [[@expression]] -To sort results, an `order by` clause can be used, optionally with `desc` to order in descending order (ascending is the default): - -```query -person where page = "{{@page.name}}" order by age desc -``` -## limit [[@expression]] -To limit the number of results, you can use a `limit` clause: - -```query -person where page = "{{@page.name}}" limit 1 -``` -## select -You can use the `select` clause to select only specific attributes from the result set. You can use it either simply as `select attribute1, attribute2` but also select the value of certain expressions and give them a name via the `select age + 1 as nextYear` syntax: - -```query -person -where page = "{{@page.name}}" -select name, age, age + 1 as nextYear -``` - -## render each `[[template]]` and render all `[[template]]` -$render -By default, results are rendered as a table. To instead render results using [[Templates|a template]], use the `render` clause, which comes in two shapes `render each` where the template is instantiated for _each_ result (the `each` keyword is optional): - -```query -person -where page = "{{@page.name}}" -render each [[internal-template/person]] -``` - -And `render all` where the entire result set is passed to the template as a list so the template can do its own iteration using `#each`, which you could then use to e.g. build a table (using this [[internal-template/people]] template, for instance): - -```query -person -where page = "{{@page.name}}" -render all [[internal-template/people]] -``` -# Expressions -$expression - -Primitives: - -* strings: `"a string"` -* numbers: `10` -* booleans: `true` or `false` -* regular expressions: `/[a-z]+/` -* null: `null` -* lists: `["value 1", 10, false]` - -Attributes can be accessed via the `attribute` syntax, and nested attributes via `attribute.subattribute.subsubattribute`. - -Logical expressions: - -* and: `name = "this" and age > 10` -* or: `name = "this" or age > 10` - -Binary expressions: -* `=` equals. - * For scalar values this performs an equivalence test (e.g. `10 = 10`) - * If the left operand is an array and the right operand is _not_, this will check if the right operand is _included_ in the left operand’s value, e.g. `[1, 2, 3] = 2` will be true. - * If both operands are arrays, they will be compared for equivalence ignoring order, so this will be true: `[1, 2, 3] = [3, 2, 1]` -* `!=` the exact inverse of the meaning of `=`, e.g. `name != "Pete"` -* `<` less than, e.g. `age < 10` -* `<=` less than or equals, e.g. `age <= 10` -* `>` greater than, e.g. `age > 10` -* `>=` greater than or equals, e.g. `age >= 10` -* `=~` to match against a regular expression, e.g. `name =~ /^template\//` -* `!=~` to not match a regular expression, e.g. `name !=~ /^template\//` -* `in` member of a list (e.g. `prop in ["foo", "bar"]`) -* `+` addition (can also concatenate strings), e.g. `10 + 12` or `name + "!!!"` -* `-` subtraction, e.g. `10 - 12` -* `/` addition, e.g. `10 / 12` -* `*` multiplication, e.g. `10 * 12` -* `%` modulo, e.g. `10 % 12` - -Operator precedence follows standard rules, use parentheses when in doubt, e.g. `(age > 10) or (name = "Pete")` +Queries are written using SilverBullet’s [[Query Language]]. \ No newline at end of file diff --git a/website/Live Templates.md b/website/Live Templates.md index 841fd610..3e003481 100644 --- a/website/Live Templates.md +++ b/website/Live Templates.md @@ -1,40 +1,46 @@ -Live Templates are a type of [[Blocks|block]] that render [[Templates]] inline in a page. +Live Templates are a type of [[Blocks|block]] that render [[Templates]] written in [[Template Language]] inline in a page. -Template blocks are specified using [[Markdown]]‘s fenced code block notation using `template` as a language. The body of the block specifies the template to use, as well as any arguments to pass to it. +There are two variants of Live Templates: -Generally you’d use it in one of two ways, either using a `page` [[Templates|template]] reference, or an inline `template`: +* `template`: where the template is specified inline in a code block. +* `include`: where an external page (template) is _included_ and rendered. + +Template blocks are specified using [[Markdown]]‘s fenced code block notation using either `template` or `include` as its language. + +# Template +To specify a template to render inline, you can use the `template` block. The body is written in [[Template Language]]. + +```template +Today is {{today}} +``` + +# Include +> **warning** Deprecated, use templates instead +> Include template are primarily here to be a drop-in replacement for the old style template blocks, see below for a suggestion how to use those instead. + +A `template` block is configured using [[YAML]] in its body. The following attributes are supported: + +* `page`: the page to use as a template +* `value`: an (optional) value to pass to the template +* `raw`: a page reference to include in the page without processing it as a template. Here’s an example using `page`: -```template +```include page: "[[internal-template/today]]" ``` -And here’s an example using `template`: -```template -template: | - Today is {{today}}! -``` - -To pass a literal value to the template, you can specify the optional `value` attribute: -```template -template: | - Hello, {{name}}! Today is _{{today}}_ -value: - name: Pete -``` - -You can also pass in the result of a [[Live Queries|query]] as a value by setting the `query` attribute: - -```template -query: | - tag where parent = "page" select name -template: | - {{#each .}} - * #{{name}} - {{/each}} -``` - -If you want to include another _page_ (not necessarily a template) unprocessed (so without replacing template placeholders), you can use `raw`: -```template +If you want to include another _page_ (not necessarily a template). unprocessed (so without replacing template placeholders), you can use `raw`: +```include raw: "[[internal/test page]]" ``` + +## Recommended alternative +Instead of using the `include` block we recommend you use a `{{template([[template]])}}` directive in a `template` instead as it is more flexible and more natural. + +Look for yourself: +```template +{{template([[internal-template/today]])}} +``` + + + diff --git a/website/Objects.md b/website/Objects.md index 79347870..bf1e30f7 100644 --- a/website/Objects.md +++ b/website/Objects.md @@ -30,7 +30,7 @@ Every page in your space is available via the `page` tag. You can attach _additi In addition to `ref` and `tags`, the `page` tag defines a bunch of additional attributes as can be seen in this example query: ```query -page where name = "{{@page.name}}" +page where name = @page.name ``` Note that you can also query this page using the `example-tag` directly: @@ -69,7 +69,7 @@ upnext render [[Library/Core/Query/Task]] And can be queried as follows: ```query -taskstate where page = "{{@page.name}}" +taskstate where page = @page.name ``` ## template @@ -90,7 +90,7 @@ Here is an example of a #quote item using a custom [[Attributes|attribute]]: And then queried via the #quote tag: ```query -quote where page = "{{@page.name}}" and tag = "item" select name, by +quote where page = @page.name and tag = "item" select name, by ``` ## paragraph @@ -129,7 +129,7 @@ _Note_: this is the data source used for the {[Mentions: Toggle]} feature as wel Here is a query that shows all links that appear in this particular page: ```query -link where page = "{{@page.name}}" +link where page = @page.name ``` ## anchor @@ -138,14 +138,14 @@ link where page = "{{@page.name}}" Here is an example query: ```query -anchor where page = "{{@page.name}}" +anchor where page = @page.name ``` ## header Headers (lines starting with `#`, `##` etc.) are indexed as well and queriable. ```query -header where page = "{{@page.name}}" limit 3 +header where page = @page.name limit 3 ``` @@ -155,12 +155,12 @@ The ultimate meta tag is _tag_ itself, which indexes for all tags used, in which Here are the tags used/defined in this page: ```query -tag where page = "{{@page.name}}" select name, parent +tag where page = @page.name select name, parent ``` ## attribute This is another meta tag, which is used to index all [[Attributes]] used in your space. This is used by e.g. attribute completion in various contexts. You likely don’t need to use this tag directly, but it’s there. ```query -attribute where page = "{{@page.name}}" limit 1 +attribute where page = @page.name limit 1 ``` diff --git a/website/Plugs/Tasks.md b/website/Plugs/Tasks.md index 9d7e68cf..43f7d90b 100644 --- a/website/Plugs/Tasks.md +++ b/website/Plugs/Tasks.md @@ -41,7 +41,7 @@ When the cursor is positioned inside of a due date, the {[Task: Postpone]} comma All meta data (`done` status, `state`, `tags`, `deadline` and custom attributes) is extracted and available via the `task` query source to [[Live Queries]]: ```query -task where page = "{{@page.name}}" +task where page = @page.name ``` ## Rendering @@ -50,7 +50,7 @@ There is a [[!silverbullet.md/template/tasks/task]] template you can use to rend Try it (by clicking on the checkbox inside of the query): ```query -task where page = "{{@page.name}}" and name = "Remote toggle me" render [[Library/Core/Query/Task]] +task where page = @page.name and name = "Remote toggle me" render [[Library/Core/Query/Task]] ``` * [ ] Remote toggle me diff --git a/website/Query Language.md b/website/Query Language.md new file mode 100644 index 00000000..3c283d32 --- /dev/null +++ b/website/Query Language.md @@ -0,0 +1,78 @@ +The syntax of queries is inspired by [SQL](https://en.wikipedia.org/wiki/SQL). It can be used using [[Live Queries]] as well as via nested queries part of template expressions: [[Expression Language#Queries]]. + +Let’s start with a query that demonstrates some of the supported clauses. Hover over the result and click the edit icon to show the code that generates the view: + +```query +page +order by lastModified desc +where size > 100 +select name +limit 10 +render [[Library/Core/Query/Page]] +``` + +It’s most convenient to use the `/query` [[Snippets|snippet]] to insert a query in a page. + +A query is formulated specifying a `querySource` followed by a number of clauses that modify or restrict the result set. If you haven’t already, check out how [[Objects]] work in SilverBullet. + +Every time `` is referenced here, an expression using the [[Expression Language]] can be used. + +# Clauses +## where +A `where` clause filters out all objects that do not match a certain condition. You can have multiple `where` clauses if you like, which will have the same effect as combining them with the `and` keyword. + +Here is a simple example based on a custom tag `#person` (see [[Objects]] on how this works): + +```#person +name: John +age: 7 +--- +name: Pete +age: 25 +``` + +To query all `person`s that are above age 21, we can use the following `where` clause: + +```query +person where page = @page.name and age > 21 +``` + +## order by +To sort results, an `order by` clause can be used, optionally with `desc` to order in descending order (ascending is the default): + +```query +person where page = @page.name order by age desc +``` + +## limit +To limit the number of results, you can use a `limit` clause: + +```query +person where page = @page.name limit 1 +``` + +## select +You can use the `select` clause to select only specific attributes from the result set. You can use it either simply as `select attribute1, attribute2` but also select the value of certain expressions and give them a name via the `select age + 1 as nextYear` syntax: + +```query +person +where page = @page.name +select name, age, age + 1 as nextYear +``` + +## render each