From c157e979f0dbaaa3a28859c363119aee1fe48512 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Sun, 23 Mar 2025 12:48:25 +0100 Subject: [PATCH] WIP --- common/common_system.ts | 108 ----- common/space_index.ts | 28 -- common/space_lua/stdlib/space_lua.ts | 7 +- common/spaces/evented_space_primitives.ts | 21 +- common/spaces/plug_space_primitives.ts | 8 +- common/syscalls/command.ts | 20 +- common/syscalls/event.ts | 12 +- common/syscalls/lua.ts | 10 +- lib/data/mq.datastore.ts | 43 +- lib/data/mq.ts | 20 - lib/plugos/hooks/mq.ts | 6 +- lib/plugos/syscalls/datastore.ts | 10 +- lib/plugos/syscalls/mq.ts | 7 +- plug-api/syscalls/mq.ts | 8 + plugs/core/Library/Std/Space Overview.md | 21 + plugs/core/Library/Std/Widgets.md | 4 +- plugs/core/std.ts | 22 +- plugs/index/api.ts | 1 + plugs/index/command.ts | 19 +- plugs/index/document.ts | 2 +- web/client.ts | 90 ++-- web/client_system.ts | 162 +++++-- web/cm_plugins/lua_directive.ts | 11 +- web/components/page_navigator.tsx | 1 + web/fuse_search.ts | 4 +- web/hooks/slash_command.ts | 28 +- web/service_worker.ts | 11 +- website/CHANGELOG.md | 446 +------------------- website/Cloud Links.md | 1 - website/Expression Language.md | 116 ----- website/Federation.md | 25 -- website/Functions.md | 70 --- website/Libraries.md | 68 --- website/Library/Core/Page/Maintenance.md | 39 +- website/Library/Core/Page/Space Overview.md | 38 -- website/Library/Core/Page/Template Index.md | 52 --- website/Live Embeds.md | 13 - website/Live Queries.md | 16 - website/Live Template Widgets.md | 26 -- website/Live Templates.md | 45 -- website/Markdown/Command links.md | 10 - website/Object Decorators.md | 81 ---- website/Plug Management.md | 11 - website/Query Language.md | 78 ---- website/Schema.md | 102 ----- website/SilverBullet.md | 9 +- website/Space Config.md | 25 -- website/Space Script.md | 243 ----------- website/Template Language.md | 83 ---- website/Templates.md | 57 --- website/internal-template/people.md | 6 - website/internal-template/person.md | 2 - website/internal-template/plug-widget.md | 7 - website/internal-template/plug.md | 5 - website/internal-template/today.md | 2 - website/internal/test page with template.md | 1 - website/internal/test page.md | 4 - 57 files changed, 411 insertions(+), 1954 deletions(-) delete mode 100644 common/common_system.ts delete mode 100644 common/space_index.ts delete mode 100644 lib/data/mq.ts create mode 100644 plugs/core/Library/Std/Space Overview.md delete mode 100644 website/Cloud Links.md delete mode 100644 website/Expression Language.md delete mode 100644 website/Federation.md delete mode 100644 website/Functions.md delete mode 100644 website/Libraries.md delete mode 100644 website/Library/Core/Page/Space Overview.md delete mode 100644 website/Library/Core/Page/Template Index.md delete mode 100644 website/Live Embeds.md delete mode 100644 website/Live Queries.md delete mode 100644 website/Live Template Widgets.md delete mode 100644 website/Live Templates.md delete mode 100644 website/Markdown/Command links.md delete mode 100644 website/Object Decorators.md delete mode 100644 website/Plug Management.md delete mode 100644 website/Query Language.md delete mode 100644 website/Schema.md delete mode 100644 website/Space Config.md delete mode 100644 website/Space Script.md delete mode 100644 website/Template Language.md delete mode 100644 website/Templates.md delete mode 100644 website/internal-template/people.md delete mode 100644 website/internal-template/person.md delete mode 100644 website/internal-template/plug-widget.md delete mode 100644 website/internal-template/plug.md delete mode 100644 website/internal-template/today.md delete mode 100644 website/internal/test page with template.md delete mode 100644 website/internal/test page.md diff --git a/common/common_system.ts b/common/common_system.ts deleted file mode 100644 index 4de2a825..00000000 --- a/common/common_system.ts +++ /dev/null @@ -1,108 +0,0 @@ -import type { AppCommand } from "$lib/command.ts"; -import type { CommandHook } from "./hooks/command.ts"; -import type { PlugNamespaceHook } from "$common/hooks/plug_namespace.ts"; -import type { SilverBulletHooks } from "../lib/manifest.ts"; -import { ScriptEnvironment } from "./space_script.ts"; -import type { EventHook } from "./hooks/event.ts"; -import type { DataStore } from "$lib/data/datastore.ts"; -import type { System } from "$lib/plugos/system.ts"; -import type { CodeWidgetHook } from "../web/hooks/code_widget.ts"; -import type { SlashCommandHook } from "../web/hooks/slash_command.ts"; -import type { DataStoreMQ } from "$lib/data/mq.datastore.ts"; -import type { ParseTree } from "../plug-api/lib/tree.ts"; -import { SpaceLuaEnvironment } from "$common/space_lua.ts"; -import type { DocumentEditorHook } from "../web/hooks/document_editor.ts"; - -const mqTimeout = 10000; // 10s -const mqTimeoutRetry = 3; - -export abstract class CommonSystem { - system!: System; - - // Hooks - commandHook!: CommandHook; - slashCommandHook?: SlashCommandHook; - namespaceHook!: PlugNamespaceHook; - codeWidgetHook!: CodeWidgetHook; - documentEditorHook!: DocumentEditorHook; - - readonly allKnownFiles = new Set(); - readonly spaceScriptCommands = new Map(); - scriptEnv: ScriptEnvironment = new ScriptEnvironment(); - spaceLuaEnv = new SpaceLuaEnvironment(); - scriptsLoaded: boolean = false; - - constructor( - protected mq: DataStoreMQ, - public ds: DataStore, - public eventHook: EventHook, - public readOnlyMode: boolean, - protected enableSpaceScript: boolean, - ) { - setInterval(() => { - mq.requeueTimeouts(mqTimeout, mqTimeoutRetry, true).catch(console.error); - }, 20000); // Look to requeue every 20s - } - - async loadSpaceScripts() { - if (this.enableSpaceScript) { - this.scriptEnv = new ScriptEnvironment(); - try { - await this.spaceLuaEnv.reload(this.system); - } catch (e: any) { - console.error("Error loading space-script:", e.message); - } - - // Reset the space script commands - this.spaceScriptCommands.clear(); - for (const [name, command] of Object.entries(this.scriptEnv.commands)) { - this.spaceScriptCommands.set(name, command); - } - - // Inject the registered events in the event hook - this.eventHook.scriptEnvironment = this.scriptEnv; - - this.commandHook.throttledBuildAllCommands(); - if (this.slashCommandHook) { - // Only on client - this.slashCommandHook.throttledBuildAllCommands(); - } - } - - this.scriptsLoaded = true; - } - - invokeSpaceFunction(name: string, args: any[]) { - const fn = this.scriptEnv.functions[name]; - if (!fn) { - throw new Error(`Function ${name} not found`); - } - return fn(...args); - } - - async applyAttributeExtractors( - tags: string[], - text: string, - tree: ParseTree, - ): Promise> { - let resultingAttributes: Record = {}; - for (const tag of tags) { - const extractors = this.scriptEnv.attributeExtractors[tag]; - if (!extractors) { - continue; - } - for (const fn of extractors) { - const extractorResult = await fn(text, tree); - if (extractorResult) { - // Merge the attributes in - resultingAttributes = { - ...resultingAttributes, - ...extractorResult, - }; - } - } - } - - return resultingAttributes; - } -} diff --git a/common/space_index.ts b/common/space_index.ts deleted file mode 100644 index fa63bd2c..00000000 --- a/common/space_index.ts +++ /dev/null @@ -1,28 +0,0 @@ -import type { DataStore } from "$lib/data/datastore.ts"; -import type { System } from "$lib/plugos/system.ts"; - -const indexVersionKey = ["$indexVersion"]; - -// Bump this one every time a full reindex is needed -const desiredIndexVersion = 7; - -let indexOngoing = false; - -export async function ensureSpaceIndex(ds: DataStore, system: System) { - const currentIndexVersion = await ds.get(indexVersionKey); - - console.info("Current space index version", currentIndexVersion); - - if (currentIndexVersion !== desiredIndexVersion && !indexOngoing) { - console.info("Performing a full space reindex, this could take a while..."); - indexOngoing = true; - await system.invokeFunction("index.reindexSpace", [true]); // noClean = true - console.info("Full space index complete."); - await markFullSpaceIndexComplete(ds); - indexOngoing = false; - } -} - -export async function markFullSpaceIndexComplete(ds: DataStore) { - await ds.set(indexVersionKey, desiredIndexVersion); -} diff --git a/common/space_lua/stdlib/space_lua.ts b/common/space_lua/stdlib/space_lua.ts index 9ba4722f..cd7ff55b 100644 --- a/common/space_lua/stdlib/space_lua.ts +++ b/common/space_lua/stdlib/space_lua.ts @@ -9,6 +9,7 @@ import { LuaTable, luaToString, luaValueToJS, + singleResult, } from "$common/space_lua/runtime.ts"; /** @@ -83,11 +84,11 @@ export async function interpolateLuaString( try { const parsedExpr = parseExpressionString(expr); const env = createAugmentedEnv(sf, envAugmentation); - const luaResult = luaValueToJS( - await evalExpression(parsedExpr, env, sf), + const luaResult = await luaValueToJS( + singleResult(await evalExpression(parsedExpr, env, sf)), sf, ); - result += luaToString(luaResult); + result += await luaToString(luaResult); } catch (e: any) { throw new LuaRuntimeError( `Error evaluating "${expr}": ${e.message}`, diff --git a/common/spaces/evented_space_primitives.ts b/common/spaces/evented_space_primitives.ts index 70acca41..6bfec61b 100644 --- a/common/spaces/evented_space_primitives.ts +++ b/common/spaces/evented_space_primitives.ts @@ -21,11 +21,12 @@ export class EventedSpacePrimitives implements SpacePrimitives { initialFileListLoad: boolean; public enabled = true; + private spaceSnapshot: Record = {}; constructor( private wrapped: SpacePrimitives, private eventHook: EventHook, - private spaceSnapshot: Record = {}, + private ignorePathPrefixes: string[] = [], ) { // Translate file change events for documents into document:index events this.eventHook.addLocalListener( @@ -79,10 +80,13 @@ export class EventedSpacePrimitives implements SpacePrimitives { // New file scenario !oldHash && !this.initialFileListLoad ) || ( - // Changed file scenario - oldHash && - oldHash !== newHash - ) + // Changed file scenario + oldHash && + oldHash !== newHash + ) && + !this.ignorePathPrefixes.some((prefix) => + meta.name.startsWith(prefix) + ) ) { console.log("Detected file change", meta.name, oldHash, newHash); await this.dispatchEvent( @@ -98,6 +102,13 @@ export class EventedSpacePrimitives implements SpacePrimitives { } for (const deletedFile of deletedFiles) { + if ( + this.ignorePathPrefixes.some((prefix) => + deletedFile.startsWith(prefix) + ) + ) { + continue; + } delete this.spaceSnapshot[deletedFile]; await this.dispatchEvent("file:deleted", deletedFile); diff --git a/common/spaces/plug_space_primitives.ts b/common/spaces/plug_space_primitives.ts index fba1fdcd..5bc34d1d 100644 --- a/common/spaces/plug_space_primitives.ts +++ b/common/spaces/plug_space_primitives.ts @@ -52,14 +52,12 @@ export class PlugSpacePrimitives implements SpacePrimitives { async fetchFileList(): Promise { const allFiles: FileMeta[] = []; const alreadySeenFiles = new Set(); - for (const { plug, name, operation, env } of this.hook.spaceFunctions) { - if ( - operation === "listFiles" && - (!this.env || !env || (env && env === this.env)) - ) { + for (const { plug, name, operation } of this.hook.spaceFunctions) { + if (operation === "listFiles") { try { for (const pm of await plug.invoke(name, [])) { allFiles.push(pm); + // console.log("Adding file from plug space", pm.name); alreadySeenFiles.add(pm.name); } } catch (e: any) { diff --git a/common/syscalls/command.ts b/common/syscalls/command.ts index a2b20502..408bc556 100644 --- a/common/syscalls/command.ts +++ b/common/syscalls/command.ts @@ -8,14 +8,14 @@ import { LuaStackFrame, luaValueToJS, } from "$common/space_lua/runtime.ts"; -import type { CommonSystem } from "$common/common_system.ts"; +import type { ClientSystem } from "../../web/client_system.ts"; export type CallbackCommandDef = CommandDef & { run: ILuaFunction; }; export function commandSyscalls( - commonSystem: CommonSystem, + clientSystem: ClientSystem, ): SysCallMapping { return { /** @@ -28,12 +28,12 @@ export function commandSyscalls( def: CallbackCommandDef, ) => { console.log("Registering Lua command: ", def.name); - commonSystem.scriptEnv.registerCommand( + clientSystem.scriptEnv.registerCommand( def, async (...args: any[]) => { const tl = await buildThreadLocalEnv( - commonSystem.system, - commonSystem.spaceLuaEnv.env, + clientSystem.system, + clientSystem.spaceLuaEnv.env, ); const sf = new LuaStackFrame(tl, null); try { @@ -42,7 +42,7 @@ export function commandSyscalls( sf, ); } catch (e: any) { - await handleLuaError(e, commonSystem.system); + await handleLuaError(e, clientSystem.system); } }, ); @@ -51,12 +51,12 @@ export function commandSyscalls( _ctx, def: CallbackCommandDef, ) => { - commonSystem.scriptEnv.registerSlashCommand( + clientSystem.scriptEnv.registerSlashCommand( def, async (...args: any[]) => { const tl = await buildThreadLocalEnv( - commonSystem.system, - commonSystem.spaceLuaEnv.env, + clientSystem.system, + clientSystem.spaceLuaEnv.env, ); const sf = new LuaStackFrame(tl, null); try { @@ -65,7 +65,7 @@ export function commandSyscalls( sf, ); } catch (e: any) { - await handleLuaError(e, commonSystem.system); + await handleLuaError(e, clientSystem.system); } }, ); diff --git a/common/syscalls/event.ts b/common/syscalls/event.ts index 35f85e42..c07ebc20 100644 --- a/common/syscalls/event.ts +++ b/common/syscalls/event.ts @@ -8,14 +8,14 @@ import { LuaStackFrame, luaValueToJS, } from "$common/space_lua/runtime.ts"; -import type { CommonSystem } from "$common/common_system.ts"; +import type { ClientSystem } from "../../web/client_system.ts"; export type CallbackEventListener = EventListenerDef & { run: ILuaFunction; }; export function eventListenerSyscalls( - commonSystem: CommonSystem, + clientSystem: ClientSystem, ): SysCallMapping { return { /** @@ -26,12 +26,12 @@ export function eventListenerSyscalls( def: CallbackEventListener, ) => { console.log("Registering Lua event listener: ", def.name); - commonSystem.scriptEnv.registerEventListener( + clientSystem.scriptEnv.registerEventListener( def, async (...args: any[]) => { const tl = await buildThreadLocalEnv( - commonSystem.system, - commonSystem.spaceLuaEnv.env, + clientSystem.system, + clientSystem.spaceLuaEnv.env, ); const sf = new LuaStackFrame(tl, null); try { @@ -41,7 +41,7 @@ export function eventListenerSyscalls( ); return val; } catch (e: any) { - await handleLuaError(e, commonSystem.system); + await handleLuaError(e, clientSystem.system); } }, ); diff --git a/common/syscalls/lua.ts b/common/syscalls/lua.ts index aa804724..b390281f 100644 --- a/common/syscalls/lua.ts +++ b/common/syscalls/lua.ts @@ -1,7 +1,6 @@ import type { SysCallMapping } from "$lib/plugos/system.ts"; import { evalExpression } from "$common/space_lua/eval.ts"; import { parse, parseExpressionString } from "../space_lua/parse.ts"; -import type { CommonSystem } from "$common/common_system.ts"; import { LuaStackFrame, luaToString, @@ -10,8 +9,9 @@ import { import { buildThreadLocalEnv } from "$common/space_lua_api.ts"; import { isSendable } from "$lib/plugos/util.ts"; import type { LuaBlock, LuaExpression } from "$common/space_lua/ast.ts"; +import type { ClientSystem } from "../../web/client_system.ts"; -export function luaSyscalls(commonSystem: CommonSystem): SysCallMapping { +export function luaSyscalls(clientSystem: ClientSystem): SysCallMapping { return { "lua.parse": (_ctx, code: string): LuaBlock => { return parse(code); @@ -29,13 +29,13 @@ export function luaSyscalls(commonSystem: CommonSystem): SysCallMapping { try { const ast = parseExpressionString(expression); const env = await buildThreadLocalEnv( - commonSystem.system, - commonSystem.spaceLuaEnv.env, + clientSystem.system, + clientSystem.spaceLuaEnv.env, ); const sf = new LuaStackFrame(env, null); const luaResult = await evalExpression( ast, - commonSystem.spaceLuaEnv.env, + clientSystem.spaceLuaEnv.env, sf, ); const jsResult = luaValueToJS(luaResult, sf); diff --git a/lib/data/mq.datastore.ts b/lib/data/mq.datastore.ts index 28e6bb76..e753d451 100644 --- a/lib/data/mq.datastore.ts +++ b/lib/data/mq.datastore.ts @@ -5,7 +5,6 @@ import type { MQStats, MQSubscribeOptions, } from "../../plug-api/types.ts"; -import type { MessageQueue } from "./mq.ts"; import type { DataStore } from "./datastore.ts"; import { parseExpressionString } from "$common/space_lua/parse.ts"; import { LuaEnv } from "$common/space_lua/runtime.ts"; @@ -18,7 +17,7 @@ const queuedPrefix = ["mq", "queued"]; const processingPrefix = ["mq", "processing"]; const dlqPrefix = ["mq", "dlq"]; -export class DataStoreMQ implements MessageQueue { +export class DataStoreMQ { // queue -> set of run() functions localSubscriptions = new Map void>>(); @@ -105,6 +104,10 @@ export class DataStoreMQ implements MessageQueue { const batchSize = options.batchSize || 1; const run = async () => { try { + // We're running, so let's make sure we're not running multiple times + if (timeout) { + clearTimeout(timeout); + } if (!running) { return; } @@ -116,9 +119,6 @@ export class DataStoreMQ implements MessageQueue { if (messages.length === batchSize) { await run(); } - if (timeout) { - clearTimeout(timeout); - } timeout = setTimeout(run, options.pollInterval || 5000); } catch (e: any) { console.error("Error in MQ subscription handler", e); @@ -242,6 +242,39 @@ export class DataStoreMQ implements MessageQueue { await this.ds.batchDelete(ids); } + /** + * Flushes a queue, including all queued, processing and DLQ messages + * @param queue + */ + async flushQueue(queue: string): Promise { + const ids: KvKey[] = []; + for ( + const item of await this.ds.luaQuery( + [...queuedPrefix, queue], + {}, + ) + ) { + ids.push([...queuedPrefix, item.id]); + } + for ( + const item of await this.ds.luaQuery([ + ...processingPrefix, + queue, + ], {}) + ) { + ids.push([...processingPrefix, item.id]); + } + for ( + const item of await this.ds.luaQuery([ + ...dlqPrefix, + queue, + ], {}) + ) { + ids.push([...dlqPrefix, item.id]); + } + await this.ds.batchDelete(ids); + } + async getQueueStats(queue: string): Promise { const queued = (await (this.ds.luaQuery([...queuedPrefix, queue], {}))).length; diff --git a/lib/data/mq.ts b/lib/data/mq.ts deleted file mode 100644 index d73245c0..00000000 --- a/lib/data/mq.ts +++ /dev/null @@ -1,20 +0,0 @@ -import type { - MQMessage, - MQStats, - MQSubscribeOptions, -} from "../../plug-api/types.ts"; - -export interface MessageQueue { - batchSend(queue: string, bodies: any[]): Promise; - send(queue: string, body: any): Promise; - subscribe( - queue: string, - options: MQSubscribeOptions, - callback: (messages: MQMessage[]) => Promise | void, - ): () => void; - ack(queue: string, id: string): Promise; - batchAck(queue: string, ids: string[]): Promise; - - getQueueStats(queue: string): Promise; - getAllQueueStats(): Promise>; -} diff --git a/lib/plugos/hooks/mq.ts b/lib/plugos/hooks/mq.ts index 7604c8e2..6f34dc0d 100644 --- a/lib/plugos/hooks/mq.ts +++ b/lib/plugos/hooks/mq.ts @@ -1,14 +1,14 @@ import type { Hook, Manifest } from "../types.ts"; import type { System } from "../system.ts"; import type { MQMessage } from "../../../plug-api/types.ts"; -import type { MessageQueue } from "../../data/mq.ts"; import { throttle } from "../../async.ts"; import type { MQHookT } from "$lib/manifest.ts"; +import type { DataStoreMQ } from "$lib/data/mq.datastore.ts"; export class MQHook implements Hook { subscriptions: (() => void)[] = []; - constructor(private system: System, readonly mq: MessageQueue) { + constructor(private system: System, readonly mq: DataStoreMQ) { } apply(system: System): void { @@ -26,7 +26,6 @@ export class MQHook implements Hook { } stop() { - // console.log("Unsubscribing from all queues"); this.subscriptions.forEach((sub) => sub()); this.subscriptions = []; } @@ -52,7 +51,6 @@ export class MQHook implements Hook { const subscriptions = functionDef.mqSubscriptions; for (const subscriptionDef of subscriptions) { const queue = subscriptionDef.queue; - // console.log("Subscribing to queue", queue, subscriptionDef); this.subscriptions.push( this.mq.subscribe( queue, diff --git a/lib/plugos/syscalls/datastore.ts b/lib/plugos/syscalls/datastore.ts index 35181aae..502d2226 100644 --- a/lib/plugos/syscalls/datastore.ts +++ b/lib/plugos/syscalls/datastore.ts @@ -2,7 +2,6 @@ import { type LuaCollectionQuery, queryLua, } from "$common/space_lua/query_collection.ts"; -import type { CommonSystem } from "$common/common_system.ts"; import type { KV, KvKey } from "../../../plug-api/types.ts"; import type { DataStore } from "../../data/datastore.ts"; import type { SysCallMapping } from "../system.ts"; @@ -13,6 +12,7 @@ import { luaValueToJS, } from "$common/space_lua/runtime.ts"; import type { KvQueryOptions } from "$lib/data/kv_primitives.ts"; +import type { ClientSystem } from "../../../web/client_system.ts"; /** * Exposes the datastore API to plugs, but scoping everything to a prefix based on the plug's name @@ -21,7 +21,7 @@ import type { KvQueryOptions } from "$lib/data/kv_primitives.ts"; */ export function dataStoreReadSyscalls( ds: DataStore, - commonSystem: CommonSystem, + clientSystem: ClientSystem, ): SysCallMapping { return { "datastore.batchGet": ( @@ -50,11 +50,11 @@ export function dataStoreReadSyscalls( scopeVariables?: Record, ): Promise => { const sf = LuaStackFrame.createWithGlobalEnv( - commonSystem.spaceLuaEnv.env, + clientSystem.spaceLuaEnv.env, ); - let env = commonSystem.spaceLuaEnv.env; + let env = clientSystem.spaceLuaEnv.env; if (scopeVariables) { - env = new LuaEnv(commonSystem.spaceLuaEnv.env); + env = new LuaEnv(clientSystem.spaceLuaEnv.env); for (const [key, value] of Object.entries(scopeVariables)) { env.setLocal(key, jsToLuaValue(value)); } diff --git a/lib/plugos/syscalls/mq.ts b/lib/plugos/syscalls/mq.ts index 158f1150..8b5e2e58 100644 --- a/lib/plugos/syscalls/mq.ts +++ b/lib/plugos/syscalls/mq.ts @@ -1,8 +1,8 @@ +import type { DataStoreMQ } from "$lib/data/mq.datastore.ts"; import type { SysCallMapping } from "../system.ts"; -import type { MessageQueue } from "../../data/mq.ts"; export function mqSyscalls( - mq: MessageQueue, + mq: DataStoreMQ, ): SysCallMapping { return { "mq.send": (_ctx, queue: string, body: any) => { @@ -17,6 +17,9 @@ export function mqSyscalls( "mq.batchAck": (_ctx, queue: string, ids: string[]) => { return mq.batchAck(queue, ids); }, + "mq.flushQueue": (_ctx, queue: string) => { + return mq.flushQueue(queue); + }, "mq.getQueueStats": (_ctx, queue: string) => { return mq.getQueueStats(queue); }, diff --git a/plug-api/syscalls/mq.ts b/plug-api/syscalls/mq.ts index 46072c82..279f6bf9 100644 --- a/plug-api/syscalls/mq.ts +++ b/plug-api/syscalls/mq.ts @@ -24,6 +24,14 @@ export function batchSend(queue: string, bodies: any[]): Promise { return syscall("mq.batchSend", queue, bodies); } +/** + * Flushes all messages from a queue. + * @param queue the name of the queue to subscribe to + */ +export function flushQueue(queue: string): Promise { + return syscall("mq.flushQueue", queue); +} + /** * Acknowledges a message from a queue, in case it needs to be explicitly acknowledged. * @param queue the name of the queue the message came from diff --git a/plugs/core/Library/Std/Space Overview.md b/plugs/core/Library/Std/Space Overview.md new file mode 100644 index 00000000..3e9c2f45 --- /dev/null +++ b/plugs/core/Library/Std/Space Overview.md @@ -0,0 +1,21 @@ +#meta + +This page compiles some useful stats about your space and may also be useful for debugging. + +# Stats +**Total pages:** ${#query[[from index.tag "page" select name]]} +**Total documents:** ${#query[[from index.tag "document" select name]]} + +# Active Space Lua +These are all Space Lua scripts and the order in which they are loaded: +${template.each(query[[ + from index.tag "space-lua" + order by _.priority desc + select { name=ref } +]], templates.pageItem)} + +# Active Space Style +${template.each(query[[ + from index.tag "space-style" + select {name=ref} +]], templates.pageItem)} diff --git a/plugs/core/Library/Std/Widgets.md b/plugs/core/Library/Std/Widgets.md index 985525ff..2011b5c8 100644 --- a/plugs/core/Library/Std/Widgets.md +++ b/plugs/core/Library/Std/Widgets.md @@ -44,9 +44,9 @@ function widgets.toc(options) local lines = string.split(text, "\n") local tocTexts = {} for _, line in ipairs(lines) do - local headerStart, headerEnd = string.find(line, "^(%#)+") + local headerStart, headerEnd = string.find(line, "^(%#+%s+)") if headerStart then - local headerText = line:sub(headerEnd+2) + local headerText = line:sub(headerEnd + 1) table.insert(tocTexts, string.rep(" ", (headerEnd-1) * 2) .. "* [[" .. pageName .. "#" .. headerText .. "|" .. headerText .. "]]") end diff --git a/plugs/core/std.ts b/plugs/core/std.ts index beda4a8e..14a170e4 100644 --- a/plugs/core/std.ts +++ b/plugs/core/std.ts @@ -39,15 +39,23 @@ const stdLibCacheKey = ["stdLibCache"]; type StdLibCache = Record; // page name -> last modified time -export async function init() { +/** + * Ensure indexing all Library/Std files + * @param initialSync set to true when this is triggered after the initial sync + * @returns + */ +export async function init(initialSync = false) { // Check if in read-only mode if (await system.getMode() === "ro") { return; } + + console.log("Doing stdlib init"); + let stdLibCache: StdLibCache | undefined = await datastore.get( stdLibCacheKey, ); - if (!stdLibCache) { + if (!stdLibCache || initialSync) { stdLibCache = {}; } // Iterate over the current file listing, check if any new files have been added, removed or modified @@ -79,7 +87,6 @@ export async function init() { // Save updated cache await datastore.set(stdLibCacheKey, stdLibCache); - // If anything changed and we're in sync mode, we can auto trigger reloading of the system after the index queue is processed if (anythingChanged) { console.log("Waiting for index queue to be processed"); let queueProcessed = false; @@ -90,8 +97,11 @@ export async function init() { } await sleep(100); } - console.log("Index queue processed, reloading system"); - await editor.reloadConfigAndCommands(); - await codeWidget.refreshAll(); + if (!initialSync) { + // If this is not a + console.log("Index queue processed, reloading system"); + await editor.reloadConfigAndCommands(); + await codeWidget.refreshAll(); + } } } diff --git a/plugs/index/api.ts b/plugs/index/api.ts index 8a56d5b2..add5f4cb 100644 --- a/plugs/index/api.ts +++ b/plugs/index/api.ts @@ -41,6 +41,7 @@ export async function clearFileIndex(file: string): Promise { if (file.endsWith(".md")) { file = file.replace(/\.md$/, ""); } + // console.log("Clearing index for", file); const allKeys: KvKey[] = []; for ( const { key } of await datastore.query({ diff --git a/plugs/index/command.ts b/plugs/index/command.ts index 92f34645..dcddeaa1 100644 --- a/plugs/index/command.ts +++ b/plugs/index/command.ts @@ -17,25 +17,19 @@ export async function reindexCommand() { await editor.flashNotification("Done with page index!"); } -export async function reindexSpace(noClear = false) { +export async function reindexSpace() { if (await system.getMode() === "ro") { console.info("Not reindexing because we're in read-only mode"); return; } - if (!noClear) { - console.log("Clearing page index..."); - // Executed this way to not have to embed the search plug code here - await system.invokeFunction("index.clearIndex"); - } - // Pre-index SETTINGS page to get useful settings + console.log("Clearing page index..."); + await system.invokeFunction("index.clearIndex"); + const files = await space.listFiles(); - if (files.find((file) => file.name === "SETTINGS.md")) { - console.log("Indexing SETTINGS page"); - await indexPage("SETTINGS"); - } + + console.log("All files to reindex", files.map((f) => f.name)); console.log("Queing", files.length, "pages to be indexed."); - // Queue all file names to be indexed await mq.batchSend("indexQueue", files.map((file) => file.name)); @@ -84,6 +78,7 @@ export async function parseIndexTextRepublish({ name, text }: IndexEvent) { // First clear the old file index entries await clearFileIndex(name); + // console.log("Indexing page", name); await events.dispatchEvent("page:index", { name, tree: parsed, diff --git a/plugs/index/document.ts b/plugs/index/document.ts index e4d08a98..d8e845c1 100644 --- a/plugs/index/document.ts +++ b/plugs/index/document.ts @@ -7,7 +7,7 @@ export async function indexDocument(name: string) { if (await system.getMode() === "ro") { return; } - console.log("Indexing document", name); + // console.log("Indexing document", name); const fileMeta = await space.getDocumentMeta(name); await indexObjects(fileMeta.name, [fileMeta]); } diff --git a/web/client.ts b/web/client.ts index d0ea8133..f3e37e08 100644 --- a/web/client.ts +++ b/web/client.ts @@ -60,7 +60,6 @@ import { IndexedDBKvPrimitives } from "$lib/data/indexeddb_kv_primitives.ts"; import { DataStoreMQ } from "$lib/data/mq.datastore.ts"; import { DataStoreSpacePrimitives } from "$common/spaces/datastore_space_primitives.ts"; -import { ensureSpaceIndex } from "$common/space_index.ts"; import { ReadOnlySpacePrimitives } from "$common/spaces/ro_space_primitives.ts"; import type { KvPrimitives } from "$lib/data/kv_primitives.ts"; import { LimitedMap } from "$lib/limited_map.ts"; @@ -185,9 +184,8 @@ export class Client { this.eventHook, (path) => { // isSyncCandidate // Exclude all plug space primitives paths - return !this.plugSpaceRemotePrimitives.isLikelyHandled(path) || - // Also exclude Library/Std - path.startsWith("Library/Std"); + return !this.plugSpaceRemotePrimitives.isLikelyHandled(path) && + !path.startsWith("Library/Std/"); }, ); @@ -262,11 +260,15 @@ export class Client { this.updateDocumentListCache().catch(console.error); } + public hasInitialSyncCompleted(): Promise { + return this.syncService.hasInitialSyncCompleted(); + } + private async initSync() { this.syncService.start(); // We're still booting, if a initial sync has already been completed we know this is the initial sync - let initialSync = !await this.syncService.hasInitialSyncCompleted(); + let initialSync = !await this.hasInitialSyncCompleted(); this.eventHook.addLocalListener("sync:success", async (operations) => { if (operations > 0) { @@ -282,18 +284,14 @@ export class Client { // A full sync just completed if (!initialSync) { // If this was NOT the initial sync let's check if we need to perform a space reindex - ensureSpaceIndex(this.stateDataStore, this.clientSystem.system).catch( + this.clientSystem.ensureSpaceIndex().catch( console.error, ); } else { // initialSync - // Let's load space scripts again, which probably weren't loaded before - await this.clientSystem.loadSpaceScripts(); - await this.loadCustomStyles(); - this.rebuildEditorState(); console.log( - "Initial sync completed, now need to do a full space index to ensure all pages are indexed using any custom space script indexers", + "Initial sync completed, now need to do a full space index to ensure all pages are indexed using any custom indexers", ); - ensureSpaceIndex(this.stateDataStore, this.clientSystem.system).catch( + this.clientSystem.ensureSpaceIndex().catch( console.error, ); initialSync = false; @@ -503,6 +501,8 @@ export class Client { this.plugSpaceRemotePrimitives, ), this.eventHook, + // Don't trigger events for these paths, they're handled in a custom way + ["Library/Std/"], ); this.spaceKV = spaceKvPrimitives; @@ -540,7 +540,9 @@ export class Client { if ( this.space.watchInterval && this.currentPath(true) === path && // Avoid reloading if the page was just saved (5s window) - (!lastSaveTimestamp || (lastSaveTimestamp < Date.now() - 5000)) + (!lastSaveTimestamp || (lastSaveTimestamp < Date.now() - 5000)) && + // Avoid reloading if the previous hash was undefined (first load) + (oldHash !== undefined) ) { console.log( "Page changed elsewhere, reloading. Old hash", @@ -756,6 +758,7 @@ export class Client { const allAspiringPages = (await this.clientSystem.queryLuaObjects("aspiring-page", { select: parseExpressionString("name"), + distinct: true, })).map((name): PageMeta => ({ ref: name, tag: "page", @@ -774,6 +777,10 @@ export class Client { async updateDocumentListCache() { console.log("Updating document list cache"); + if (!this.clientSystem.system.loadedPlugs.has("index")) { + console.warn("Index plug not loaded, won't update document list cache"); + return; + } const allDocuments = await this.clientSystem.queryLuaObjects( "document", @@ -1266,31 +1273,33 @@ export class Client { }); // Fetch (possibly) enriched meta data asynchronously - this.clientSystem.getObjectByRef< - PageMeta - >( - this.currentPage, - "page", - this.currentPage, - ).then((enrichedMeta) => { - if (!enrichedMeta) { - // Nothing in the store, revert to default - enrichedMeta = doc.meta; - } - - const bodyEl = this.parent.parentElement; - if (bodyEl) { - bodyEl.removeAttribute("class"); - if (enrichedMeta.pageDecoration?.cssClasses) { - bodyEl.className = enrichedMeta.pageDecoration.cssClasses.join(" ") - .replaceAll(/[^a-zA-Z0-9-_ ]/g, ""); + if (await this.hasInitialSyncCompleted()) { + this.clientSystem.getObjectByRef< + PageMeta + >( + this.currentPage, + "page", + this.currentPage, + ).then((enrichedMeta) => { + if (!enrichedMeta) { + // Nothing in the store, revert to default + enrichedMeta = doc.meta; } - } - this.ui.viewDispatch({ - type: "update-current-page-meta", - meta: enrichedMeta, - }); - }).catch(console.error); + + const bodyEl = this.parent.parentElement; + if (bodyEl) { + bodyEl.removeAttribute("class"); + if (enrichedMeta.pageDecoration?.cssClasses) { + bodyEl.className = enrichedMeta.pageDecoration.cssClasses.join(" ") + .replaceAll(/[^a-zA-Z0-9-_ ]/g, ""); + } + } + this.ui.viewDispatch({ + type: "update-current-page-meta", + meta: enrichedMeta, + }); + }).catch(console.error); + } // When loading a different page OR if the page is read-only (in which case we don't want to apply local patches, because there's no point) if (loadingDifferentPage || doc.meta.perm === "ro") { @@ -1413,6 +1422,13 @@ export class Client { } async loadCustomStyles() { + if (!await this.hasInitialSyncCompleted()) { + console.info( + "Not loading custom styles yet, since initial synca has not completed yet", + ); + return; + } + const spaceStyles = await this.clientSystem.queryLuaObjects( "space-style", {}, diff --git a/web/client_system.ts b/web/client_system.ts index 8b365cb3..77bd313b 100644 --- a/web/client_system.ts +++ b/web/client_system.ts @@ -34,7 +34,6 @@ import { codeWidgetSyscalls } from "./syscalls/code_widget.ts"; import { clientCodeWidgetSyscalls } from "./syscalls/client_code_widget.ts"; import { KVPrimitivesManifestCache } from "$lib/plugos/manifest_cache.ts"; import { createKeyBindings } from "./editor_state.ts"; -import { CommonSystem } from "$common/common_system.ts"; import type { DataStoreMQ } from "$lib/data/mq.datastore.ts"; import { plugPrefix } from "$common/spaces/constants.ts"; import { jsonschemaSyscalls } from "$common/syscalls/jsonschema.ts"; @@ -44,27 +43,44 @@ import { commandSyscalls } from "$common/syscalls/command.ts"; import { eventListenerSyscalls } from "$common/syscalls/event.ts"; import { DocumentEditorHook } from "./hooks/document_editor.ts"; import type { LuaCollectionQuery } from "$common/space_lua/query_collection.ts"; +import type { AppCommand } from "$lib/command.ts"; +import { ScriptEnvironment } from "$common/space_script.ts"; +import { SpaceLuaEnvironment } from "$common/space_lua.ts"; +import type { ParseTree } from "@silverbulletmd/silverbullet/lib/tree"; const plugNameExtractRegex = /\/(.+)\.plug\.js$/; +const indexVersionKey = ["$indexVersion"]; +// Bump this one every time a full reindex is needed +const desiredIndexVersion = 7; +const mqTimeout = 10000; // 10s +const mqTimeoutRetry = 3; /** - * Wrapper around a System, used by the client + * Wrapper around a System */ -export class ClientSystem extends CommonSystem { +export class ClientSystem { + system!: System; + + // Hooks + commandHook!: CommandHook; + slashCommandHook!: SlashCommandHook; + namespaceHook!: PlugNamespaceHook; + codeWidgetHook!: CodeWidgetHook; + documentEditorHook!: DocumentEditorHook; + + readonly allKnownFiles = new Set(); + readonly spaceScriptCommands = new Map(); + scriptEnv: ScriptEnvironment = new ScriptEnvironment(); + spaceLuaEnv = new SpaceLuaEnvironment(); + scriptsLoaded: boolean = false; + constructor( private client: Client, - mq: DataStoreMQ, - ds: DataStore, - eventHook: EventHook, - readOnlyMode: boolean, + protected mq: DataStoreMQ, + public ds: DataStore, + public eventHook: EventHook, + public readOnlyMode: boolean, ) { - super( - mq, - ds, - eventHook, - readOnlyMode, - client.clientConfig.enableSpaceScript, - ); this.system = new System(undefined, { manifestCache: new KVPrimitivesManifestCache( ds.kv, @@ -72,6 +88,10 @@ export class ClientSystem extends CommonSystem { ), }); + setInterval(() => { + mq.requeueTimeouts(mqTimeout, mqTimeoutRetry, true).catch(console.error); + }, 20000); // Look to requeue every 20s + this.system.addHook(this.eventHook); // Plug page namespace hook @@ -86,9 +106,6 @@ export class ClientSystem extends CommonSystem { this.documentEditorHook = new DocumentEditorHook(); this.system.addHook(this.documentEditorHook); - // MQ hook - this.system.addHook(new MQHook(this.system, this.mq)); - // Command hook this.commandHook = new CommandHook( this.readOnlyMode, @@ -108,11 +125,11 @@ export class ClientSystem extends CommonSystem { }); }, }); - this.system.addHook(this.commandHook); - // Slash command hook - this.slashCommandHook = new SlashCommandHook(this.client, this); - this.system.addHook(this.slashCommandHook); + this.slashCommandHook = new SlashCommandHook(this.client); + + // MQ hook + this.system.addHook(new MQHook(this.system, this.mq)); // Syscall hook this.system.addHook(new SyscallHook()); @@ -135,8 +152,8 @@ export class ClientSystem extends CommonSystem { } init() { - // Slash command hook - this.slashCommandHook = new SlashCommandHook(this.client, this); + // Init is called after the editor is initialized, so we can safely add the command hook + this.system.addHook(this.commandHook); this.system.addHook(this.slashCommandHook); // Syscalls available to all plugs @@ -184,11 +201,75 @@ export class ClientSystem extends CommonSystem { } } + async loadSpaceScripts() { + if (!await this.client.hasInitialSyncCompleted()) { + console.info( + "Not loading space scripts, since initial synca has not completed yet", + ); + return; + } + this.scriptEnv = new ScriptEnvironment(); + try { + await this.spaceLuaEnv.reload(this.system); + } catch (e: any) { + console.error("Error loading space-script:", e.message); + } + + // Reset the space script commands + this.spaceScriptCommands.clear(); + for (const [name, command] of Object.entries(this.scriptEnv.commands)) { + this.spaceScriptCommands.set(name, command); + } + + // Inject the registered events in the event hook + this.eventHook.scriptEnvironment = this.scriptEnv; + + this.commandHook.throttledBuildAllCommands(); + this.slashCommandHook.throttledBuildAllCommands(); + + this.scriptsLoaded = true; + } + + invokeSpaceFunction(name: string, args: any[]) { + const fn = this.scriptEnv.functions[name]; + if (!fn) { + throw new Error(`Function ${name} not found`); + } + return fn(...args); + } + + async applyAttributeExtractors( + tags: string[], + text: string, + tree: ParseTree, + ): Promise> { + let resultingAttributes: Record = {}; + for (const tag of tags) { + const extractors = this.scriptEnv.attributeExtractors[tag]; + if (!extractors) { + continue; + } + for (const fn of extractors) { + const extractorResult = await fn(text, tree); + if (extractorResult) { + // Merge the attributes in + resultingAttributes = { + ...resultingAttributes, + ...extractorResult, + }; + } + } + } + + return resultingAttributes; + } + async reloadPlugsFromSpace(space: Space) { console.log("Loading plugs"); await this.system.unloadAll(); console.log("(Re)loading plugs"); - await Promise.all((await space.listPlugs()).map(async (plugMeta) => { + const allPlugs = await space.listPlugs(); + await Promise.all(allPlugs.map(async (plugMeta) => { try { const plugName = plugNameExtractRegex.exec(plugMeta.name)![1]; await this.system.load( @@ -221,4 +302,37 @@ export class ClientSystem extends CommonSystem { ["index.getObjectByRef", page, tag, ref], ); } + + private indexOngoing = false; + + async ensureSpaceIndex() { + const currentIndexVersion = await this.ds.get(indexVersionKey); + + console.info("Current space index version", currentIndexVersion); + + if (currentIndexVersion !== desiredIndexVersion && !this.indexOngoing) { + console.info( + "Performing a full space reindex, this could take a while...", + ); + // First let's fetch all pages to make sure we have a cache of known pages + // await this.client.space.fetchPageList(); + this.indexOngoing = true; + await this.system.invokeFunction("index.reindexSpace", []); + await this.system.invokeFunction("core.init", [true]); // initialSync = true + console.info("Full space index complete."); + await this.markFullSpaceIndexComplete(this.ds); + this.indexOngoing = false; + // Let's load space scripts again, which probably weren't loaded before + console.log( + "Now loading space scripts, custom styles and rebuilding editor state", + ); + await this.loadSpaceScripts(); + await this.client.loadCustomStyles(); + this.client.rebuildEditorState(); + } + } + + async markFullSpaceIndexComplete(ds: DataStore) { + await ds.set(indexVersionKey, desiredIndexVersion); + } } diff --git a/web/cm_plugins/lua_directive.ts b/web/cm_plugins/lua_directive.ts index c9074a93..bed263e4 100644 --- a/web/cm_plugins/lua_directive.ts +++ b/web/cm_plugins/lua_directive.ts @@ -17,6 +17,7 @@ import { LuaEnv, LuaStackFrame, luaValueToJS, + singleResult, } from "$common/space_lua/runtime.ts"; import { LuaRuntimeError } from "$common/space_lua/runtime.ts"; import { encodeRef } from "@silverbulletmd/silverbullet/lib/page_ref"; @@ -105,10 +106,12 @@ export function luaDirectivePlugin(client: Client) { ); threadLocalizedEnv.setLocal("_CTX", tl); const result = luaValueToJS( - await evalExpression( - expr, - threadLocalizedEnv, - sf, + singleResult( + await evalExpression( + expr, + threadLocalizedEnv, + sf, + ), ), sf, ); diff --git a/web/components/page_navigator.tsx b/web/components/page_navigator.tsx index a3b30200..5205a2e7 100644 --- a/web/components/page_navigator.tsx +++ b/web/components/page_navigator.tsx @@ -88,6 +88,7 @@ export function PageNavigator({ currentPath && currentPath === pageMeta.name || pageMeta._isAspiring ) { // ... then we put it all the way to the end + console.log("Got an aspiring page", pageMeta.name); orderId = Infinity; } const cssClass = (pageMeta.pageDecoration?.cssClasses || []).join(" ") diff --git a/web/fuse_search.ts b/web/fuse_search.ts index 9b77e116..ddd76281 100644 --- a/web/fuse_search.ts +++ b/web/fuse_search.ts @@ -16,13 +16,11 @@ export const fuzzySearchAndSort = ( const enrichedArr: FuseOption[] = arr.map((item) => { return { - name: item.name, + ...item, // Only relevant for pages and or documents and not commands baseName: item.name.split("/").pop()!, displayName: item?.meta?.displayName, aliases: item?.meta?.aliases?.join(" "), - description: item.description, - meta: item.meta, }; }); diff --git a/web/hooks/slash_command.ts b/web/hooks/slash_command.ts index 27e9fc51..982c51c8 100644 --- a/web/hooks/slash_command.ts +++ b/web/hooks/slash_command.ts @@ -13,7 +13,6 @@ import type { } from "../../plug-api/types.ts"; import { safeRun, throttle } from "$lib/async.ts"; import type { SlashCommandDef, SlashCommandHookT } from "$lib/manifest.ts"; -import type { CommonSystem } from "$common/common_system.ts"; import type { Shortcut } from "@silverbulletmd/silverbullet/type/client"; export type AppSlashCommand = { @@ -25,10 +24,8 @@ const slashCommandRegexp = /([^\w:]|^)\/[\w#\-]*/; export class SlashCommandHook implements Hook { slashCommands: AppSlashCommand[] = []; - private editor: Client; - constructor(editor: Client, private commonSystem: CommonSystem) { - this.editor = editor; + constructor(private client: Client) { } throttledBuildAllCommands = throttle(() => { @@ -36,7 +33,8 @@ export class SlashCommandHook implements Hook { }, 200); buildAllCommands() { - const system = this.commonSystem.system; + const clientSystem = this.client.clientSystem; + const system = clientSystem.system; this.slashCommands = []; for (const plug of system.loadedPlugs.values()) { @@ -60,7 +58,7 @@ export class SlashCommandHook implements Hook { // Iterate over script defined slash commands for ( const command of Object.values( - this.commonSystem.scriptEnv.slashCommands, + clientSystem.scriptEnv.slashCommands, ) ) { this.slashCommands.push(command); @@ -68,7 +66,7 @@ export class SlashCommandHook implements Hook { // Iterate over all shortcuts // Add slash commands for shortcuts that configure them for ( - const shortcut of this.editor.config.get("shortcuts", []) + const shortcut of this.client.config.get("shortcuts", []) ) { if (shortcut.slashCommand) { this.slashCommands.push({ @@ -76,7 +74,7 @@ export class SlashCommandHook implements Hook { name: shortcut.slashCommand, description: shortcut.command, }, - run: () => this.editor.runCommandByName(shortcut.command), + run: () => this.client.runCommandByName(shortcut.command), }); } } @@ -103,7 +101,7 @@ export class SlashCommandHook implements Hook { } // Check if the slash command is available in the current context - const parentNodes = this.editor.extractParentNodes(ctx.state, currentNode); + const parentNodes = this.client.extractParentNodes(ctx.state, currentNode); for (const def of this.slashCommands) { if ( def.slashCommand.onlyContexts && !def.slashCommand.onlyContexts.some( @@ -125,7 +123,7 @@ export class SlashCommandHook implements Hook { boost: def.slashCommand.boost, apply: () => { // Delete slash command part - this.editor.editorView.dispatch({ + this.client.editorView.dispatch({ changes: { from: prefix!.from + prefixText.indexOf("/"), to: ctx.pos, @@ -135,14 +133,14 @@ export class SlashCommandHook implements Hook { // Replace with whatever the completion is safeRun(async () => { await def.run(); - this.editor.focus(); + this.client.focus(); }); }, }); } const slashCompletions: CompletionResult | SlashCompletions | null = - await this.editor + await this.client .completeWithEvent( ctx, "slash:complete", @@ -159,7 +157,7 @@ export class SlashCommandHook implements Hook { boost: slashCompletion.order && -slashCompletion.order, apply: () => { // Delete slash command part - this.editor.editorView.dispatch({ + this.client.editorView.dispatch({ changes: { from: prefix!.from + prefixText.indexOf("/"), to: ctx.pos, @@ -168,11 +166,11 @@ export class SlashCommandHook implements Hook { }); // Replace with whatever the completion is safeRun(async () => { - await this.editor.clientSystem.system.invokeFunction( + await this.client.clientSystem.system.invokeFunction( slashCompletion.invoke, [slashCompletion], ); - this.editor.focus(); + this.client.focus(); }); }, }); diff --git a/web/service_worker.ts b/web/service_worker.ts index 2038144e..fb18ba57 100644 --- a/web/service_worker.ts +++ b/web/service_worker.ts @@ -153,7 +153,7 @@ self.addEventListener("fetch", (event: any) => { requestUrl.searchParams.get("raw") === "true" ) { // If this is a /*.* request, this can either be a plug worker load or an document load - return handleLocalFileRequest(pathname); + return handleLocalFileRequest(pathname, request); } else { // Must be a page URL, let's serve index.html which will handle it return (await caches.match(precacheFiles["/"])) || fetch(request); @@ -169,6 +169,7 @@ self.addEventListener("fetch", (event: any) => { async function handleLocalFileRequest( pathname: string, + request: Request, ): Promise { const path = decodePageURI(pathname.slice(1)); const data = await ds?.get([...filesContentPrefix, path]); @@ -187,10 +188,16 @@ async function handleLocalFileRequest( }, ); } else { - console.error( + console.warn( "Did not find file in locally synced space", path, ); + // If this is a _plug request and we don't have it, we may not have performed an initial sync yet + if (path.startsWith("_plug/")) { + console.info("Proxying _plug fetch to server", path); + return fetch(request); + } + return new Response("Not found", { status: 404, headers: { diff --git a/website/CHANGELOG.md b/website/CHANGELOG.md index d60e2297..ab684f8f 100644 --- a/website/CHANGELOG.md +++ b/website/CHANGELOG.md @@ -3,431 +3,23 @@ An attempt at documenting the changes/new features introduced in each release. --- ## Edge -_These features are not yet properly released, you need to use [the edge builds](https://community.silverbullet.md/t/living-on-the-edge-builds/27) to try them._ +* You’ve entered the brave new world of SilverBullet 2.0. Welcome +* The following features have been removed: + * Online mode + * Query language + * Template language + * Command links + * Anchor support + * Space Script + * Schemas + * Space Config + * Attribute extractors + * Object decorators + * Unfurl functionality + * Live embeds + * Federation + * Markdown widgets (plug feature) + * Cron hook (plug feature) + * Widget hooks (plug feature) -* **Breaking (configuration) changes:** - * `indexPage` and `spaceIgnore` have been removed from [[SETTINGS]] and are now server environment variables: [[Install/Configuration]] -* **Breaking (Lua) changes:** - * You now need to use `widget.new` to create Lua widgets, see [[Space Lua#Widgets]] -* [[Space Lua#Widgets]] now support events - -## 0.10.4 2025-02-25 -* **Lua**: _Tons_ of fixes, changes and enhancements, see [[Space Lua]] and related for up-to-date docs. Some notable things: - * [[Space Lua/Lua Integrated Query]] is now here - * Code completion of APIs in `space-lua` and Lua directive syntax, implemented in Lua itself: [[^Library/Std/Lua]]. - * [[API]] docs (not fully complete, but getting there) - * WIP Lua [[^Library/Std]] [[Libraries]], distributed with SilverBullet core. - * [[^Library/Std/Config]] (future Lua replacement for [[^SETTINGS]]) - * [[^Library/Std/Template]] (Lua replacement of [[Templates]]) with a few samples here [[^Library/Std/Query Templates]]. - * Breaking change: API convention changed moving from `snake_case` to `camelCase` everywhere. -* `indentMultiplier` setting added (by [Mufeed Ali](https://github.com/silverbulletmd/silverbullet/pull/1212)) -* Open external links in new tab (by [Ohad Lutzky](https://github.com/silverbulletmd/silverbullet/pull/1229)) -* URL parse fixes (by [Ohad Lutzky](https://github.com/silverbulletmd/silverbullet/pull/1175)) -* [[Plugs]] are now ideally configured via [[Space Config]] (by [Marek S. Łukasiewicz](https://github.com/silverbulletmd/silverbullet/pull/1122)) -* Fixed styling for completed tasks with page links (by [Max Richter](https://github.com/silverbulletmd/silverbullet/pull/1225)) -* New design for [[Authentication]] page (by [majjejjam](https://github.com/silverbulletmd/silverbullet/pull/1124)) -* Tags are now easier to style (by [TR Staake](https://github.com/silverbulletmd/silverbullet/pull/1179)) -* Swift syntax highlighting now actually highlights Swift code (instead of Rust) (by [David Chiles](https://github.com/silverbulletmd/silverbullet/pull/1183)) -* (Security) Implemented a lockout mechanism after a number of failed login attempts for [[Authentication]] (configured via [[Install/Configuration#Authentication]]) (by [Peter Weston](https://github.com/silverbulletmd/silverbullet/pull/1152)) -* Move lines up/down with {[Outline: Move Up]} and {[Outline: Move Down]} commands (by [Simon Cambier](https://github.com/silverbulletmd/silverbullet/pull/1232)) -* Added more options for [[Table of Contents]] rendering (by [Nico S.](https://github.com/silverbulletmd/silverbullet/pull/1258)). -* Disable spellcheck for more code-like blocks (by [MrMugame](https://github.com/silverbulletmd/silverbullet/pull/1244)) - -## 0.10.1 2024-11-14 -This is a “major” release primarily because of the underlying migration to rely on Deno 2 and a bunch of foundational work that’s not really leveraged yet. Stay tuned for more. - -* Migrated underlying system to Deno 2.0, please upgrade to a recent Deno 2 build! -* Another attempt at supporting authentication proxies better -* {[Editor: New Window]} command (bound to Ctrl-n/Cmd-n) which should make creating new windows work better on non-mac OSes -* Fixes for automatic page reload -* Automatically convert item into task when using {[Task: Cycle State]} (by [Abin Simon](https://github.com/silverbulletmd/silverbullet/pull/1099)) -* Reduce ranking of non-existing pages in completion and picker (by [Abin Simon](https://github.com/silverbulletmd/silverbullet/issues/1100)) -* Better configurability of automatic bracket closing in [[^SETTINGS]] (`autoCloseBrackets` setting) (by [Fernando Serboncini](https://github.com/silverbulletmd/silverbullet/pull/1110)) -* Smart quote configuration (by [Marek](https://github.com/silverbulletmd/silverbullet/pull/1121)) -* Index page links in [[Frontmatter]] (by [Marek S. Łukasiewicz](https://github.com/silverbulletmd/silverbullet/pull/1066)) -* Template parser fixes (by [MrMugame](https://github.com/silverbulletmd/silverbullet/pull/1102)) -* You can now hit `Tab` to accept the current auto complete option (by [Vighnesh Shenoy](https://github.com/silverbulletmd/silverbullet/pull/1150)) -* Some dark mode fixes and top bar improvements (by [Mufeed Ali](https://github.com/silverbulletmd/silverbullet/pull/1137)) -* A few Vim mode fixes (by [Ohad](https://github.com/silverbulletmd/silverbullet/pull/1146) [Lutzky](https://github.com/silverbulletmd/silverbullet/pull/1145)) -* Support for [tel: links](tel:12345) (by [Peter Weston](https://github.com/silverbulletmd/silverbullet/pull/1139)) -* “Remember me” functionality for [[Authentication]] (by [aekaisato](https://github.com/silverbulletmd/silverbullet/pull/1132)) -* (Highly experimental) work started on [[Space Lua]] (you can observe, but don’t use just yet) - -## 0.9.4 2024-09-04 -* Slashes in page names are no no longer URL encoded (in the URL) -* `task` and `item` [[Objects]] now have a `parent` attribute when nested, and inherit their ancestors’ tags in `itags` — see [[Objects#item]] for an example. -* When a page changed on disk, it now does a diff-based reload, largely retaining local changes, edit history and cursor position (by [jcgurango](https://github.com/silverbulletmd/silverbullet/pull/1056)) -* The default generated SETTINGS uses `yaml` again until more plugs are updated to support [[Space Config]] -* Markdown preview now has a print button that appears on hover (by [Ruibin Xing](https://github.com/silverbulletmd/silverbullet/pull/1065)) -* Some more iterations on the [[Schema]] feature (still experimental) - -## 0.9.2 2024-08-18 -* Widget buttons for [[Transclusions]] (by [onespaceman](https://github.com/silverbulletmd/silverbullet/pull/1013)) as well as some other transclusion-related fixes -* At long last, date and time functions and attributes now return local time instead of UTC - * _Local_ means the time of the place where code runs, so this may be your server or client depending on the [[Client Modes]]. Ideally you set the timezones the same on both. -* In [[Space Script]], all syscalls are now exposed via “globals”, so they can be called directly using e.g. `editor.flashNotification("hello")` instead of `syscall("editor.flashNotification", "hello")`. The old way still works, but the new way is way cleaner. -* [MVP](https://en.wikipedia.org/wiki/Minimum_viable_product) of [[Schema]] support. This is still very nascent, future iterations are to come. -* SETTINGS is now indexed first, when a full space reindex needs to happen. -* New {[Page: Rename Linked Page]} command -* [[Live Queries]] are now also server-side rendered, just like [[Live Templates]] -* `Ctrl-n` and `Ctrl-p` shortcuts now work in the filter box (by [Ruibin Xing](https://github.com/silverbulletmd/silverbullet/pull/1036)) -* Internal refactor, and more leverage of [JSR](https://jsr.io/). The SilverBullet [plug API](https://jsr.io/@silverbulletmd/silverbullet) is now published on JSR as well, and soon this will be the preferred way of importing the plug APIs. -* And these syscalls are now [fully documented](https://jsr.io/@silverbulletmd/silverbullet/doc/syscalls/~) - -## 0.9.0 2024-08-04 -This is a more major version bump due to [[Space Config]], which is a bigger architectural change. It shouldn’t break anything for existing users — but you know — famous last words. - -* [[Space Config]] is here! We hear you like Space stuff ([[Space Script]] and [[Space Style]]) so we keep adding more! This allows you to distribute your [[^SETTINGS]] across pages. A few things of note: - * This is quite a _deep architectural change_ so I may have accidentally broken things I didn’t intend to, please report issues. - * There is now a `@config` global variable that you can reference from anywhere containing the space’s currently active full configuration - * There is now linting (checking) of `space-config` blocks, checking that you use built-in configuration keys correctly and reporting errors if not. Later this will be extended to allow plugs and space scripts to define custom validators as well (based on JSON schema). - * Possible side-effect: if you configured a custom `indexPage` and are upgrading, it may be that your initial load brings you back to `index`. Just let the space reindex finish and reload and you should be good. - * **Plug developers**: If you were using the `readSetting` library call, please switch to using the new `system.getSpaceConfig()` syscall instead once this is released. `readSetting` read the `SETTINGS` page only, this may now no longer give the full picture. -* [[Transclusions]] now support audio, video and PDFs in addition to images (by [onespaceman](https://github.com/silverbulletmd/silverbullet/pull/1008)) -* New [[^Library/Core/Page/Space Overview]] library page part of the [[Library/Core]] Library that lists all of your [[Space Config]], [[Space Style]] and [[Space Script]] as well as some other fun stats. -* Support for adding CSS classes in [[Page Decorations]] (by [onespaceman](https://github.com/silverbulletmd/silverbullet/pull/945)). Check out [[Page Decorations#Use case: pimp my page]] 🤣 -* Added caching to file listings (useful for large spaces or slow disks) (by [Justyn Shull](https://github.com/silverbulletmd/silverbullet/pull/1012)) - -## 0.8.5 2024-07-30 -* New power-user feature: [[Object Decorators]]. Its primary use case is to apply [[Page Decorations]], but you may find other uses as well. Speaking of which... -* **Breaking change:** The way [[Page Decorations]] are specified has changed. It has now been replaced with [[Object Decorators]](as just mentioned), which are a more generic mechanism. Your existing `pageDecorations` will stop working. You’ll have to rewrite based on the new format. See [[Page Decorations]] for examples. -* New [[CHANGELOG@L20]] syntax (by [Marek S. Łukasiewicz](https://github.com/silverbulletmd/silverbullet/pull/988)) -* [[Library/Core]] library updates (run {[Libraries: Update]} to get it): - * [[^Library/Core/Widget/Table of Contents]] widget can now be disabled using a [[Page Decorations|Page Decoration]] - * [[Library/Core/Quick Notes]] now orders quick notes in reverse-chronological order -* **Fix:** Spellchecker correction was broken in Chrome -* **Fix:** Markdown preview was broken - -## 0.8.4 2024-07-27 -* [[Page Picker#Keyboard shortcuts]]: allow folder completion using Shift-Space (by [Marek S. Łukasiewicz](https://github.com/silverbulletmd/silverbullet/pull/961)) -* New [[Page Decorations]]: ability to hide pages from the page picker and auto complete (by [Marek S. Łukasiewicz](https://github.com/silverbulletmd/silverbullet/pull/962)) -* The [[Expression Language]] now supports decimal numbers 🤯 (e.g. `3.14`) -* Non-existing pages that have been previously linked to (so effectively: broken links) now appear in the [[Page Picker]] (with a “Create page” hint) as well as in [[Links]] auto complete (marked as “Linked but not created”). This should make it easier to “fill in the gaps”: you can liberally create page links, and create those pages later easily via the page picker. -* New [[Space Script]] feature: [[Space Script#Custom HTTP endpoints]]: create custom endpoints on your SilverBullet server (under the `/_/` prefix). -* Updated the [[^Library/Core/New Page/Quick Note]] template to now use a (Windows and Android) naming pattern using the new `safeTime` function, also tweaked the naming pattern slightly (put a `/` in between the date and time so that these are organized by date), updated [[Library/Core/Quick Notes]] to show full path. -* Added Ruby [[Markdown/Syntax Highlighting]] (by [Bo Jeanes](https://github.com/silverbulletmd/silverbullet/pull/966)) -* **Fix**: Due to some race conditions some situations pages would disappear from the index (e.g. from the [[Page Picker]]), this should now be fixed -* New [[SETTINGS]]: `pwaOpenLastPage` and `useSmartQuotes` (latter by [Marek S. Łukasiewicz](https://github.com/silverbulletmd/silverbullet/pull/960)) -* **Fix**: CORS issues in Firefox (admonition related) (by [onespaceman](https://github.com/silverbulletmd/silverbullet/pull/964)) -* **Technical:** Did another round of dependency upgrades and cleaned up some dead code. - ---- - -## 0.8.2 2024-07-17 -* [[Page Decorations]] are here (initial implementation by [Deepak Narayan](https://github.com/silverbulletmd/silverbullet/pull/940), later refined by Zef) -* New type of [[Shortcuts|shortcut]]: `slashCommand` -* Naming is hard. Renamed the `source` attribute of [[Libraries]] to `import`. egacy references to `source` will keep working. -* Added support for [[Links#Caret page links]] making it slightly more convenient to link to [[Meta Pages]] -* **Fix:** Remove redundant vertical space for some tables (by [Eric Yang](https://github.com/silverbulletmd/silverbullet/pull/952)) -* **Fix:** very large spaces would let the server blow up when saving snapshots. This is now fixed. -* **Fix:** Conflict copies could no longer be edited, whoops (initial fix by [Semyon Novikov](https://github.com/silverbulletmd/silverbullet/pull/939), later refined by Zef) -* **Fix**: `silverbullet upgrade` should now work again (or at least on the next upgrade) - ---- - -## 0.8.1 2024-07-11 -* The old **Template Picker** has now been rebranded to [[Meta Picker]] and surfaces pages in your space tagged as `#template` or `#meta`. Read more about this in [[Meta Pages]]. -* [[Transclusions]] has now been implemented, allowing inline embedding of other pages as well as images (by onespaceman) using the convenient `![[link]]` syntax. -* [[Libraries]] management has been rethought. You can now decoratively specify them in [[SETTINGS]] and keep them up to date with the {[Libraries: Update]} command. -* For new spaces, the default [[SETTINGS]] page is now tagged with `#meta`, which means it will only appear in the [[Meta Picker]]. There is also a new {[Navigate: Open SETTINGS]} command (bound to `Ctrl-,` and `Cmd-,`). -* Attachments are now indexed, and smartly moved when pages are renamed (by onespaceman) -* Images can now be resized: [[Attachments#Embedding]] (initial work done by [Florent](https://github.com/silverbulletmd/silverbullet/pull/833), later adapted by onespaceman) -* To make pure reading and browsing on touch devices a nicer experience, there is now a new **edit toggle** (top right). When _disabled_, you switch to _reader mode_ which makes sure your software keyboard doesn’t block the screen when navigating your space. This button is only visible on mobile devices (no physical keyboard attached) only. Can be disabled via the `hideEditButton` [[SETTINGS]] (and is disabled on this website, that’s why you don’t see it). -* Super^script^ and Sub~script~ are now supported (by [MrMugame](https://github.com/silverbulletmd/silverbullet/pull/879)) -* Added a {[Delete Line]} command (by [pihentagy](https://github.com/silverbulletmd/silverbullet/pull/866)) -* Improved selection behavior (by [MrMugame](https://github.com/silverbulletmd/silverbullet/pull/904)) -* Hide `\` escapes in [[Live Preview]] (by [MrMugame](https://github.com/silverbulletmd/silverbullet/pull/901)) -* Added Erlang [[Markdown/Syntax Highlighting]] -* Dates (formatted as e.g. `2023-07-01` or `2023-07-01 23:33`) in [[Frontmatter]] are now converted into strings (rather than empty objects) -* **Breaking change:** The legacy handlebars template syntax that supported `{{escapeRegex "something"}}` function calls (rather than `{{escapeRegex("something")}}`) has now been removed. -* `task` and `item` objects now have an additional `text` attribute that contains the full text of the item and/or task, with any [[Attributes]] and [[Tags]] intact (whereas they are removed from `name`) -* The `#boot` PWA loading the last opened page feature is back (put `#boot` at the end of your SilverBullet URL to auto load the last opened page) -* Numerous other bug fixes (thanks MrMugame and onespaceman) ---- - -## 0.7.7 2024-04-30 -* Added ability to configure PWA app name and description (web app manifest) via [[Install/Configuration]] variables (by [s1gnate-sync](https://github.com/silverbulletmd/silverbullet/pull/854)) -* Improved styling for modals (by [Daniel](https://github.com/silverbulletmd/silverbullet/pull/840)) -* Allow middle click to open links (by [Daniel](https://github.com/silverbulletmd/silverbullet/pull/841)) -* Various theme fixes (by [onespaceman](https://github.com/silverbulletmd/silverbullet/pull/831)) -* Fix Int polyfill (by [Daniel](https://github.com/silverbulletmd/silverbullet/pull/836)) -* Make maximum attachment size configurable (by [Thomas](https://github.com/silverbulletmd/silverbullet/pull/832)) -* Fix frontmatter tag indexing when tags are numbers (by [Justyn](https://github.com/silverbulletmd/silverbullet/pull/830)) -* Fix for supporting page names with spaces at the tail or front end (by [Florent](https://github.com/silverbulletmd/silverbullet/pull/817)) -* On the technical side: - * Radically improved dependency management in the code base (by [Maks](https://github.com/silverbulletmd/silverbullet/pull/770)) -* Dependency bumps for a lot of codemirror-related packages as well as Deno itself - ---- - -## 0.7.6 2024-03-15 -* We now have [[Space Style]] in addition to [[Space Script]], enabling CSS tweaks to SilverBullet itself from any page (by [onespaceman](https://github.com/silverbulletmd/silverbullet/pull/796)) -* Added [[Functions#contains(str, substr)]] function -* Properly report errors when calling non-existing [[Functions]] in queries -* It’s now possible to call [[Space Script#Custom functions]] in `indexPage` in [[SETTINGS]] -* Support custom [[Markdown/Admonitions]] (by [onespaceman](https://github.com/silverbulletmd/silverbullet/pull/786)) -* Improved snippets for [[Links]] indexing (visible in [[Linked Mentions]]) (a {[Space: Reindex]} is required to see this), attempts to include the entire surrounding sentence, and if that’s too long cuts it down, but not mid-word. -* [[Objects#header]] containing [[Tags]] or [[Attributes]] are now properly indexed -* (Hopefully) fixed [weird rendering bugs](https://github.com/silverbulletmd/silverbullet/issues/745) for markdown in templates -* New {[Editor: Undo]} and {[Editor: Redo]} commands (so you can customize their keybindings or add them as actionButtons in [[SETTINGS]]). -* Various bugfixes around frontmatter in [[Snippets]]. -* More attempts at fixing unnecessary “Page has changed elsewhere, reloading” reloads. -* Various other bug fixes - ---- - -## 0.7.5 2024-02-29 -* [[Plugs/Share]] using the {[Share: Page Or Selection]} command (bound to Ctrl-s/Cmd-s by default): allowing you to quickly share the current page (or selection) to the clipboard as: - * Clean markdown (to paste into other markdown supporting tools) - * Rich text (to paste into unenlightened rich text tools like Google Docs, Confluence, Word etc.) -* Various improvements in tests and input checks (by Maks [here](https://github.com/silverbulletmd/silverbullet/pull/754) and [here](https://github.com/silverbulletmd/silverbullet/pull/751)) -* Better sizing of the top bar (was buggy before and poorly implemented) (by [onespaceman](https://github.com/silverbulletmd/silverbullet/pull/753)) -* Hashtag can now contain more different characters, including emoji (by [Matthew Pietz](https://github.com/silverbulletmd/silverbullet/pull/752)) -* New [[Space Script]] APIs: `registerAttributeExtractor` and `registerEventListener` - ---- - -## 0.7.3 2024-02-24 - -* We had a big influx of people and contributions from the community, which is amazing. Here are the highlights: - * Ability to “bake” templates and query results (with the new “bake” button or {[Page: Bake live blocks]}), that is: replace these blocks with their rendered results, and therefore freeze them in time, by [Marek S. Łukasiewicz](https://github.com/silverbulletmd/silverbullet/pull/719) - * Pre-fill a new page with heading title in {[Page: Extract]} by [Patrik Stenmark](https://github.com/silverbulletmd/silverbullet/pull/744) - * Added custom data field to template plug by [Michael Kolb](https://github.com/silverbulletmd/silverbullet/pull/716) - * Markdown tables are now indexed [[Objects#table]], and queryable by [Michael Kolb](https://github.com/silverbulletmd/silverbullet/pull/729) - * Added “copy to clipboard” button to code blocks, by [Joe Krill](https://github.com/silverbulletmd/silverbullet/pull/735) - * Fenced code can now also use `~~~` in addition to triple backticks, by [Marek S. Łukasiewicz](https://github.com/silverbulletmd/silverbullet/pull/694) - * Markdown preview pane now uses custom styles and dark mode, by [Joe Krill](https://github.com/silverbulletmd/silverbullet/pull/741) - * Correctly skip adding default shortcuts for overridden commands, by [Joe Krill](https://github.com/silverbulletmd/silverbullet/pull/739) - * Fix {[Link: Unfurl]} command, by [Joe Krill](https://github.com/silverbulletmd/silverbullet/pull/738) - * [[Expression Language]] now supports unary minus (e.g. `-3`), by [Marek S. Łukasiewicz](https://github.com/silverbulletmd/silverbullet/pull/732) - * Added [[Markdown/Syntax Highlighting]] for Diff, Powershell, Perl, TCL, Verilos, VHDL, Docker and CMake), by [Giovanni Pollo](https://github.com/silverbulletmd/silverbullet/pull/718) and Go by [Viktoras](https://github.com/silverbulletmd/silverbullet/pull/709) - * Fixed dark mode for templates, by [Ashish Dhama](https://github.com/silverbulletmd/silverbullet/pull/698) -* There are also two very notable new plugs you may be interested in trying: - * [[Plugs/TreeView]]: a sidebar showing (and allowing you to manipulate) your space’s folder tree (at long last) - * [[Plugs/AI]]: various clever AI integrations (supporting many different LLMs, including locally hosted ones) -* [[Snippets]] using `matchRegex` can now use the `|^|` caret to wrap text around the replacement, see the [[Snippets#Examples]] -* Changed the signature of `silverbullet.registerFunction` to make the first argument an object, see [[Space Script#Custom functions]]. Old string-based scripts still work, for backwards compatibility. -* The [[Functions#replace(str, match, replacement)]] function now supports multiple replacements -* You can now use backticks (`) around identifiers in [[Expression Language]], to e.g. use names with spaces or other weird characters as attribute names. -* [[Link Unfurl]] now supports unfurling youtube videos -* Fixed edit button on code widgets after they have shifted -* Fixed page completion in template blocks -* Giant code reorganization (hopefully resulting in 0 regressions) - ---- - -## 0.7.1 2024-02-08 -* Numerous bug fixes and significant performance improvements in template rendering (which now happen server-side), including code completion fixes. -* New `{{#each @varname in }}` syntax in [[Template Language#each directive]]. -* **Experimental feature**: [[Space Script]], the ability to extend SilverBullet from within your SilverBullet space with JavaScript. -* New [[Functions#readPage(name)]] function. -* New query sources: [[Objects#command]] and [[Objects#syscall]]. The [[Keyboard Shortcuts]] and [[Commands]] pages now use these to list all key bindings and available commands automatically. -* You can now create emoji aliases (implemented by [Maarrk](https://github.com/silverbulletmd/silverbullet/pull/679)), see [[SETTINGS]] for an example 😅 -* You can now conditionally show action buttons (see [[SETTINGS]]) _only_ on mobile devices (implemented by [Maarrk](https://github.com/silverbulletmd/silverbullet/pull/686)) - ---- - -## 0.7.0 2024-02-02 -I know what you’re thinking: another “major” minor release, so quickly? - -Yeah... we’re swapping out some of the guts of SilverBullet here, so this warrants the version bump. Honestly, this should have been 100.0, but you know... - -This is the one where the template engine SilverBullet is swapped out and no longer using [Handlebars.js](https://handlebarsjs.com/) under the hood. We’d like to thank handlebars for its effort, but everything has to end some day. It’s time to take our destiny into our own hands. - -SilverBullet now uses its own custom [[Template Language]] that is _way, way, way_ dare I say _way_ more powerful than Handlebars. I put in a lot of effort to ensure backwards compatibility, so unless you did some obscure things in your templates, everything should keep working as is. If not, please reach out on our [community](https://community.silverbullet.md) and I’ll do my best to support you in this journey that for sure will be worth it. - -Trust me, I’m an engineer. - -> **note** **Upgrade note** -> While the [[Library/Core]] library that you (probably) imported previously will keep working (I hope), I would recommend you do another import overwriting the old files. Mainly because many templates have been rebuilt to leverage the new features of our new [[Template Language]] and therefore much cleaner. So please follow the [[Library/Core#Installation]] instructions, and hit “Ok” about 30 times. - -So, what’s the fuss all about? - -* **Soft breaking change**: `include` is the new name of the old `template` [[Blocks]]. Previously, a template block’s primary function was to include an external template or page — that role has now been replaced with [[Live Templates#Include]]. If you used `template` blocks in the past, you will see a warning squiggly warning appear suggesting you turn it into an `include` block, which is functionally equivalent to the old `template.` To fix this across your space, simply replace triple-backtick `template` with triple-backtick `include` across all your files and you should be done. That said, SilverBullet will detect if you’re using a legacy syntax `template` blocks and automatically interpret them as `include` blocks instead so _nothing should break_. Famous last words. -* So, funny story, we’re also **instantly deprecating** these new `include` blocks and suggest you `template` all the things, see [[Live Templates#Recommended alternative]] for the rationale behind this insanity. You will ask: “So what is he going to deprecate next? [[Live Queries]]? _Hahaha!_” Honestly, probably yes, because everything those can do, templates do better — basically. But let’s take it slow for now. One step at a time, as my mom says. -* **`template`s are the new hotness**. The body of a `template` block is now interpreted as the new and shiny [[Template Language]]! This means you get syntax highlighting and (best effort) code completion for all your markdown and template directives endeavors. -* The [[Template Language]] and [[Query Language]] have been unified. It may not have been visible to everybody, but the handlebars syntax and SB’s own query language were different before. No longer. -* The [[Expression Language]] used as part of the new [[Template Language]] has been significantly expanded. Some notable new features: - * It now supports some basics like a `not` operator, and ternary operator (`bla ? true : false`). I know, the level of innovation here is crazy. - * It now supports (sub) queries as values, simply wrap any query in `{` and `}` and you’re good to go: [[Expression Language#Queries]] - * It now supports _page references_ as values: [[Expression Language#Page references]] - * It now (officially) supports function calls, and while I’ve kept the set of built-in [[Functions]] deliberately small for now — this is where the big wins are going to be gained in future releases: [[Expression Language#Function calls]]. Let me know in the [community](https://community.silverbullet.md) what functions you’d like to see. _Even the ability to expand the set of functions using JavaScript from within your space_ is now on the table for future iterations. -* A [[CHANGELOG]] entry is never not going to do this justice. I recommend you simply (re)read the following pages and check out the examples to have your 🤯 and start to think what you can do with this: - * [[Templates]] - * [[Template Language]] - * [[Query Language]] - * [[Expression Language]] -* Ah yes, you can also still take notes with SilverBullet. That still works. I think. - -## 0.6.1 2024-01-29 -* Tag pages: when you click on a `#tag` you will now be directed to a page that shows all pages, tasks, items and paragraphs tagged with that tag. -* Action buttons (top right buttons) can now be configured; see [[SETTINGS]] for how to do this. -* Headers are now indexed, meaning you can query them [[Objects#header]] and also reference them by name via page links using `#` that I just demonstrated 👈. See [[Links]] for more information on all the types of link formats that SilverBullet now supports. -* New {[Task: Remove Completed]} command to remove all completed tasks from a page -* **Read-only mode** (experimental) is here; see [[Install/Configuration#Run mode]] on how to enable it. This allows you to expose your space to the outside world in all its glory but without allowing anybody to edit anything. - * [silverbullet.md](https://silverbullet.md) has now been redeployed to run in this mode - * Pages in this mode are _server-side rendered_ so their content can actually be indexed by search engines. In addition, some [OpenGraph](https://ogp.me/) attributes are put in the HTML to enable nice embedding links, e.g. in our [community](https://community.silverbullet.md). -* New {[Clear Local Storage & Logout]} command to wipe out any locally synced data (and log you out if you use [[Authentication]]). -* Bug fixes: - * Improved Ctrl/Cmd-click (to open links in a new window) behavior: now actually follow `@pos` and `$anchor` links. - * Right-clicking links now opens the browser's native context menu again - * `tags` in [[Frontmatter]] are now properly indexed again when listed as an array -* Internal changes: - * Big refactor: of navigation and browser history, fixed some {[Page: Rename]} bugs along the way - * Plugs now can no longer define their own markdown syntax, migrated all plug-specific syntax into the main parser. This should remove a bunch of editor “flashing”, especially during sync. - ---- - -## 0.6.0 2024-01-22 - -* **Templates 2.0**: templates are now turbocharged (that’s a technical term) and have replaced a lot of previously built-in (slash) commands. There’s more to this than will fit this CHANGELOG, have a look at [[Templates]]: and more specifically [[Page Templates]], [[Snippets]], [[Live Template Widgets]] and [[Libraries]], and read the items below. -* **Upgrade instructions**: to get the best experience after upgrading to 0.6.0 as an existing user, do the following: - * Upgrade your docker image/deno version to 0.6.0 (or `latest`). - * Reload your page 2-3x to be sure you have the latest front-end code running. - * Run the {[Library: Import]} command in your space, and enter the following federation URL: `!silverbullet.md/Library/` This will import both the [[Library/Core]] and [[Library/Journal]] libraries into your space, bringing you roughly on par with 0.5.x versions in terms of functionality (this will include the daily note, weekly note, various slash commands etc.) -* A **quick FAQ** on the new template system: - * **Where did my templates go!?** They have now moved to the [[Meta Picker]], run {[Navigate: Page Picker]} (or press `Cmd-Shift-t` on Mac or `Ctrl-Shift-t` on Windows/Linux) to get to them. - * **Where did all my slash commands go?!** They are now distributed via [[Libraries]]. Yep, Libraries are here, enabling an easier way to distribute templates and pages. Read [[Libraries]] for more info. - * **But, what about slash templates etc.?!** Yeah, we did some rebranding and changed how these are defined. Slash templates are now [[Snippets]] and cannot _just_ be instantiated via [[Slash Commands]], but through [[Commands]] and custom keybindings as well. Awesomeness. - * **And my page templates broke!?** Yeah, same story as with [[Snippets]]: the format for defining these changed a bit, but it should be easy to update to the new format: check [[Page Templates]]. -* The [[Getting Started]] page (that is embedded in the `index` page that is auto-generated when creating a new space) has been updated to include instructions on how to import the [[Library/Core]] library. -* **Directives have now been removed** from the code base. Please use [[Live Queries]] and [[Live Templates]] instead. If you haven’t migrated yet and want to auto-migrate, downgrade your SilverBullet version to 0.5.11 (e.g. using the `zefhemel/silverbullet:0.5.11` docker image) and run the {[Directive: Convert Entire Space to Live/Templates]} command with that version. -* (Hopefully subtle) **breaking change** in how tags work (see [[Objects]]): - * Every object now has a `tag` attribute, signifying the “main” tag for that object (e.g. `page`, `item`) - * The `tags` attribute will now _only_ contain explicitly assigned tags (so not the built-in tag, which moved to `tag`) - * The new `itags` attribute (available in many objects) includes both the `tag` and `tags` as well as any tags inherited from the page the object appears in. - * Page tags now no longer need to appear at the top of the page, but can appear anywhere as long as they are the only thing appearing in a paragraph with no additional text; see [[Objects#page]]. -* New [[Markdown/Code Widgets|Code Widget]]: `toc` to manually include a [[Table of Contents]] -* Filter list (used by [[Page Picker]], [[Meta Picker]] and [[Command Palette]]) improvements: - * Better ranking - * Better positioning of modal (especially on mobile) - * Better mouse behavior -* Templates: - * Somewhat nicer rendering of {{templateVars}} (notice the gray background) - * Rendering of [[Markdown/Code Widgets]] (such as live queries and templates) **is now disabled** on template pages, which should make them less confusing to read and interpret. -* The `indexPage` [[SETTINGS]] can now contain template variables, such as `{{today}}` -* Backend work in preparation for supporting more “serverless” deployments (e.g. Cloudflare workers and Deno Deploy) in the future - * Move from [Oak](https://oakserver.github.io/oak/) to [Hono](https://hono.dev/) - * Support for in-process plug loading (without workers) - ---- - -## 0.5.11 2024-01-02 -* Keyboard shortcuts as well as priority (order in which they appear in the [[Command Palette]]) can now be configured for [[Commands]] in [[SETTINGS]]. The `priority` enables you to put frequently used commands at the top. -* The rendering of [[Live Templates]], [[Live Queries]], [[Table of Contents]] and [[Linked Mentions]] has been re-implemented. Rendering should now be near-instant, and the “flappy” behavior should be largely gone, especially after an initial load (results are cached). There may still be some visual regressions. Please report them if you find them. - ---- - -## 0.5.10 2023-12-22 -* **Breaking change**: Local attachment URLs (`[page](url)` syntax and `![alt](url)` image syntax) are now interpreted relative to the page's folder unless their URL starts with a `/`, then they're relative to the space root (as per [this issue](https://github.com/silverbulletmd/silverbullet/issues/363)) -* **Breaking change:** Revamped [[Templates]], specifically changed the format of [[Page Templates]]. The “Template: Instantiate Page” has been renamed to {[Page: From Template]}. -* It is now even more recommended to tag your [[Templates]] with the “template” tag because completion in [[Live Queries]] and [[Live Templates]] will now only suggest `#template` tagged pages. -* New [[Frontmatter]] attributes with special meaning: `displayName` and `aliases` (allowing to specify alternative names for pages) -* The [[Page Picker]] now also shows (and matches against) tags, aliases and display names for pages. -* It is now possible to filter pages based on tags in the [[Page Picker]] by typing a hashtag in the filter phrase, e.g. `#template` to filter pages that have a `template` tag. -* Added new commands to manage [[Outlines]]. Note this resulted in changing names and keyboard shortcuts for managing folds as well, to be more consistent with the other outline commands. -* Removed built-in multi-user [[Authentication]], `SB_AUTH` is no longer supported; use `--user` or `SB_USER` instead or an authentication layer such as [[Authelia]]. -* Background and more experimental work: - * Work on supporting multiple database and storage backends, reviving [[Install/Deno Deploy]] support. - * This is now documented on the brand-new [[Install/Configuration]] page. - * A new `silverbullet sync` command to [[Sync]] spaces (early days, use with caution) - * Technical refactoring in preparation for multi-tenant deployment support (allowing you to run a single SB instance and serve multiple spaces and users at the same time) - * Lazy everything: plugs are now lazily loaded (after a first load, manifests are cached). On the server side, a whole lot of infrastructure is now only booted once the first HTTP request comes in - ---- - -## 0.5.8 2023-12-07 -* Various bugfixes, primarily related to the new way of running docker containers, which broke things for some people. Be sure to have a look at the new [[Install/Configuration]] configuration options - ---- - -## 0.5.7 2023-11-28 -* New {[Upload: File]} command to upload files and pictures (particularly valuable for mobile use cases). Implemented by [prcrst](https://github.com/silverbulletmd/silverbullet/pull/571). -* General support for highlighting errors (underlined) in the editor. Currently implemented for: - * All YAML fenced code blocks (and [[Frontmatter]]): will now highlight YAML parse errors - * [[Live Queries]]: will highlight non-existing query sources and non-existing template references in `render` clauses -* Basic [[Table of Contents]] support: any page _with 3 headers or more_ now has a “Table of Contents” widget appear (see this very page). You can toggle this feature using the {[Table of Contents: Toggle]} command. -* Tapping/clicking the top bar (outside of the page name and action buttons) now scrolls your page to the very top. -* Slightly more gracious error reporting on load when using the Online [[Client Modes]] and the server is offline. -* Any page tagged with `#template` is no longer indexed (besides as a `template`) -* Upgraded set of emoji (completed via the :thinking_face: syntax) to 15.1 (so more emoji) -* Various bug fixes - ---- -## 0.5.6 2023-11-19 -* Various optimization and bug fixes -* Experimental idea: [[Libraries]] -* The `Alt-Shift-n` key was previously bound to both {[Page: New]} and {[Quick Note]}. That won’t work, so now it’s just bound to {[Quick Note]} -* The `Alt-q` command is now bound to the new {[Live Queries and Templates: Refresh All]} command, refreshing all [[Live Queries]] and [[Live Templates]] on the page. This is to get y’all prepared to move away from directives. -* It’s likely that version 0.6.0 **will remove directives**, so please switch over to live queries and templates, e.g. using... - * The new {[Directive: Convert Entire Space to Live/Templates]} command, which will (attempt) to convert all uses of directives in your space automatically (backup your space before, though, just in case) - - ---- -## 0.5.5 2023-11-13 -* Bugfix: on some filesystems that don't report file creation time (like some NASes), SilverBullet crash. This should now be fixed. -* Performance improvements the loading of code widgets (e.g. Live Queries, Live Templates) - ---- -## 0.5.4 2023-11-11 -* We’re on a journey to rethink [[Templates]]: - * It is now _recommended_ you tag all your templates with a `#template` tag, this will exclude them from [[Objects]] indexing and may in the future be used to do better template name completion (but not yet). - * New feature: Introducing [[Snippets]], allowing you to create custom [[Slash Commands]]. This deprecates snippets and page templates, because [[Snippets]] are awesomer. -* Many styling fixes and improvements to [[Live Queries]] and [[Live Templates]] -* Added a “source” button to [[Live Queries]] and [[Live Templates]] for better debugging (showing you the markdown code rendered by the template so you can more easily detect issues) -* [[Live Queries]]: - * Support for `render all` where the entire result set is passed to a single template allowing you to e.g. dynamically build up tables, see [[Live Queries$render]] for an example. -* The default generated [[SETTINGS]] page now contains a link to [[SETTINGS]] on silverbullet.md for documentation purposes. -* The syntax to reference [[Markdown/Anchors]] has now changed to use `$`, instead of `@` (e.g. [[Live Queries$render]]), the old syntax still works but is deprecated. The reason for this change is consistency: you define an anchor using the `$myanchor` syntax, referencing it the same way makes more sense. -* [[Page Name Rules]] are now documented - ---- -## 0.5.3 2023-10-13 -* Changes to [[Objects]]: - * Paragraphs are now indexed, see [[Objects#paragraph]] (thanks to [Ian Shehadeh](https://github.com/silverbulletmd/silverbullet/pull/528)) - * For consistency, list items are now always indexed as well (whether they contain a [[Tags|tag]] or not) see [[Objects#item]]. -* The {[Directive: Convert to Live Query/Template]} now also converts `#use` and `#include` directives -* Styling improvements for Linked Mentions -* SilverBullet now fully works when added as PWA on Safari 17 (via the “Add to Dock” option). -* Fix support for handlebars variables in [[Live Queries]] and [[Live Templates]] -* Plug robustness improvements (SB shouldn’t fully break when loading plugs that rely on disabled syscalls) -* Various other bug fixes - ---- - -## 0.5.1 2023-10-05 -* Fixes to auto-sizing of [[Live Queries]] and [[Live Templates]] widgets -* Fixed the combination of `limit` and `order by` not working well -* Auto complete for queries now works for queries split across multiple lines -* Auto complete for fenced code block languages (use the `/code` slash command) -* Slightly tweaked semantics for the `=` operator on arrays, see [[Live Queries]] for details - ---- - -## 0.5.0 2023-10-03 -Oh boy, this is a big one. This release brings you the following: - -* [[Objects]]: a more generic system for indexing and querying content in your space, including the ability to define your own custom object “types” (dubbed [[Tags]]). See the referenced pages for examples. -* [[Live Queries]] and [[Live Templates]]: ultimately will replace directives in future versions and **directives are now deprecated.** They differ from directives in that they don’t materialize their output into the page itself, but rather render them on the fly so only the query/template instantiation is kept on disk. All previous directive examples on this website how now been replaced with [[Live Templates]] and [[Live Queries]]. To ease the conversion there is {[Directive: Convert Query to Live Query]} command: just put your cursor inside of an existing (query) directive and run it to auto-convert. -* The query syntax used in [[Live Queries]] (but also used in directives) has been significantly expanded, although there may still be bugs. There’s still more value to be unlocked here in future releases. -* The previous “backlinks” plug is now built into SilverBullet as [[Linked Mentions]] and appears at the bottom of every page (if there are incoming links). You can toggle linked mentions via {[Mentions: Toggle]}. -* A whole bunch of [[PlugOS]] syscalls have been updated. I’ll do my best update known existing plugs, but if you built existing ones some things may have broken. Please report anything broken in [Github issues](https://github.com/silverbulletmd/silverbullet/issues). - -**Important**: -* If you have plugs such as “backlinks” or “graphview” installed, please remove them (or to be safe: all plugs) from the `_plug` folder in your space after the upgrade. Then, also remove them from your `PLUGS` page. The backlinks plug is now included by default (named [[Linked Mentions]]), and GraphView still needs to be updated (although it’s been kind of abandoned by the author). - -Due to significant changes in how data is stored, likely your space will be resynced to all your clients once you upgrade. Just in case you may also want to {[Space: Reindex]} your space. If things are really broken, try the {[Debug: Reset Client]} command. - ---- - -## 0.4.0 2023-09-09 -The big change in this release is that SilverBullet now supports two [[Client Modes|client modes]]: _online_ mode and _sync_ mode. Read more about them here: [[Client Modes]]. - -Other notable changes: -* Massive reshuffling of built-in [[Plugs]], splitting the old “core” plug into [[Plugs/Editor]], [[Plugs/Template]] and [[Plugs/Index]]. -* Directives in [[Live Preview]] now always take up a single line height. -* [[Plugs/Tasks]] now support custom states (not just `[x]` and `[ ]`), for example: - * [IN PROGRESS] An in progress task - * [BLOCKED] A task that’s blocked - [[Plugs/Tasks|Read more]] -* Removed [[Cloud Links]] support in favor of [[Federation]]. If you still have legacy cloud links, simply replace the 🌩️ with a `!` and things should work as before. +For previous versions, see [the v1 CHANGELOG](https://v1.silverbullet.md/CHANGELOG) \ No newline at end of file diff --git a/website/Cloud Links.md b/website/Cloud Links.md deleted file mode 100644 index 0003f04c..00000000 --- a/website/Cloud Links.md +++ /dev/null @@ -1 +0,0 @@ -Deprecated in favor of [[Federation]] \ No newline at end of file diff --git a/website/Expression Language.md b/website/Expression Language.md deleted file mode 100644 index b773ba61..00000000 --- a/website/Expression Language.md +++ /dev/null @@ -1,116 +0,0 @@ -SilverBullet has a simple expression language that is used by the [[Query Language]], [[Template Language]] and `where` condition in [[Live Template Widgets]]. - -Examples in this page will be demonstrated by embedding expressions inside of a [[Templates|template]]. - -While a custom language, it takes a lot of inspiration from JavaScript and SQL, but includes some features very specific to SilverBullet, including syntax for embedded queries and page references. - -# Primitive values -* strings: `"a string"`, or `'a string'`. Escaping is not currently supported, so use your quotes wisely. -* numbers: `10` -* booleans: `true` or `false` -* regular expressions: `/[a-z]+/` -* null: `null` -* lists: `["value 1", 10, false]` -* objects: `{"name": "Jack", "age": 1232}` -* identifiers: starting with a letter, followed by alphanumerics, `_` or `-`. Identifiers can also be surrounded with backticks: ` and in that case contain any non-backtick characters, including spaces. -* attributes: - * `.` for the current object - * `attr` for the current object’s attribute with the name `attr` - * `.attr` to access an attribute from the object that `` evaluates to -* variables: - * `@page` points to the currently open page - * `@config` contains the space’s current [[Space Config]] - * Variables defined through `#let` or `#each` directives in [[Template Language]] - -## Examples -```template -String expression: {{"This is a string"}} -List expression: {{[1, 2, 3]}} -Attribute of variable: {{@page.name}} -``` - -# Function calls -* `functionName(argument1, argument2, ...)`: call function `functionName` -* `functionName()` or `functionName`: call function `functionName` without arguments - -## Examples -```template -contains with argument list: {{contains("FizzBuzz", "Fizz")}} -Today with argument list: {{today()}} -Today without an argument list: {{today}} -``` - -## Supported functions -[[Functions]] - -# Queries -Expression can include [[Query Language|queries]] using the `{ page limit 1 }` syntax. Queries evaluate to one of two things: - -* Queries with a `render` or `render all` clause will evaluate a string, where the referenced template is applied (similar to [[Live Queries]]). -* Queries _without_ a `clause` will evaluate to a list of results. - -Note that you can reference variables inside your query as well. - -## Example -Incomplete task: -* [ ] This task is not yet complete - -```template -Number of incomplete tasks on this page: {{count({task where not done and page = @page.name})}} - -A rendered query: -{{{page limit 3 render [[Library/Core/Query/Page]]}}} -``` - -# Page references -Page references use the `[[page name]]` syntax and evaluate to the content of the referenced page (as a string), this makes them a good candidate to be used in conjunction with [[Functions#template(text, value)]] or to simply inline another page: - -```template -Including another page directly, without template rendering: {{[[internal/test page with template]]}} - -And rendered as a template: {{template([[internal/test page with template]], "actual value")}} -``` - -# Logical expressions -* and: `name = "this" and age > 10` -* or: `name = "this" or age > 10` - -## Examples -```template -Simple boolean expression: {{"pete" = "pete" or "hank" = "pete"}} -``` - -# Operators -* `=` equals - * For scalar values this performs an equivalence test (e.g. `10 = 10`) - * If the left operand is an array and the right operand is _not_, this will check if the right operand is _included_ in the left operand’s value, e.g. `[1, 2, 3] = 2` will be true. - * If both operands are arrays, they will be compared for equivalence ignoring order, so this will be true: `[1, 2, 3] = [3, 2, 1]` -* `!=` the exact inverse of the meaning of `=`, e.g. `name != "Pete"` -* `<` less than, e.g. `age < 10` -* `<=` less than or equals, e.g. `age <= 10` -* `>` greater than, e.g. `age > 10` -* `>=` greater than or equals, e.g. `age >= 10` -* `=~` to match against a regular expression, e.g. `name =~ /^template\//` (the operand can either be a regular expression or a string expression that will be turned into a regular expression) -* `!=~` to not match a regular expression, e.g. `name !=~ /^template\//` -* `in` member of a list (e.g. `prop in ["foo", "bar"]`) -* `+` addition (can also concatenate strings), e.g. `10 + 12` or `name + "!!!"` -* `-` subtraction, e.g. `10 - 12` -* `/` division, e.g. `10 / 12` -* `*` multiplication, e.g. `10 * 12` -* `%` modulo, e.g. `10 % 12` -* `not ` or `!` to negate the truthiness value -* ` ? : ` - -## Operator precedence -Operator precedence follows standard rules, use parentheses when in doubt, e.g. `(age > 10) or (name = "Pete")` - -## Examples -```template -Some arithmatic: {{10 * 100 / 1000}} -Some regexp match: {{"hello" =~ /ell/}} -Some list check: {{["template", "other"] = "template"}} -{{8 > 3 ? "8 is larger than 3" : "8 is not larger than 3"}} -``` - -# Comments -Line comments are supported via `# this is a line comment` diff --git a/website/Federation.md b/website/Federation.md deleted file mode 100644 index 549aab13..00000000 --- a/website/Federation.md +++ /dev/null @@ -1,25 +0,0 @@ -Federation enables _browsing_ content from spaces _outside_ the user’s space, specified by quasi-URLs in the shape of `!domain.tld/path`. An example would be: [[!silverbullet.md/CHANGELOG]]. - -This enables a few things: - -* **Browsing** other publicly hosted SilverBullet spaces (or websites adhering to its [[HTTP API]]) within the comfort of your own SilverBullet client. One use case of this is [[Transclusions|transcluding]] the [[Getting Started]] page in the user’s automatically generated index page when setting up a fresh space. -* **Referencing** other spaces for other purposes, which is leveraged in [[Libraries]]. - -# How it works -Effectively, any page [[Links|link]] starting with `!` is rewritten as follows: - -* Replace the initial `!` with `https://` -* Append `.md` at the end - -The resulting URL is then fetched and displayed in the editor in read-only mode. This means that you can navigate to _any markdown_ file on the (public) Internet. - -For example: `https://raw.githubusercontent.com/silverbulletmd/silverbullet/main/README.md` -Can be written to federation syntax as follows: `!raw.githubusercontent.com/silverbulletmd/silverbullet/main/README` -And used as a link: [[!raw.githubusercontent.com/silverbulletmd/silverbullet/main/README]] - -If the target server supports the SilverBullet [[HTTP API]] (specifically its `/index.json` endpoint), page completion will be provided as well. - -Upon fetching of the page content, a best effort attempt will be made to rewrite any local page links in the page to the appropriate federated paths. - -## Hosting -Tooling to make hosting public spaces easier is still a work in progress. The way it is enabled on `silverbullet.md` is by running a dedicated SilverBullet instance in read-only mode (see [[Install/Configuration#Run mode]]). \ No newline at end of file diff --git a/website/Functions.md b/website/Functions.md deleted file mode 100644 index 7b0ef7fa..00000000 --- a/website/Functions.md +++ /dev/null @@ -1,70 +0,0 @@ -Here is a list of all functions you can use in SilverBullet’s [[Expression Language]]: - -# Lists -## count(list) -Count the number of elements in a list: - -```template -There are **{{count({page})}}** pages in this space. -``` - -## at(list, index) -Returns the `index`th element of `list` (starting from 0 of course). - -```template -Index 1 of our count to three is {{at([1, 2, 3], 1)}} -``` - -# Templates -## template(text, value) -Renders a template with an optional value: - -```template -{{template([[Library/Core/Query/Page]], {"name": "Some page"})}} -``` - - -# Date and time -## today() -Today’s date in a `YYYY-MM-dd` format. - -## tomorrow() -Tomorrow’s date in a `YYYY-MM-dd` format. - -## yesterday() -Yesterday’s date in `YYYY-MM-dd` format. - -## niceDate(timestamp) -Convert a unix timestamp to a `YYYY-MM-dd` format. - -## time() -Current time. - -# String manipulation -## replace(str, match, replacement) -Replace text in a string. `match` can either be a literal string or a regular expression: `replace("hello", "ell", "all")` would produce “hallo”, and `replace("hello", /l/, "b")` would produce “hebbo”. - -This function supports an infinite number of replacements, so you can keep adding more, e.g. `replace(str, match1, replacement1, match2, replacement2, match3, replacement3)` - -## contains(str, substr) -Returns whether `str` contains `substr` as a substring. - -## json(obj) -Convert the argument to a JSON string (for debugging purposes). - -```template -The current page object: {{json(@page)}} -``` - -# Space related -## pageExists(name) -Checks if the page `name` exists: - -```template -This very page exists: {{pageExists(@page.name)}} - -And this one: {{pageExists("non-existing")}} -``` - -## readPage(name) -Reads in the content of the page `name` and returns it as a string. \ No newline at end of file diff --git a/website/Libraries.md b/website/Libraries.md deleted file mode 100644 index 946dd166..00000000 --- a/website/Libraries.md +++ /dev/null @@ -1,68 +0,0 @@ -A lot of useful functionality in SilverBullet is implemented through [[Templates]], as well as regular [[Pages]]. Some of these you will create yourself for your own specific use, but many are generic and generally useful. Libraries offer a way to _distribute_ sets of templates and pages easily, but also help to give some guidance on how to structure templates for your personal use. - -# What’s in a library -Here are some things that a library may provide: -* Various [[Slash Commands]], such as `/today`, `/task`, `/table`. -* Useful [[Page Templates]] -* Useful widgets such as [[Table of Contents]] or [[Linked Mentions]] -* Useful pages that help you perform maintenance on your space, like detecting broken links, such as [[Library/Core/Page/Maintenance]]. - -# Configuring libraries -Libraries are managed by specifying them in [[SETTINGS]] under the `libraries` attribute. The paths used to libraries are using [[Federation]] page reference syntax. - -When you set up a fresh space, the [[Library/Core]] is automatically configured: - -```yaml -libraries: -- import: "[[!silverbullet.md/Library/Core/*]]" -``` - -If you would like to _exclude_ specific pages, for instance [[Library/Core/Widget/Table of Contents]], you can do so using the library’s `exclude` attribute - -```yaml -libraries: -- import: "[[!silverbullet.md/Library/Core/*]]" - exclude: - - "[[!silverbullet.md/Library/Core/Widget/Table of Contents]]" -``` - -In these paths, you can also use the `*` wildcard: - -```yaml - - "[[!silverbullet.md/Library/Core/Widget/*]]" -``` - -This would exclude all widgets specified in the Core library. - -# Updating libraries -To update all libraries from their original source (we recommend doing this regularly to get the latest updates), run the {[Libraries: Update]} command. - -Updating does the following for each library: - -1. It _deletes_ all pages under its local source prefix in your space. For instance for the Core library, this would be `Library/Core/*`. This means that you should **not make local changes** to these files, because they will be wiped out on every update. -2. It downloads fresh copies from the `source` and puts them in the local source prefix (for `!silverbullet.md/Library/Core/*` this would be `Library/Core/`). -3. It performs a {[System: Reload]} so all new commands and configurations are immediately available. - -# Where are libraries stored? -Libraries are kept in your space’s `Library/` folder. Since these are just regular [[Meta Pages]], the just live along the rest of your content. For instance, the [[Library/Core]] is kept under `Library/Core`. This is _largely_ a convention, but one worth to sticking to. - -If you would like to keep your own templates as a library as well, we recommend putting them in `Library/Personal` as a convention. Then, when you identify a set that may be valuable to other people, you can move them elsewhere under `Library/`. - -# Can I change libraries locally? -Technically there is nothing stopping you from making local changes to library content, however **these changes would be reset on every library update**. - -Therefore, the recommended approach when requiring to make local adjustments is: -1. Make a copy (copy & paste or {[Page: Copy]}) of the page or template you want to change _outside_ the original library path, e.g. under `Library/Personal`. -2. _Exclude_ the page in question using the `exclude` attribute explained under [[#Configuring libraries]] and then run {[Libraries: Update]} again, to remove the excluded (non-modified) version of the page from your space. -3. Profit - -# How to host a library? -This is a topic for exploration. The way it is done here at `silverbullet.md` is by running a SilverBullet instance with the libraries in read-only mode (see [[Install/Configuration#Run mode]]). However, we should explore easier ways to do this. Come talk to us in [the SilverBullet Community](https://community.silverbullet.md/) if you’re interested. - -# What libraries exist? -Libraries are still a young concept in SilverBullet and therefore we’re still exploring how to organize and structure these. - -Currently, we have the following libraries available: -* [[Library/Core]]: this is the library you want to use _for sure_. Just do it. -* [[Library/Journal]]: for the journalers among us. - diff --git a/website/Library/Core/Page/Maintenance.md b/website/Library/Core/Page/Maintenance.md index dad17605..9dfd7b1c 100644 --- a/website/Library/Core/Page/Maintenance.md +++ b/website/Library/Core/Page/Maintenance.md @@ -3,35 +3,20 @@ We would like to keep our space clean. These are some tools that help you do that. # Aspiring pages -This shows all page links that link to a page that does not (yet) exist. These could be broken links or just pages _aspiring_ to be created. +This shows page links (max 20 to keep things sensible) that link to a page that does not (yet) exist. These could be broken links or just pages _aspiring_ to be created. -```template -{{#let @brokenLinks = {aspiring-page}}} -{{#if @brokenLinks}} -{{#each @brokenLinks}} -* [[{{ref}}]]: broken link to [[{{name}}]] -{{/each}} -{{else}} -No aspiring pages, all good! -{{/if}} -{{/let}} -``` +${template.each(query[[ + from index.tag "aspiring-page" + limit 20 +]], template.new[==[ + * [[${ref}]]: broken link to [[${name}]] +]==])} # Conflicting copies These are pages that have conflicting copies (as a result of sync). Have a look at them as well as their original (non-conflicting) versions and decide which one to keep. -```template -{{#let @conflictPages = { - page where name =~ /\.conflicted\.\d+$/ - select name as conflictedName, - replace(name, /\.conflicted\.\d+$/, "") as cleanName -}}} -{{#if @conflictPages}} -{{#each @conflictPages}} -* [[{{cleanName}}]]: confict copy [[{{conflictedName}}]] -{{/each}} -{{else}} -No conflicting pages! -{{/if}} -{{/let}} -``` +${template.each(query[[ + from index.tag "page" where name:find("%.conflicted%.") +]], template.new[==[ + * [[${name:gsub("%.conflicted%..+$", "")}]]: conflict copy [[${name}]] +]==])} diff --git a/website/Library/Core/Page/Space Overview.md b/website/Library/Core/Page/Space Overview.md deleted file mode 100644 index 662f3bbe..00000000 --- a/website/Library/Core/Page/Space Overview.md +++ /dev/null @@ -1,38 +0,0 @@ -#meta - -This page compiles some useful things about your space and may also be useful for debugging things. - -# Stats -```template -**Total pages:** {{count({page select name})}} -**Total documents:** {{count({document select name})}} -**Total tags:** {{count({tag select name})}} -``` - -# Active [[!silverbullet.md/Space Lua]] -${template.each(query[[ - from index.tag "space-lua" - select {name= ref} -]], templates.pageItem)} - -# Active [[!silverbullet.md/Space Style]] -```template -{{#each {space-style}}} -* [[{{ref}}]] -{{/each}} -``` - -# Active [[!silverbullet.md/Space Config]] -You have space config defined on the following pages: -```template -{{#each {space-config select replace(ref, /@.+/, "") as page}}} -* [[{{page}}]] -{{/each}} -``` - -Composed, this leads to the following active configuration: -```template -~~~yaml -{{yaml(@config)}} -~~~ -``` diff --git a/website/Library/Core/Page/Template Index.md b/website/Library/Core/Page/Template Index.md deleted file mode 100644 index a79446f6..00000000 --- a/website/Library/Core/Page/Template Index.md +++ /dev/null @@ -1,52 +0,0 @@ -#meta - -This page lists all templates currently available in your space. You can also navigate to them via the {[Navigate: Meta Picker]}. More information on templates can be found [[!silverbullet.md/Templates]] - -# New Page -These [[!silverbullet.md/Page Templates]] are available through the {[Page: From Template]} command: - -```query -template where hooks.newPage render [[Library/Core/Query/Template]] -``` - -# Snippets -These can be used as [[!silverbullet.md/Snippets]] via [[!silverbullet.md/Slash Commands]]: - -```template -{{#each { - template - where hooks.snippet - order by hooks.snippet.slashCommand -}}} -* [[{{ref}}|/{{hooks.snippet.slashCommand}}]] {{description}} -{{/each}} -``` - -# Widgets -Widgets can either be automatically attached to the top or bottom of pages (matching certain criteria) or used inline via [[!silverbullet.md/Live Templates]]. - -## Top -```query -template -where hooks.top -order by `order` -render [[Library/Core/Query/Template]] -``` - -## Bottom -```query -template -where hooks.bottom -order by `order` -render [[Library/Core/Query/Template]] -``` - -## Inline -Use with [[!silverbullet.md/Live Templates#Include]] to render useful things in your pages: - -```query -template -where name =~ /\/Widget\// and not hooks.top and not hooks.bottom -order by `order` -render [[Library/Core/Query/Template]] -``` diff --git a/website/Live Embeds.md b/website/Live Embeds.md deleted file mode 100644 index 80f428a1..00000000 --- a/website/Live Embeds.md +++ /dev/null @@ -1,13 +0,0 @@ -Live Embeds allow you to embed internet content into your page inside of an iframe. This is useful to embed youtube videos or other websites. - -```embed -url: https://youtu.be/8btx9HeuZ4s -``` - -Note, there is specific support for YouTube videos — it automatically sets the width and height, and replaces the URL with an embed URL. - -The body of an `embed` block is written in [[YAML]] and supports the following attributes: - -* `url` (mandatory): the URL of the content to embed -* `height` (optional): the height of the embedded page in pixels -* `width` (optional): the width of the embedded page in pixels diff --git a/website/Live Queries.md b/website/Live Queries.md deleted file mode 100644 index f5cfb4d9..00000000 --- a/website/Live Queries.md +++ /dev/null @@ -1,16 +0,0 @@ -#level/intermediate - -Live Queries are a [[Blocks|block]] that generates a (quasi) live view on various data sources, usually [[Objects]], and renders their results inline via [[Live Preview]] either as a table or using [[Templates]]. - -The syntax used is: - - ```query - page limit 3 - ``` - -Queries are written using SilverBullet’s [[Query Language]]. - -# Baking -A query block can be replaced with its current output by clicking the "Bake result" button in the top right corner. This will make it a regular part of page, which will not respond to changes in the data source anymore. - -You can also use the {[Page: Bake live blocks]} command to do it in-place for every [[Blocks|block]] in this page which is rendered as Markdown (but not [[Live Template Widgets]]). You can undo the replacement with `Ctrl+Z`/`Cmd+Z` as usual, but if you want to keep your queries you should make a copy of the page first. diff --git a/website/Live Template Widgets.md b/website/Live Template Widgets.md deleted file mode 100644 index 4084ff5c..00000000 --- a/website/Live Template Widgets.md +++ /dev/null @@ -1,26 +0,0 @@ -#level/hardcore - -Live Template Widgets allow you to automatically render templated markdown widgets at the top or bottom of pages matching specific criteria. - -> **warning** Warning -> This feature is still _experimental_, aspects of it may change, or it could be removed altogether. - -# Defining -Live Template Widgets are regular [[Templates]] that set a `hooks.top` or `hooks.bottom` attribute (depending on where you would like them to appear), specifying: - -* `where`: should contain an [[Live Queries$expression]] that evaluates to true for the _pages_ you would like to apply this template to, usually this checks for a specific tag, but it can be any expression. Think of this as a `where` clause that should match for the pages this template is applied to. -* `order` (optional): in case you have multiple templates that have matching `where` expression, the one with the priority set to the lowest number wins. - -# Example -The following widget template applies to all pages tagged with `person` (see the `where`). It uses the [[Library/Core/Widget/Linked Tasks]] template to show all incomplete tasks that contain a link to the current page. - - --- - tags: template - hooks.top.where: 'tags = "person"' - --- - ## Incoming tasks - ```template - page: "[[!silverbullet.md/Library/Core/Widget/Linked Tasks]]" - ``` - -More examples can be found [[Library/Core/Page/Template Index$widgets|here]]. diff --git a/website/Live Templates.md b/website/Live Templates.md deleted file mode 100644 index 9b8421a0..00000000 --- a/website/Live Templates.md +++ /dev/null @@ -1,45 +0,0 @@ -#level/intermediate - -Live Templates are a type of [[Blocks|block]] that render [[Templates]] written in [[Template Language]] inline in a page. - -There are two variants of Live Templates: - -* `template`: where the template is specified inline in a code block. -* `include`: where an external page (template) is _included_ and rendered. - -Template blocks are specified using [[Markdown]]‘s fenced code block notation using either `template` or `include` as its language. They can also be [[Live Queries#Baking|baked]]. - -# Template -To specify a template to render inline, you can use the `template` block. The body is written in [[Template Language]]. - -```template -Today is {{today}} -``` - -# Include -> **warning** Deprecated, use templates instead -> Include template are primarily here to be a drop-in replacement for the old style template blocks, see below for a suggestion how to use those instead. - -A `template` block is configured using [[YAML]] in its body. The following attributes are supported: - -* `page`: the page to use as a template -* `value`: an (optional) value to pass to the template -* `raw`: a page reference to include in the page without processing it as a template. - -Here’s an example using `page`: -```include -page: "[[internal-template/today]]" -``` - -If you want to include another _page_ (not necessarily a template). unprocessed (so without replacing template placeholders), you can use `raw`: -```include -raw: "[[internal/test page]]" -``` - -## Recommended alternative -Instead of using the `include` block we recommend you use a `{{template([[template]])}}` directive in a `template` instead as it is more flexible and more natural. - -Look for yourself: -```template -{{template([[internal-template/today]])}} -``` diff --git a/website/Markdown/Command links.md b/website/Markdown/Command links.md deleted file mode 100644 index 5db64413..00000000 --- a/website/Markdown/Command links.md +++ /dev/null @@ -1,10 +0,0 @@ -Command links allow you to create buttons in your pages that trigger commands. - -# Basic use -{[Stats: Show]} or {[Open Daily Note]} - -# Aliasing -{[Stats: Show|Show me stats]} - -# Passing arguments -{[Flash: Custom Message|Say hello]("hello there")} \ No newline at end of file diff --git a/website/Object Decorators.md b/website/Object Decorators.md deleted file mode 100644 index 10650ffc..00000000 --- a/website/Object Decorators.md +++ /dev/null @@ -1,81 +0,0 @@ -#level/hardcore - -Object decorators are an **advanced technique** that can be used to add attributes to [[Objects]] dynamically whose values are _calculated dynamically_ (on-the-fly) based on an [[Expression Language|expression]]. - -> **warning** Warning -> This feature is still experimental and may change in the (near) future. - -The primary use case is [[Page Decorations]], but it is a powerful mechanism that probably has wider applications. As always, with great power comes great responsibility. - -# Syntax -Object decorations are specified using [[Space Config]] (either in [[^SETTINGS]] or elsewhere): - -~~~markdown -```space-config -objectDecorators: -- where: '<>' - attributes: - <>: '<>' -``` -~~~ - -> **note** Note -> For changes to take effect you may have to reload your client (just refresh the page). - -A few things of note: - -* `<>` is a [[YAML]] string-encoded expression using SilverBullet’s [[Expression Language]]. Some examples: - * `where: 'tags = "book"'` to make this apply to all objects that has `book` as one of its tags. -* `<>` can either be a simple attribute name, or a nested one using the `attribute.subAttribute` syntax. Some examples: - * `fullName` - * `pageDecoration.prefix` -* `<>` like `<>` must be a YAML string-encoded expression using the [[Expression Language]], some examples together with the attribute path: - * `alwaysTen: '10'` (attaches an attribute named `alwaysTen` with the numeric value `10` to all objects matching the `where` clause) - * `alwaysTrue: 'true'` (same as `alwaysTen` but with a boolean value) - * `fullName: 'firstName + " " + lastName'` (attaches a `fullName` attribute that concatenates the `firstName` and `lastName` attributes with a space in between) - * `nameLength: 'count(name)'` (attaches an attribute `nameLength` with the string length of `name` — not particularly useful, but to demonstrate you can call [[Functions]] here too). - -## Rules -A few rules and best practices to keep things civil: - -* It is recommended to _always filter based on `tag`_ (so by adding e.g. `tag = "page"` to your `where` clause) to limit the radius of impact. Otherwise you may accidentally apply new attributes of all your [[Objects]] (items, tasks, pages, etc.). -* Dynamic attributes _cannot override already existing attributes_. If the object already has an attribute with the same name, this value will be kept as is. -* For performance reasons, all expressions (both filter and value expressions) need to be _synchronously evaluatable_. - * Generally, this means they need to be “simple expressions” that require no expensive calls. - * Simple expressions include simple things like literals, arithmetic, calling some of the cheap [[Functions]] such as `today()` or string manipulation functions. - * Expensive calls include any additional database queries, or any function call (custom or otherwise) that are _asynchronous_. These are _not supported_. - * This requirement **will be checked at runtime**. Watch your server logs and browser’s JavaScript JavaScript console to see these errors. - -# Example -Let’s say that you use the `human` tag to track various humans in your space, as follows: - -```#human -firstName: Steve -lastName: Bee ---- -firstName: Stephanie -lastName: Bee -``` - -This will get you the following data set: - -```query -human select firstName, lastName -``` - -However, you would like to dynamically compute an additional attribute for all humans, namely `fullName`. This can be done as follows with [[Space Config]]: - -```space-config -objectDecorators: -- where: 'tag = "human"' - attributes: - fullName: 'firstName + " " + lastName' -``` - -Which will give you the following: - -```query -human select fullName, firstName, lastName -``` - -As the French say: _Voila!_ diff --git a/website/Plug Management.md b/website/Plug Management.md deleted file mode 100644 index a6fe5af5..00000000 --- a/website/Plug Management.md +++ /dev/null @@ -1,11 +0,0 @@ -Plug management using the `PLUGS` file is also implemented in the [[Plugs/Editor]] plug. - -The optional `PLUGS` file is only processed when running the {[Plugs: Update]} command, in which case it will fetch all the listed plugs and copy them into the (hidden) `_plug/` folder in the user’s space. SilverBullet loads these files on boot (or on demand after running the {[Plugs: Update]} command). - -You can also use the {[Plugs: Add]} to add a plug, which will automatically create a `PLUGS` if it does not yet exist. - -The [[Plugs/Editor]] plug has support for the following URI prefixes for plugs: - -* `https:` loading plugs via HTTPS, e.g. `[https://](https://raw.githubusercontent.com/silverbulletmd/silverbullet-github/main/github.plug.js)` -* `github:org/repo/file.plug.js` internally rewritten to a `https` url as above. -* `ghr:org/repo/version` to fetch a plug from a Github release. Will fetch latest version, if it was omitted. diff --git a/website/Query Language.md b/website/Query Language.md deleted file mode 100644 index f34a1d94..00000000 --- a/website/Query Language.md +++ /dev/null @@ -1,78 +0,0 @@ -The syntax of queries is inspired by [SQL](https://en.wikipedia.org/wiki/SQL). It can be used using [[Live Queries]] as well as via nested queries part of template expressions: [[Expression Language#Queries]]. - -Let’s start with a query that demonstrates some of the supported clauses. Hover over the result and click the edit icon to show the code that generates the view: - -```query -page -order by lastModified desc -where size > 100 -select name -limit 10 -render [[Library/Core/Query/Page]] -``` - -It’s most convenient to use the `/query` [[Snippets|snippet]] to insert a query in a page. - -A query is formulated specifying a `querySource` followed by a number of clauses that modify or restrict the result set. If you haven’t already, check out how [[Objects]] work in SilverBullet. - -Every time `` is referenced here, an expression using the [[Expression Language]] can be used. - -# Clauses -## where -A `where` clause filters out all objects that do not match a certain condition. You can have multiple `where` clauses if you like, which will have the same effect as combining them with the `and` keyword. - -Here is a simple example based on a custom tag `#person` (see [[Objects]] on how this works): - -```#person -name: John -age: 7 ---- -name: Pete -age: 25 -``` - -To query all `person`s that are above age 21, we can use the following `where` clause: - -```query -person where page = @page.name and age > 21 -``` - -## order by -To sort results, an `order by` clause can be used, optionally with `desc` to order in descending order (ascending is the default): - -```query -person where page = @page.name order by age desc -``` - -## limit -To limit the number of results, you can use a `limit` clause: - -```query -person where page = @page.name limit 1 -``` - -## select -You can use the `select` clause to select only specific attributes from the result set. You can use it either simply as `select attribute1, attribute2` but also select the value of certain expressions and give them a name, even one containing spaces using the backtick identifier syntax: - -```query -person -where page = @page.name -select name, age, age + 1 as `next year` -``` - -## render each