From 1910ffce3e48bfeaeb3fc6600c044d91666a34e4 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Thu, 12 Jun 2025 13:52:13 +0200 Subject: [PATCH] Index handling refinements --- web/client.ts | 33 ++++++----------- web/client_system.ts | 82 +++++++++++++++--------------------------- web/syscalls/system.ts | 6 +++- website/API/system.md | 9 ----- 4 files changed, 45 insertions(+), 85 deletions(-) diff --git a/web/client.ts b/web/client.ts index c8d79644..f2f6a39a 100644 --- a/web/client.ts +++ b/web/client.ts @@ -228,7 +228,7 @@ export class Client { // Load plugs await this.loadPlugs(); - await this.clientSystem.loadSpaceScripts(); + await this.clientSystem.loadScripts(); await this.initNavigator(); await this.initSync(); await this.eventHook.dispatchEvent("system:ready"); @@ -292,7 +292,7 @@ export class Client { // A full sync just completed if (!initialSync) { // If this was NOT the initial sync let's check if we need to perform a space reindex - this.clientSystem.ensureSpaceIndex().catch( + this.clientSystem.ensureFullIndex().catch( console.error, ); } else { // initialSync @@ -301,7 +301,7 @@ export class Client { ); console.log("Enabling eventing on space primitives"); this.space.spacePrimitives.enablePageEvents = true; - this.clientSystem.ensureSpaceIndex().catch( + this.clientSystem.ensureFullIndex().catch( console.error, ); initialSync = false; @@ -742,32 +742,24 @@ export class Client { async updatePageListCache() { console.log("Updating page list cache"); - // **** Check if the initial sync has been completed **** - const initialSyncCompleted = await this.hasInitialSyncCompleted(); + // Check if the initial sync has been completed + const initialIndexCompleted = await this.clientSystem + .hasFullIndexCompleted(); let allPages: PageMeta[] = []; if ( - initialSyncCompleted && this.clientSystem.system.loadedPlugs.has("index") + initialIndexCompleted && this.clientSystem.system.loadedPlugs.has("index") ) { - // **** Logic when initial sync IS completed AND index plug is loaded **** console.log( "Initial sync complete and index plug loaded, loading full page list via index.", ); // Fetch actual indexed pages - allPages = await this.clientSystem.queryLuaObjects( - "page", - {}, - ); + allPages = await this.clientSystem.queryLuaObjects("page", {}); // Fetch aspiring pages only when using the index - const aspiringPageNames = await this.clientSystem.queryLuaObjects< - string - >( + const aspiringPageNames = await this.clientSystem.queryLuaObjects( "aspiring-page", - { - select: parseExpressionString("name"), - distinct: true, - }, + { select: parseExpressionString("name"), distinct: true }, ); // Map and push aspiring pages directly into allPages allPages.push( @@ -782,8 +774,6 @@ export class Client { })), ); } else { - // **** Logic when initial sync is NOT YET completed OR index plug isn't ready **** - // Use space.fetchPageList() directly console.log( "Initial sync not complete or index plug not loaded. Fetching page list directly using space.fetchPageList().", ); @@ -791,7 +781,7 @@ export class Client { // Call fetchPageList directly allPages = await this.space.fetchPageList(); - // Let's do some heuristic based post processing + // Let's do some heuristic-based post processing for (const page of allPages) { // These are _mostly_ meta pages, let's add a tag for them if (page.name.startsWith("Library/")) { @@ -806,7 +796,6 @@ export class Client { "error", ); } - // Note: Aspiring pages are not available through fetchPageList() } this.ui.viewDispatch({ diff --git a/web/client_system.ts b/web/client_system.ts index ba4afd20..c06ea972 100644 --- a/web/client_system.ts +++ b/web/client_system.ts @@ -46,7 +46,6 @@ import type { LuaCollectionQuery } from "./space_lua/query_collection.ts"; import type { AppCommand } from "$lib/command.ts"; import { ScriptEnvironment } from "./space_script.ts"; import { SpaceLuaEnvironment } from "./space_lua.ts"; -import type { ParseTree } from "@silverbulletmd/silverbullet/lib/tree"; const plugNameExtractRegex = /\/(.+)\.plug\.js$/; const indexVersionKey = ["$indexVersion"]; @@ -69,7 +68,7 @@ export class ClientSystem { documentEditorHook!: DocumentEditorHook; readonly allKnownFiles = new Set(); - readonly spaceScriptCommands = new Map(); + readonly scriptCommands = new Map(); scriptEnv: ScriptEnvironment = new ScriptEnvironment(); spaceLuaEnv = new SpaceLuaEnvironment(); scriptsLoaded: boolean = false; @@ -109,7 +108,7 @@ export class ClientSystem { // Command hook this.commandHook = new CommandHook( this.readOnlyMode, - this.spaceScriptCommands, + this.scriptCommands, ); this.commandHook.on({ commandsUpdated: (commandMap) => { @@ -201,10 +200,10 @@ export class ClientSystem { } } - async loadSpaceScripts() { - if (!await this.client.hasInitialSyncCompleted()) { + async loadScripts() { + if (!await this.hasFullIndexCompleted()) { console.info( - "Not loading space scripts, since initial synca has not completed yet", + "Not loading space scripts, since initial indexing has not completed yet", ); return; } @@ -212,60 +211,26 @@ export class ClientSystem { try { await this.spaceLuaEnv.reload(this.system); } catch (e: any) { - console.error("Error loading space-script:", e.message); + console.error("Error loading Lua script:", e.message); } // Reset the space script commands - this.spaceScriptCommands.clear(); + this.scriptCommands.clear(); for (const [name, command] of Object.entries(this.scriptEnv.commands)) { - this.spaceScriptCommands.set(name, command); + this.scriptCommands.set(name, command); } // Inject the registered events in the event hook this.eventHook.scriptEnvironment = this.scriptEnv; + // Make scripted (slash) commands available this.commandHook.throttledBuildAllCommands(); this.slashCommandHook.throttledBuildAllCommands(); this.scriptsLoaded = true; } - invokeSpaceFunction(name: string, args: any[]) { - const fn = this.scriptEnv.functions[name]; - if (!fn) { - throw new Error(`Function ${name} not found`); - } - return fn(...args); - } - - async applyAttributeExtractors( - tags: string[], - text: string, - tree: ParseTree, - ): Promise> { - let resultingAttributes: Record = {}; - for (const tag of tags) { - const extractors = this.scriptEnv.attributeExtractors[tag]; - if (!extractors) { - continue; - } - for (const fn of extractors) { - const extractorResult = await fn(text, tree); - if (extractorResult) { - // Merge the attributes in - resultingAttributes = { - ...resultingAttributes, - ...extractorResult, - }; - } - } - } - - return resultingAttributes; - } - async reloadPlugsFromSpace(space: Space) { - console.log("Loading plugs"); await this.system.unloadAll(); console.log("(Re)loading plugs"); const allPlugs = await space.listPlugs(); @@ -310,33 +275,44 @@ export class ClientSystem { private indexOngoing = false; - async ensureSpaceIndex() { - const currentIndexVersion = await this.ds.get(indexVersionKey); + async ensureFullIndex() { + const currentIndexVersion = await this.getCurrentIndexVersion(); - console.info("Current space index version", currentIndexVersion); + console.info( + "Current space index version", + currentIndexVersion, + "index ongoing", + this.indexOngoing, + ); if (currentIndexVersion !== desiredIndexVersion && !this.indexOngoing) { console.info( "Performing a full space reindex, this could take a while...", ); - // First let's fetch all pages to make sure we have a cache of known pages - // await this.client.space.fetchPageList(); this.indexOngoing = true; await this.system.invokeFunction("index.reindexSpace", []); console.info("Full space index complete."); - await this.markFullSpaceIndexComplete(this.ds); + await this.markFullSpaceIndexComplete(); this.indexOngoing = false; // Let's load space scripts again, which probably weren't loaded before console.log( "Now loading space scripts, custom styles and rebuilding editor state", ); - await this.loadSpaceScripts(); + await this.loadScripts(); await this.client.loadCustomStyles(); this.client.rebuildEditorState(); } } - async markFullSpaceIndexComplete(ds: DataStore) { - await ds.set(indexVersionKey, desiredIndexVersion); + private getCurrentIndexVersion() { + return this.ds.get(indexVersionKey); + } + + private async markFullSpaceIndexComplete() { + await this.ds.set(indexVersionKey, desiredIndexVersion); + } + + public async hasFullIndexCompleted() { + return (await this.ds.get(indexVersionKey)) === desiredIndexVersion; } } diff --git a/web/syscalls/system.ts b/web/syscalls/system.ts index 313bf6e6..b9ed900d 100644 --- a/web/syscalls/system.ts +++ b/web/syscalls/system.ts @@ -90,7 +90,11 @@ export function systemSyscalls( return client.config.values; }, "system.loadSpaceScripts": async () => { - await client.clientSystem.loadSpaceScripts(); + console.warn("DEPRECATED: used system.loadScripts instead"); + await client.clientSystem.loadScripts(); + }, + "system.loadScripts": async () => { + await client.clientSystem.loadScripts(); }, "system.loadSpaceStyles": async () => { if (!client) { diff --git a/website/API/system.md b/website/API/system.md index aaaee403..114a71f4 100644 --- a/website/API/system.md +++ b/website/API/system.md @@ -21,15 +21,6 @@ Example: system.invokeCommand("editor.save", {}) ``` -### system.invokeSpaceFunction(name, ...) -Invokes a space function by name. - -Example: -```lua -local result = system.invokeSpaceFunction("customFunction", "arg1", "arg2") -print("Function result:", result) -``` - ## System Information ### system.listCommands()