From 658f6e4e182413acb7959555ab01cbdd4e22b34d Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Mon, 5 Jan 2026 15:34:45 +0100 Subject: [PATCH] Undoes the 2 stage index This caused too many edge cases. Will work on optimizing the one-pass index instead (not part of this commit). --- client/client.ts | 36 ++++++++--------------- client/client_system.ts | 10 +++---- client/spaces/evented_space_primitives.ts | 8 ++--- plugs/index/index.plug.yaml | 16 ++-------- plugs/index/page.ts | 16 +++------- plugs/index/queue.ts | 34 +-------------------- 6 files changed, 29 insertions(+), 91 deletions(-) diff --git a/client/client.ts b/client/client.ts index 48dbcafd..03b40c27 100644 --- a/client/client.ts +++ b/client/client.ts @@ -332,13 +332,7 @@ export class Client { ) => { console.log("Queueing index for", name); await this.eventHook.dispatchEvent("file:clearindex", name); - await this.mq.send("preIndexQueue", name); - if (await this.clientSystem.hasPreIndexCompleted()) { - // When the client is clean booted (no index yet), we skip indexing on the first pass - // and then schedule it once pre-index has finished for all files - // However, if pre indexing has already completed, then we schedule it immediately - await this.mq.send("indexQueue", name); - } + await this.mq.send("indexQueue", name); }, ); @@ -350,35 +344,31 @@ export class Client { this.space = space; let startTime = -1; - this.eventHook.addLocalListener("file:initial", () => { + this.eventHook.addLocalListener("file:initial", async () => { startTime = Date.now(); + await this.clientSystem.setIndexOngoing(true); }); const emptyQueueHandler = async () => { - if (!await this.clientSystem.hasPreIndexCompleted()) { - // Pre indexing has just finished for the first time + await this.clientSystem.setIndexOngoing(false); + if (!await this.clientSystem.hasInitialIndexCompleted()) { + // Indexing has just finished for the first time console.info( - "Pre-indexing complete after", + "Initial index complete after", (Date.now() - startTime) / 1000, "s", ); // Unsubscribe myself this.eventHook.removeLocalListener( - "mq:emptyQueue:preIndexQueue", + "mq:emptyQueue:indexQueue", emptyQueueHandler, ); - await this.clientSystem.markPreIndexComplete(); + await this.clientSystem.markInitialIndexComplete(); await this.clientSystem.reloadState(); - // Queue all pages for full indexing - await this.mq.batchSend( - "indexQueue", - (await space.fetchPageList()).map((pm) => pm.name + ".md"), - ); - await this.mq.awaitEmptyQueue("indexQueue"); } }; this.eventHook.addLocalListener( - "mq:emptyQueue:preIndexQueue", + "mq:emptyQueue:indexQueue", emptyQueueHandler, ); @@ -611,7 +601,7 @@ export class Client { console.log("Updating page list cache"); // Check if the initial sync has been completed const initialIndexCompleted = await this.clientSystem - .hasPreIndexCompleted(); + .hasInitialIndexCompleted(); let allPages: PageMeta[] = []; @@ -1132,7 +1122,7 @@ export class Client { // Fetch the meta which includes the possibly indexed stuff, like page // decorations - if (await this.clientSystem.hasPreIndexCompleted()) { + if (await this.clientSystem.hasInitialIndexCompleted()) { try { const enrichedMeta = await this.clientSystem.getObjectByRef( pageName, @@ -1280,7 +1270,7 @@ export class Client { console.warn("Not loading custom styles, since space style is disabled"); return; } - if (!await this.clientSystem.hasPreIndexCompleted()) { + if (!await this.clientSystem.hasInitialIndexCompleted()) { return; } diff --git a/client/client_system.ts b/client/client_system.ts index 7e2cdf89..b18d8ccc 100644 --- a/client/client_system.ts +++ b/client/client_system.ts @@ -201,7 +201,7 @@ export class ClientSystem { console.info("Space Lua scripts are disabled, skipping loading scripts"); return; } - if (!await this.hasPreIndexCompleted()) { + if (!await this.hasInitialIndexCompleted()) { console.info( "Not loading space scripts, since initial indexing has not completed yet", ); @@ -305,10 +305,10 @@ export class ClientSystem { "[index]", "Performing a full space reindex, this could take a while...", ); - await this.setIndexOngoing(); + await this.setIndexOngoing(true); await this.system.invokeFunction("index.reindexSpace", []); console.info("[index]", "Full space index complete."); - await this.markPreIndexComplete(); + await this.markInitialIndexComplete(); await this.setIndexOngoing(false); // Let's load space scripts again, which probably weren't loaded before await this.reloadState(); @@ -324,7 +324,7 @@ export class ClientSystem { this.client.rebuildEditorState(); } - public async hasPreIndexCompleted() { + public async hasInitialIndexCompleted() { return (await this.ds.get(indexVersionKey)) === desiredIndexVersion; } @@ -332,7 +332,7 @@ export class ClientSystem { return this.ds.get(indexVersionKey); } - async markPreIndexComplete() { + async markInitialIndexComplete() { await this.ds.set(indexVersionKey, desiredIndexVersion); } } diff --git a/client/spaces/evented_space_primitives.ts b/client/spaces/evented_space_primitives.ts index aba4c907..e65dcdc8 100644 --- a/client/spaces/evented_space_primitives.ts +++ b/client/spaces/evented_space_primitives.ts @@ -21,6 +21,10 @@ export class EventedSpacePrimitives implements SpacePrimitives { private enabled = false; + // Snapshot state management + private spaceSnapshot: Record = {}; + private snapshotChanged = false; + constructor( private wrapped: SpacePrimitives, private eventHook: EventHook, @@ -44,10 +48,6 @@ export class EventedSpacePrimitives implements SpacePrimitives { }); } - // Snapshot state management - private spaceSnapshot: Record = {}; - private snapshotChanged = false; - private updateInSnapshot(key: string, value: number) { const oldValue = this.spaceSnapshot[key]; this.spaceSnapshot[key] = value; diff --git a/plugs/index/index.plug.yaml b/plugs/index/index.plug.yaml index f93343f9..fcfbcca4 100644 --- a/plugs/index/index.plug.yaml +++ b/plugs/index/index.plug.yaml @@ -47,13 +47,6 @@ functions: batchSize: 3 autoAck: true - processPreIndexQueue: - path: ./queue.ts:processPreIndexQueue - mqSubscriptions: - - queue: preIndexQueue - batchSize: 5 - autoAck: true - reindexSpace: path: "./queue.ts:reindexSpace" @@ -67,11 +60,6 @@ functions: events: - page:index - preIndexPage: - path: page.ts:preIndexPage - events: - - page:preindex - indexParagraphs: path: "./paragraph.ts:indexParagraphs" events: @@ -115,13 +103,13 @@ functions: indexSpaceLua: path: script.ts:indexSpaceLua events: - - page:preindex + - page:index # Style indexSpaceStyle: path: style.ts:indexSpaceStyle events: - - page:preindex + - page:index # Hashtags indexTags: diff --git a/plugs/index/page.ts b/plugs/index/page.ts index 051bc72d..b0b5efff 100644 --- a/plugs/index/page.ts +++ b/plugs/index/page.ts @@ -25,7 +25,7 @@ import type { AspiringPageObject } from "./page_links.ts"; import type { PageMeta } from "@silverbulletmd/silverbullet/type/index"; import type { LintDiagnostic } from "@silverbulletmd/silverbullet/type/client"; -export async function preIndexPage({ name, tree }: IndexTreeEvent) { +export async function indexPage({ name, tree }: IndexTreeEvent) { if (name.startsWith("_")) { // Don't index pages starting with _ return; @@ -66,17 +66,6 @@ export async function preIndexPage({ name, tree }: IndexTreeEvent) { updateITags(combinedPageMeta, frontmatter); - // console.log("Page object", combinedPageMeta); - await indexObjects(name, [combinedPageMeta]); -} - -// Most interesting indexing for pages happens in the preindex phase, here we're just cleaning up aspiring pages -export async function indexPage({ name }: IndexTreeEvent) { - if (name.startsWith("_")) { - // Don't index pages starting with _ - return; - } - // Make sure this page is no (longer) in the aspiring pages list // TODO: This can possibly done more optimally const aspiringPages = await queryLuaObjects( @@ -91,6 +80,9 @@ export async function indexPage({ name }: IndexTreeEvent) { console.log("Deleting aspiring page", aspiringPage); await deleteObject("aspiring-page", aspiringPage.page, aspiringPage.ref); } + + // console.log("Page object", combinedPageMeta); + await indexObjects(name, [combinedPageMeta]); } export async function lintFrontmatter(): Promise { diff --git a/plugs/index/queue.ts b/plugs/index/queue.ts index 4f14f083..f20ec0fb 100644 --- a/plugs/index/queue.ts +++ b/plugs/index/queue.ts @@ -20,21 +20,13 @@ export async function reindexSpace() { console.log("Clearing page index..."); await system.invokeFunction("index.clearIndex"); - const startTime = Date.now(); - const files = await space.listFiles(); console.log("Queing", files.length, "pages to be indexed."); // Queue all file names to be indexed - await mq.batchSend("preIndexQueue", files.map((file) => file.name)); await mq.batchSend("indexQueue", files.map((file) => file.name)); - await editor.showProgress(0, "index"); - // We'll assume this one completes last await mq.awaitEmptyQueue("indexQueue"); - - // And notify the user - console.log(`Indexing completed in ${(Date.now() - startTime) / 1000}s`); - await editor.showProgress(); + console.log("And done with full index!"); } setTimeout(updateIndexProgressInUI, uiUpdateInterval); @@ -94,30 +86,6 @@ async function indexFile(path: string) { name, tree, } as IndexTreeEvent); - } -} - -export async function processPreIndexQueue(messages: MQMessage[]) { - for (const message of messages) { - const path: string = message.body; - console.log("[pre-index]", `Pre-indexing file ${path}`); - await preIndexFile(path); - } -} - -async function preIndexFile(path: string) { - if (path.endsWith(".md")) { - // Page - const name = path.slice(0, -3); - // Read file - const text = await space.readPage(name); - const tree = await markdown.parseMarkdown(text); - - // Emit the event which will be picked up by indexers - await events.dispatchEvent("page:preindex", { - name, - tree, - } as IndexTreeEvent); } else { await events.dispatchEvent("document:index", path); }