Undoes the 2 stage index

This caused too many edge cases. Will work on optimizing the one-pass
index instead (not part of this commit).
This commit is contained in:
Zef Hemel
2026-01-05 15:34:45 +01:00
parent fa2d6d9a2d
commit 658f6e4e18
6 changed files with 29 additions and 91 deletions
+13 -23
View File
@@ -332,13 +332,7 @@ export class Client {
) => {
console.log("Queueing index for", name);
await this.eventHook.dispatchEvent("file:clearindex", name);
await this.mq.send("preIndexQueue", name);
if (await this.clientSystem.hasPreIndexCompleted()) {
// When the client is clean booted (no index yet), we skip indexing on the first pass
// and then schedule it once pre-index has finished for all files
// However, if pre indexing has already completed, then we schedule it immediately
await this.mq.send("indexQueue", name);
}
await this.mq.send("indexQueue", name);
},
);
@@ -350,35 +344,31 @@ export class Client {
this.space = space;
let startTime = -1;
this.eventHook.addLocalListener("file:initial", () => {
this.eventHook.addLocalListener("file:initial", async () => {
startTime = Date.now();
await this.clientSystem.setIndexOngoing(true);
});
const emptyQueueHandler = async () => {
if (!await this.clientSystem.hasPreIndexCompleted()) {
// Pre indexing has just finished for the first time
await this.clientSystem.setIndexOngoing(false);
if (!await this.clientSystem.hasInitialIndexCompleted()) {
// Indexing has just finished for the first time
console.info(
"Pre-indexing complete after",
"Initial index complete after",
(Date.now() - startTime) / 1000,
"s",
);
// Unsubscribe myself
this.eventHook.removeLocalListener(
"mq:emptyQueue:preIndexQueue",
"mq:emptyQueue:indexQueue",
emptyQueueHandler,
);
await this.clientSystem.markPreIndexComplete();
await this.clientSystem.markInitialIndexComplete();
await this.clientSystem.reloadState();
// Queue all pages for full indexing
await this.mq.batchSend(
"indexQueue",
(await space.fetchPageList()).map((pm) => pm.name + ".md"),
);
await this.mq.awaitEmptyQueue("indexQueue");
}
};
this.eventHook.addLocalListener(
"mq:emptyQueue:preIndexQueue",
"mq:emptyQueue:indexQueue",
emptyQueueHandler,
);
@@ -611,7 +601,7 @@ export class Client {
console.log("Updating page list cache");
// Check if the initial sync has been completed
const initialIndexCompleted = await this.clientSystem
.hasPreIndexCompleted();
.hasInitialIndexCompleted();
let allPages: PageMeta[] = [];
@@ -1132,7 +1122,7 @@ export class Client {
// Fetch the meta which includes the possibly indexed stuff, like page
// decorations
if (await this.clientSystem.hasPreIndexCompleted()) {
if (await this.clientSystem.hasInitialIndexCompleted()) {
try {
const enrichedMeta = await this.clientSystem.getObjectByRef<PageMeta>(
pageName,
@@ -1280,7 +1270,7 @@ export class Client {
console.warn("Not loading custom styles, since space style is disabled");
return;
}
if (!await this.clientSystem.hasPreIndexCompleted()) {
if (!await this.clientSystem.hasInitialIndexCompleted()) {
return;
}
+5 -5
View File
@@ -201,7 +201,7 @@ export class ClientSystem {
console.info("Space Lua scripts are disabled, skipping loading scripts");
return;
}
if (!await this.hasPreIndexCompleted()) {
if (!await this.hasInitialIndexCompleted()) {
console.info(
"Not loading space scripts, since initial indexing has not completed yet",
);
@@ -305,10 +305,10 @@ export class ClientSystem {
"[index]",
"Performing a full space reindex, this could take a while...",
);
await this.setIndexOngoing();
await this.setIndexOngoing(true);
await this.system.invokeFunction("index.reindexSpace", []);
console.info("[index]", "Full space index complete.");
await this.markPreIndexComplete();
await this.markInitialIndexComplete();
await this.setIndexOngoing(false);
// Let's load space scripts again, which probably weren't loaded before
await this.reloadState();
@@ -324,7 +324,7 @@ export class ClientSystem {
this.client.rebuildEditorState();
}
public async hasPreIndexCompleted() {
public async hasInitialIndexCompleted() {
return (await this.ds.get(indexVersionKey)) === desiredIndexVersion;
}
@@ -332,7 +332,7 @@ export class ClientSystem {
return this.ds.get(indexVersionKey);
}
async markPreIndexComplete() {
async markInitialIndexComplete() {
await this.ds.set(indexVersionKey, desiredIndexVersion);
}
}
+4 -4
View File
@@ -21,6 +21,10 @@ export class EventedSpacePrimitives implements SpacePrimitives {
private enabled = false;
// Snapshot state management
private spaceSnapshot: Record<string, number> = {};
private snapshotChanged = false;
constructor(
private wrapped: SpacePrimitives,
private eventHook: EventHook,
@@ -44,10 +48,6 @@ export class EventedSpacePrimitives implements SpacePrimitives {
});
}
// Snapshot state management
private spaceSnapshot: Record<string, number> = {};
private snapshotChanged = false;
private updateInSnapshot(key: string, value: number) {
const oldValue = this.spaceSnapshot[key];
this.spaceSnapshot[key] = value;
+2 -14
View File
@@ -47,13 +47,6 @@ functions:
batchSize: 3
autoAck: true
processPreIndexQueue:
path: ./queue.ts:processPreIndexQueue
mqSubscriptions:
- queue: preIndexQueue
batchSize: 5
autoAck: true
reindexSpace:
path: "./queue.ts:reindexSpace"
@@ -67,11 +60,6 @@ functions:
events:
- page:index
preIndexPage:
path: page.ts:preIndexPage
events:
- page:preindex
indexParagraphs:
path: "./paragraph.ts:indexParagraphs"
events:
@@ -115,13 +103,13 @@ functions:
indexSpaceLua:
path: script.ts:indexSpaceLua
events:
- page:preindex
- page:index
# Style
indexSpaceStyle:
path: style.ts:indexSpaceStyle
events:
- page:preindex
- page:index
# Hashtags
indexTags:
+4 -12
View File
@@ -25,7 +25,7 @@ import type { AspiringPageObject } from "./page_links.ts";
import type { PageMeta } from "@silverbulletmd/silverbullet/type/index";
import type { LintDiagnostic } from "@silverbulletmd/silverbullet/type/client";
export async function preIndexPage({ name, tree }: IndexTreeEvent) {
export async function indexPage({ name, tree }: IndexTreeEvent) {
if (name.startsWith("_")) {
// Don't index pages starting with _
return;
@@ -66,17 +66,6 @@ export async function preIndexPage({ name, tree }: IndexTreeEvent) {
updateITags(combinedPageMeta, frontmatter);
// console.log("Page object", combinedPageMeta);
await indexObjects<PageMeta>(name, [combinedPageMeta]);
}
// Most interesting indexing for pages happens in the preindex phase, here we're just cleaning up aspiring pages
export async function indexPage({ name }: IndexTreeEvent) {
if (name.startsWith("_")) {
// Don't index pages starting with _
return;
}
// Make sure this page is no (longer) in the aspiring pages list
// TODO: This can possibly done more optimally
const aspiringPages = await queryLuaObjects<AspiringPageObject>(
@@ -91,6 +80,9 @@ export async function indexPage({ name }: IndexTreeEvent) {
console.log("Deleting aspiring page", aspiringPage);
await deleteObject("aspiring-page", aspiringPage.page, aspiringPage.ref);
}
// console.log("Page object", combinedPageMeta);
await indexObjects<PageMeta>(name, [combinedPageMeta]);
}
export async function lintFrontmatter(): Promise<LintDiagnostic[]> {
+1 -33
View File
@@ -20,21 +20,13 @@ export async function reindexSpace() {
console.log("Clearing page index...");
await system.invokeFunction("index.clearIndex");
const startTime = Date.now();
const files = await space.listFiles();
console.log("Queing", files.length, "pages to be indexed.");
// Queue all file names to be indexed
await mq.batchSend("preIndexQueue", files.map((file) => file.name));
await mq.batchSend("indexQueue", files.map((file) => file.name));
await editor.showProgress(0, "index");
// We'll assume this one completes last
await mq.awaitEmptyQueue("indexQueue");
// And notify the user
console.log(`Indexing completed in ${(Date.now() - startTime) / 1000}s`);
await editor.showProgress();
console.log("And done with full index!");
}
setTimeout(updateIndexProgressInUI, uiUpdateInterval);
@@ -94,30 +86,6 @@ async function indexFile(path: string) {
name,
tree,
} as IndexTreeEvent);
}
}
export async function processPreIndexQueue(messages: MQMessage[]) {
for (const message of messages) {
const path: string = message.body;
console.log("[pre-index]", `Pre-indexing file ${path}`);
await preIndexFile(path);
}
}
async function preIndexFile(path: string) {
if (path.endsWith(".md")) {
// Page
const name = path.slice(0, -3);
// Read file
const text = await space.readPage(name);
const tree = await markdown.parseMarkdown(text);
// Emit the event which will be picked up by indexers
await events.dispatchEvent("page:preindex", {
name,
tree,
} as IndexTreeEvent);
} else {
await events.dispatchEvent("document:index", path);
}