Undoes the 2 stage index
This caused too many edge cases. Will work on optimizing the one-pass index instead (not part of this commit).
This commit is contained in:
+13
-23
@@ -332,13 +332,7 @@ export class Client {
|
||||
) => {
|
||||
console.log("Queueing index for", name);
|
||||
await this.eventHook.dispatchEvent("file:clearindex", name);
|
||||
await this.mq.send("preIndexQueue", name);
|
||||
if (await this.clientSystem.hasPreIndexCompleted()) {
|
||||
// When the client is clean booted (no index yet), we skip indexing on the first pass
|
||||
// and then schedule it once pre-index has finished for all files
|
||||
// However, if pre indexing has already completed, then we schedule it immediately
|
||||
await this.mq.send("indexQueue", name);
|
||||
}
|
||||
await this.mq.send("indexQueue", name);
|
||||
},
|
||||
);
|
||||
|
||||
@@ -350,35 +344,31 @@ export class Client {
|
||||
this.space = space;
|
||||
|
||||
let startTime = -1;
|
||||
this.eventHook.addLocalListener("file:initial", () => {
|
||||
this.eventHook.addLocalListener("file:initial", async () => {
|
||||
startTime = Date.now();
|
||||
await this.clientSystem.setIndexOngoing(true);
|
||||
});
|
||||
|
||||
const emptyQueueHandler = async () => {
|
||||
if (!await this.clientSystem.hasPreIndexCompleted()) {
|
||||
// Pre indexing has just finished for the first time
|
||||
await this.clientSystem.setIndexOngoing(false);
|
||||
if (!await this.clientSystem.hasInitialIndexCompleted()) {
|
||||
// Indexing has just finished for the first time
|
||||
console.info(
|
||||
"Pre-indexing complete after",
|
||||
"Initial index complete after",
|
||||
(Date.now() - startTime) / 1000,
|
||||
"s",
|
||||
);
|
||||
// Unsubscribe myself
|
||||
this.eventHook.removeLocalListener(
|
||||
"mq:emptyQueue:preIndexQueue",
|
||||
"mq:emptyQueue:indexQueue",
|
||||
emptyQueueHandler,
|
||||
);
|
||||
await this.clientSystem.markPreIndexComplete();
|
||||
await this.clientSystem.markInitialIndexComplete();
|
||||
await this.clientSystem.reloadState();
|
||||
// Queue all pages for full indexing
|
||||
await this.mq.batchSend(
|
||||
"indexQueue",
|
||||
(await space.fetchPageList()).map((pm) => pm.name + ".md"),
|
||||
);
|
||||
await this.mq.awaitEmptyQueue("indexQueue");
|
||||
}
|
||||
};
|
||||
this.eventHook.addLocalListener(
|
||||
"mq:emptyQueue:preIndexQueue",
|
||||
"mq:emptyQueue:indexQueue",
|
||||
emptyQueueHandler,
|
||||
);
|
||||
|
||||
@@ -611,7 +601,7 @@ export class Client {
|
||||
console.log("Updating page list cache");
|
||||
// Check if the initial sync has been completed
|
||||
const initialIndexCompleted = await this.clientSystem
|
||||
.hasPreIndexCompleted();
|
||||
.hasInitialIndexCompleted();
|
||||
|
||||
let allPages: PageMeta[] = [];
|
||||
|
||||
@@ -1132,7 +1122,7 @@ export class Client {
|
||||
|
||||
// Fetch the meta which includes the possibly indexed stuff, like page
|
||||
// decorations
|
||||
if (await this.clientSystem.hasPreIndexCompleted()) {
|
||||
if (await this.clientSystem.hasInitialIndexCompleted()) {
|
||||
try {
|
||||
const enrichedMeta = await this.clientSystem.getObjectByRef<PageMeta>(
|
||||
pageName,
|
||||
@@ -1280,7 +1270,7 @@ export class Client {
|
||||
console.warn("Not loading custom styles, since space style is disabled");
|
||||
return;
|
||||
}
|
||||
if (!await this.clientSystem.hasPreIndexCompleted()) {
|
||||
if (!await this.clientSystem.hasInitialIndexCompleted()) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -201,7 +201,7 @@ export class ClientSystem {
|
||||
console.info("Space Lua scripts are disabled, skipping loading scripts");
|
||||
return;
|
||||
}
|
||||
if (!await this.hasPreIndexCompleted()) {
|
||||
if (!await this.hasInitialIndexCompleted()) {
|
||||
console.info(
|
||||
"Not loading space scripts, since initial indexing has not completed yet",
|
||||
);
|
||||
@@ -305,10 +305,10 @@ export class ClientSystem {
|
||||
"[index]",
|
||||
"Performing a full space reindex, this could take a while...",
|
||||
);
|
||||
await this.setIndexOngoing();
|
||||
await this.setIndexOngoing(true);
|
||||
await this.system.invokeFunction("index.reindexSpace", []);
|
||||
console.info("[index]", "Full space index complete.");
|
||||
await this.markPreIndexComplete();
|
||||
await this.markInitialIndexComplete();
|
||||
await this.setIndexOngoing(false);
|
||||
// Let's load space scripts again, which probably weren't loaded before
|
||||
await this.reloadState();
|
||||
@@ -324,7 +324,7 @@ export class ClientSystem {
|
||||
this.client.rebuildEditorState();
|
||||
}
|
||||
|
||||
public async hasPreIndexCompleted() {
|
||||
public async hasInitialIndexCompleted() {
|
||||
return (await this.ds.get(indexVersionKey)) === desiredIndexVersion;
|
||||
}
|
||||
|
||||
@@ -332,7 +332,7 @@ export class ClientSystem {
|
||||
return this.ds.get(indexVersionKey);
|
||||
}
|
||||
|
||||
async markPreIndexComplete() {
|
||||
async markInitialIndexComplete() {
|
||||
await this.ds.set(indexVersionKey, desiredIndexVersion);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,10 @@ export class EventedSpacePrimitives implements SpacePrimitives {
|
||||
|
||||
private enabled = false;
|
||||
|
||||
// Snapshot state management
|
||||
private spaceSnapshot: Record<string, number> = {};
|
||||
private snapshotChanged = false;
|
||||
|
||||
constructor(
|
||||
private wrapped: SpacePrimitives,
|
||||
private eventHook: EventHook,
|
||||
@@ -44,10 +48,6 @@ export class EventedSpacePrimitives implements SpacePrimitives {
|
||||
});
|
||||
}
|
||||
|
||||
// Snapshot state management
|
||||
private spaceSnapshot: Record<string, number> = {};
|
||||
private snapshotChanged = false;
|
||||
|
||||
private updateInSnapshot(key: string, value: number) {
|
||||
const oldValue = this.spaceSnapshot[key];
|
||||
this.spaceSnapshot[key] = value;
|
||||
|
||||
@@ -47,13 +47,6 @@ functions:
|
||||
batchSize: 3
|
||||
autoAck: true
|
||||
|
||||
processPreIndexQueue:
|
||||
path: ./queue.ts:processPreIndexQueue
|
||||
mqSubscriptions:
|
||||
- queue: preIndexQueue
|
||||
batchSize: 5
|
||||
autoAck: true
|
||||
|
||||
reindexSpace:
|
||||
path: "./queue.ts:reindexSpace"
|
||||
|
||||
@@ -67,11 +60,6 @@ functions:
|
||||
events:
|
||||
- page:index
|
||||
|
||||
preIndexPage:
|
||||
path: page.ts:preIndexPage
|
||||
events:
|
||||
- page:preindex
|
||||
|
||||
indexParagraphs:
|
||||
path: "./paragraph.ts:indexParagraphs"
|
||||
events:
|
||||
@@ -115,13 +103,13 @@ functions:
|
||||
indexSpaceLua:
|
||||
path: script.ts:indexSpaceLua
|
||||
events:
|
||||
- page:preindex
|
||||
- page:index
|
||||
|
||||
# Style
|
||||
indexSpaceStyle:
|
||||
path: style.ts:indexSpaceStyle
|
||||
events:
|
||||
- page:preindex
|
||||
- page:index
|
||||
|
||||
# Hashtags
|
||||
indexTags:
|
||||
|
||||
+4
-12
@@ -25,7 +25,7 @@ import type { AspiringPageObject } from "./page_links.ts";
|
||||
import type { PageMeta } from "@silverbulletmd/silverbullet/type/index";
|
||||
import type { LintDiagnostic } from "@silverbulletmd/silverbullet/type/client";
|
||||
|
||||
export async function preIndexPage({ name, tree }: IndexTreeEvent) {
|
||||
export async function indexPage({ name, tree }: IndexTreeEvent) {
|
||||
if (name.startsWith("_")) {
|
||||
// Don't index pages starting with _
|
||||
return;
|
||||
@@ -66,17 +66,6 @@ export async function preIndexPage({ name, tree }: IndexTreeEvent) {
|
||||
|
||||
updateITags(combinedPageMeta, frontmatter);
|
||||
|
||||
// console.log("Page object", combinedPageMeta);
|
||||
await indexObjects<PageMeta>(name, [combinedPageMeta]);
|
||||
}
|
||||
|
||||
// Most interesting indexing for pages happens in the preindex phase, here we're just cleaning up aspiring pages
|
||||
export async function indexPage({ name }: IndexTreeEvent) {
|
||||
if (name.startsWith("_")) {
|
||||
// Don't index pages starting with _
|
||||
return;
|
||||
}
|
||||
|
||||
// Make sure this page is no (longer) in the aspiring pages list
|
||||
// TODO: This can possibly done more optimally
|
||||
const aspiringPages = await queryLuaObjects<AspiringPageObject>(
|
||||
@@ -91,6 +80,9 @@ export async function indexPage({ name }: IndexTreeEvent) {
|
||||
console.log("Deleting aspiring page", aspiringPage);
|
||||
await deleteObject("aspiring-page", aspiringPage.page, aspiringPage.ref);
|
||||
}
|
||||
|
||||
// console.log("Page object", combinedPageMeta);
|
||||
await indexObjects<PageMeta>(name, [combinedPageMeta]);
|
||||
}
|
||||
|
||||
export async function lintFrontmatter(): Promise<LintDiagnostic[]> {
|
||||
|
||||
+1
-33
@@ -20,21 +20,13 @@ export async function reindexSpace() {
|
||||
console.log("Clearing page index...");
|
||||
await system.invokeFunction("index.clearIndex");
|
||||
|
||||
const startTime = Date.now();
|
||||
|
||||
const files = await space.listFiles();
|
||||
|
||||
console.log("Queing", files.length, "pages to be indexed.");
|
||||
// Queue all file names to be indexed
|
||||
await mq.batchSend("preIndexQueue", files.map((file) => file.name));
|
||||
await mq.batchSend("indexQueue", files.map((file) => file.name));
|
||||
await editor.showProgress(0, "index");
|
||||
// We'll assume this one completes last
|
||||
await mq.awaitEmptyQueue("indexQueue");
|
||||
|
||||
// And notify the user
|
||||
console.log(`Indexing completed in ${(Date.now() - startTime) / 1000}s`);
|
||||
await editor.showProgress();
|
||||
console.log("And done with full index!");
|
||||
}
|
||||
|
||||
setTimeout(updateIndexProgressInUI, uiUpdateInterval);
|
||||
@@ -94,30 +86,6 @@ async function indexFile(path: string) {
|
||||
name,
|
||||
tree,
|
||||
} as IndexTreeEvent);
|
||||
}
|
||||
}
|
||||
|
||||
export async function processPreIndexQueue(messages: MQMessage[]) {
|
||||
for (const message of messages) {
|
||||
const path: string = message.body;
|
||||
console.log("[pre-index]", `Pre-indexing file ${path}`);
|
||||
await preIndexFile(path);
|
||||
}
|
||||
}
|
||||
|
||||
async function preIndexFile(path: string) {
|
||||
if (path.endsWith(".md")) {
|
||||
// Page
|
||||
const name = path.slice(0, -3);
|
||||
// Read file
|
||||
const text = await space.readPage(name);
|
||||
const tree = await markdown.parseMarkdown(text);
|
||||
|
||||
// Emit the event which will be picked up by indexers
|
||||
await events.dispatchEvent("page:preindex", {
|
||||
name,
|
||||
tree,
|
||||
} as IndexTreeEvent);
|
||||
} else {
|
||||
await events.dispatchEvent("document:index", path);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user