From 2efeb93b47f4892753e03d5feedc1cb2b6134447 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Wed, 10 Dec 2025 09:47:24 +0100 Subject: [PATCH] Implements #1707 This splits indexing into two phases: * preindex (where only pages, space lua, space styles are indexed) * index (where everything else is indexed) The goal is to get to more-or-less operational state quicker this way, because probably most functionality can already be enabled after the pre-index phase. --- client/client.ts | 68 +- client/client_system.ts | 9 +- client/data/encrypted_kv_primitives.ts | 5 + client/data/indexeddb_kv_primitives.ts | 9 + client/data/kv_primitives.ts | 5 + client/data/memory_kv_primitives.ts | 8 + client/data/mq.datastore.test.ts | 15 +- client/data/mq.datastore.ts | 73 +- client/editor_ui.tsx | 4 +- client/styles/editor.scss | 1278 ++++++++++++------------ plug-api/syscalls/mq.ts | 2 +- plug-api/types/event.ts | 1 - plugs/index/index.plug.yaml | 24 +- plugs/index/page.ts | 18 +- plugs/index/queue.ts | 93 +- plugs/index/script.ts | 26 - 16 files changed, 845 insertions(+), 793 deletions(-) diff --git a/client/client.ts b/client/client.ts index b08ef566..73ef3e16 100644 --- a/client/client.ts +++ b/client/client.ts @@ -202,7 +202,7 @@ export class Client { this.commandAugmenter = new Augmenter(this.ds, ["aug", "command"]); // Setup message queue on top of that - this.mq = new DataStoreMQ(this.ds); + this.mq = new DataStoreMQ(this.ds, this.eventHook); // Instantiate a PlugOS system this.clientSystem = new ClientSystem( @@ -330,30 +330,58 @@ export class Client { async ( name: string, ) => { - // TODO: Optimization opportunity here: dispatch the page:index here directly rather than sending it off to a queue which will refetch the file console.log("Queueing index for", name); - - await this.mq.send("indexQueue", name); + await this.eventHook.dispatchEvent("file:clearindex", name); + await this.mq.send("preIndexQueue", name); + if (await this.clientSystem.hasPreIndexCompleted()) { + // When the client is clean booted (no index yet), we skip indexing on the first pass + // and then schedule it once pre-index has finished for all files + // However, if pre indexing has already completed, then we schedule it immediately + await this.mq.send("indexQueue", name); + } }, ); - this.eventHook.addLocalListener("file:initial", async () => { - const startTime = Date.now(); - await this.mq.awaitEmptyQueue("indexQueue"); - console.info( - "Initial indexing complete after", - (Date.now() - startTime) / 1000, - "s", - ); - await this.clientSystem.markFullSpaceIndexComplete(); - await this.clientSystem.reloadState(); - }); - - this.space = new Space( + const space = new Space( this.eventedSpacePrimitives, this.eventHook, ); + this.space = space; + + let startTime = -1; + this.eventHook.addLocalListener("file:initial", () => { + startTime = Date.now(); + }); + + const emptyQueueHandler = async () => { + if (!await this.clientSystem.hasPreIndexCompleted()) { + // Pre indexing has just finished for the first time + console.info( + "Pre-indexing complete after", + (Date.now() - startTime) / 1000, + "s", + ); + // Unsubscribe myself + this.eventHook.removeLocalListener( + "mq:emptyQueue:preIndexQueue", + emptyQueueHandler, + ); + await this.clientSystem.markPreIndexComplete(); + await this.clientSystem.reloadState(); + // Queue all pages for full indexing + await this.mq.batchSend( + "indexQueue", + (await space.fetchPageList()).map((pm) => pm.name + ".md"), + ); + await this.mq.awaitEmptyQueue("indexQueue"); + } + }; + this.eventHook.addLocalListener( + "mq:emptyQueue:preIndexQueue", + emptyQueueHandler, + ); + let lastSaveTimestamp: number | undefined; const updateLastSaveTimestamp = () => { @@ -580,7 +608,7 @@ export class Client { console.log("Updating page list cache"); // Check if the initial sync has been completed const initialIndexCompleted = await this.clientSystem - .hasFullIndexCompleted(); + .hasPreIndexCompleted(); let allPages: PageMeta[] = []; @@ -1101,7 +1129,7 @@ export class Client { // Fetch the meta which includes the possibly indexed stuff, like page // decorations - if (await this.clientSystem.hasFullIndexCompleted()) { + if (await this.clientSystem.hasPreIndexCompleted()) { try { const enrichedMeta = await this.clientSystem.getObjectByRef( pageName, @@ -1246,7 +1274,7 @@ export class Client { console.warn("Not loading custom styles, since space style is disabled"); return; } - if (!await this.clientSystem.hasFullIndexCompleted()) { + if (!await this.clientSystem.hasPreIndexCompleted()) { return; } diff --git a/client/client_system.ts b/client/client_system.ts index 501857f6..7e2cdf89 100644 --- a/client/client_system.ts +++ b/client/client_system.ts @@ -201,7 +201,7 @@ export class ClientSystem { console.info("Space Lua scripts are disabled, skipping loading scripts"); return; } - if (!await this.hasFullIndexCompleted()) { + if (!await this.hasPreIndexCompleted()) { console.info( "Not loading space scripts, since initial indexing has not completed yet", ); @@ -294,7 +294,6 @@ export class ClientSystem { if (!currentIndexVersion) { console.log("No index version found, assuming fresh install"); - await this.markFullSpaceIndexComplete(); return; } @@ -309,7 +308,7 @@ export class ClientSystem { await this.setIndexOngoing(); await this.system.invokeFunction("index.reindexSpace", []); console.info("[index]", "Full space index complete."); - await this.markFullSpaceIndexComplete(); + await this.markPreIndexComplete(); await this.setIndexOngoing(false); // Let's load space scripts again, which probably weren't loaded before await this.reloadState(); @@ -325,7 +324,7 @@ export class ClientSystem { this.client.rebuildEditorState(); } - public async hasFullIndexCompleted() { + public async hasPreIndexCompleted() { return (await this.ds.get(indexVersionKey)) === desiredIndexVersion; } @@ -333,7 +332,7 @@ export class ClientSystem { return this.ds.get(indexVersionKey); } - async markFullSpaceIndexComplete() { + async markPreIndexComplete() { await this.ds.set(indexVersionKey, desiredIndexVersion); } } diff --git a/client/data/encrypted_kv_primitives.ts b/client/data/encrypted_kv_primitives.ts index c4d04425..1d6846d8 100644 --- a/client/data/encrypted_kv_primitives.ts +++ b/client/data/encrypted_kv_primitives.ts @@ -109,6 +109,11 @@ export class EncryptedKvPrimitives implements KvPrimitives { } } + async countQuery({ prefix }: KvQueryOptions): Promise { + const encryptedPrefix = prefix ? await this.encryptKey(prefix) : undefined; + return this.wrapped.countQuery({ prefix: encryptedPrefix }); + } + close() { this.wrapped.close(); } diff --git a/client/data/indexeddb_kv_primitives.ts b/client/data/indexeddb_kv_primitives.ts index 1d332211..14fb0218 100644 --- a/client/data/indexeddb_kv_primitives.ts +++ b/client/data/indexeddb_kv_primitives.ts @@ -79,6 +79,15 @@ export class IndexedDBKvPrimitives implements KvPrimitives { } } + countQuery({ prefix }: KvQueryOptions): Promise { + const tx = this.db.transaction(objectStoreName, "readonly"); + prefix = prefix || []; + return tx.store.count(IDBKeyRange.bound( + this.buildKey([...prefix, ""]), + this.buildKey([...prefix, "\uffff"]), + )); + } + close() { this.db.close(); } diff --git a/client/data/kv_primitives.ts b/client/data/kv_primitives.ts index 3e75a236..c2ec90c5 100644 --- a/client/data/kv_primitives.ts +++ b/client/data/kv_primitives.ts @@ -13,6 +13,11 @@ export interface KvPrimitives { query(options: KvQueryOptions): AsyncIterableIterator>; + /** + * A more efficient way to do counts + */ + countQuery(options: KvQueryOptions): Promise; + // Completely clear all data from this datastore clear(): Promise; diff --git a/client/data/memory_kv_primitives.ts b/client/data/memory_kv_primitives.ts index 5f00f79e..04cbcc26 100644 --- a/client/data/memory_kv_primitives.ts +++ b/client/data/memory_kv_primitives.ts @@ -144,6 +144,14 @@ export class MemoryKvPrimitives implements KvPrimitives { } } + countQuery({ prefix }: KvQueryOptions): Promise { + const prefixStr = prefix ? prefix.join(memoryKeySeparator) : undefined; + const keys = [...this.store.keys()]; + return Promise.resolve( + keys.filter((key) => !prefixStr || key.startsWith(prefixStr)).length, + ); + } + async close(): Promise { // Force immediate persistence when closing if (this.filePath) { diff --git a/client/data/mq.datastore.test.ts b/client/data/mq.datastore.test.ts index d779b423..8c0ea9a6 100644 --- a/client/data/mq.datastore.test.ts +++ b/client/data/mq.datastore.test.ts @@ -5,14 +5,21 @@ import { DataStore } from "./datastore.ts"; import { FakeTime } from "@std/testing/time"; import type { MQMessage } from "../../plug-api/types/datastore.ts"; +import { EventHook } from "../plugos/hooks/event.ts"; +import { System } from "../plugos/system.ts"; +import type { EventHookT } from "@silverbulletmd/silverbullet/type/manifest"; Deno.test("DataStore MQ", async () => { const time = new FakeTime(); const db = new MemoryKvPrimitives(); // In-memory only, no persistence + const eventHook = new EventHook(); + const system = new System(); + system.addHook(eventHook); try { const mq = new DataStoreMQ( new DataStore(db), + eventHook, ); let messages: MQMessage[]; @@ -32,8 +39,8 @@ Deno.test("DataStore MQ", async () => { await time.tickAsync(20); await mq.requeueTimeouts(10); messages = await mq.poll("test", 10); - const stats = await mq.getAllQueueStats(); - assertEquals(stats["test"].processing, 1); + const stats = await mq.getQueueStats(); + assertEquals(stats.processing, 1); assertEquals(messages.length, 1); assertEquals(messages[0].retries, 1); @@ -77,10 +84,14 @@ Deno.test("DataStore MQ", async () => { Deno.test("DataStore MQ - Scale test with multiple subscribers", async () => { const time = new FakeTime(); const db = new MemoryKvPrimitives(); + const eventHook = new EventHook(); + const system = new System(); + system.addHook(eventHook); try { const mq = new DataStoreMQ( new DataStore(db), + eventHook, ); const queueName = "scale-test"; diff --git a/client/data/mq.datastore.ts b/client/data/mq.datastore.ts index 4f76521f..6b96f171 100644 --- a/client/data/mq.datastore.ts +++ b/client/data/mq.datastore.ts @@ -9,6 +9,7 @@ import type { MQSubscribeOptions, } from "../../plug-api/types/datastore.ts"; import { race, sleep } from "@silverbulletmd/silverbullet/lib/async"; +import type { EventHook } from "../plugos/hooks/event.ts"; export type ProcessingMessage = MQMessage & { ts: number; @@ -49,6 +50,10 @@ export class QueueWorker { await this.callback(messages); } else { // No messages, wait to be woken up or a timeout + this.mq.eventHook.dispatchEvent( + `mq:emptyQueue:${this.queue}`, + this.queue, + ); try { await race([ // Wait to be woken up explicitly @@ -98,6 +103,7 @@ export class DataStoreMQ { constructor( private ds: DataStore, + public eventHook: EventHook, ) { } @@ -350,12 +356,16 @@ export class DataStoreMQ { await this.ds.batchDelete(ids); } - async getQueueStats(queue: string): Promise { - const queued = - (await (this.ds.luaQuery([...queuedPrefix, queue], {}))).length; - const processing = - (await (this.ds.luaQuery([...processingPrefix, queue], {}))).length; - const dlq = (await (this.ds.luaQuery([...dlqPrefix, queue], {}))).length; + async getQueueStats(queue?: string): Promise { + const queued = await this.ds.kv.countQuery({ + prefix: queue ? [...queuedPrefix, queue] : queuedPrefix, + }); + const processing = await this.ds.kv.countQuery({ + prefix: queue ? [...processingPrefix, queue] : processingPrefix, + }); + const dlq = await this.ds.kv.countQuery({ + prefix: queue ? [...dlqPrefix, queue] : dlqPrefix, + }); return { queued, processing, @@ -372,55 +382,4 @@ export class DataStoreMQ { await sleep(200); } } - - async getAllQueueStats(): Promise> { - const allStatus: Record = {}; - for ( - const message of await this.ds.luaQuery( - queuedPrefix, - {}, - ) - ) { - if (!allStatus[message.queue]) { - allStatus[message.queue] = { - queued: 0, - processing: 0, - dlq: 0, - }; - } - allStatus[message.queue].queued++; - } - for ( - const message of await this.ds.luaQuery( - processingPrefix, - {}, - ) - ) { - if (!allStatus[message.queue]) { - allStatus[message.queue] = { - queued: 0, - processing: 0, - dlq: 0, - }; - } - allStatus[message.queue].processing++; - } - for ( - const message of await this.ds.luaQuery( - dlqPrefix, - {}, - ) - ) { - if (!allStatus[message.queue]) { - allStatus[message.queue] = { - queued: 0, - processing: 0, - dlq: 0, - }; - } - allStatus[message.queue].dlq++; - } - - return allStatus; - } } diff --git a/client/editor_ui.tsx b/client/editor_ui.tsx index 31bb2209..e1aa57e7 100644 --- a/client/editor_ui.tsx +++ b/client/editor_ui.tsx @@ -114,7 +114,9 @@ export class MainUI { updateTheme(); if (viewState.uiOptions.darkMode === undefined) { - const mediaQuery = globalThis.matchMedia("(prefers-color-scheme: dark)"); + const mediaQuery = globalThis.matchMedia( + "(prefers-color-scheme: dark)", + ); mediaQuery.addEventListener("change", updateTheme); return () => { diff --git a/client/styles/editor.scss b/client/styles/editor.scss index 3006297a..d05afed9 100644 --- a/client/styles/editor.scss +++ b/client/styles/editor.scss @@ -1,720 +1,718 @@ .cm-focused { - outline: none !important; + outline: none !important; } #sb-main .cm-editor { - font-size: 18px; - height: 100%; + font-size: 18px; + height: 100%; - .cm-content { - width: 100%; - margin-left: auto; - margin-right: auto; - max-width: var(--#{"editor-width"}); - padding: 5px 20px; + .cm-content { + width: 100%; + margin-left: auto; + margin-right: auto; + max-width: var(--#{"editor-width"}); + padding: 5px 20px; + } + + .cm-line { + padding: 0; + + &.sb-line-h1, + &.sb-line-h2, + &.sb-line-h3, + &.sb-line-h4, + &.sb-line-h5, + &.sb-line-h6 { + font-weight: bold; + padding: 2px; + } + } + + .cm-line:has(.sb-directive) { + background-color: var(--editor-directive-background-color); + } + + .sb-line-h1, + h1 { + font-size: 1.5em; + } + + .sb-line-h2, + h2 { + font-size: 1.2em; + } + + .sb-line-h3, + h3 { + font-size: 1.1em; + } + + .sb-line-h4, + h4, + .sb-line-h5, + h5, + .sb-line-h6, + h6 { + font-size: 1em; + } + + // If a header only contains a tag, it's likely a line containging "#" which may turn into a hashtag, so style it as such instead of a header + .sb-line-h1:has(> a.sb-hashtag:only-child) { + font-size: 1em; + padding: 0; + font-weight: normal; + + .sb-meta { + color: var(--editor-hashtag-color); + } + } + + .cm-panels-bottom .cm-vim-panel { + padding: 0 20px; + max-width: var(--editor-width); + margin: auto; + font-family: var(--editor-font); + height: 1.5em; + line-height: 1.5em; + + input { + font-family: var(--editor-font); + font-size: 1em; + } + } + + // Gutter styling + .cm-gutters { + background-color: transparent; + border-right: none; + } + + .cm-foldPlaceholder { + background-color: transparent; + border: 0; + } + + // Indentation of follow-up lines + @mixin lineOverflow($baseIndent, $bulletIndent: 0) { + text-indent: calc(-1 * (#{$baseIndent}ch + 2ch)); + padding-left: calc(#{$baseIndent}ch + 2ch); + + &.sb-line-task { + text-indent: calc(-1 * (#{$baseIndent}ch + 5ch)); + padding-left: calc(#{$baseIndent}ch + 5ch); + + .cm-list-bullet::after { + left: calc(#{$baseIndent}ch + 5ch); + } } - .cm-line { - padding: 0; - - &.sb-line-h1, - &.sb-line-h2, - &.sb-line-h3, - &.sb-line-h4, - &.sb-line-h5, - &.sb-line-h6 { - font-weight: bold; - padding: 2px; - } - } - - .cm-line:has(.sb-directive) { - background-color: var(--editor-directive-background-color); - } - - .sb-line-h1, - h1 { - font-size: 1.5em; - } - - .sb-line-h2, - h2 { - font-size: 1.2em; - } - - .sb-line-h3, - h3 { - font-size: 1.1em; - } - - .sb-line-h4, - h4, - .sb-line-h5, - h5, - .sb-line-h6, - h6 { - font-size: 1em; - } - - // If a header only contains a tag, it's likely a line containging "#" which may turn into a hashtag, so style it as such instead of a header - .sb-line-h1:has(> a.sb-hashtag:only-child) { - font-size: 1em; - padding: 0; - font-weight: normal; - - .sb-meta { - color: var(--editor-hashtag-color); - } - } - - .cm-panels-bottom .cm-vim-panel { - padding: 0 20px; - max-width: var(--editor-width); - margin: auto; - font-family: var(--editor-font); - height: 1.5em; - line-height: 1.5em; - - input { - font-family: var(--editor-font); - font-size: 1em; - } - } - - // Gutter styling - .cm-gutters { - background-color: transparent; - border-right: none; - } - - .cm-foldPlaceholder { - background-color: transparent; - border: 0; - } - - // Indentation of follow-up lines - @mixin lineOverflow($baseIndent, $bulletIndent: 0) { - text-indent: calc(-1 * (#{$baseIndent}ch + 2ch)); - padding-left: calc(#{$baseIndent}ch + 2ch); - - &.sb-line-task { - text-indent: calc(-1 * (#{$baseIndent}ch + 5ch)); - padding-left: calc(#{$baseIndent}ch + 5ch); - - .cm-list-bullet::after { - left: calc(#{$baseIndent}ch + 5ch); - } - } - - &.sb-line-blockquote { - text-indent: calc(-1 * (#{$baseIndent}ch + 4ch)); - padding-left: calc(#{$baseIndent}ch + 4ch); - } - - .cm-list-bullet::after { - left: calc(#{$baseIndent}ch + #{$bulletIndent}ch + 2ch); - } - } - - .sb-line-ul { - &.sb-line-li-1 { - @include lineOverflow(0); - } - - &.sb-line-li-1.sb-line-li-2 { - @include lineOverflow(2); - } - - &.sb-line-li-1.sb-line-li-2.sb-line-li-3 { - @include lineOverflow(4); - } - - &.sb-line-li-1.sb-line-li-2.sb-line-li-3.sb-line-li-4 { - @include lineOverflow(6); - } - - &.sb-line-li-1.sb-line-li-2.sb-line-li-3.sb-line-li-4.sb-line-li-5 { - @include lineOverflow(8); - } - } - - .sb-line-ol { - &.sb-line-li-1 { - @include lineOverflow(1); - } - - &.sb-line-li-1.sb-line-li-2 { - @include lineOverflow(4); - } - - &.sb-line-li-1.sb-line-li-2.sb-line-li-3 { - @include lineOverflow(7); - } - - &.sb-line-li-1.sb-line-li-2.sb-line-li-3.sb-line-li-4 { - @include lineOverflow(10); - } - - &.sb-line-li-1.sb-line-li-2.sb-line-li-3.sb-line-li-4.sb-line-li-5 { - @include lineOverflow(13); - } - } - - .sb-line-ol.sb-line-ul { - // &.sb-line-li-1 { - // @include lineOverflow(1); - // } - - &.sb-line-li-1.sb-line-li-2 { - @include lineOverflow(3); - } - - &.sb-line-li-1.sb-line-li-2.sb-line-li-3 { - @include lineOverflow(6); - } - - &.sb-line-li-1.sb-line-li-2.sb-line-li-3.sb-line-li-4 { - @include lineOverflow(9); - } - - &.sb-line-li-1.sb-line-li-2.sb-line-li-3.sb-line-li-4.sb-line-li-5 { - @include lineOverflow(12); - } - } - - .sb-line-blockquote { - &.sb-line-li-1 { - @include lineOverflow(-1, 2); - } - - &.sb-line-li-1.sb-line-li-2 { - @include lineOverflow(1, 2); - } - - &.sb-line-li-1.sb-line-li-2.sb-line-li-3 { - @include lineOverflow(4); - } - - &.sb-line-li-1.sb-line-li-2.sb-line-li-3.sb-line-li-4 { - @include lineOverflow(6); - } - - &.sb-line-li-1.sb-line-li-2.sb-line-li-3.sb-line-li-4.sb-line-li-5 { - @include lineOverflow(8); - } - } - - .sb-line-comment { - text-indent: -1 * 3ch; - padding-left: 3ch; - } - - .cm-list-bullet { - position: relative; - visibility: hidden; + &.sb-line-blockquote { + text-indent: calc(-1 * (#{$baseIndent}ch + 4ch)); + padding-left: calc(#{$baseIndent}ch + 4ch); } .cm-list-bullet::after { - visibility: visible; - position: absolute; - content: "\2022"; - /* U+2022 BULLET */ + left: calc(#{$baseIndent}ch + #{$bulletIndent}ch + 2ch); + } + } + + .sb-line-ul { + &.sb-line-li-1 { + @include lineOverflow(0); } - .cm-task-checked, - .sb-line-task:has(.cm-task-checked) .sb-wiki-link { - text-decoration: line-through !important; + &.sb-line-li-1.sb-line-li-2 { + @include lineOverflow(2); } - .cm-tooltip-autocomplete { - .cm-completionDetail { - font-style: normal; - display: block; - font-size: 80%; - margin-left: 5px; - } - - .cm-completionLabel { - display: block; - margin-left: 5px; - } - - .cm-completionIcon { - display: none; - } + &.sb-line-li-1.sb-line-li-2.sb-line-li-3 { + @include lineOverflow(4); } - .sb-header-inside.sb-line-h1 { - text-indent: -2ch; + &.sb-line-li-1.sb-line-li-2.sb-line-li-3.sb-line-li-4 { + @include lineOverflow(6); } - .sb-header-inside.sb-line-h2 { - text-indent: -3ch; + &.sb-line-li-1.sb-line-li-2.sb-line-li-3.sb-line-li-4.sb-line-li-5 { + @include lineOverflow(8); + } + } + + .sb-line-ol { + &.sb-line-li-1 { + @include lineOverflow(1); } - .sb-header-inside.sb-line-h3 { - text-indent: -4ch; + &.sb-line-li-1.sb-line-li-2 { + @include lineOverflow(4); } - .sb-header-inside.sb-line-h4 { - text-indent: -5ch; + &.sb-line-li-1.sb-line-li-2.sb-line-li-3 { + @include lineOverflow(7); } - .sb-header-inside.sb-line-h5 { - text-indent: -6ch; + &.sb-line-li-1.sb-line-li-2.sb-line-li-3.sb-line-li-4 { + @include lineOverflow(10); } - .sb-header-inside.sb-line-h6 { - text-indent: -7ch; + &.sb-line-li-1.sb-line-li-2.sb-line-li-3.sb-line-li-4.sb-line-li-5 { + @include lineOverflow(13); + } + } + + .sb-line-ol.sb-line-ul { + // &.sb-line-li-1 { + // @include lineOverflow(1); + // } + + &.sb-line-li-1.sb-line-li-2 { + @include lineOverflow(3); } - .sb-checkbox > input[type="checkbox"] { - width: 3ch; + &.sb-line-li-1.sb-line-li-2.sb-line-li-3 { + @include lineOverflow(6); } - .sb-hashtag, - .hashtag { - border-radius: 6px; - padding: 0 3px; - margin: 0 1px 0 0; - font-size: 0.9em; + &.sb-line-li-1.sb-line-li-2.sb-line-li-3.sb-line-li-4 { + @include lineOverflow(9); } - a.sb-hashtag { - text-decoration: none; + &.sb-line-li-1.sb-line-li-2.sb-line-li-3.sb-line-li-4.sb-line-li-5 { + @include lineOverflow(12); + } + } + + .sb-line-blockquote { + &.sb-line-li-1 { + @include lineOverflow(-1, 2); } - .sb-strikethrough { - text-decoration: line-through; - - &.sb-meta { - text-decoration: none; - } + &.sb-line-li-1.sb-line-li-2 { + @include lineOverflow(1, 2); } - .sb-line-hr { - margin-top: 1em; - margin-bottom: -1em; + &.sb-line-li-1.sb-line-li-2.sb-line-li-3 { + @include lineOverflow(4); } - .sb-hr { - font-weight: bold; + &.sb-line-li-1.sb-line-li-2.sb-line-li-3.sb-line-li-4 { + @include lineOverflow(6); } - .sb-naked-url { - cursor: pointer; + &.sb-line-li-1.sb-line-li-2.sb-line-li-3.sb-line-li-4.sb-line-li-5 { + @include lineOverflow(8); + } + } + + .sb-line-comment { + text-indent: -1 * 3ch; + padding-left: 3ch; + } + + .cm-list-bullet { + position: relative; + visibility: hidden; + } + + .cm-list-bullet::after { + visibility: visible; + position: absolute; + content: "\2022"; + /* U+2022 BULLET */ + } + + .cm-task-checked, + .sb-line-task:has(.cm-task-checked) .sb-wiki-link { + text-decoration: line-through !important; + } + + .cm-tooltip-autocomplete { + .cm-completionDetail { + font-style: normal; + display: block; + font-size: 80%; + margin-left: 5px; } - .sb-link:not(.sb-url) { - cursor: pointer; + .cm-completionLabel { + display: block; + margin-left: 5px; } - .sb-link:not(.sb-meta, .sb-url) { - text-decoration: underline; + .cm-completionIcon { + display: none; + } + } + + .sb-header-inside.sb-line-h1 { + text-indent: -2ch; + } + + .sb-header-inside.sb-line-h2 { + text-indent: -3ch; + } + + .sb-header-inside.sb-line-h3 { + text-indent: -4ch; + } + + .sb-header-inside.sb-line-h4 { + text-indent: -5ch; + } + + .sb-header-inside.sb-line-h5 { + text-indent: -6ch; + } + + .sb-header-inside.sb-line-h6 { + text-indent: -7ch; + } + + .sb-checkbox > input[type="checkbox"] { + width: 3ch; + } + + .sb-hashtag, + .hashtag { + border-radius: 6px; + padding: 0 3px; + margin: 0 1px 0 0; + font-size: 0.9em; + } + + a.sb-hashtag { + text-decoration: none; + } + + .sb-strikethrough { + text-decoration: line-through; + + &.sb-meta { + text-decoration: none; + } + } + + .sb-line-hr { + margin-top: 1em; + margin-bottom: -1em; + } + + .sb-hr { + font-weight: bold; + } + + .sb-naked-url { + cursor: pointer; + } + + .sb-link:not(.sb-url) { + cursor: pointer; + } + + .sb-link:not(.sb-meta, .sb-url) { + text-decoration: underline; + } + + .sb-url:not(.sb-link) { + text-decoration: underline; + cursor: pointer; + } + + .sb-wiki-link { + border-radius: 5px; + padding: 0 5px; + // white-space: nowrap; + text-decoration: none; + cursor: pointer; + } + + .sb-lua-directive-inline { + display: inline; + padding: 2px; + } + + .sb-lua-directive-block { + display: block; + margin: -1em 0; + } + + .sb-widget-array { + display: flex; + flex-direction: column; + } + + .sb-task-mark { + font-size: 91%; + } + + .sb-task-state { + font-size: 91%; + } + + .sb-task-deadline { + background-color: rgba(22, 22, 22, 0.07); + } + + .sb-line-frontmatter-outside, + .sb-line-code-outside { + .sb-meta { + color: transparent; } - .sb-url:not(.sb-link) { - text-decoration: underline; - cursor: pointer; + color: transparent; + } + + .sb-line-blockquote { + text-indent: -2ch; + } + + .sb-blockquote-outside { + text-indent: -1ch; + min-height: 1em; + } + + .sb-line-table-outside { + display: none; + } + + .sb-line-tbl-header { + font-weight: bold; + } + + .sb-line-tbl-header .meta { + font-weight: normal; + } + + .sb-table-widget { + display: block; + font-weight: normal; + margin-bottom: -3rem; + overflow: auto; + } + + table { + width: 100%; + border-spacing: 0; + } + + thead tr { + font-weight: bold; + } + + thead tr td { + // Primarily to make the column headers look reasonable with the button bar that may be put on top + padding: 11px 8px; + } + + th, + td { + padding: 8px; + white-space: nowrap; + } + + // Table alignment classes for justified tables + .sb-table-align-left { + text-align: left; + } + + .sb-table-align-right { + text-align: right; + } + + .sb-table-align-center { + text-align: center; + } + + // dont apply background color twice for (fenced) code blocks + .sb-line-code .sb-code { + background-color: transparent; + } + + .sb-line-code-outside .sb-code-info { + display: block; + float: right; + font-size: 90%; + padding-right: 7px; + } + + .sb-code-copy-button { + float: right; + cursor: pointer; + margin: 0 3px; + } + + .sb-code-copy-button > svg { + height: 1rem; + width: 1rem; + } + + .sb-line-fenced-code .sb-code { + background-color: transparent; + } + + .sb-line-fenced-code .sb-comment { + border-radius: 0; + font-style: inherit; + font-size: inherit; + line-height: inherit; + } + + .sb-keyword { + font-weight: bold; + } + + // Allow for cursor movement into a code widget fenced code block from bottom-up + .sb-fenced-code-hide { + background-color: transparent; + line-height: 0; + } + + .sb-lua-wrapper { + ::selection { + background-color: var(--editor-selection-background-color) !important; } + } - .sb-wiki-link { - border-radius: 5px; - padding: 0 5px; - // white-space: nowrap; - text-decoration: none; - cursor: pointer; - } + .sb-lua-top-widget, + .sb-lua-bottom-widget { + border: 1px solid var(--editor-widget-background-color); + border-radius: 5px; + white-space: normal; + position: relative; + min-height: 48px; + } - .sb-lua-directive-inline { - display: inline; - padding: 2px; - } + .sb-lua-top-widget { + margin: 0 0 10px 0; + } - .sb-lua-directive-block { - display: block; - margin: -1em 0; - } + .sb-lua-bottom-widget { + margin: 10px 0 0 0; + } - .sb-widget-array { - display: flex; - flex-direction: column; - } + .sb-lua-top-widget h1, + .sb-lua-bottom-widget h1 { + margin: -10px -10px 10px -10px !important; + padding: 15px 10px !important; + background-color: var(--editor-widget-background-color); + font-size: 1.2em; + } - .sb-task-mark { - font-size: 91%; - } + .sb-markdown-top-widget h1, + .sb-markdown-bottom-widget h1 { + margin: 0 0 5px 0; + padding: 10px !important; + background-color: var(--editor-widget-background-color); + font-size: 1.2em; + } - .sb-task-state { - font-size: 91%; - } + .sb-markdown-top-widget:has(*) { + margin-bottom: 10px; + } - .sb-task-deadline { - background-color: rgba(22, 22, 22, 0.07); - } + .sb-markdown-bottom-widget { + margin-top: 10px; + } - .sb-line-frontmatter-outside, - .sb-line-code-outside { - .sb-meta { - color: transparent; - } + .sb-lua-top-widget .content, + .sb-lua-bottom-widget .content { + max-height: 500px; + padding: 10px; + } - color: transparent; - } + .sb-lua-bottom-widget p strong { + display: block; + padding: 10px 12px; + background-color: var(--editor-widget-background-color); + border: 1px solid var(--editor-widget-background-color); + border-radius: 8px 8px 0 0; + font-weight: 600; + } - .sb-line-blockquote { - text-indent: -2ch; - } + .sb-lua-bottom-widget blockquote { + margin: 0 5px; + padding: 12px; + border: 1px solid var(--editor-widget-background-color); + border-radius: 8px; + background-color: var(--editor-widget-background-color); + color: var(--editor-text-color); + border-top: 1px solid var(--editor-widget-background-color); + } - .sb-blockquote-outside { - text-indent: -1ch; - min-height: 1em; - } - - .sb-line-table-outside { - display: none; - } - - .sb-line-tbl-header { - font-weight: bold; - } - - .sb-line-tbl-header .meta { - font-weight: normal; - } - - .sb-table-widget { - display: block; - font-weight: normal; - margin-bottom: -3rem; - overflow: auto; - } - - table { - width: 100%; - border-spacing: 0; - } - - thead tr { - font-weight: bold; - } - - thead tr td { - // Primarily to make the column headers look reasonable with the button bar that may be put on top - padding: 11px 8px; - } - - th, - td { - padding: 8px; - white-space: nowrap; - } - - // Table alignment classes for justified tables - .sb-table-align-left { - text-align: left; - } - - .sb-table-align-right { - text-align: right; - } - - .sb-table-align-center { - text-align: center; - } - - // dont apply background color twice for (fenced) code blocks - .sb-line-code .sb-code { - background-color: transparent; - } - - .sb-line-code-outside .sb-code-info { - display: block; - float: right; - font-size: 90%; - padding-right: 7px; - } - - .sb-code-copy-button { - float: right; - cursor: pointer; - margin: 0 3px; - } - - .sb-code-copy-button > svg { - height: 1rem; - width: 1rem; - } - - .sb-line-fenced-code .sb-code { - background-color: transparent; - } - - .sb-line-fenced-code .sb-comment { - border-radius: 0; - font-style: inherit; - font-size: inherit; - line-height: inherit; - } - - .sb-keyword { - font-weight: bold; - } - - // Allow for cursor movement into a code widget fenced code block from bottom-up - .sb-fenced-code-hide { - background-color: transparent; - line-height: 0; - } - - .sb-lua-wrapper { - ::selection { - background-color: var( - --editor-selection-background-color - ) !important; - } - } - - .sb-lua-top-widget, - .sb-lua-bottom-widget { - border: 1px solid var(--editor-widget-background-color); - border-radius: 5px; - white-space: normal; - position: relative; - min-height: 48px; - } - - .sb-lua-top-widget { - margin: 0 0 10px 0; - } - - .sb-lua-bottom-widget { - margin: 10px 0 0 0; - } - - .sb-lua-top-widget h1, - .sb-lua-bottom-widget h1 { - margin: -10px -10px 10px -10px !important; - padding: 15px 10px !important; - background-color: var(--editor-widget-background-color); - font-size: 1.2em; - } - - .sb-markdown-top-widget h1, - .sb-markdown-bottom-widget h1 { - margin: 0 0 5px 0; - padding: 10px !important; - background-color: var(--editor-widget-background-color); - font-size: 1.2em; - } - - .sb-markdown-top-widget:has(*) { - margin-bottom: 10px; - } - - .sb-markdown-bottom-widget { - margin-top: 10px; - } - - .sb-lua-top-widget .content, - .sb-lua-bottom-widget .content { - max-height: 500px; - padding: 10px; - } - - .sb-lua-bottom-widget p strong { - display: block; - padding: 10px 12px; - background-color: var(--editor-widget-background-color); - border: 1px solid var(--editor-widget-background-color); - border-radius: 8px 8px 0 0; - font-weight: 600; - } - - .sb-lua-bottom-widget blockquote { - margin: 0 5px; - padding: 12px; - border: 1px solid var(--editor-widget-background-color); - border-radius: 8px; - background-color: var(--editor-widget-background-color); - color: var(--editor-text-color); - border-top: 1px solid var(--editor-widget-background-color); - } + .sb-markdown-top-widget:has(*) .content { + max-height: 500px; + } + @media screen and (max-height: 1000px) { .sb-markdown-top-widget:has(*) .content { - max-height: 500px; + max-height: 300px; + } + } + + .sb-markdown-widget, + .sb-lua-directive-block, + .sb-lua-directive-inline, + .sb-markdown-top-widget:has(*), + .sb-markdown-bottom-widget:has(*) { + border: 1px solid var(--editor-widget-background-color); + border-radius: 5px; + white-space: normal; + position: relative; + min-height: 48px; + + .content { + overflow-y: auto; } - @media screen and (max-height: 1000px) { - .sb-markdown-top-widget:has(*) .content { - max-height: 300px; - } + ul, + ol { + margin-top: 0; + margin-bottom: 0; } - .sb-markdown-widget, - .sb-lua-directive-block, - .sb-lua-directive-inline, - .sb-markdown-top-widget:has(*), - .sb-markdown-bottom-widget:has(*) { - border: 1px solid var(--editor-widget-background-color); - border-radius: 5px; - white-space: normal; - position: relative; - min-height: 48px; - - .content { - overflow-y: auto; - } - - ul, - ol { - margin-top: 0; - margin-bottom: 0; - } - - ul { - list-style: none; - // padding-left: 1ch; - } - - ul li::before { - content: "\2022"; - /* Add content: \2022 is the CSS Code/unicode for a bullet */ - color: var(--editor-list-bullet-color); - display: inline-block; - /* Needed to add space between the bullet and the text */ - width: 1em; - /* Also needed for space (tweak if needed) */ - margin-left: -1em; - } - - h1, - h2, - h3, - h4, - h5 { - margin: 0; - } - - a.wiki-link { - border-radius: 5px; - padding: 0 5px; - color: var(--editor-wiki-link-page-color); - background-color: var(--editor-wiki-link-page-background-color); - text-decoration: none; - } - - span.task-deadline { - background-color: rgba(22, 22, 22, 0.07); - } - - tt { - background-color: var(--editor-code-background-color); - } - - // Only show the button bar on hover on non-touch devices - &:hover .button-bar { - display: block; - } - - // Always show button bar on touch devices - @media (hover: none) and (pointer: coarse) { - .button-bar { - display: block !important; - } - } - - .button-bar { - position: absolute; - right: 0; - top: 0; - display: none; - background: var(--editor-widget-background-color); - padding-inline: 3px; - padding: 4px 0; - border-radius: 0 0 0 5px; - - button { - border: none; - background: none; - cursor: pointer; - color: var(--root-color); - margin-right: -8px; - padding: 11px 8px; - } - - button:last-of-type { - margin-right: 2px; - } - } + ul { + list-style: none; + // padding-left: 1ch; } - .sb-lua-directive-inline:has(> button) { + ul li::before { + content: "\2022"; + /* Add content: \2022 is the CSS Code/unicode for a bullet */ + color: var(--editor-list-bullet-color); + display: inline-block; + /* Needed to add space between the bullet and the text */ + width: 1em; + /* Also needed for space (tweak if needed) */ + margin-left: -1em; + } + + h1, + h2, + h3, + h4, + h5 { + margin: 0; + } + + a.wiki-link { + border-radius: 5px; + padding: 0 5px; + color: var(--editor-wiki-link-page-color); + background-color: var(--editor-wiki-link-page-background-color); + text-decoration: none; + } + + span.task-deadline { + background-color: rgba(22, 22, 22, 0.07); + } + + tt { + background-color: var(--editor-code-background-color); + } + + // Only show the button bar on hover on non-touch devices + &:hover .button-bar { + display: block; + } + + // Always show button bar on touch devices + @media (hover: none) and (pointer: coarse) { + .button-bar { + display: block !important; + } + } + + .button-bar { + position: absolute; + right: 0; + top: 0; + display: none; + background: var(--editor-widget-background-color); + padding-inline: 3px; + padding: 4px 0; + border-radius: 0 0 0 5px; + + button { border: none; - border-radius: 0; + background: none; + cursor: pointer; + color: var(--root-color); + margin-right: -8px; + padding: 11px 8px; + } + + button:last-of-type { + margin-right: 2px; + } } + } - .sb-inline-content > div > .content > * { - max-width: 100%; - display: block; + .sb-lua-directive-inline:has(> button) { + border: none; + border-radius: 0; + } + + .sb-inline-content > div > .content > * { + max-width: 100%; + display: block; + } + + .sb-inline-content { + margin: 0; + overflow: hidden; + } + + .sb-fenced-code-iframe { + background-color: transparent; + + iframe { + border: 0; + width: 100%; + max-width: 100%; + padding: 0; + margin: 0 0 -2ch 0; + border: 1px solid var(--editor-widget-background-color); + border-radius: 5px; } + } - .sb-inline-content { - margin: 0; - overflow: hidden; - } + .sb-line-blockquote.sb-line-ul.sb-line-li > .sb-quote.sb-meta:first-child { + margin-left: -1ch; + } - .sb-fenced-code-iframe { - background-color: transparent; + .sb-admonition { + border-left-width: 4px !important; + border-left-style: solid; + padding-left: 5px; + } - iframe { - border: 0; - width: 100%; - max-width: 100%; - padding: 0; - margin: 0 0 -2ch 0; - border: 1px solid var(--editor-widget-background-color); - border-radius: 5px; - } - } + .sb-admonition-type::before { + content: ""; + mask: var(--admonition-icon) no-repeat; + -webkit-mask: var(--admonition-icon) no-repeat; + mask-size: cover; + -webkit-mask-size: cover; - .sb-line-blockquote.sb-line-ul.sb-line-li > .sb-quote.sb-meta:first-child { - margin-left: -1ch; - } + /* used in (custom) admonition definitions */ + --admonition-width: 1.1em; + width: 0em; + height: 1.1em; + display: inline-flex; + vertical-align: middle; + } - .sb-admonition { - border-left-width: 4px !important; - border-left-style: solid; - padding-left: 5px; - } + .sb-frontmatter-marker { + float: right; + font-size: 80%; + padding-right: 7px; + } - .sb-admonition-type::before { - content: ""; - mask: var(--admonition-icon) no-repeat; - -webkit-mask: var(--admonition-icon) no-repeat; - mask-size: cover; - -webkit-mask-size: cover; - - /* used in (custom) admonition definitions */ - --admonition-width: 1.1em; - width: 0em; - height: 1.1em; - display: inline-flex; - vertical-align: middle; - } - - .sb-frontmatter-marker { - float: right; - font-size: 80%; - padding-right: 7px; - } - - .cm-scroller { - // Give some breathing space at the bottom of the screen - padding-bottom: 20em; - } + .cm-scroller { + // Give some breathing space at the bottom of the screen + padding-bottom: 20em; + } } div:not(.cm-focused).cm-fat-cursor { - outline: none !important; + outline: none !important; } diff --git a/plug-api/syscalls/mq.ts b/plug-api/syscalls/mq.ts index 6cb2b73e..540300d1 100644 --- a/plug-api/syscalls/mq.ts +++ b/plug-api/syscalls/mq.ts @@ -58,7 +58,7 @@ export function batchAck(queue: string, ids: string[]): Promise { * Retrieves stats on a particular queue. * @param queue the name of the queue */ -export function getQueueStats(queue: string): Promise { +export function getQueueStats(queue?: string): Promise { return syscall("mq.getQueueStats", queue); } diff --git a/plug-api/types/event.ts b/plug-api/types/event.ts index 010af508..28638dbf 100644 --- a/plug-api/types/event.ts +++ b/plug-api/types/event.ts @@ -3,7 +3,6 @@ import type { EventSubscriptionDef } from "./manifest.ts"; export type IndexTreeEvent = { name: string; - tree: ParseTree; }; diff --git a/plugs/index/index.plug.yaml b/plugs/index/index.plug.yaml index 6fa07ba5..f93343f9 100644 --- a/plugs/index/index.plug.yaml +++ b/plugs/index/index.plug.yaml @@ -32,6 +32,7 @@ functions: path: api.ts:clearFileIndex events: - file:deleted + - file:clearindex reindexSpaceCommand: path: "./command.ts:reindexCommand" @@ -46,6 +47,13 @@ functions: batchSize: 3 autoAck: true + processPreIndexQueue: + path: ./queue.ts:processPreIndexQueue + mqSubscriptions: + - queue: preIndexQueue + batchSize: 5 + autoAck: true + reindexSpace: path: "./queue.ts:reindexSpace" @@ -59,6 +67,11 @@ functions: events: - page:index + preIndexPage: + path: page.ts:preIndexPage + events: + - page:preindex + indexParagraphs: path: "./paragraph.ts:indexParagraphs" events: @@ -98,27 +111,24 @@ functions: events: - page:index - # Script - indexSpaceScript: - path: script.ts:indexSpaceScript - events: - - page:index + # Lua indexSpaceLua: path: script.ts:indexSpaceLua events: - - page:index + - page:preindex # Style indexSpaceStyle: path: style.ts:indexSpaceStyle events: - - page:index + - page:preindex # Hashtags indexTags: path: tags.ts:indexTags events: - page:index + tagComplete: path: tags.ts:tagComplete events: diff --git a/plugs/index/page.ts b/plugs/index/page.ts index edb5b857..051bc72d 100644 --- a/plugs/index/page.ts +++ b/plugs/index/page.ts @@ -25,11 +25,12 @@ import type { AspiringPageObject } from "./page_links.ts"; import type { PageMeta } from "@silverbulletmd/silverbullet/type/index"; import type { LintDiagnostic } from "@silverbulletmd/silverbullet/type/client"; -export async function indexPage({ name, tree }: IndexTreeEvent) { +export async function preIndexPage({ name, tree }: IndexTreeEvent) { if (name.startsWith("_")) { // Don't index pages starting with _ return; } + const pageMeta = await space.getPageMeta(name); const frontmatter = await extractFrontMatter(tree); const toplevelAttributes = await extractAttributes(tree); @@ -65,7 +66,19 @@ export async function indexPage({ name, tree }: IndexTreeEvent) { updateITags(combinedPageMeta, frontmatter); + // console.log("Page object", combinedPageMeta); + await indexObjects(name, [combinedPageMeta]); +} + +// Most interesting indexing for pages happens in the preindex phase, here we're just cleaning up aspiring pages +export async function indexPage({ name }: IndexTreeEvent) { + if (name.startsWith("_")) { + // Don't index pages starting with _ + return; + } + // Make sure this page is no (longer) in the aspiring pages list + // TODO: This can possibly done more optimally const aspiringPages = await queryLuaObjects( "aspiring-page", { @@ -78,9 +91,6 @@ export async function indexPage({ name, tree }: IndexTreeEvent) { console.log("Deleting aspiring page", aspiringPage); await deleteObject("aspiring-page", aspiringPage.page, aspiringPage.ref); } - - // console.log("Page object", combinedPageMeta); - await indexObjects(name, [combinedPageMeta]); } export async function lintFrontmatter(): Promise { diff --git a/plugs/index/queue.ts b/plugs/index/queue.ts index 4ad78729..4f14f083 100644 --- a/plugs/index/queue.ts +++ b/plugs/index/queue.ts @@ -9,7 +9,8 @@ import { import { sleep } from "@silverbulletmd/silverbullet/lib/async"; import type { MQMessage } from "@silverbulletmd/silverbullet/type/datastore"; import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; -import { clearFileIndex } from "./api.ts"; + +const uiUpdateInterval = 5000; export async function reindexSpace() { if (await system.getMode() === "ro") { @@ -25,8 +26,10 @@ export async function reindexSpace() { console.log("Queing", files.length, "pages to be indexed."); // Queue all file names to be indexed + await mq.batchSend("preIndexQueue", files.map((file) => file.name)); await mq.batchSend("indexQueue", files.map((file) => file.name)); await editor.showProgress(0, "index"); + // We'll assume this one completes last await mq.awaitEmptyQueue("indexQueue"); // And notify the user @@ -34,56 +37,88 @@ export async function reindexSpace() { await editor.showProgress(); } -setTimeout(updateIndexProgressInUI, 5000); +setTimeout(updateIndexProgressInUI, uiUpdateInterval); + +async function totalItemsQueued() { + const queueStats = await mq.getQueueStats(); + return queueStats.queued + queueStats.processing; +} async function updateIndexProgressInUI() { // Let's see if there's anything in the index queue - let queueStats = await mq.getQueueStats("indexQueue"); - if (queueStats.queued > 0 || queueStats.processing > 0) { + let totalQueued = await totalItemsQueued(); + if (totalQueued > 0) { // Something's queued, likely it makes sense to compare this to the total number of files (progress wise) const fileList = await space.listFiles(); - while (queueStats.queued > 0 || queueStats.processing > 0) { - queueStats = await mq.getQueueStats("indexQueue"); - const percentage = Math.round( - (fileList.length - queueStats.queued) / fileList.length * 100, + while (totalQueued > 0) { + totalQueued = await totalItemsQueued(); + let percentage = Math.round( + (fileList.length - totalQueued) / fileList.length * 100, ); + if (percentage < 0) { + // The assumption that the queue size is related tot he file list length turns out to be wrong + // // Let's show some sort of number + percentage = 67; + } if (percentage > 99) { // Hide progress circle await editor.showProgress(); } else { await editor.showProgress(percentage, "index"); } - // Update UI every second await sleep(1000); } } // Schedule again - setTimeout(updateIndexProgressInUI, 5000); + setTimeout(updateIndexProgressInUI, uiUpdateInterval); } export async function processIndexQueue(messages: MQMessage[]) { for (const message of messages) { - let name: string = message.body; - console.log(`Indexing file ${name}`); - if (name.endsWith(".md")) { - name = name.slice(0, -3); - await indexPage(name); - } else { - await events.dispatchEvent("document:index", name); - } + const path: string = message.body; + console.log("[index]", `Indexing file ${path}`); + await indexFile(path); } } -async function indexPage(name: string) { - // Clear any previous index entries for this file - await clearFileIndex(name); - // Read and parse the file - const text = await space.readPage(name); - const parsed = await markdown.parseMarkdown(text); +async function indexFile(path: string) { + if (path.endsWith(".md")) { + // Page + const name = path.slice(0, -3); + // Read and parse the file + const text = await space.readPage(name); + const tree = await markdown.parseMarkdown(text); - // Emit the event which will be picked up by indexers - await events.dispatchEvent("page:index", { - name, - tree: parsed, - } as IndexTreeEvent); + // Emit the event which will be picked up by indexers + await events.dispatchEvent("page:index", { + name, + tree, + } as IndexTreeEvent); + } +} + +export async function processPreIndexQueue(messages: MQMessage[]) { + for (const message of messages) { + const path: string = message.body; + console.log("[pre-index]", `Pre-indexing file ${path}`); + await preIndexFile(path); + } +} + +async function preIndexFile(path: string) { + if (path.endsWith(".md")) { + // Page + const name = path.slice(0, -3); + // Read file + const text = await space.readPage(name); + const tree = await markdown.parseMarkdown(text); + + // Emit the event which will be picked up by indexers + await events.dispatchEvent("page:preindex", { + name, + tree, + } as IndexTreeEvent); + } else { + await events.dispatchEvent("document:index", path); + } } diff --git a/plugs/index/script.ts b/plugs/index/script.ts index 312b91cd..e70ada8f 100644 --- a/plugs/index/script.ts +++ b/plugs/index/script.ts @@ -11,32 +11,6 @@ export type ScriptObject = ObjectValue<{ priority?: number; }>; -export async function indexSpaceScript({ name, tree }: IndexTreeEvent) { - const allScripts: ScriptObject[] = []; - collectNodesOfType(tree, "FencedCode").map((t) => { - const codeInfoNode = findNodeOfType(t, "CodeInfo"); - if (!codeInfoNode) { - return; - } - const fenceType = codeInfoNode.children![0].text!; - if (fenceType !== "space-script") { - return; - } - const codeTextNode = findNodeOfType(t, "CodeText"); - if (!codeTextNode) { - // Honestly, this shouldn't happen - return; - } - const codeText = codeTextNode.children![0].text!; - allScripts.push({ - ref: `${name}@${t.from!}`, - tag: "space-script", - script: codeText, - }); - }); - await indexObjects(name, allScripts); -} - export async function indexSpaceLua({ name, tree }: IndexTreeEvent) { const allScripts: ScriptObject[] = []; collectNodesOfType(tree, "FencedCode").map((t) => {