diff --git a/client/client.ts b/client/client.ts index 6831eb8d..03d3f9f3 100644 --- a/client/client.ts +++ b/client/client.ts @@ -80,6 +80,8 @@ import type { KvPrimitives } from "./data/kv_primitives.ts"; import { deriveDbName } from "@silverbulletmd/silverbullet/lib/crypto"; import { LuaRuntimeError } from "./space_lua/runtime.ts"; import { resolveASTReference } from "./space_lua.ts"; +import { ObjectIndex } from "./data/object_index.ts"; +import type { LuaCollectionQuery } from "./space_lua/query_collection.ts"; const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/; @@ -158,6 +160,7 @@ export class Client { console.error, ); }, 2000); + objectIndex!: ObjectIndex; constructor( private parent: Element, @@ -204,12 +207,20 @@ export class Client { // Setup message queue on top of that this.mq = new DataStoreMQ(this.ds, this.eventHook); + this.objectIndex = new ObjectIndex( + this.ds, + this.config, + this.eventHook, + this.mq, + ); + // Instantiate a PlugOS system this.clientSystem = new ClientSystem( this, this.mq, this.ds, this.eventHook, + this.objectIndex, this.bootConfig.readOnly, ); @@ -331,7 +342,7 @@ export class Client { name: string, ) => { console.log("Queueing index for", name); - await this.eventHook.dispatchEvent("file:clearindex", name); + await this.objectIndex.clearFileIndex(name); await this.mq.send("indexQueue", name); }, ); @@ -343,37 +354,6 @@ export class Client { this.space = space; - let startTime = -1; - this.eventHook.addLocalListener("file:initial", async () => { - startTime = Date.now(); - await this.clientSystem.setIndexOngoing(true); - }); - - const emptyQueueHandler = async () => { - await this.clientSystem.setIndexOngoing(false); - if ( - startTime !== -1 && !await this.clientSystem.hasInitialIndexCompleted() - ) { - // Indexing has just finished for the first time - console.info( - "Initial index complete after", - (Date.now() - startTime) / 1000, - "s", - ); - // Unsubscribe myself - this.eventHook.removeLocalListener( - "mq:emptyQueue:indexQueue", - emptyQueueHandler, - ); - await this.clientSystem.markInitialIndexComplete(); - await this.clientSystem.reloadState(); - } - }; - this.eventHook.addLocalListener( - "mq:emptyQueue:indexQueue", - emptyQueueHandler, - ); - let lastSaveTimestamp: number | undefined; const updateLastSaveTimestamp = () => { @@ -523,7 +503,7 @@ export class Client { resolve(); // In the background we'll fetch any enriched meta data, if any - const enrichedMeta = await this.clientSystem.getObjectByRef< + const enrichedMeta = await this.objectIndex.getObjectByRef< PageMeta >( this.currentName(), @@ -599,26 +579,37 @@ export class Client { this.updateDocumentListCache().catch(console.error); } + queryLuaObjects( + tag: string, + query: LuaCollectionQuery, + scopedVariables?: Record, + ): Promise { + return this.objectIndex.queryLuaObjects( + this.clientSystem.spaceLuaEnv.env, + tag, + query, + scopedVariables, + ); + } + async updatePageListCache() { console.log("Updating page list cache"); // Check if the initial sync has been completed - const initialIndexCompleted = await this.clientSystem + const initialIndexCompleted = await this.objectIndex .hasInitialIndexCompleted(); let allPages: PageMeta[] = []; - if ( - initialIndexCompleted && this.clientSystem.system.loadedPlugs.has("index") - ) { + if (initialIndexCompleted) { console.log( - "Initial index complete and index plug loaded, loading full page list via index.", + "Initial index complete, loading full page list via index.", ); // Fetch indexed pages - allPages = await this.clientSystem.queryLuaObjects("page", {}); + allPages = await this.queryLuaObjects("page", {}); // Overlay augmented meta values await this.pageMetaAugmenter.augmentObjectArray(allPages, "ref"); // Fetch aspiring pages - const aspiringPageNames = await this.clientSystem.queryLuaObjects( + const aspiringPageNames = await this.queryLuaObjects( "aspiring-page", { select: parseExpressionString("name"), distinct: true }, ); @@ -677,7 +668,7 @@ export class Client { return; } - const allDocuments = await this.clientSystem.queryLuaObjects( + const allDocuments = await this.queryLuaObjects( "document", {}, ); @@ -1125,9 +1116,9 @@ export class Client { // Fetch the meta which includes the possibly indexed stuff, like page // decorations - if (await this.clientSystem.hasInitialIndexCompleted()) { + if (await this.objectIndex.hasInitialIndexCompleted()) { try { - const enrichedMeta = await this.clientSystem.getObjectByRef( + const enrichedMeta = await this.objectIndex.getObjectByRef( pageName, "page", pageName, @@ -1291,11 +1282,11 @@ export class Client { console.warn("Not loading custom styles, since space style is disabled"); return; } - if (!await this.clientSystem.hasInitialIndexCompleted()) { + if (!await this.objectIndex.hasInitialIndexCompleted()) { return; } - const spaceStyles = await this.clientSystem.queryLuaObjects( + const spaceStyles = await this.queryLuaObjects( "space-style", { objectVariable: "_", diff --git a/client/client_system.ts b/client/client_system.ts index 8c268f64..4e1b875b 100644 --- a/client/client_system.ts +++ b/client/client_system.ts @@ -41,17 +41,13 @@ import { indexSyscalls } from "./plugos/syscalls/index.ts"; import { configSyscalls } from "./plugos/syscalls/config.ts"; import { eventSyscalls } from "./plugos/syscalls/event.ts"; import { DocumentEditorHook } from "./plugos/hooks/document_editor.ts"; -import type { LuaCollectionQuery } from "./space_lua/query_collection.ts"; import type { Command } from "./types/command.ts"; import { SpaceLuaEnvironment } from "./space_lua.ts"; import { builtinPlugPaths } from "../plugs/builtin_plugs.ts"; import { ServiceRegistry } from "./service_registry.ts"; import { serviceRegistrySyscalls } from "./plugos/syscalls/service_registry.ts"; +import type { ObjectIndex } from "./data/object_index.ts"; -const indexVersionKey = ["$indexVersion"]; -const indexQueuedKey = ["$indexQueued"]; -// Bump this one every time a full reindex is needed -const desiredIndexVersion = 9; const mqTimeout = 10000; // 10s const mqTimeoutRetry = 3; @@ -85,6 +81,7 @@ export class ClientSystem { protected mq: DataStoreMQ, public ds: DataStore, public eventHook: EventHook, + private objectIndex: ObjectIndex, public readOnlyMode: boolean, ) { this.system = new System(undefined, { @@ -94,7 +91,7 @@ export class ClientSystem { ), }); - this.spaceLuaEnv = new SpaceLuaEnvironment(this.system); + this.spaceLuaEnv = new SpaceLuaEnvironment(this.system, objectIndex); this.serviceRegistry = new ServiceRegistry(this.eventHook, client.config); setInterval(() => { @@ -143,6 +140,10 @@ export class ClientSystem { // Syscall hook this.system.addHook(new SyscallHook()); + + this.eventHook.addLocalListener("editor:reloadState", async () => { + await this.reloadState(); + }); } init() { @@ -163,7 +164,7 @@ export class ClientSystem { clientCodeWidgetSyscalls(), languageSyscalls(), jsonschemaSyscalls(), - indexSyscalls(client), + indexSyscalls(this.objectIndex, this.client), //commandSyscalls(client), luaSyscalls(this), mqSyscalls(this.mq), @@ -199,7 +200,7 @@ export class ClientSystem { console.info("Space Lua scripts are disabled, skipping loading scripts"); return; } - if (!await this.hasInitialIndexCompleted()) { + if (!await this.objectIndex.hasInitialIndexCompleted()) { console.info( "Not loading space scripts, since initial indexing has not completed yet", ); @@ -260,61 +261,6 @@ export class ClientSystem { return this.system.localSyscall(name, args); } - queryLuaObjects(tag: string, query: LuaCollectionQuery): Promise { - return this.system.invokeFunction("index.queryLuaObjects", [tag, query]); - } - - // Index handling - - getObjectByRef(page: string, tag: string, ref: string) { - return this.localSyscall( - "system.invokeFunction", - ["index.getObjectByRef", page, tag, ref], - ); - } - - async isIndexOngoing() { - return !!(await this.ds.get(indexQueuedKey)); - } - - async setIndexOngoing(val: boolean = true) { - await this.ds.set(indexQueuedKey, val); - } - - async ensureFullIndex() { - if (!this.client.fullSyncCompleted) { - console.info( - "Initial full sync not completed, skipping index check", - ); - return; - } - const currentIndexVersion = await this.getCurrentIndexVersion(); - - if (!currentIndexVersion) { - console.log("No index version found, assuming fresh install"); - return; - } - - if ( - currentIndexVersion !== desiredIndexVersion && - !await this.isIndexOngoing() - ) { - console.info( - "[index]", - "Performing a full space reindex, this could take a while...", - currentIndexVersion, - desiredIndexVersion, - ); - await this.setIndexOngoing(true); - await this.system.invokeFunction("index.reindexSpace", []); - console.info("[index]", "Full space index complete."); - await this.markInitialIndexComplete(); - await this.setIndexOngoing(false); - // Let's load space scripts again, which probably weren't loaded before - await this.reloadState(); - } - } - public async reloadState() { console.log( "Now loading space scripts, custom styles and rebuilding editor state", @@ -323,16 +269,4 @@ export class ClientSystem { await this.client.loadCustomStyles(); this.client.rebuildEditorState(); } - - public async hasInitialIndexCompleted() { - return (await this.ds.get(indexVersionKey)) === desiredIndexVersion; - } - - private getCurrentIndexVersion() { - return this.ds.get(indexVersionKey); - } - - async markInitialIndexComplete() { - await this.ds.set(indexVersionKey, desiredIndexVersion); - } } diff --git a/client/data/object_index.ts b/client/data/object_index.ts new file mode 100644 index 00000000..d66fac36 --- /dev/null +++ b/client/data/object_index.ts @@ -0,0 +1,304 @@ +import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; +import type { Config } from "../config.ts"; +import type { + LuaCollectionQuery, + LuaQueryCollection, +} from "../space_lua/query_collection.ts"; +import { + jsToLuaValue, + LuaEnv, + LuaStackFrame, + type LuaTable, +} from "../space_lua/runtime.ts"; +import type { DataStore } from "./datastore.ts"; +import type { KV, KvKey } from "@silverbulletmd/silverbullet/type/datastore"; +import type { EventHook } from "../plugos/hooks/event.ts"; +import type { DataStoreMQ } from "./mq.datastore.ts"; +import type { Space } from "../space.ts"; + +const indexKey = "idx"; +const pageKey = "ridx"; + +const indexVersionKey = ["$indexVersion"]; +const indexQueuedKey = ["$indexQueued"]; +// Bump this one every time a full reindex is needed +const desiredIndexVersion = 9; + +export class ObjectIndex { + constructor( + private ds: DataStore, + private config: Config, + private eventHook: EventHook, + private mq: DataStoreMQ, + ) { + let startTime = -1; + this.eventHook.addLocalListener("file:initial", async () => { + startTime = Date.now(); + await this.setIndexOngoing(true); + }); + + this.eventHook.addLocalListener("file:deleted", async (path: string) => { + await this.clearFileIndex(path); + }); + + const emptyQueueHandler = async () => { + await this.setIndexOngoing(false); + if ( + startTime !== -1 && !await this.hasInitialIndexCompleted() + ) { + // Indexing has just finished for the first time + console.info( + "Initial index complete after", + (Date.now() - startTime) / 1000, + "s", + ); + // Unsubscribe myself + this.eventHook.removeLocalListener( + "mq:emptyQueue:indexQueue", + emptyQueueHandler, + ); + await this.markInitialIndexComplete(); + this.eventHook.dispatchEvent("editor:reloadState"); + } + }; + this.eventHook.addLocalListener( + "mq:emptyQueue:indexQueue", + emptyQueueHandler, + ); + } + + tag(tagName: string): LuaQueryCollection { + if (!tagName) { + throw new Error("Tag name is required"); + } + return { + query: ( + query: LuaCollectionQuery, + env: LuaEnv, + sf: LuaStackFrame, + ): Promise => { + return this.ds.luaQuery( + ["idx", tagName], + query, + env, + sf, + (key, value: any) => { + const tag = key[1]; + const tagDef = this.config.get( + ["tagDefinitions", tag], + undefined, + ); + if (!tagDef || !tagDef.has("metatable")) { + // Return as is + return value; + } + // Convert to LuaTable + value = jsToLuaValue(value); + value.metatable = tagDef.get("metatable"); + return value; + }, + ); + }, + }; + } + + getObjectByRef(page: string, tag: string, ref: string) { + return this.ds.get([indexKey, tag, this.cleanKey(ref, page), page]); + } + + async isIndexOngoing() { + return !!(await this.ds.get(indexQueuedKey)); + } + + async setIndexOngoing(val: boolean = true) { + await this.ds.set(indexQueuedKey, val); + } + + async ensureFullIndex(space: Space) { + // Commenting out this check because I think it always holds when calling this API + // if (!this.client.fullSyncCompleted) { + // console.info( + // "Initial full sync not completed, skipping index check", + // ); + // return; + // } + const currentIndexVersion = await this.getCurrentIndexVersion(); + + if (!currentIndexVersion) { + console.log("No index version found, assuming fresh install"); + return; + } + + if ( + currentIndexVersion !== desiredIndexVersion && + !await this.isIndexOngoing() + ) { + console.info( + "[index]", + "Performing a full space reindex, this could take a while...", + currentIndexVersion, + desiredIndexVersion, + ); + await this.setIndexOngoing(true); + await this.reindexSpace(space); + console.info("[index]", "Full space index complete."); + await this.markInitialIndexComplete(); + await this.setIndexOngoing(false); + // Let's load space scripts again, which probably weren't loaded before + this.eventHook.dispatchEvent("editor:reloadState"); + } + } + + async reindexSpace(space: Space) { + console.log("Clearing page index..."); + await this.clearIndex(); + + const files = await space.deduplicatedFileList(); + + console.log("Queing", files.length, "pages to be indexed."); + // Queue all file names to be indexed + const startTime = Date.now(); + await this.mq.batchSend("indexQueue", files.map((file) => file.name)); + await this.mq.awaitEmptyQueue("indexQueue"); + console.log("Done with full index after", Date.now() - startTime, "ms"); + } + + public async hasInitialIndexCompleted() { + return (await this.ds.get(indexVersionKey)) === desiredIndexVersion; + } + + private getCurrentIndexVersion() { + return this.ds.get(indexVersionKey); + } + + async markInitialIndexComplete() { + await this.ds.set(indexVersionKey, desiredIndexVersion); + } + + cleanKey(ref: string, page: string) { + if (ref.startsWith(`${page}@`)) { + return ref.substring(page.length + 1); + } else { + return ref; + } + } + + queryLuaObjects( + globalEnv: LuaEnv, + tag: string, + query: LuaCollectionQuery, + scopedVariables?: Record, + ): Promise[]> { + const sf = LuaStackFrame.createWithGlobalEnv( + globalEnv, + ); + let env = globalEnv; + if (scopedVariables) { + env = new LuaEnv(globalEnv); + for (const [key, value] of Object.entries(scopedVariables)) { + env.setLocal(key, jsToLuaValue(value)); + } + } + return this.ds.luaQuery([indexKey, tag], query, env, sf); + } + + batchSet(page: string, kvs: KV[]): Promise { + const finalBatch: KV[] = []; + for (const { key, value } of kvs) { + finalBatch.push({ + key: [indexKey, ...key, page], + value, + }, { + key: [pageKey, page, ...key], + value: true, + }); + } + return this.ds.batchSet(finalBatch); + } + + batchDelete(page: string, keys: KvKey[]): Promise { + const finalBatch: KvKey[] = []; + for (const key of keys) { + finalBatch.push([indexKey, ...key, page]); + } + return this.ds.batchDelete(finalBatch); + } + + /** + * Clears all keys for a given file + * @param file + */ + public async clearFileIndex(file: string): Promise { + if (file.endsWith(".md")) { + file = file.replace(/\.md$/, ""); + } + // console.log("Clearing index for", file); + const allKeys: KvKey[] = []; + for await ( + const { key } of this.ds.query({ + prefix: [pageKey, file], + }) + ) { + allKeys.push(key); + allKeys.push([indexKey, ...key.slice(2), file]); + } + await this.ds.batchDelete(allKeys); + } + + /** + * Clears the entire index + */ + public async clearIndex(): Promise { + const allKeys: KvKey[] = []; + for await ( + const { key } of this.ds.query({ prefix: [indexKey] }) + ) { + allKeys.push(key); + } + for await ( + const { key } of this.ds.query({ prefix: [pageKey] }) + ) { + allKeys.push(key); + } + await this.ds.batchDelete(allKeys); + console.log("Deleted", allKeys.length, "keys from the index"); + } + + /** + * Indexes entities in the data store + */ + public indexObjects( + page: string, + objects: ObjectValue[], + ): Promise { + const kvs: KV[] = []; + for (const obj of objects) { + if (!obj.tag) { + console.error("Object has no tag", obj, "this shouldn't happen"); + continue; + } + // Index as all the tag + any additional tags specified + const allTags = [obj.tag, ...obj.tags || []]; + for (const tag of allTags) { + // The object itself + kvs.push({ + key: [tag, this.cleanKey(obj.ref, page)], + value: obj, + }); + } + } + if (kvs.length > 0) { + return this.batchSet(page, kvs); + } else { + return Promise.resolve(); + } + } + + deleteObject( + page: string, + tag: string, + ref: string, + ): Promise { + return this.batchDelete(page, [[tag, this.cleanKey(ref, page)]]); + } +} diff --git a/client/plugos/syscalls/index.ts b/client/plugos/syscalls/index.ts index bdf654ac..92ce8c5f 100644 --- a/client/plugos/syscalls/index.ts +++ b/client/plugos/syscalls/index.ts @@ -3,53 +3,62 @@ import type { LuaCollectionQuery, LuaQueryCollection, } from "../../space_lua/query_collection.ts"; -import { - jsToLuaValue, - type LuaEnv, - type LuaStackFrame, - type LuaTable, -} from "../../space_lua/runtime.ts"; +import type { LuaTable } from "../../space_lua/runtime.ts"; +import type { ObjectIndex } from "../../data/object_index.ts"; +import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; import type { Client } from "../../client.ts"; -export function indexSyscalls(client: Client): SysCallMapping { +export function indexSyscalls( + objectIndex: ObjectIndex, + client: Client, +): SysCallMapping { return { "index.tag": (_ctx, tagName: string): LuaQueryCollection => { - if (!tagName) { - throw new Error("Tag name is required"); - } - return { - query: ( - query: LuaCollectionQuery, - env: LuaEnv, - sf: LuaStackFrame, - ): Promise => { - return client.ds.luaQuery( - ["idx", tagName], - query, - env, - sf, - (key, value: any) => { - const tag = key[1]; - const tagDef = client.config.get( - ["tagDefinitions", tag], - undefined, - ); - if (!tagDef || !tagDef.has("metatable")) { - // Return as is - return value; - } - // Convert to LuaTable - value = jsToLuaValue(value); - value.metatable = tagDef.get("metatable"); - return value; - }, - ); - }, - }; + return objectIndex.tag(tagName); }, "index.ensureFullIndex": (_ctx) => { - return client.clientSystem.ensureFullIndex(); + return objectIndex.ensureFullIndex(client.space); + }, + "index.reindexSpace": () => { + return objectIndex.reindexSpace(client.space); + }, + "index.indexObjects": ( + _ctx, + page: string, + objects: ObjectValue[], + ): Promise => { + return objectIndex.indexObjects(page, objects); + }, + "index.getObjectByRef": ( + _ctx, + page: string, + tag: string, + ref: string, + ): Promise => { + return objectIndex.getObjectByRef(ref, page, tag); + }, + "index.queryLuaObjects": ( + _ctx, + tag: string, + query: LuaCollectionQuery, + scopedVariables?: Record, + ): Promise => { + return objectIndex.queryLuaObjects( + client.clientSystem.spaceLuaEnv.env, + tag, + query, + scopedVariables, + ); + }, + + "index.deleteObject": ( + _ctx, + page: string, + tag: string, + ref: string, + ): Promise => { + return objectIndex.deleteObject(page, tag, ref); }, "lua:index.defineTag": (_ctx, tagDef: LuaTable) => { // Using 'lua:' prefix to _not_ convert tagDef to a JS version (but keep original LuaTable) diff --git a/client/space_lua.ts b/client/space_lua.ts index fa204344..aa5d7c14 100644 --- a/client/space_lua.ts +++ b/client/space_lua.ts @@ -11,11 +11,12 @@ import { import type { ASTCtx } from "./space_lua/ast.ts"; import { buildLuaEnv } from "./space_lua_api.ts"; import type { LuaCollectionQuery } from "./space_lua/query_collection.ts"; +import type { ObjectIndex } from "./data/object_index.ts"; export class SpaceLuaEnvironment { env: LuaEnv; - constructor(private system: System) { + constructor(private system: System, private objectIndex: ObjectIndex) { this.env = buildLuaEnv(system); } @@ -24,15 +25,16 @@ export class SpaceLuaEnvironment { * @param system */ async reload() { - const allScripts: SpaceLuaObject[] = await this.system.invokeFunction( - "index.queryLuaObjects", - ["space-lua", { + const allScripts: SpaceLuaObject[] = await this.objectIndex.queryLuaObjects( + this.env, + "space-lua", + { objectVariable: "script", orderBy: [{ expr: parseExpressionString("script.priority"), desc: true, }], - } as LuaCollectionQuery], + } as LuaCollectionQuery, ); try { this.env = buildLuaEnv(this.system); diff --git a/plug-api/syscalls/index.ts b/plug-api/syscalls/index.ts index 08c88c38..991c5b78 100644 --- a/plug-api/syscalls/index.ts +++ b/plug-api/syscalls/index.ts @@ -31,9 +31,8 @@ export function queryLuaObjects( tag: string, query: LuaCollectionQuery, scopedVariables?: Record, - ttlSecs?: number, ): Promise[]> { - return syscall("index.queryLuaObjects", tag, query, scopedVariables, ttlSecs); + return syscall("index.queryLuaObjects", tag, query, scopedVariables); } /** @@ -57,3 +56,15 @@ export function getObjectByRef( export function ensureFullIndex(): Promise { return syscall("index.ensureFullIndex"); } + +export function reindexSpace(): Promise { + return syscall("index.reindexSpace"); +} + +export function deleteObject( + page: string, + tag: string, + ref: string, +): Promise { + return syscall("index.deleteObject", page, tag, ref); +} diff --git a/plug-api/system_mock.ts b/plug-api/system_mock.ts index 367dcfb4..9d7276b7 100644 --- a/plug-api/system_mock.ts +++ b/plug-api/system_mock.ts @@ -2,6 +2,7 @@ import { Config } from "../client/config.ts"; import { DataStore } from "../client/data/datastore.ts"; import { MemoryKvPrimitives } from "../client/data/memory_kv_primitives.ts"; import { DataStoreMQ } from "../client/data/mq.datastore.ts"; +import { ObjectIndex } from "../client/data/object_index.ts"; import { EventHook } from "../client/plugos/hooks/event.ts"; import { dataStoreReadSyscalls, @@ -46,9 +47,11 @@ export function createMockSystem() { eventedSpacePrimitives: spacePrimitives, }; + const objectIndex = new ObjectIndex(ds, config, eventHook, mq); + const clientSystemMock: any = { system: system, - spaceLuaEnv: new SpaceLuaEnvironment(system), + spaceLuaEnv: new SpaceLuaEnvironment(system, objectIndex), }; clientMock.clientSystem = clientSystemMock; @@ -60,7 +63,7 @@ export function createMockSystem() { markdownSyscalls(clientMock), languageSyscalls(), jsonschemaSyscalls(), - indexSyscalls(clientMock), + indexSyscalls(objectIndex, clientMock), luaSyscalls(clientSystemMock), mqSyscalls(mq), dataStoreReadSyscalls(ds, clientSystemMock), diff --git a/plug-api/types/index.ts b/plug-api/types/index.ts index 696392e1..1c05e25f 100644 --- a/plug-api/types/index.ts +++ b/plug-api/types/index.ts @@ -47,7 +47,7 @@ export type SyscallMeta = { * - ref: a unique reference (id) for the object, ideally a page reference * - tags: a list of tags that the object belongs to */ -export type ObjectValue = { +export type ObjectValue = { ref: string; tag: string; // main tag tags?: string[]; diff --git a/plugs/editor/complete.ts b/plugs/editor/complete.ts index 861d776d..ed31d4c1 100644 --- a/plugs/editor/complete.ts +++ b/plugs/editor/complete.ts @@ -46,12 +46,7 @@ export async function pageComplete(completeEvent: CompleteEvent) { if (prefix.startsWith("^")) { // A carrot prefix means we're looking for a meta page - allPages = await index.queryLuaObjects( - "page", - isMetaPageQuery!, - {}, - 5, - ); + allPages = await index.queryLuaObjects("page", isMetaPageQuery); // Let's prefix the names with a caret to make them match allPages = allPages.map((page) => ({ ...page, @@ -61,9 +56,9 @@ export async function pageComplete(completeEvent: CompleteEvent) { // This is the most common case, we're combining three types of completions here: allPages = (await Promise.all([ // All non-meta pages - index.queryLuaObjects("page", isntMetaPageQuery, {}, 5), + index.queryLuaObjects("page", isntMetaPageQuery), // All documents - index.queryLuaObjects("document", isDocumentQuery, {}, 5), + index.queryLuaObjects("document", isDocumentQuery), // And all links to non-existing pages (to augment the existing ones) index.queryLuaObjects( "aspiring-page", @@ -71,8 +66,6 @@ export async function pageComplete(completeEvent: CompleteEvent) { select: { type: "Variable", name: "name", ctx: {} as any }, distinct: true, }, - {}, - 5, ).then((aspiringPages) => // Rewrite them to PageMeta shaped objects aspiringPages.map((aspiringPage: string): PageMeta => ({ diff --git a/plugs/index/api.ts b/plugs/index/api.ts index 213c54e1..87fcd0b8 100644 --- a/plugs/index/api.ts +++ b/plugs/index/api.ts @@ -1,6 +1,4 @@ -import { datastore, markdown } from "@silverbulletmd/silverbullet/syscalls"; -import { ttlCache } from "../../plug-api/lib/memory_cache.ts"; -import type { LuaCollectionQuery } from "../../client/space_lua/query_collection.ts"; +import { markdown } from "@silverbulletmd/silverbullet/syscalls"; import { extractFrontMatter as extractFrontmatterFromTree, type FrontMatter, @@ -11,12 +9,6 @@ import { renderToText, } from "@silverbulletmd/silverbullet/lib/tree"; import { applyPatches, type YamlPatch } from "../../plug-api/lib/yaml.ts"; -import type { ObjectValue } from "../../plug-api/types/index.ts"; - -import type { KV, KvKey, KvQuery } from "../../plug-api/types/datastore.ts"; - -const indexKey = "idx"; -const pageKey = "ridx"; /* * Key namespace: @@ -25,150 +17,12 @@ const pageKey = "ridx"; * ["type", type] -> true // for fast type listing */ -export function batchSet(page: string, kvs: KV[]): Promise { - const finalBatch: KV[] = []; - for (const { key, value } of kvs) { - finalBatch.push({ - key: [indexKey, ...key, page], - value, - }, { - key: [pageKey, page, ...key], - value: true, - }); - } - return datastore.batchSet(finalBatch); -} - -export function batchDelete(page: string, keys: KvKey[]): Promise { - const finalBatch: KvKey[] = []; - for (const key of keys) { - finalBatch.push([indexKey, ...key, page]); - } - return datastore.batchDel(finalBatch); -} - -/** - * Clears all keys for a given file - * @param file - */ -export async function clearFileIndex(file: string): Promise { - if (file.endsWith(".md")) { - file = file.replace(/\.md$/, ""); - } - // console.log("Clearing index for", file); - const allKeys: KvKey[] = []; - for ( - const { key } of await datastore.query({ - prefix: [pageKey, file], - }) - ) { - allKeys.push(key); - allKeys.push([indexKey, ...key.slice(2), file]); - } - await datastore.batchDel(allKeys); -} - -/** - * Clears the entire index - */ -export async function clearIndex(): Promise { - const allKeys: KvKey[] = []; - for ( - const { key } of await datastore.query({ prefix: [indexKey] }) - ) { - allKeys.push(key); - } - for ( - const { key } of await datastore.query({ prefix: [pageKey] }) - ) { - allKeys.push(key); - } - await datastore.batchDel(allKeys); - console.log("Deleted", allKeys.length, "keys from the index"); -} - -// OBJECTS API - -/** - * Indexes entities in the data store - */ -export function indexObjects( - page: string, - objects: ObjectValue[], -): Promise { - const kvs: KV[] = []; - for (const obj of objects) { - if (!obj.tag) { - console.error("Object has no tag", obj, "this shouldn't happen"); - continue; - } - // Index as all the tag + any additional tags specified - const allTags = [obj.tag, ...obj.tags || []]; - for (const tag of allTags) { - // The object itself - kvs.push({ - key: [tag, cleanKey(obj.ref, page)], - value: obj, - }); - } - } - if (kvs.length > 0) { - return batchSet(page, kvs); - } else { - return Promise.resolve(); - } -} - -function cleanKey(ref: string, page: string) { - if (ref.startsWith(`${page}@`)) { - return ref.substring(page.length + 1); - } else { - return ref; - } -} - -export function queryLuaObjects( - tag: string, - query: LuaCollectionQuery, - scopedVariables?: Record, - ttlSecs?: number, -): Promise[]> { - return ttlCache(query, () => { - return datastore.queryLua([indexKey, tag], query, scopedVariables); - }, ttlSecs); -} - -export function deleteObject( - tag: string, - page: string, - ref: string, -): Promise { - return batchDelete(page, [[tag, cleanKey(ref, page)]]); -} - -export async function query( - query: KvQuery, -): Promise { - return (await datastore.query({ - ...query, - prefix: [indexKey, ...query.prefix ? query.prefix : []], - })).map(({ key, value }) => ({ key: key.slice(1), value })); -} - -export function getObjectByRef( - page: string, - tag: string, - ref: string, -): Promise | undefined> { - return datastore.get([indexKey, tag, cleanKey(ref, page), page]); -} - export async function extractFrontmatter( text: string, extractOptions: FrontMatterExtractOptions = {}, ): Promise<{ frontmatter: FrontMatter; text: string }> { const tree = await markdown.parseMarkdown(text); - const frontmatter = await extractFrontmatterFromTree(tree, extractOptions); + const frontmatter = extractFrontmatterFromTree(tree, extractOptions); return { frontmatter, text: renderToText(tree) }; } diff --git a/plugs/index/command.ts b/plugs/index/command.ts index 609633b6..94ec65e6 100644 --- a/plugs/index/command.ts +++ b/plugs/index/command.ts @@ -1,8 +1,7 @@ -import { editor } from "@silverbulletmd/silverbullet/syscalls"; -import { reindexSpace } from "./queue.ts"; +import { editor, index } from "@silverbulletmd/silverbullet/syscalls"; export async function reindexCommand() { await editor.flashNotification("Performing full page reindex..."); - await reindexSpace(); + await index.reindexSpace(); await editor.flashNotification("Done with page index!"); } diff --git a/plugs/index/document.ts b/plugs/index/document.ts index ee3f1f3f..6a9d5c64 100644 --- a/plugs/index/document.ts +++ b/plugs/index/document.ts @@ -1,5 +1,4 @@ -import { space, system } from "@silverbulletmd/silverbullet/syscalls"; -import { indexObjects } from "./api.ts"; +import { index, space, system } from "@silverbulletmd/silverbullet/syscalls"; import type { DocumentMeta } from "../../plug-api/types/index.ts"; // Note: clearFileIndex is not called but since this is the only document:index listener, this should be fine (famous last words) @@ -9,5 +8,5 @@ export async function indexDocument(name: string) { } // console.log("Indexing document", name); const fileMeta = await space.getDocumentMeta(name); - await indexObjects(fileMeta.name, [fileMeta]); + await index.indexObjects(fileMeta.name, [fileMeta]); } diff --git a/plugs/index/header.ts b/plugs/index/header.ts index 8247f0a2..19c1539e 100644 --- a/plugs/index/header.ts +++ b/plugs/index/header.ts @@ -4,12 +4,11 @@ import { type ParseTree, renderToText, } from "@silverbulletmd/silverbullet/lib/tree"; -import { queryLuaObjects } from "./api.ts"; import { getNameFromPath, parseToRef, } from "@silverbulletmd/silverbullet/lib/ref"; -import { lua } from "@silverbulletmd/silverbullet/syscalls"; +import { index, lua } from "@silverbulletmd/silverbullet/syscalls"; import type { ObjectValue, PageMeta, @@ -80,14 +79,13 @@ export async function headerComplete(completeEvent: CompleteEvent) { return; } - const headers = await queryLuaObjects( + const headers = await index.queryLuaObjects( "header", { objectVariable: "_", where: await lua.parseExpression(`_.page == name`), }, { name: getNameFromPath(ref.path) || completeEvent.pageName }, - 5, ); return { diff --git a/plugs/index/index.plug.yaml b/plugs/index/index.plug.yaml index e63c3c40..3ae4c529 100644 --- a/plugs/index/index.plug.yaml +++ b/plugs/index/index.plug.yaml @@ -1,20 +1,6 @@ name: index functions: # Public API - batchSet: - path: api.ts:batchSet - indexObjects: - path: api.ts:indexObjects - syscall: index.indexObjects - query: - path: api.ts:query - syscall: index.query - queryLuaObjects: - path: api.ts:queryLuaObjects - syscall: index.queryLuaObjects - getObjectByRef: - path: api.ts:getObjectByRef - syscall: index.getObjectByRef extractFrontmatter: path: api.ts:extractFrontmatter syscall: index.extractFrontmatter @@ -34,16 +20,6 @@ functions: path: yaml.ts:stringify syscall: yaml.stringify - # Index management - clearIndex: - path: api.ts:clearIndex - - clearDSIndex: - path: api.ts:clearFileIndex - events: - - file:deleted - - file:clearindex - reindexSpaceCommand: path: "./command.ts:reindexCommand" command: @@ -57,9 +33,6 @@ functions: batchSize: 3 autoAck: true - reindexSpace: - path: "./queue.ts:reindexSpace" - indexDocument: path: "./document.ts:indexDocument" events: diff --git a/plugs/index/index.ts b/plugs/index/index.ts index a598df43..7abe16fe 100644 --- a/plugs/index/index.ts +++ b/plugs/index/index.ts @@ -5,7 +5,6 @@ import type { import { extractFrontMatter, type FrontMatter } from "./frontmatter.ts"; import type { ParseTree } from "@silverbulletmd/silverbullet/lib/tree"; import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; -import { indexObjects } from "./api.ts"; import { indexPage as pageIndexPage } from "./page.ts"; import { indexData } from "./data.ts"; import { indexItems } from "./item.ts"; @@ -16,7 +15,7 @@ import { indexTables } from "./table.ts"; import { indexSpaceLua } from "./space_lua.ts"; import { indexSpaceStyle } from "./space_style.ts"; import { indexTags } from "./tags.ts"; -import { markdown } from "@silverbulletmd/silverbullet/syscalls"; +import { index, markdown } from "@silverbulletmd/silverbullet/syscalls"; export type IndexerFunction = ( pageMeta: PageMeta, @@ -68,11 +67,11 @@ export async function indexPage({ name, tree, meta, text }: IndexTreeEvent) { // console.log("Now going to index page", name); // Index the page - const index = await Promise.all(allIndexers.map((indexer) => { + const indexResults = await Promise.all(allIndexers.map((indexer) => { return indexer(meta, frontmatter, tree, text); })); // console.log("Found these objects", index.flat()); - await indexObjects(name, index.flat()); + await index.indexObjects(name, indexResults.flat()); } diff --git a/plugs/index/link.ts b/plugs/index/link.ts index c581523b..b84b33a1 100644 --- a/plugs/index/link.ts +++ b/plugs/index/link.ts @@ -9,7 +9,6 @@ import { isLocalURL, resolveMarkdownLink, } from "@silverbulletmd/silverbullet/lib/resolve"; -import { queryLuaObjects } from "./api.ts"; import type { FrontMatter } from "./frontmatter.ts"; import { updateITags } from "./tags.ts"; import { @@ -21,7 +20,7 @@ import { mdLinkRegex, wikiLinkRegex, } from "../../client/markdown_parser/constants.ts"; -import { lua, space } from "@silverbulletmd/silverbullet/syscalls"; +import { index, lua, space } from "@silverbulletmd/silverbullet/syscalls"; import type { ObjectValue, PageMeta, @@ -247,7 +246,7 @@ export function collectPageLinks(n: ParseTree): string[] { export async function getBackLinks( name: string, ): Promise { - return (await queryLuaObjects("link", { + return (await index.queryLuaObjects("link", { objectVariable: "_", where: await lua.parseExpression(`_.toPage == name or _.toFile == name`), }, { diff --git a/plugs/index/page.ts b/plugs/index/page.ts index f2968ef5..411490da 100644 --- a/plugs/index/page.ts +++ b/plugs/index/page.ts @@ -1,7 +1,11 @@ -import { editor, lua, markdown } from "@silverbulletmd/silverbullet/syscalls"; +import { + editor, + index, + lua, + markdown, +} from "@silverbulletmd/silverbullet/syscalls"; import type { FrontMatter } from "./frontmatter.ts"; -import { deleteObject, getObjectByRef, queryLuaObjects } from "./api.ts"; import { findNodeOfType, type ParseTree, @@ -47,7 +51,7 @@ export async function indexPage( // Make sure this page is no (longer) in the aspiring pages list // TODO: This can possibly done more optimally - const aspiringPages = await queryLuaObjects( + const aspiringPages = await index.queryLuaObjects( "aspiring-page", { objectVariable: "_", @@ -57,7 +61,11 @@ export async function indexPage( ); for (const aspiringPage of aspiringPages) { console.log("Deleting aspiring page", aspiringPage); - await deleteObject("aspiring-page", aspiringPage.page, aspiringPage.ref); + await index.deleteObject( + "aspiring-page", + aspiringPage.page, + aspiringPage.ref, + ); } return [combinedPageMeta]; @@ -145,7 +153,7 @@ export async function loadPageObject(pageName?: string): Promise { created: "", } as PageMeta; } - return (await getObjectByRef( + return (await index.getObjectByRef( pageName, "page", pageName, diff --git a/plugs/index/plug_api.ts b/plugs/index/plug_api.ts index aa6dda0d..3d361091 100644 --- a/plugs/index/plug_api.ts +++ b/plugs/index/plug_api.ts @@ -1,7 +1,7 @@ import { system } from "@silverbulletmd/silverbullet/syscalls"; import type { ObjectValue } from "../../plug-api/types/index.ts"; -import type { KV, KvQuery } from "../../plug-api/types/datastore.ts"; +import type { KV } from "../../plug-api/types/datastore.ts"; export function indexObjects( page: string, @@ -14,12 +14,6 @@ export function batchSet(page: string, kvs: KV[]): Promise { return system.invokeFunction("index.batchSet", page, kvs); } -export function query( - query: KvQuery, -): Promise { - return system.invokeFunction("index.query", query); -} - export function getObjectByRef( page: string, tag: string, diff --git a/plugs/index/queue.ts b/plugs/index/queue.ts index 075d3bcb..ae81e33d 100644 --- a/plugs/index/queue.ts +++ b/plugs/index/queue.ts @@ -4,7 +4,6 @@ import { markdown, mq, space, - system, } from "@silverbulletmd/silverbullet/syscalls"; import { sleep } from "@silverbulletmd/silverbullet/lib/async"; import type { MQMessage } from "@silverbulletmd/silverbullet/type/datastore"; @@ -12,24 +11,6 @@ import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; const uiUpdateInterval = 5000; -export async function reindexSpace() { - if (await system.getMode() === "ro") { - console.info("Not reindexing because we're in read-only mode"); - return; - } - console.log("Clearing page index..."); - await system.invokeFunction("index.clearIndex"); - - const files = await space.listFiles(); - - console.log("Queing", files.length, "pages to be indexed."); - // Queue all file names to be indexed - const startTime = Date.now(); - await mq.batchSend("indexQueue", files.map((file) => file.name)); - await mq.awaitEmptyQueue("indexQueue"); - console.log("Done with full index after", Date.now() - startTime, "ms"); -} - setTimeout(updateIndexProgressInUI, uiUpdateInterval); async function totalItemsQueued() { diff --git a/plugs/index/refactor.ts b/plugs/index/refactor.ts index 2fd5326c..ac6aeb41 100644 --- a/plugs/index/refactor.ts +++ b/plugs/index/refactor.ts @@ -1,5 +1,6 @@ import { editor, + index, lua, markdown, space, @@ -16,7 +17,6 @@ import { findParentMatching, nodeAtPos, } from "@silverbulletmd/silverbullet/lib/tree"; -import { queryLuaObjects } from "./api.ts"; import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; import { isValidPath } from "@silverbulletmd/silverbullet/lib/ref"; import { notFoundError } from "@silverbulletmd/silverbullet/constants"; @@ -169,7 +169,7 @@ async function renamePage(oldName: string, newName: string) { const documentsToMove = new Set(); // Links only need to be updated if the folder changes if (oldFolder !== newFolder) { - const linksInPage = await queryLuaObjects("link", { + const linksInPage = await index.queryLuaObjects("link", { objectVariable: "_", where: await lua.parseExpression(`_.page == oldName`), }, { diff --git a/plugs/index/tags.ts b/plugs/index/tags.ts index 9b955280..b5c8cfc0 100644 --- a/plugs/index/tags.ts +++ b/plugs/index/tags.ts @@ -1,5 +1,4 @@ import type { FrontMatter } from "./frontmatter.ts"; -import { queryLuaObjects } from "./api.ts"; import { collectNodesOfType, findParentMatching, @@ -14,6 +13,7 @@ import type { import type { CompleteEvent } from "@silverbulletmd/silverbullet/type/client"; import { tagRegex } from "../../client/markdown_parser/constants.ts"; import { extractHashtag } from "@silverbulletmd/silverbullet/lib/tags"; +import { index } from "@silverbulletmd/silverbullet/syscalls"; export type TagObject = ObjectValue<{ name: string; @@ -73,14 +73,12 @@ export async function tagComplete(completeEvent: CompleteEvent) { } // Query all tags with a matching parent - const allTags: string[] = await queryLuaObjects( + const allTags: string[] = await index.queryLuaObjects( "tag", { distinct: true, select: { type: "Variable", name: "name", ctx: {} as any }, }, - {}, - 5, ); return { diff --git a/website/CHANGELOG.md b/website/CHANGELOG.md index 72cc96cd..0423c520 100644 --- a/website/CHANGELOG.md +++ b/website/CHANGELOG.md @@ -3,6 +3,9 @@ An attempt at documenting the changes/new features introduced in each release. ## Edge Whenever a commit is pushed to the `main` branch, within ~10 minutes, it will be released as a docker image with the `:v2` tag, and a binary in the [edge release](https://github.com/silverbulletmd/silverbullet/releases/tag/edge). If you want to live on the bleeding edge of SilverBullet goodness (or regression) this is where to do it. +* Migrated core indexer APIs to SB core˜ + + ## 2.4.0 * Indexer rework (note: upgrading will start a full space reindex automatically): * Performance: up to 2x faster