wushenghua/plainleaf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
Zef Hemel
2026-01-20 13:48:15 +01:00
parent 55bb0f0a0a
commit c25a073e81
22 changed files with 461 additions and 409 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/client.ts
+36 -45
View File
@@ -80,6 +80,8 @@ import type { KvPrimitives } from "./data/kv_primitives.ts";
import { deriveDbName } from "@silverbulletmd/silverbullet/lib/crypto";
import { LuaRuntimeError } from "./space_lua/runtime.ts";
import { resolveASTReference } from "./space_lua.ts";
import { ObjectIndex } from "./data/object_index.ts";
import type { LuaCollectionQuery } from "./space_lua/query_collection.ts";
const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/;
@@ -158,6 +160,7 @@ export class Client {
console.error,
);
}, 2000);
objectIndex!: ObjectIndex;
constructor(
private parent: Element,
@@ -204,12 +207,20 @@ export class Client {
// Setup message queue on top of that
this.mq = new DataStoreMQ(this.ds, this.eventHook);
this.objectIndex = new ObjectIndex(
this.ds,
this.config,
this.eventHook,
this.mq,
);
// Instantiate a PlugOS system
this.clientSystem = new ClientSystem(
this,
this.mq,
this.ds,
this.eventHook,
this.objectIndex,
this.bootConfig.readOnly,
);
@@ -331,7 +342,7 @@ export class Client {
name: string,
) => {
console.log("Queueing index for", name);
await this.eventHook.dispatchEvent("file:clearindex", name);
await this.objectIndex.clearFileIndex(name);
await this.mq.send("indexQueue", name);
},
);
@@ -343,37 +354,6 @@ export class Client {
this.space = space;
let startTime = -1;
this.eventHook.addLocalListener("file:initial", async () => {
startTime = Date.now();
await this.clientSystem.setIndexOngoing(true);
});
const emptyQueueHandler = async () => {
await this.clientSystem.setIndexOngoing(false);
if (
startTime !== -1 && !await this.clientSystem.hasInitialIndexCompleted()
) {
// Indexing has just finished for the first time
console.info(
"Initial index complete after",
(Date.now() - startTime) / 1000,
"s",
);
// Unsubscribe myself
this.eventHook.removeLocalListener(
"mq:emptyQueue:indexQueue",
emptyQueueHandler,
);
await this.clientSystem.markInitialIndexComplete();
await this.clientSystem.reloadState();
}
};
this.eventHook.addLocalListener(
"mq:emptyQueue:indexQueue",
emptyQueueHandler,
);
let lastSaveTimestamp: number | undefined;
const updateLastSaveTimestamp = () => {
@@ -523,7 +503,7 @@ export class Client {
resolve();
// In the background we'll fetch any enriched meta data, if any
const enrichedMeta = await this.clientSystem.getObjectByRef<
const enrichedMeta = await this.objectIndex.getObjectByRef<
PageMeta
>(
this.currentName(),
@@ -599,26 +579,37 @@ export class Client {
this.updateDocumentListCache().catch(console.error);
}
queryLuaObjects<T>(
tag: string,
query: LuaCollectionQuery,
scopedVariables?: Record<string, any>,
): Promise<T[]> {
return this.objectIndex.queryLuaObjects(
this.clientSystem.spaceLuaEnv.env,
tag,
query,
scopedVariables,
);
}
async updatePageListCache() {
console.log("Updating page list cache");
// Check if the initial sync has been completed
const initialIndexCompleted = await this.clientSystem
const initialIndexCompleted = await this.objectIndex
.hasInitialIndexCompleted();
let allPages: PageMeta[] = [];
if (
initialIndexCompleted && this.clientSystem.system.loadedPlugs.has("index")
) {
if (initialIndexCompleted) {
console.log(
"Initial index complete and index plug loaded, loading full page list via index.",
"Initial index complete, loading full page list via index.",
);
// Fetch indexed pages
allPages = await this.clientSystem.queryLuaObjects<PageMeta>("page", {});
allPages = await this.queryLuaObjects<PageMeta>("page", {});
// Overlay augmented meta values
await this.pageMetaAugmenter.augmentObjectArray(allPages, "ref");
// Fetch aspiring pages
const aspiringPageNames = await this.clientSystem.queryLuaObjects<string>(
const aspiringPageNames = await this.queryLuaObjects<string>(
"aspiring-page",
{ select: parseExpressionString("name"), distinct: true },
);
@@ -677,7 +668,7 @@ export class Client {
return;
}
const allDocuments = await this.clientSystem.queryLuaObjects<DocumentMeta>(
const allDocuments = await this.queryLuaObjects<DocumentMeta>(
"document",
{},
);
@@ -1125,9 +1116,9 @@ export class Client {
// Fetch the meta which includes the possibly indexed stuff, like page
// decorations
if (await this.clientSystem.hasInitialIndexCompleted()) {
if (await this.objectIndex.hasInitialIndexCompleted()) {
try {
const enrichedMeta = await this.clientSystem.getObjectByRef<PageMeta>(
const enrichedMeta = await this.objectIndex.getObjectByRef<PageMeta>(
pageName,
"page",
pageName,
@@ -1291,11 +1282,11 @@ export class Client {
console.warn("Not loading custom styles, since space style is disabled");
return;
}
if (!await this.clientSystem.hasInitialIndexCompleted()) {
if (!await this.objectIndex.hasInitialIndexCompleted()) {
return;
}
const spaceStyles = await this.clientSystem.queryLuaObjects<StyleObject>(
const spaceStyles = await this.queryLuaObjects<StyleObject>(
"space-style",
{
objectVariable: "_",
client/client_system.ts
+9 -75
View File
@@ -41,17 +41,13 @@ import { indexSyscalls } from "./plugos/syscalls/index.ts";
import { configSyscalls } from "./plugos/syscalls/config.ts";
import { eventSyscalls } from "./plugos/syscalls/event.ts";
import { DocumentEditorHook } from "./plugos/hooks/document_editor.ts";
import type { LuaCollectionQuery } from "./space_lua/query_collection.ts";
import type { Command } from "./types/command.ts";
import { SpaceLuaEnvironment } from "./space_lua.ts";
import { builtinPlugPaths } from "../plugs/builtin_plugs.ts";
import { ServiceRegistry } from "./service_registry.ts";
import { serviceRegistrySyscalls } from "./plugos/syscalls/service_registry.ts";
import type { ObjectIndex } from "./data/object_index.ts";
const indexVersionKey = ["$indexVersion"];
const indexQueuedKey = ["$indexQueued"];
// Bump this one every time a full reindex is needed
const desiredIndexVersion = 9;
const mqTimeout = 10000; // 10s
const mqTimeoutRetry = 3;
@@ -85,6 +81,7 @@ export class ClientSystem {
protected mq: DataStoreMQ,
public ds: DataStore,
public eventHook: EventHook,
private objectIndex: ObjectIndex,
public readOnlyMode: boolean,
) {
this.system = new System(undefined, {
@@ -94,7 +91,7 @@ export class ClientSystem {
),
});
this.spaceLuaEnv = new SpaceLuaEnvironment(this.system);
this.spaceLuaEnv = new SpaceLuaEnvironment(this.system, objectIndex);
this.serviceRegistry = new ServiceRegistry(this.eventHook, client.config);
setInterval(() => {
@@ -143,6 +140,10 @@ export class ClientSystem {
// Syscall hook
this.system.addHook(new SyscallHook());
this.eventHook.addLocalListener("editor:reloadState", async () => {
await this.reloadState();
});
}
init() {
@@ -163,7 +164,7 @@ export class ClientSystem {
clientCodeWidgetSyscalls(),
languageSyscalls(),
jsonschemaSyscalls(),
indexSyscalls(client),
indexSyscalls(this.objectIndex, this.client),
//commandSyscalls(client),
luaSyscalls(this),
mqSyscalls(this.mq),
@@ -199,7 +200,7 @@ export class ClientSystem {
console.info("Space Lua scripts are disabled, skipping loading scripts");
return;
}
if (!await this.hasInitialIndexCompleted()) {
if (!await this.objectIndex.hasInitialIndexCompleted()) {
console.info(
"Not loading space scripts, since initial indexing has not completed yet",
);
@@ -260,61 +261,6 @@ export class ClientSystem {
return this.system.localSyscall(name, args);
}
queryLuaObjects<T>(tag: string, query: LuaCollectionQuery): Promise<T[]> {
return this.system.invokeFunction("index.queryLuaObjects", [tag, query]);
}
// Index handling
getObjectByRef<T>(page: string, tag: string, ref: string) {
return this.localSyscall(
"system.invokeFunction",
["index.getObjectByRef", page, tag, ref],
);
}
async isIndexOngoing() {
return !!(await this.ds.get(indexQueuedKey));
}
async setIndexOngoing(val: boolean = true) {
await this.ds.set(indexQueuedKey, val);
}
async ensureFullIndex() {
if (!this.client.fullSyncCompleted) {
console.info(
"Initial full sync not completed, skipping index check",
);
return;
}
const currentIndexVersion = await this.getCurrentIndexVersion();
if (!currentIndexVersion) {
console.log("No index version found, assuming fresh install");
return;
}
if (
currentIndexVersion !== desiredIndexVersion &&
!await this.isIndexOngoing()
) {
console.info(
"[index]",
"Performing a full space reindex, this could take a while...",
currentIndexVersion,
desiredIndexVersion,
);
await this.setIndexOngoing(true);
await this.system.invokeFunction("index.reindexSpace", []);
console.info("[index]", "Full space index complete.");
await this.markInitialIndexComplete();
await this.setIndexOngoing(false);
// Let's load space scripts again, which probably weren't loaded before
await this.reloadState();
}
}
public async reloadState() {
console.log(
"Now loading space scripts, custom styles and rebuilding editor state",
@@ -323,16 +269,4 @@ export class ClientSystem {
await this.client.loadCustomStyles();
this.client.rebuildEditorState();
}
public async hasInitialIndexCompleted() {
return (await this.ds.get(indexVersionKey)) === desiredIndexVersion;
}
private getCurrentIndexVersion() {
return this.ds.get(indexVersionKey);
}
async markInitialIndexComplete() {
await this.ds.set(indexVersionKey, desiredIndexVersion);
}
}
client/data/object_index.ts
+304
View File
@@ -0,0 +1,304 @@
import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index";
import type { Config } from "../config.ts";
import type {
LuaCollectionQuery,
LuaQueryCollection,
} from "../space_lua/query_collection.ts";
import {
jsToLuaValue,
LuaEnv,
LuaStackFrame,
type LuaTable,
} from "../space_lua/runtime.ts";
import type { DataStore } from "./datastore.ts";
import type { KV, KvKey } from "@silverbulletmd/silverbullet/type/datastore";
import type { EventHook } from "../plugos/hooks/event.ts";
import type { DataStoreMQ } from "./mq.datastore.ts";
import type { Space } from "../space.ts";
const indexKey = "idx";
const pageKey = "ridx";
const indexVersionKey = ["$indexVersion"];
const indexQueuedKey = ["$indexQueued"];
// Bump this one every time a full reindex is needed
const desiredIndexVersion = 9;
export class ObjectIndex {
constructor(
private ds: DataStore,
private config: Config,
private eventHook: EventHook,
private mq: DataStoreMQ,
) {
let startTime = -1;
this.eventHook.addLocalListener("file:initial", async () => {
startTime = Date.now();
await this.setIndexOngoing(true);
});
this.eventHook.addLocalListener("file:deleted", async (path: string) => {
await this.clearFileIndex(path);
});
const emptyQueueHandler = async () => {
await this.setIndexOngoing(false);
if (
startTime !== -1 && !await this.hasInitialIndexCompleted()
) {
// Indexing has just finished for the first time
console.info(
"Initial index complete after",
(Date.now() - startTime) / 1000,
"s",
);
// Unsubscribe myself
this.eventHook.removeLocalListener(
"mq:emptyQueue:indexQueue",
emptyQueueHandler,
);
await this.markInitialIndexComplete();
this.eventHook.dispatchEvent("editor:reloadState");
}
};
this.eventHook.addLocalListener(
"mq:emptyQueue:indexQueue",
emptyQueueHandler,
);
}
tag(tagName: string): LuaQueryCollection {
if (!tagName) {
throw new Error("Tag name is required");
}
return {
query: (
query: LuaCollectionQuery,
env: LuaEnv,
sf: LuaStackFrame,
): Promise<any[]> => {
return this.ds.luaQuery(
["idx", tagName],
query,
env,
sf,
(key, value: any) => {
const tag = key[1];
const tagDef = this.config.get<LuaTable | undefined>(
["tagDefinitions", tag],
undefined,
);
if (!tagDef || !tagDef.has("metatable")) {
// Return as is
return value;
}
// Convert to LuaTable
value = jsToLuaValue(value);
value.metatable = tagDef.get("metatable");
return value;
},
);
},
};
}
getObjectByRef<T>(page: string, tag: string, ref: string) {
return this.ds.get([indexKey, tag, this.cleanKey(ref, page), page]);
}
async isIndexOngoing() {
return !!(await this.ds.get(indexQueuedKey));
}
async setIndexOngoing(val: boolean = true) {
await this.ds.set(indexQueuedKey, val);
}
async ensureFullIndex(space: Space) {
// Commenting out this check because I think it always holds when calling this API
// if (!this.client.fullSyncCompleted) {
// console.info(
// "Initial full sync not completed, skipping index check",
// );
// return;
// }
const currentIndexVersion = await this.getCurrentIndexVersion();
if (!currentIndexVersion) {
console.log("No index version found, assuming fresh install");
return;
}
if (
currentIndexVersion !== desiredIndexVersion &&
!await this.isIndexOngoing()
) {
console.info(
"[index]",
"Performing a full space reindex, this could take a while...",
currentIndexVersion,
desiredIndexVersion,
);
await this.setIndexOngoing(true);
await this.reindexSpace(space);
console.info("[index]", "Full space index complete.");
await this.markInitialIndexComplete();
await this.setIndexOngoing(false);
// Let's load space scripts again, which probably weren't loaded before
this.eventHook.dispatchEvent("editor:reloadState");
}
}
async reindexSpace(space: Space) {
console.log("Clearing page index...");
await this.clearIndex();
const files = await space.deduplicatedFileList();
console.log("Queing", files.length, "pages to be indexed.");
// Queue all file names to be indexed
const startTime = Date.now();
await this.mq.batchSend("indexQueue", files.map((file) => file.name));
await this.mq.awaitEmptyQueue("indexQueue");
console.log("Done with full index after", Date.now() - startTime, "ms");
}
public async hasInitialIndexCompleted() {
return (await this.ds.get(indexVersionKey)) === desiredIndexVersion;
}
private getCurrentIndexVersion() {
return this.ds.get(indexVersionKey);
}
async markInitialIndexComplete() {
await this.ds.set(indexVersionKey, desiredIndexVersion);
}
cleanKey(ref: string, page: string) {
if (ref.startsWith(`${page}@`)) {
return ref.substring(page.length + 1);
} else {
return ref;
}
}
queryLuaObjects<T>(
globalEnv: LuaEnv,
tag: string,
query: LuaCollectionQuery,
scopedVariables?: Record<string, any>,
): Promise<ObjectValue<T>[]> {
const sf = LuaStackFrame.createWithGlobalEnv(
globalEnv,
);
let env = globalEnv;
if (scopedVariables) {
env = new LuaEnv(globalEnv);
for (const [key, value] of Object.entries(scopedVariables)) {
env.setLocal(key, jsToLuaValue(value));
}
}
return this.ds.luaQuery([indexKey, tag], query, env, sf);
}
batchSet(page: string, kvs: KV[]): Promise<void> {
const finalBatch: KV[] = [];
for (const { key, value } of kvs) {
finalBatch.push({
key: [indexKey, ...key, page],
value,
}, {
key: [pageKey, page, ...key],
value: true,
});
}
return this.ds.batchSet(finalBatch);
}
batchDelete(page: string, keys: KvKey[]): Promise<void> {
const finalBatch: KvKey[] = [];
for (const key of keys) {
finalBatch.push([indexKey, ...key, page]);
}
return this.ds.batchDelete(finalBatch);
}
/**
* Clears all keys for a given file
* @param file
*/
public async clearFileIndex(file: string): Promise<void> {
if (file.endsWith(".md")) {
file = file.replace(/\.md$/, "");
}
// console.log("Clearing index for", file);
const allKeys: KvKey[] = [];
for await (
const { key } of this.ds.query({
prefix: [pageKey, file],
})
) {
allKeys.push(key);
allKeys.push([indexKey, ...key.slice(2), file]);
}
await this.ds.batchDelete(allKeys);
}
/**
* Clears the entire index
*/
public async clearIndex(): Promise<void> {
const allKeys: KvKey[] = [];
for await (
const { key } of this.ds.query({ prefix: [indexKey] })
) {
allKeys.push(key);
}
for await (
const { key } of this.ds.query({ prefix: [pageKey] })
) {
allKeys.push(key);
}
await this.ds.batchDelete(allKeys);
console.log("Deleted", allKeys.length, "keys from the index");
}
/**
* Indexes entities in the data store
*/
public indexObjects<T>(
page: string,
objects: ObjectValue<T>[],
): Promise<void> {
const kvs: KV<T>[] = [];
for (const obj of objects) {
if (!obj.tag) {
console.error("Object has no tag", obj, "this shouldn't happen");
continue;
}
// Index as all the tag + any additional tags specified
const allTags = [obj.tag, ...obj.tags || []];
for (const tag of allTags) {
// The object itself
kvs.push({
key: [tag, this.cleanKey(obj.ref, page)],
value: obj,
});
}
}
if (kvs.length > 0) {
return this.batchSet(page, kvs);
} else {
return Promise.resolve();
}
}
deleteObject(
page: string,
tag: string,
ref: string,
): Promise<void> {
return this.batchDelete(page, [[tag, this.cleanKey(ref, page)]]);
}
}
client/plugos/syscalls/index.ts
+49 -40
View File
@@ -3,53 +3,62 @@ import type {
LuaCollectionQuery,
LuaQueryCollection,
} from "../../space_lua/query_collection.ts";
import {
jsToLuaValue,
type LuaEnv,
type LuaStackFrame,
type LuaTable,
} from "../../space_lua/runtime.ts";
import type { LuaTable } from "../../space_lua/runtime.ts";
import type { ObjectIndex } from "../../data/object_index.ts";
import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index";
import type { Client } from "../../client.ts";
export function indexSyscalls(client: Client): SysCallMapping {
export function indexSyscalls(
objectIndex: ObjectIndex,
client: Client,
): SysCallMapping {
return {
"index.tag": (_ctx, tagName: string): LuaQueryCollection => {
if (!tagName) {
throw new Error("Tag name is required");
}
return {
query: (
query: LuaCollectionQuery,
env: LuaEnv,
sf: LuaStackFrame,
): Promise<any[]> => {
return client.ds.luaQuery(
["idx", tagName],
query,
env,
sf,
(key, value: any) => {
const tag = key[1];
const tagDef = client.config.get<LuaTable | undefined>(
["tagDefinitions", tag],
undefined,
);
if (!tagDef || !tagDef.has("metatable")) {
// Return as is
return value;
}
// Convert to LuaTable
value = jsToLuaValue(value);
value.metatable = tagDef.get("metatable");
return value;
},
);
},
};
return objectIndex.tag(tagName);
},
"index.ensureFullIndex": (_ctx) => {
return client.clientSystem.ensureFullIndex();
return objectIndex.ensureFullIndex(client.space);
},
"index.reindexSpace": () => {
return objectIndex.reindexSpace(client.space);
},
"index.indexObjects": (
_ctx,
page: string,
objects: ObjectValue[],
): Promise<void> => {
return objectIndex.indexObjects(page, objects);
},
"index.getObjectByRef": (
_ctx,
page: string,
tag: string,
ref: string,
): Promise<ObjectValue | undefined> => {
return objectIndex.getObjectByRef(ref, page, tag);
},
"index.queryLuaObjects": (
_ctx,
tag: string,
query: LuaCollectionQuery,
scopedVariables?: Record<string, any>,
): Promise<ObjectValue[]> => {
return objectIndex.queryLuaObjects(
client.clientSystem.spaceLuaEnv.env,
tag,
query,
scopedVariables,
);
},
"index.deleteObject": (
_ctx,
page: string,
tag: string,
ref: string,
): Promise<void> => {
return objectIndex.deleteObject(page, tag, ref);
},
"lua:index.defineTag": (_ctx, tagDef: LuaTable) => {
// Using 'lua:' prefix to _not_ convert tagDef to a JS version (but keep original LuaTable)
client/space_lua.ts
+7 -5
View File
@@ -11,11 +11,12 @@ import {
import type { ASTCtx } from "./space_lua/ast.ts";
import { buildLuaEnv } from "./space_lua_api.ts";
import type { LuaCollectionQuery } from "./space_lua/query_collection.ts";
import type { ObjectIndex } from "./data/object_index.ts";
export class SpaceLuaEnvironment {
env: LuaEnv;
constructor(private system: System<any>) {
constructor(private system: System<any>, private objectIndex: ObjectIndex) {
this.env = buildLuaEnv(system);
}
@@ -24,15 +25,16 @@ export class SpaceLuaEnvironment {
* @param system
*/
async reload() {
const allScripts: SpaceLuaObject[] = await this.system.invokeFunction(
"index.queryLuaObjects",
["space-lua", {
const allScripts: SpaceLuaObject[] = await this.objectIndex.queryLuaObjects(
this.env,
"space-lua",
{
objectVariable: "script",
orderBy: [{
expr: parseExpressionString("script.priority"),
desc: true,
}],
} as LuaCollectionQuery],
} as LuaCollectionQuery,
);
try {
this.env = buildLuaEnv(this.system);
plug-api/syscalls/index.ts
+13 -2
View File
@@ -31,9 +31,8 @@ export function queryLuaObjects<T>(
tag: string,
query: LuaCollectionQuery,
scopedVariables?: Record<string, any>,
ttlSecs?: number,
): Promise<ObjectValue<T>[]> {
return syscall("index.queryLuaObjects", tag, query, scopedVariables, ttlSecs);
return syscall("index.queryLuaObjects", tag, query, scopedVariables);
}
/**
@@ -57,3 +56,15 @@ export function getObjectByRef<T>(
export function ensureFullIndex(): Promise<void> {
return syscall("index.ensureFullIndex");
}
export function reindexSpace(): Promise<void> {
return syscall("index.reindexSpace");
}
export function deleteObject(
page: string,
tag: string,
ref: string,
): Promise<void> {
return syscall("index.deleteObject", page, tag, ref);
}
plug-api/system_mock.ts
+5 -2
View File
@@ -2,6 +2,7 @@ import { Config } from "../client/config.ts";
import { DataStore } from "../client/data/datastore.ts";
import { MemoryKvPrimitives } from "../client/data/memory_kv_primitives.ts";
import { DataStoreMQ } from "../client/data/mq.datastore.ts";
import { ObjectIndex } from "../client/data/object_index.ts";
import { EventHook } from "../client/plugos/hooks/event.ts";
import {
dataStoreReadSyscalls,
@@ -46,9 +47,11 @@ export function createMockSystem() {
eventedSpacePrimitives: spacePrimitives,
};
const objectIndex = new ObjectIndex(ds, config, eventHook, mq);
const clientSystemMock: any = {
system: system,
spaceLuaEnv: new SpaceLuaEnvironment(system),
spaceLuaEnv: new SpaceLuaEnvironment(system, objectIndex),
};
clientMock.clientSystem = clientSystemMock;
@@ -60,7 +63,7 @@ export function createMockSystem() {
markdownSyscalls(clientMock),
languageSyscalls(),
jsonschemaSyscalls(),
indexSyscalls(clientMock),
indexSyscalls(objectIndex, clientMock),
luaSyscalls(clientSystemMock),
mqSyscalls(mq),
dataStoreReadSyscalls(ds, clientSystemMock),
plug-api/types/index.ts
+1 -1
View File
@@ -47,7 +47,7 @@ export type SyscallMeta = {
* - ref: a unique reference (id) for the object, ideally a page reference
* - tags: a list of tags that the object belongs to
*/
export type ObjectValue<T> = {
export type ObjectValue<T = any> = {
ref: string;
tag: string; // main tag
tags?: string[];
plugs/editor/complete.ts
+3 -10
View File
@@ -46,12 +46,7 @@ export async function pageComplete(completeEvent: CompleteEvent) {
if (prefix.startsWith("^")) {
// A carrot prefix means we're looking for a meta page
allPages = await index.queryLuaObjects<PageMeta>(
"page",
isMetaPageQuery!,
{},
5,
);
allPages = await index.queryLuaObjects<PageMeta>("page", isMetaPageQuery);
// Let's prefix the names with a caret to make them match
allPages = allPages.map((page) => ({
...page,
@@ -61,9 +56,9 @@ export async function pageComplete(completeEvent: CompleteEvent) {
// This is the most common case, we're combining three types of completions here:
allPages = (await Promise.all([
// All non-meta pages
index.queryLuaObjects<PageMeta>("page", isntMetaPageQuery, {}, 5),
index.queryLuaObjects<PageMeta>("page", isntMetaPageQuery),
// All documents
index.queryLuaObjects<DocumentMeta>("document", isDocumentQuery, {}, 5),
index.queryLuaObjects<DocumentMeta>("document", isDocumentQuery),
// And all links to non-existing pages (to augment the existing ones)
index.queryLuaObjects<string>(
"aspiring-page",
@@ -71,8 +66,6 @@ export async function pageComplete(completeEvent: CompleteEvent) {
select: { type: "Variable", name: "name", ctx: {} as any },
distinct: true,
},
{},
5,
).then((aspiringPages) =>
// Rewrite them to PageMeta shaped objects
aspiringPages.map((aspiringPage: string): PageMeta => ({
plugs/index/api.ts
+2 -148
View File
@@ -1,6 +1,4 @@
import { datastore, markdown } from "@silverbulletmd/silverbullet/syscalls";
import { ttlCache } from "../../plug-api/lib/memory_cache.ts";
import type { LuaCollectionQuery } from "../../client/space_lua/query_collection.ts";
import { markdown } from "@silverbulletmd/silverbullet/syscalls";
import {
extractFrontMatter as extractFrontmatterFromTree,
type FrontMatter,
@@ -11,12 +9,6 @@ import {
renderToText,
} from "@silverbulletmd/silverbullet/lib/tree";
import { applyPatches, type YamlPatch } from "../../plug-api/lib/yaml.ts";
import type { ObjectValue } from "../../plug-api/types/index.ts";
import type { KV, KvKey, KvQuery } from "../../plug-api/types/datastore.ts";
const indexKey = "idx";
const pageKey = "ridx";
/*
* Key namespace:
@@ -25,150 +17,12 @@ const pageKey = "ridx";
* ["type", type] -> true // for fast type listing
*/
export function batchSet(page: string, kvs: KV[]): Promise<void> {
const finalBatch: KV[] = [];
for (const { key, value } of kvs) {
finalBatch.push({
key: [indexKey, ...key, page],
value,
}, {
key: [pageKey, page, ...key],
value: true,
});
}
return datastore.batchSet(finalBatch);
}
export function batchDelete(page: string, keys: KvKey[]): Promise<void> {
const finalBatch: KvKey[] = [];
for (const key of keys) {
finalBatch.push([indexKey, ...key, page]);
}
return datastore.batchDel(finalBatch);
}
/**
* Clears all keys for a given file
* @param file
*/
export async function clearFileIndex(file: string): Promise<void> {
if (file.endsWith(".md")) {
file = file.replace(/\.md$/, "");
}
// console.log("Clearing index for", file);
const allKeys: KvKey[] = [];
for (
const { key } of await datastore.query({
prefix: [pageKey, file],
})
) {
allKeys.push(key);
allKeys.push([indexKey, ...key.slice(2), file]);
}
await datastore.batchDel(allKeys);
}
/**
* Clears the entire index
*/
export async function clearIndex(): Promise<void> {
const allKeys: KvKey[] = [];
for (
const { key } of await datastore.query({ prefix: [indexKey] })
) {
allKeys.push(key);
}
for (
const { key } of await datastore.query({ prefix: [pageKey] })
) {
allKeys.push(key);
}
await datastore.batchDel(allKeys);
console.log("Deleted", allKeys.length, "keys from the index");
}
// OBJECTS API
/**
* Indexes entities in the data store
*/
export function indexObjects<T>(
page: string,
objects: ObjectValue<T>[],
): Promise<void> {
const kvs: KV<T>[] = [];
for (const obj of objects) {
if (!obj.tag) {
console.error("Object has no tag", obj, "this shouldn't happen");
continue;
}
// Index as all the tag + any additional tags specified
const allTags = [obj.tag, ...obj.tags || []];
for (const tag of allTags) {
// The object itself
kvs.push({
key: [tag, cleanKey(obj.ref, page)],
value: obj,
});
}
}
if (kvs.length > 0) {
return batchSet(page, kvs);
} else {
return Promise.resolve();
}
}
function cleanKey(ref: string, page: string) {
if (ref.startsWith(`${page}@`)) {
return ref.substring(page.length + 1);
} else {
return ref;
}
}
export function queryLuaObjects<T>(
tag: string,
query: LuaCollectionQuery,
scopedVariables?: Record<string, any>,
ttlSecs?: number,
): Promise<ObjectValue<T>[]> {
return ttlCache(query, () => {
return datastore.queryLua([indexKey, tag], query, scopedVariables);
}, ttlSecs);
}
export function deleteObject(
tag: string,
page: string,
ref: string,
): Promise<void> {
return batchDelete(page, [[tag, cleanKey(ref, page)]]);
}
export async function query(
query: KvQuery,
): Promise<KV[]> {
return (await datastore.query({
...query,
prefix: [indexKey, ...query.prefix ? query.prefix : []],
})).map(({ key, value }) => ({ key: key.slice(1), value }));
}
export function getObjectByRef<T>(
page: string,
tag: string,
ref: string,
): Promise<ObjectValue<T> | undefined> {
return datastore.get([indexKey, tag, cleanKey(ref, page), page]);
}
export async function extractFrontmatter(
text: string,
extractOptions: FrontMatterExtractOptions = {},
): Promise<{ frontmatter: FrontMatter; text: string }> {
const tree = await markdown.parseMarkdown(text);
const frontmatter = await extractFrontmatterFromTree(tree, extractOptions);
const frontmatter = extractFrontmatterFromTree(tree, extractOptions);
return { frontmatter, text: renderToText(tree) };
}
plugs/index/command.ts
+2 -3
View File
@@ -1,8 +1,7 @@
import { editor } from "@silverbulletmd/silverbullet/syscalls";
import { reindexSpace } from "./queue.ts";
import { editor, index } from "@silverbulletmd/silverbullet/syscalls";
export async function reindexCommand() {
await editor.flashNotification("Performing full page reindex...");
await reindexSpace();
await index.reindexSpace();
await editor.flashNotification("Done with page index!");
}
plugs/index/document.ts
+2 -3
View File
@@ -1,5 +1,4 @@
import { space, system } from "@silverbulletmd/silverbullet/syscalls";
import { indexObjects } from "./api.ts";
import { index, space, system } from "@silverbulletmd/silverbullet/syscalls";
import type { DocumentMeta } from "../../plug-api/types/index.ts";
// Note: clearFileIndex is not called but since this is the only document:index listener, this should be fine (famous last words)
@@ -9,5 +8,5 @@ export async function indexDocument(name: string) {
}
// console.log("Indexing document", name);
const fileMeta = await space.getDocumentMeta(name);
await indexObjects<DocumentMeta>(fileMeta.name, [fileMeta]);
await index.indexObjects<DocumentMeta>(fileMeta.name, [fileMeta]);
}
plugs/index/header.ts
+2 -4
View File
@@ -4,12 +4,11 @@ import {
type ParseTree,
renderToText,
} from "@silverbulletmd/silverbullet/lib/tree";
import { queryLuaObjects } from "./api.ts";
import {
getNameFromPath,
parseToRef,
} from "@silverbulletmd/silverbullet/lib/ref";
import { lua } from "@silverbulletmd/silverbullet/syscalls";
import { index, lua } from "@silverbulletmd/silverbullet/syscalls";
import type {
ObjectValue,
PageMeta,
@@ -80,14 +79,13 @@ export async function headerComplete(completeEvent: CompleteEvent) {
return;
}
const headers = await queryLuaObjects<HeaderObject>(
const headers = await index.queryLuaObjects<HeaderObject>(
"header",
{
objectVariable: "_",
where: await lua.parseExpression(`_.page == name`),
},
{ name: getNameFromPath(ref.path) || completeEvent.pageName },
5,
);
return {
plugs/index/index.plug.yaml
-27
View File
@@ -1,20 +1,6 @@
name: index
functions:
# Public API
batchSet:
path: api.ts:batchSet
indexObjects:
path: api.ts:indexObjects
syscall: index.indexObjects
query:
path: api.ts:query
syscall: index.query
queryLuaObjects:
path: api.ts:queryLuaObjects
syscall: index.queryLuaObjects
getObjectByRef:
path: api.ts:getObjectByRef
syscall: index.getObjectByRef
extractFrontmatter:
path: api.ts:extractFrontmatter
syscall: index.extractFrontmatter
@@ -34,16 +20,6 @@ functions:
path: yaml.ts:stringify
syscall: yaml.stringify
# Index management
clearIndex:
path: api.ts:clearIndex
clearDSIndex:
path: api.ts:clearFileIndex
events:
- file:deleted
- file:clearindex
reindexSpaceCommand:
path: "./command.ts:reindexCommand"
command:
@@ -57,9 +33,6 @@ functions:
batchSize: 3
autoAck: true
reindexSpace:
path: "./queue.ts:reindexSpace"
indexDocument:
path: "./document.ts:indexDocument"
events:
plugs/index/index.ts
+3 -4
View File
@@ -5,7 +5,6 @@ import type {
import { extractFrontMatter, type FrontMatter } from "./frontmatter.ts";
import type { ParseTree } from "@silverbulletmd/silverbullet/lib/tree";
import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event";
import { indexObjects } from "./api.ts";
import { indexPage as pageIndexPage } from "./page.ts";
import { indexData } from "./data.ts";
import { indexItems } from "./item.ts";
@@ -16,7 +15,7 @@ import { indexTables } from "./table.ts";
import { indexSpaceLua } from "./space_lua.ts";
import { indexSpaceStyle } from "./space_style.ts";
import { indexTags } from "./tags.ts";
import { markdown } from "@silverbulletmd/silverbullet/syscalls";
import { index, markdown } from "@silverbulletmd/silverbullet/syscalls";
export type IndexerFunction = (
pageMeta: PageMeta,
@@ -68,11 +67,11 @@ export async function indexPage({ name, tree, meta, text }: IndexTreeEvent) {
// console.log("Now going to index page", name);
// Index the page
const index = await Promise.all(allIndexers.map((indexer) => {
const indexResults = await Promise.all(allIndexers.map((indexer) => {
return indexer(meta, frontmatter, tree, text);
}));
// console.log("Found these objects", index.flat());
await indexObjects<any>(name, index.flat());
await index.indexObjects<any>(name, indexResults.flat());
}
plugs/index/link.ts
+2 -3
View File
@@ -9,7 +9,6 @@ import {
isLocalURL,
resolveMarkdownLink,
} from "@silverbulletmd/silverbullet/lib/resolve";
import { queryLuaObjects } from "./api.ts";
import type { FrontMatter } from "./frontmatter.ts";
import { updateITags } from "./tags.ts";
import {
@@ -21,7 +20,7 @@ import {
mdLinkRegex,
wikiLinkRegex,
} from "../../client/markdown_parser/constants.ts";
import { lua, space } from "@silverbulletmd/silverbullet/syscalls";
import { index, lua, space } from "@silverbulletmd/silverbullet/syscalls";
import type {
ObjectValue,
PageMeta,
@@ -247,7 +246,7 @@ export function collectPageLinks(n: ParseTree): string[] {
export async function getBackLinks(
name: string,
): Promise<LinkObject[]> {
return (await queryLuaObjects<LinkObject>("link", {
return (await index.queryLuaObjects<LinkObject>("link", {
objectVariable: "_",
where: await lua.parseExpression(`_.toPage == name or _.toFile == name`),
}, {
plugs/index/page.ts
+13 -5
View File
@@ -1,7 +1,11 @@
import { editor, lua, markdown } from "@silverbulletmd/silverbullet/syscalls";
import {
editor,
index,
lua,
markdown,
} from "@silverbulletmd/silverbullet/syscalls";
import type { FrontMatter } from "./frontmatter.ts";
import { deleteObject, getObjectByRef, queryLuaObjects } from "./api.ts";
import {
findNodeOfType,
type ParseTree,
@@ -47,7 +51,7 @@ export async function indexPage(
// Make sure this page is no (longer) in the aspiring pages list
// TODO: This can possibly done more optimally
const aspiringPages = await queryLuaObjects<AspiringPageObject>(
const aspiringPages = await index.queryLuaObjects<AspiringPageObject>(
"aspiring-page",
{
objectVariable: "_",
@@ -57,7 +61,11 @@ export async function indexPage(
);
for (const aspiringPage of aspiringPages) {
console.log("Deleting aspiring page", aspiringPage);
await deleteObject("aspiring-page", aspiringPage.page, aspiringPage.ref);
await index.deleteObject(
"aspiring-page",
aspiringPage.page,
aspiringPage.ref,
);
}
return [combinedPageMeta];
@@ -145,7 +153,7 @@ export async function loadPageObject(pageName?: string): Promise<PageMeta> {
created: "",
} as PageMeta;
}
return (await getObjectByRef<PageMeta>(
return (await index.getObjectByRef<PageMeta>(
pageName,
"page",
pageName,
plugs/index/plug_api.ts
+1 -7
View File
@@ -1,7 +1,7 @@
import { system } from "@silverbulletmd/silverbullet/syscalls";
import type { ObjectValue } from "../../plug-api/types/index.ts";
import type { KV, KvQuery } from "../../plug-api/types/datastore.ts";
import type { KV } from "../../plug-api/types/datastore.ts";
export function indexObjects<T>(
page: string,
@@ -14,12 +14,6 @@ export function batchSet(page: string, kvs: KV[]): Promise<void> {
return system.invokeFunction("index.batchSet", page, kvs);
}
export function query(
query: KvQuery,
): Promise<KV[]> {
return system.invokeFunction("index.query", query);
}
export function getObjectByRef<T>(
page: string,
tag: string,
plugs/index/queue.ts
-19
View File
@@ -4,7 +4,6 @@ import {
markdown,
mq,
space,
system,
} from "@silverbulletmd/silverbullet/syscalls";
import { sleep } from "@silverbulletmd/silverbullet/lib/async";
import type { MQMessage } from "@silverbulletmd/silverbullet/type/datastore";
@@ -12,24 +11,6 @@ import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event";
const uiUpdateInterval = 5000;
export async function reindexSpace() {
if (await system.getMode() === "ro") {
console.info("Not reindexing because we're in read-only mode");
return;
}
console.log("Clearing page index...");
await system.invokeFunction("index.clearIndex");
const files = await space.listFiles();
console.log("Queing", files.length, "pages to be indexed.");
// Queue all file names to be indexed
const startTime = Date.now();
await mq.batchSend("indexQueue", files.map((file) => file.name));
await mq.awaitEmptyQueue("indexQueue");
console.log("Done with full index after", Date.now() - startTime, "ms");
}
setTimeout(updateIndexProgressInUI, uiUpdateInterval);
async function totalItemsQueued() {
plugs/index/refactor.ts
+2 -2
View File
@@ -1,5 +1,6 @@
import {
editor,
index,
lua,
markdown,
space,
@@ -16,7 +17,6 @@ import {
findParentMatching,
nodeAtPos,
} from "@silverbulletmd/silverbullet/lib/tree";
import { queryLuaObjects } from "./api.ts";
import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index";
import { isValidPath } from "@silverbulletmd/silverbullet/lib/ref";
import { notFoundError } from "@silverbulletmd/silverbullet/constants";
@@ -169,7 +169,7 @@ async function renamePage(oldName: string, newName: string) {
const documentsToMove = new Set<string>();
// Links only need to be updated if the folder changes
if (oldFolder !== newFolder) {
const linksInPage = await queryLuaObjects<LinkObject>("link", {
const linksInPage = await index.queryLuaObjects<LinkObject>("link", {
objectVariable: "_",
where: await lua.parseExpression(`_.page == oldName`),
}, {
plugs/index/tags.ts
+2 -4
View File
@@ -1,5 +1,4 @@
import type { FrontMatter } from "./frontmatter.ts";
import { queryLuaObjects } from "./api.ts";
import {
collectNodesOfType,
findParentMatching,
@@ -14,6 +13,7 @@ import type {
import type { CompleteEvent } from "@silverbulletmd/silverbullet/type/client";
import { tagRegex } from "../../client/markdown_parser/constants.ts";
import { extractHashtag } from "@silverbulletmd/silverbullet/lib/tags";
import { index } from "@silverbulletmd/silverbullet/syscalls";
export type TagObject = ObjectValue<{
name: string;
@@ -73,14 +73,12 @@ export async function tagComplete(completeEvent: CompleteEvent) {
}
// Query all tags with a matching parent
const allTags: string[] = await queryLuaObjects<string>(
const allTags: string[] = await index.queryLuaObjects<string>(
"tag",
{
distinct: true,
select: { type: "Variable", name: "name", ctx: {} as any },
},
{},
5,
);
return {
website/CHANGELOG.md
+3
View File
@@ -3,6 +3,9 @@ An attempt at documenting the changes/new features introduced in each release.
## Edge
Whenever a commit is pushed to the `main` branch, within ~10 minutes, it will be released as a docker image with the `:v2` tag, and a binary in the [edge release](https://github.com/silverbulletmd/silverbullet/releases/tag/edge). If you want to live on the bleeding edge of SilverBullet goodness (or regression) this is where to do it.
* Migrated core indexer APIs to SB core˜
## 2.4.0
* Indexer rework (note: upgrading will start a full space reindex automatically):
* Performance: up to 2x faster