import { datastore, markdown } from "@silverbulletmd/silverbullet/syscalls"; import { ttlCache } from "../../lib/memory_cache.ts"; import type { LuaCollectionQuery } from "../../lib/space_lua/query_collection.ts"; import { extractFrontMatter as extractFrontmatterFromTree, type FrontMatter, type FrontMatterExtractOptions, } from "../../plug-api/lib/frontmatter.ts"; import { collectNodesOfType, renderToText, } from "@silverbulletmd/silverbullet/lib/tree"; import { applyPatches, type SetKeyPatch } from "../../lib/yaml.ts"; import type { ObjectValue } from "../../type/index.ts"; import type { KV, KvKey, KvQuery } from "../../type/datastore.ts"; const indexKey = "idx"; const pageKey = "ridx"; /* * Key namespace: * [indexKey, type, ...key, page] -> value * [pageKey, page, ...key] -> true // for fast page clearing * ["type", type] -> true // for fast type listing */ export function batchSet(page: string, kvs: KV[]): Promise { const finalBatch: KV[] = []; for (const { key, value } of kvs) { finalBatch.push({ key: [indexKey, ...key, page], value, }, { key: [pageKey, page, ...key], value: true, }); } return datastore.batchSet(finalBatch); } export function batchDelete(page: string, keys: KvKey[]): Promise { const finalBatch: KvKey[] = []; for (const key of keys) { finalBatch.push([indexKey, ...key, page]); } return datastore.batchDel(finalBatch); } /** * Clears all keys for a given file * @param file */ export async function clearFileIndex(file: string): Promise { if (file.endsWith(".md")) { file = file.replace(/\.md$/, ""); } // console.log("Clearing index for", file); const allKeys: KvKey[] = []; for ( const { key } of await datastore.query({ prefix: [pageKey, file], }) ) { allKeys.push(key); allKeys.push([indexKey, ...key.slice(2), file]); } await datastore.batchDel(allKeys); } /** * Clears the entire index */ export async function clearIndex(): Promise { const allKeys: KvKey[] = []; for ( const { key } of await datastore.query({ prefix: [indexKey] }) ) { allKeys.push(key); } for ( const { key } of await datastore.query({ prefix: [pageKey] }) ) { allKeys.push(key); } await datastore.batchDel(allKeys); console.log("Deleted", allKeys.length, "keys from the index"); } // OBJECTS API /** * Indexes entities in the data store */ export function indexObjects( page: string, objects: ObjectValue[], ): Promise { const kvs: KV[] = []; for (const obj of objects) { if (!obj.tag) { console.error("Object has no tag", obj, "this shouldn't happen"); continue; } // Index as all the tag + any additional tags specified const allTags = [obj.tag, ...obj.tags || []]; for (const tag of allTags) { // The object itself kvs.push({ key: [tag, cleanKey(obj.ref, page)], value: obj, }); } } if (kvs.length > 0) { return batchSet(page, kvs); } else { return Promise.resolve(); } } function cleanKey(ref: string, page: string) { if (ref.startsWith(`${page}@`)) { return ref.substring(page.length + 1); } else { return ref; } } export function queryLuaObjects( tag: string, query: LuaCollectionQuery, scopedVariables?: Record, ttlSecs?: number, ): Promise[]> { return ttlCache(query, () => { return datastore.queryLua([indexKey, tag], query, scopedVariables); }, ttlSecs); } export function deleteObject( tag: string, page: string, ref: string, ): Promise { return batchDelete(page, [[tag, cleanKey(ref, page)]]); } export async function query( query: KvQuery, ): Promise { return (await datastore.query({ ...query, prefix: [indexKey, ...query.prefix ? query.prefix : []], })).map(({ key, value }) => ({ key: key.slice(1), value })); } export function getObjectByRef( page: string, tag: string, ref: string, ): Promise | undefined> { return datastore.get([indexKey, tag, cleanKey(ref, page), page]); } export async function extractFrontmatter( text: string, extractOptions: FrontMatterExtractOptions = {}, ): Promise<{ frontmatter: FrontMatter; text: string }> { const tree = await markdown.parseMarkdown(text); const frontmatter = await extractFrontmatterFromTree(tree, extractOptions); return { frontmatter, text: renderToText(tree) }; } export async function patchFrontmatter( text: string, patches: SetKeyPatch[], ): Promise { const tree = await markdown.parseMarkdown(text); const frontmatter = collectNodesOfType(tree, "FrontMatterCode"); if (frontmatter.length === 0) { // No frontmatter found, create from the patches const patchedFrontmatter = applyPatches("", patches); return "---\n" + patchedFrontmatter + "---\n\n" + text; } else { // Existing frontmatter found, patch it const frontmatterText = renderToText(frontmatter[0]); const patchedFrontmatter = applyPatches(frontmatterText, patches); // Replace the frontmatter with the patched frontmatter in the original string return text.slice(0, frontmatter[0].from) + patchedFrontmatter + text.slice(frontmatter[0].to); } }