From 596b92cd559fa55c8e4a4492460cd28a709c0163 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Wed, 7 Jan 2026 15:44:04 +0100 Subject: [PATCH] Reindex refactor feature complete --- client/client.ts | 2 +- client/space_lua.ts | 4 +- plug-api/types/event.ts | 1 + plugs/index/index.plug.yaml | 39 +----------------- plugs/index/index.ts | 21 ++++++++-- plugs/index/item.test.ts | 2 +- plugs/index/page.test.ts | 2 +- plugs/index/page_links.test.ts | 44 ++++++++++++++++++++ plugs/index/page_links.ts | 52 +++++++++++------------- plugs/index/paragraph.test.ts | 47 +++++++++++++++++++++ plugs/index/paragraph.ts | 44 +++++++++++--------- plugs/index/queue.ts | 4 +- plugs/index/space_lua.test.ts | 41 +++++++++++++++++++ plugs/index/{script.ts => space_lua.ts} | 25 ++++++++---- plugs/index/space_style.test.ts | 39 ++++++++++++++++++ plugs/index/{style.ts => space_style.ts} | 19 ++++++--- plugs/index/table.test.ts | 36 ++++++++++++++++ plugs/index/table.ts | 22 ++++++---- plugs/index/tags.test.ts | 38 +++++++++++++++++ plugs/index/tags.ts | 45 +++++++++++--------- website/CHANGELOG.md | 2 +- 21 files changed, 390 insertions(+), 139 deletions(-) create mode 100644 plugs/index/page_links.test.ts create mode 100644 plugs/index/paragraph.test.ts create mode 100644 plugs/index/space_lua.test.ts rename plugs/index/{script.ts => space_lua.ts} (64%) create mode 100644 plugs/index/space_style.test.ts rename plugs/index/{style.ts => space_style.ts} (73%) create mode 100644 plugs/index/table.test.ts create mode 100644 plugs/index/tags.test.ts diff --git a/client/client.ts b/client/client.ts index 03b40c27..99a8fd67 100644 --- a/client/client.ts +++ b/client/client.ts @@ -35,7 +35,7 @@ import type { PageCreatingContent, PageCreatingEvent, } from "@silverbulletmd/silverbullet/type/event"; -import type { StyleObject } from "../plugs/index/style.ts"; +import type { StyleObject } from "../plugs/index/space_style.ts"; import { jitter, throttle } from "@silverbulletmd/silverbullet/lib/async"; import { EventedSpacePrimitives } from "./spaces/evented_space_primitives.ts"; import { HttpSpacePrimitives } from "./spaces/http_space_primitives.ts"; diff --git a/client/space_lua.ts b/client/space_lua.ts index 2b6c3cc7..fde63a60 100644 --- a/client/space_lua.ts +++ b/client/space_lua.ts @@ -1,5 +1,5 @@ import type { System } from "./plugos/system.ts"; -import type { ScriptObject } from "../plugs/index/script.ts"; +import type { SpaceLuaObject } from "../plugs/index/space_lua.ts"; import { LuaEnv, LuaRuntimeError, LuaStackFrame } from "./space_lua/runtime.ts"; import { parse as parseLua, parseExpressionString } from "./space_lua/parse.ts"; import { evalStatement } from "./space_lua/eval.ts"; @@ -24,7 +24,7 @@ export class SpaceLuaEnvironment { * @param system */ async reload() { - const allScripts: ScriptObject[] = await this.system.invokeFunction( + const allScripts: SpaceLuaObject[] = await this.system.invokeFunction( "index.queryLuaObjects", ["space-lua", { objectVariable: "script", diff --git a/plug-api/types/event.ts b/plug-api/types/event.ts index 28638dbf..e3b1e529 100644 --- a/plug-api/types/event.ts +++ b/plug-api/types/event.ts @@ -4,6 +4,7 @@ import type { EventSubscriptionDef } from "./manifest.ts"; export type IndexTreeEvent = { name: string; tree: ParseTree; + text: string; }; export type PageCreatingEvent = { diff --git a/plugs/index/index.plug.yaml b/plugs/index/index.plug.yaml index 4975de1b..3679d340 100644 --- a/plugs/index/index.plug.yaml +++ b/plugs/index/index.plug.yaml @@ -67,47 +67,13 @@ functions: events: - page:index - indexParagraphs: - path: "./paragraph.ts:indexParagraphs" - events: - - page:index - - # Backlinks - indexLinks: - path: "./page_links.ts:indexLinks" - events: - - page:index - - # Tables - indexTables: - path: "table.ts:indexTables" - events: - - page:index - # Headers headerComplete: path: header.ts:headerComplete events: - editor:complete - # Lua - indexSpaceLua: - path: script.ts:indexSpaceLua - events: - - page:index - - # Style - indexSpaceStyle: - path: style.ts:indexSpaceStyle - events: - - page:index - # Hashtags - indexTags: - path: tags.ts:indexTags - events: - - page:index - tagComplete: path: tags.ts:tagComplete events: @@ -136,6 +102,8 @@ functions: requireMode: rw requireEditor: page + # Refactoring Commands + renameDocumentCommand: path: "./refactor.ts:renameDocumentCommand" command: @@ -143,7 +111,6 @@ functions: requireMode: rw requireEditor: notpage - # Refactoring Commands extractToPageCommand: path: ./refactor.ts:extractToPageCommand command: @@ -437,8 +404,6 @@ config: type: number alias: type: string - asTemplate: - type: boolean header: type: object properties: diff --git a/plugs/index/index.ts b/plugs/index/index.ts index 628f2a1d..9b5219a9 100644 --- a/plugs/index/index.ts +++ b/plugs/index/index.ts @@ -11,11 +11,18 @@ import { indexPage as pageIndexPage } from "./page.ts"; import { indexData } from "./data.ts"; import { indexItems } from "./item.ts"; import { indexHeaders } from "./header.ts"; +import { indexParagraphs } from "./paragraph.ts"; +import { indexLinks } from "./page_links.ts"; +import { indexTables } from "./table.ts"; +import { indexSpaceLua } from "./space_lua.ts"; +import { indexSpaceStyle } from "./space_style.ts"; +import { indexTags } from "./tags.ts"; export type IndexerFunction = ( pageMeta: PageMeta, frontmatter: FrontMatter, tree: ParseTree, + text: string, ) => Promise[]>; const allIndexers: IndexerFunction[] = [ @@ -23,20 +30,26 @@ const allIndexers: IndexerFunction[] = [ indexData, indexItems, indexHeaders, + indexParagraphs, + indexLinks, + indexTables, + indexSpaceLua, + indexSpaceStyle, + indexTags, ]; -export async function indexPage({ name, tree }: IndexTreeEvent) { +export async function indexPage({ name, tree, text }: IndexTreeEvent) { const pageMeta = await space.getPageMeta(name); const frontmatter = extractFrontMatter(tree); - console.log("Now going to index page", name); + // console.log("Now going to index page", name); // Index the page const index = await Promise.all(allIndexers.map((indexer) => { - return indexer(pageMeta, frontmatter, tree); + return indexer(pageMeta, frontmatter, tree, text); })); - console.log("Found these objects", index.flat()); + // console.log("Found these objects", index.flat()); await indexObjects(name, index.flat()); } diff --git a/plugs/index/item.test.ts b/plugs/index/item.test.ts index ef22d52f..cd7f0dcd 100644 --- a/plugs/index/item.test.ts +++ b/plugs/index/item.test.ts @@ -15,7 +15,7 @@ const itemsMd = ` * [ ] Sub task `; -Deno.test("Test item extraction", async () => { +Deno.test("Test item indexing", async () => { createMockSystem(); const tree = parseMarkdown(itemsMd); const frontmatter = extractFrontMatter(tree); diff --git a/plugs/index/page.test.ts b/plugs/index/page.test.ts index c5570c37..371708e8 100644 --- a/plugs/index/page.test.ts +++ b/plugs/index/page.test.ts @@ -15,7 +15,7 @@ hello: attribute `.trim(); -Deno.test("Test page indexer", async () => { +Deno.test("Test page indexing", async () => { createMockSystem(); const tree = parseMarkdown(testPage); const frontmatter = extractFrontMatter(tree); diff --git a/plugs/index/page_links.test.ts b/plugs/index/page_links.test.ts new file mode 100644 index 00000000..fc37cc55 --- /dev/null +++ b/plugs/index/page_links.test.ts @@ -0,0 +1,44 @@ +import { parseMarkdown } from "../../client/markdown_parser/parser.ts"; +import { createMockSystem } from "../../plug-api/system_mock.ts"; +import type { PageMeta } from "@silverbulletmd/silverbullet/type/index"; +import { extractFrontMatter } from "./frontmatter.ts"; +import { assertEquals } from "@std/assert"; +import { indexLinks } from "./page_links.ts"; + +const testPage = ` +--- +attribute: "[[fm-link]]" +--- +This is a [[page-link]] to [[aliased-link|aliased]], or [this](md-link), and [[broken]] +`.trim(); + +Deno.test("Test page link indexing", async () => { + const { space } = createMockSystem(); + + // Create dummy targets to avoid a lot of aspiring pages + await space.writePage("page-link", ""); + await space.writePage("fm-link", ""); + await space.writePage("aliased-link", ""); + await space.writePage("folder/md-link", ""); + + const tree = parseMarkdown(testPage); + const frontmatter = extractFrontMatter(tree); + + const pageMeta: PageMeta = { + ref: "folder/test", + name: "folder/test", + tag: "page", + created: "", + lastModified: "", + perm: "rw", + }; + + const objects = await indexLinks(pageMeta, frontmatter, tree, testPage); + // console.log(objects); + assertEquals(objects.length, 6); // 5 links + 1 aspiring page + assertEquals(objects[0].toPage, "fm-link"); + assertEquals(objects[1].toPage, "page-link"); + assertEquals(objects[2].toPage, "aliased-link"); + assertEquals(objects[2].alias, "aliased"); + assertEquals(objects[3].toPage, "folder/md-link"); // relative +}); diff --git a/plugs/index/page_links.ts b/plugs/index/page_links.ts index e4e45fbb..8f49b1b2 100644 --- a/plugs/index/page_links.ts +++ b/plugs/index/page_links.ts @@ -1,16 +1,16 @@ import { collectNodesOfType, findNodeOfType, + type ParseTree, renderToText, traverseTree, } from "@silverbulletmd/silverbullet/lib/tree"; -import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; import { isLocalURL, resolveMarkdownLink, } from "@silverbulletmd/silverbullet/lib/resolve"; -import { indexObjects, queryLuaObjects } from "./api.ts"; -import { extractFrontMatter } from "./frontmatter.ts"; +import { queryLuaObjects } from "./api.ts"; +import type { FrontMatter } from "./frontmatter.ts"; import { updateITags } from "./tags.ts"; import { getNameFromPath, @@ -23,7 +23,10 @@ import { wikiLinkRegex, } from "../../client/markdown_parser/constants.ts"; import { lua, space } from "@silverbulletmd/silverbullet/syscalls"; -import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; +import type { + ObjectValue, + PageMeta, +} from "@silverbulletmd/silverbullet/type/index"; export type LinkObject = ObjectValue< { @@ -35,7 +38,6 @@ export type LinkObject = ObjectValue< pos: number; snippet: string; alias?: string; - asTemplate: boolean; toFile?: never; } | { // Document Link @@ -46,7 +48,6 @@ export type LinkObject = ObjectValue< pos: number; snippet: string; alias?: string; - asTemplate: boolean; toPage?: never; } >; @@ -64,16 +65,21 @@ export type AspiringPageObject = ObjectValue<{ name: string; }>; -export async function indexLinks({ name, tree }: IndexTreeEvent) { - const links: ObjectValue[] = []; - const frontmatter = await extractFrontMatter(tree); - const pageText = renderToText(tree); +export async function indexLinks( + pageMeta: PageMeta, + frontmatter: FrontMatter, + tree: ParseTree, + pageText: string, +) { + const objects: ObjectValue[] = []; // If this is a meta template page, we don't want to index links if (frontmatter.tags?.find((t) => t.startsWith("meta/template"))) { - return; + return []; } + const name = pageMeta.name; + traverseTree(tree, (n): boolean => { // Index [[WikiLinks]] if (n.type === "WikiLink") { @@ -88,7 +94,6 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) { snippet: extractSnippetAroundIndex(pageText, pos), pos, page: name, - asTemplate: false, }; const ref = parseToRef(url); @@ -105,7 +110,7 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) { link.alias = wikiLinkAlias.children![0].text!; } updateITags(link, frontmatter); - links.push(link); + objects.push(link); return true; } @@ -135,7 +140,6 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) { snippet: extractSnippetAroundIndex(pageText, pos), pos, page: name, - asTemplate: false, }; const ref = parseToRef(url); @@ -152,7 +156,7 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) { link.alias = alias; } updateITags(link, frontmatter); - links.push(link); + objects.push(link); return true; } @@ -176,7 +180,6 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) { page: name, snippet: extractSnippetAroundIndex(pageText, pos), pos: pos, - asTemplate: false, }; const ref = parseToRef(stringRef); @@ -193,24 +196,19 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) { link.alias = alias; } updateITags(link, frontmatter); - links.push(link); + objects.push(link); } } } return false; }); - // console.log("Found", links, "page link(s)"); - if (links.length > 0) { - await indexObjects(name, links); - } - // Now let's check which are aspiring pages - const aspiringPages: ObjectValue[] = []; - for (const link of links) { + for (const link of objects.slice()) { if (link.toPage) { + // TODO: Optimize fileExists to be more efficient if (!await space.fileExists(`${link.toPage}.md`)) { - aspiringPages.push({ + objects.push({ ref: `${name}@${link.pos}`, tag: "aspiring-page", page: name, @@ -228,9 +226,7 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) { } } - if (aspiringPages.length > 0) { - await indexObjects(name, aspiringPages); - } + return objects; } export async function getBackLinks( diff --git a/plugs/index/paragraph.test.ts b/plugs/index/paragraph.test.ts new file mode 100644 index 00000000..3e659091 --- /dev/null +++ b/plugs/index/paragraph.test.ts @@ -0,0 +1,47 @@ +import { parseMarkdown } from "../../client/markdown_parser/parser.ts"; +import { createMockSystem } from "../../plug-api/system_mock.ts"; +import type { PageMeta } from "@silverbulletmd/silverbullet/type/index"; +import { extractFrontMatter } from "./frontmatter.ts"; +import { assertEquals } from "@std/assert"; +import { indexParagraphs } from "./paragraph.ts"; + +const testPage = ` +#tag-only + +Some paragraph + +Another paragraph that is #tagged + +* Item paragraph (don't index) +`.trim(); + +Deno.test("Test paragraph indexing", async () => { + const { config } = createMockSystem(); + + config.set("index.paragraph.all", true); + let tree = parseMarkdown(testPage); + let frontmatter = extractFrontMatter(tree); + + const pageMeta: PageMeta = { + ref: "test", + name: "test", + tag: "page", + created: "", + lastModified: "", + perm: "rw", + }; + + let paragraphs = await indexParagraphs(pageMeta, frontmatter, tree); + console.log(paragraphs); + + assertEquals(paragraphs.length, 2); + assertEquals([...new Set(paragraphs[0].itags)], ["paragraph", "tag-only"]); + assertEquals(paragraphs[1].tags, ["tagged"]); + + config.set("index.paragraph.all", false); + tree = parseMarkdown(testPage); + frontmatter = extractFrontMatter(tree); + + paragraphs = await indexParagraphs(pageMeta, frontmatter, tree); + assertEquals(paragraphs.length, 1); +}); diff --git a/plugs/index/paragraph.ts b/plugs/index/paragraph.ts index 801c8622..240f1a10 100644 --- a/plugs/index/paragraph.ts +++ b/plugs/index/paragraph.ts @@ -1,15 +1,18 @@ -import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; -import { indexObjects } from "./api.ts"; import { findParentMatching, + type ParseTree, renderToText, traverseTree, } from "@silverbulletmd/silverbullet/lib/tree"; import { cleanTags, collectTags, updateITags } from "./tags.ts"; -import { extractFrontMatter } from "./frontmatter.ts"; -import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; +import type { FrontMatter } from "./frontmatter.ts"; +import type { + ObjectValue, + PageMeta, +} from "@silverbulletmd/silverbullet/type/index"; import { system } from "@silverbulletmd/silverbullet/syscalls"; import { cleanAttributes, collectAttributes } from "./attribute.ts"; +import { deepClone } from "@silverbulletmd/silverbullet/lib/json"; /** ParagraphObject An index object for the top level text nodes */ export type ParagraphObject = ObjectValue< @@ -20,16 +23,18 @@ export type ParagraphObject = ObjectValue< } & Record >; -export async function indexParagraphs({ name: page, tree }: IndexTreeEvent) { +export async function indexParagraphs( + pageMeta: PageMeta, + frontmatter: FrontMatter, + tree: ParseTree, +) { const shouldIndexAll = await system.getConfig( "index.paragraph.all", - true, + false, ); const objects: ParagraphObject[] = []; - const frontmatter = extractFrontMatter(tree); - traverseTree(tree, (p) => { if (p.type !== "Paragraph") { return false; @@ -37,26 +42,27 @@ export async function indexParagraphs({ name: page, tree }: IndexTreeEvent) { if (findParentMatching(p, (n) => n.type === "ListItem")) { // Not looking at paragraphs nested in a list - return false; + return true; } const fullText = renderToText(p); - // Collect tags and remove from the tree + // Collect tags const tags = collectTags(p); if (tags.length === 0 && !shouldIndexAll) { // Don't index paragraphs without a hashtag - return false; + return true; } // Extract attributes const attrs = collectAttributes(p); - // Clean tree - cleanTags(p); - cleanAttributes(p); - const text = renderToText(p); + // Clean tree, just to check if it's effectively empty or not + const cleanP = deepClone(p, ["parent"]); + cleanTags(cleanP); + cleanAttributes(cleanP); + const text = renderToText(cleanP); if (!text.trim()) { // Empty paragraph, just tags and attributes maybe @@ -66,9 +72,9 @@ export async function indexParagraphs({ name: page, tree }: IndexTreeEvent) { const pos = p.from!; const paragraph: ParagraphObject = { tag: "paragraph", - ref: `${page}@${pos}`, + ref: `${pageMeta.name}@${pos}`, text: fullText, - page, + page: pageMeta.name, pos, ...attrs, }; @@ -84,7 +90,5 @@ export async function indexParagraphs({ name: page, tree }: IndexTreeEvent) { return true; }); - // console.log("Paragraph objects", objects); - - await indexObjects(page, objects); + return objects; } diff --git a/plugs/index/queue.ts b/plugs/index/queue.ts index f20ec0fb..25fc7bc3 100644 --- a/plugs/index/queue.ts +++ b/plugs/index/queue.ts @@ -24,9 +24,10 @@ export async function reindexSpace() { console.log("Queing", files.length, "pages to be indexed."); // Queue all file names to be indexed + const startTime = Date.now(); await mq.batchSend("indexQueue", files.map((file) => file.name)); await mq.awaitEmptyQueue("indexQueue"); - console.log("And done with full index!"); + console.log("Done with full index after", Date.now() - startTime, "ms"); } setTimeout(updateIndexProgressInUI, uiUpdateInterval); @@ -85,6 +86,7 @@ async function indexFile(path: string) { await events.dispatchEvent("page:index", { name, tree, + text, } as IndexTreeEvent); } else { await events.dispatchEvent("document:index", path); diff --git a/plugs/index/space_lua.test.ts b/plugs/index/space_lua.test.ts new file mode 100644 index 00000000..87334121 --- /dev/null +++ b/plugs/index/space_lua.test.ts @@ -0,0 +1,41 @@ +import { parseMarkdown } from "../../client/markdown_parser/parser.ts"; +import { createMockSystem } from "../../plug-api/system_mock.ts"; +import type { PageMeta } from "@silverbulletmd/silverbullet/type/index"; +import { extractFrontMatter } from "./frontmatter.ts"; +import { assertEquals } from "@std/assert"; +import { indexSpaceLua } from "./space_lua.ts"; + +const testPage = ` +Hello +\`\`\`space-lua +function sup() +end +\`\`\` + +\`\`\`space-lua +-- priority: 10 +function sup() +end +\`\`\` +`.trim(); + +Deno.test("Test space lua indexing", async () => { + createMockSystem(); + + const tree = parseMarkdown(testPage); + const frontmatter = extractFrontMatter(tree); + + const pageMeta: PageMeta = { + ref: "folder/test", + name: "folder/test", + tag: "page", + created: "", + lastModified: "", + perm: "rw", + }; + + const objects = await indexSpaceLua(pageMeta, frontmatter, tree); + assertEquals(objects.length, 2); + assertEquals(objects[0].priority, undefined); + assertEquals(objects[1].priority, 10); +}); diff --git a/plugs/index/script.ts b/plugs/index/space_lua.ts similarity index 64% rename from plugs/index/script.ts rename to plugs/index/space_lua.ts index e70ada8f..6a506dbe 100644 --- a/plugs/index/script.ts +++ b/plugs/index/space_lua.ts @@ -1,18 +1,25 @@ -import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; import { collectNodesOfType, findNodeOfType, + type ParseTree, } from "@silverbulletmd/silverbullet/lib/tree"; -import { indexObjects } from "./api.ts"; -import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; +import type { + ObjectValue, + PageMeta, +} from "@silverbulletmd/silverbullet/type/index"; +import type { FrontMatter } from "./frontmatter.ts"; -export type ScriptObject = ObjectValue<{ +export type SpaceLuaObject = ObjectValue<{ script: string; priority?: number; }>; -export async function indexSpaceLua({ name, tree }: IndexTreeEvent) { - const allScripts: ScriptObject[] = []; +export function indexSpaceLua( + pageMeta: PageMeta, + _frontmatter: FrontMatter, + tree: ParseTree, +) { + const allSpaceLuas: SpaceLuaObject[] = []; collectNodesOfType(tree, "FencedCode").map((t) => { const codeInfoNode = findNodeOfType(t, "CodeInfo"); if (!codeInfoNode) { @@ -31,12 +38,12 @@ export async function indexSpaceLua({ name, tree }: IndexTreeEvent) { // Parse out "-- priority: " const priority = codeText.match(/--\s*priority:\s*(-?\d+)/)?.[1]; - allScripts.push({ - ref: `${name}@${t.from!}`, + allSpaceLuas.push({ + ref: `${pageMeta.name}@${t.from!}`, tag: "space-lua", script: codeText, priority: priority !== undefined ? +priority : undefined, }); }); - await indexObjects(name, allScripts); + return Promise.resolve(allSpaceLuas); } diff --git a/plugs/index/space_style.test.ts b/plugs/index/space_style.test.ts new file mode 100644 index 00000000..a654ca07 --- /dev/null +++ b/plugs/index/space_style.test.ts @@ -0,0 +1,39 @@ +import { parseMarkdown } from "../../client/markdown_parser/parser.ts"; +import { createMockSystem } from "../../plug-api/system_mock.ts"; +import type { PageMeta } from "@silverbulletmd/silverbullet/type/index"; +import { extractFrontMatter } from "./frontmatter.ts"; +import { assertEquals } from "@std/assert"; +import { indexSpaceStyle } from "./space_style.ts"; + +const testPage = ` +Hello +\`\`\`space-style +.hello {} +\`\`\` + +\`\`\`space-style +/* priority: 10 */ +.hello2 {} +\`\`\` +`.trim(); + +Deno.test("Test space style indexing", async () => { + createMockSystem(); + + const tree = parseMarkdown(testPage); + const frontmatter = extractFrontMatter(tree); + + const pageMeta: PageMeta = { + ref: "folder/test", + name: "folder/test", + tag: "page", + created: "", + lastModified: "", + perm: "rw", + }; + + const objects = await indexSpaceStyle(pageMeta, frontmatter, tree); + assertEquals(objects.length, 2); + assertEquals(objects[0].priority, undefined); + assertEquals(objects[1].priority, 10); +}); diff --git a/plugs/index/style.ts b/plugs/index/space_style.ts similarity index 73% rename from plugs/index/style.ts rename to plugs/index/space_style.ts index 69811981..659e9e21 100644 --- a/plugs/index/style.ts +++ b/plugs/index/space_style.ts @@ -1,17 +1,24 @@ -import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; import { collectNodesOfType, findNodeOfType, + type ParseTree, } from "@silverbulletmd/silverbullet/lib/tree"; -import { indexObjects } from "./api.ts"; -import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; +import type { + ObjectValue, + PageMeta, +} from "@silverbulletmd/silverbullet/type/index"; +import type { FrontMatter } from "./frontmatter.ts"; export type StyleObject = ObjectValue<{ style: string; priority?: number; }>; -export async function indexSpaceStyle({ name, tree }: IndexTreeEvent) { +export function indexSpaceStyle( + pageMeta: PageMeta, + _frontmatter: FrontMatter, + tree: ParseTree, +) { const allStyles: StyleObject[] = []; // Also collect CSS from custom styles in config @@ -37,12 +44,12 @@ export async function indexSpaceStyle({ name, tree }: IndexTreeEvent) { const priority = codeText.match(/\/\*+\s*priority:\s*(-?\d+)/)?.[1]; allStyles.push({ - ref: `${name}@${t.from!}`, + ref: `${pageMeta.name}@${t.from!}`, tag: "space-style", style: codeText, priority: priority !== undefined ? +priority : undefined, }); }); - await indexObjects(name, allStyles); + return Promise.resolve(allStyles); } diff --git a/plugs/index/table.test.ts b/plugs/index/table.test.ts new file mode 100644 index 00000000..13f1f5be --- /dev/null +++ b/plugs/index/table.test.ts @@ -0,0 +1,36 @@ +import { parseMarkdown } from "../../client/markdown_parser/parser.ts"; +import { createMockSystem } from "../../plug-api/system_mock.ts"; +import type { PageMeta } from "@silverbulletmd/silverbullet/type/index"; +import { extractFrontMatter } from "./frontmatter.ts"; +import { assertEquals } from "@std/assert"; +import { indexTables } from "./table.ts"; + +const testPage = ` +| name | age | +|----------|----------| +| Frank | 20 | +| Fred | 21 | +`.trim(); + +Deno.test("Test table indexing", async () => { + createMockSystem(); + + const tree = parseMarkdown(testPage); + const frontmatter = extractFrontMatter(tree); + + const pageMeta: PageMeta = { + ref: "test", + name: "test", + tag: "page", + created: "", + lastModified: "", + perm: "rw", + }; + + const datas = await indexTables(pageMeta, frontmatter, tree); + assertEquals(datas.length, 2); + assertEquals(datas[0].name, "Frank"); + assertEquals(datas[0].age, "20"); + assertEquals(datas[1].name, "Fred"); + assertEquals(datas[1].age, "21"); +}); diff --git a/plugs/index/table.ts b/plugs/index/table.ts index 7ffbc503..93d03935 100644 --- a/plugs/index/table.ts +++ b/plugs/index/table.ts @@ -1,4 +1,3 @@ -import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; import { renderToText, replaceNodesMatching, @@ -8,9 +7,12 @@ import { collectNodesOfType, type ParseTree, } from "@silverbulletmd/silverbullet/lib/tree"; -import { indexObjects } from "./api.ts"; -import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; +import type { + ObjectValue, + PageMeta, +} from "@silverbulletmd/silverbullet/type/index"; import { extractHashtag } from "@silverbulletmd/silverbullet/lib/tags"; +import type { FrontMatter } from "./frontmatter.ts"; type TableRowObject = & ObjectValue<{ @@ -47,7 +49,11 @@ function concatChildrenTextsPreserveLinks(nodes: ParseTree[]): string { return nodes.map((c) => renderToText(c)).join("").trim(); } -export async function indexTables({ name: pageName, tree }: IndexTreeEvent) { +export function indexTables( + pageMeta: PageMeta, + _frontmatter: FrontMatter, + tree: ParseTree, +) { const result: ObjectValue[] = []; collectNodesMatching( @@ -72,11 +78,11 @@ export async function indexTables({ name: pageName, tree }: IndexTreeEvent) { const cells = collectNodesOfType(row, "TableCell"); const tableRow: TableRowObject = { - tableref: `${pageName}@${table.from}`, - ref: `${pageName}@${row.from}`, + tableref: `${pageMeta.name}@${table.from}`, + ref: `${pageMeta.name}@${row.from}`, tag: "table", tags: [...tags], - page: pageName, + page: pageMeta.name, pos: row.from!, }; cells.forEach((c, i) => { @@ -94,5 +100,5 @@ export async function indexTables({ name: pageName, tree }: IndexTreeEvent) { }, ); - await indexObjects(pageName, result); + return Promise.resolve(result); } diff --git a/plugs/index/tags.test.ts b/plugs/index/tags.test.ts new file mode 100644 index 00000000..37c2f5b8 --- /dev/null +++ b/plugs/index/tags.test.ts @@ -0,0 +1,38 @@ +import { parseMarkdown } from "../../client/markdown_parser/parser.ts"; +import { createMockSystem } from "../../plug-api/system_mock.ts"; +import type { PageMeta } from "@silverbulletmd/silverbullet/type/index"; +import { extractFrontMatter } from "./frontmatter.ts"; +import { assertEquals } from "@std/assert"; +import { indexTags } from "./tags.ts"; + +const testPage = ` +#page-tag + +* Item #item-tag +* [ ] Task #task-tag +`.trim(); + +Deno.test("Test tag indexing", async () => { + createMockSystem(); + + const tree = parseMarkdown(testPage); + const frontmatter = extractFrontMatter(tree); + + const pageMeta: PageMeta = { + ref: "folder/test", + name: "folder/test", + tag: "page", + created: "", + lastModified: "", + perm: "rw", + }; + + const objects = await indexTags(pageMeta, frontmatter, tree); + assertEquals(objects.length, 3); + assertEquals(objects[0].name, "page-tag"); + assertEquals(objects[0].parent, "page"); + assertEquals(objects[1].name, "item-tag"); + assertEquals(objects[1].parent, "item"); + assertEquals(objects[2].name, "task-tag"); + assertEquals(objects[2].parent, "task"); +}); diff --git a/plugs/index/tags.ts b/plugs/index/tags.ts index 386c161d..62b34148 100644 --- a/plugs/index/tags.ts +++ b/plugs/index/tags.ts @@ -1,6 +1,5 @@ -import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; -import { extractFrontMatter, type FrontMatter } from "./frontmatter.ts"; -import { indexObjects, queryLuaObjects } from "./api.ts"; +import type { FrontMatter } from "./frontmatter.ts"; +import { queryLuaObjects } from "./api.ts"; import { addParentPointers, collectNodesOfType, @@ -9,7 +8,10 @@ import { replaceNodesMatching, traverseTree, } from "@silverbulletmd/silverbullet/lib/tree"; -import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; +import type { + ObjectValue, + PageMeta, +} from "@silverbulletmd/silverbullet/type/index"; import type { CompleteEvent } from "@silverbulletmd/silverbullet/type/client"; import { tagRegex } from "../../client/markdown_parser/constants.ts"; import { extractHashtag } from "@silverbulletmd/silverbullet/lib/tags"; @@ -20,10 +22,17 @@ export type TagObject = ObjectValue<{ parent: string; }>; -export async function indexTags({ name, tree }: IndexTreeEvent) { +/** + * Handles indexing of page, item and task level tags, data tags are handled in data.ts + */ +export function indexTags( + pageMeta: PageMeta, + frontmatter: FrontMatter, + tree: ParseTree, +) { const tags = new Set(); // name:parent addParentPointers(tree); - const pageTags: string[] = extractFrontMatter(tree).tags || []; + const pageTags: string[] = frontmatter.tags || []; for (const pageTag of pageTags) { tags.add(`${pageTag}:page`); } @@ -40,20 +49,16 @@ export async function indexTags({ name, tree }: IndexTreeEvent) { tags.add(`${tagName}:page`); } }); - // console.log("Indexing these tags", tags); - await indexObjects( - name, - [...tags].map((tag) => { - const [tagName, parent] = tag.split(":"); - return { - ref: tag, - tag: "tag", - name: tagName, - page: name, - parent, - }; - }), - ); + return Promise.resolve([...tags].map((tag) => { + const [tagName, parent] = tag.split(":"); + return { + ref: tag, + tag: "tag", + name: tagName, + page: pageMeta.name, + parent, + }; + })); } export async function tagComplete(completeEvent: CompleteEvent) { diff --git a/website/CHANGELOG.md b/website/CHANGELOG.md index 054ca06e..542c2bcb 100644 --- a/website/CHANGELOG.md +++ b/website/CHANGELOG.md @@ -6,7 +6,7 @@ Whenever a commit is pushed to the `main` branch, within ~10 minutes, it will be * Removed full-text search plug from the main distribution, this has now been moved to [a separate repo](https://github.com/silverbulletmd/basic-search) (installable via the library manager). This dramatically improves indexing speed. But actually: install [Silversearch](https://github.com/MrMugame/silversearch) instead. * Indexer refactor: * For consistency with items, `task` `refs` now point to the item’s position resulting in a slight positional shift, if you have code relying on this, you may have to adjust it - * + * `page:index` now also received `text` attribute * Disabled indexing all paragraph text (even those not tagged) by default, this caused significant indexing overhead. [See discussion](https://community.silverbullet.md/t/who-is-using-paragraph-for-queries/3686). * To re-enable: `config.set("index.paragraph.all", true)` * Production builds now include sourcemaps for easier debugging in browser DevTools. If you don't want to serve sourcemaps publicly, you can block `*.js.map` files at your reverse proxy level (see [[TLS#Blocking sourcemaps]]).