wushenghua/plainleaf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add validation and error handling for object indexing

Browse Source 
- Introduced a new `validate` callback for `tag.define`
- Added `index.validateObjects` API to validate and transform objects before indexing.
- Updated relevant documentation and examples to reflect these changes.
This commit is contained in:
Zef Hemel
2026-02-03 15:59:04 +01:00
parent 3d1e569bd2
commit 0773df31e2
21 changed files with 255 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/data/object_index.ts
+74 -15
View File
@@ -29,6 +29,7 @@ type TagDefinition = {
metatable?: any;
mustValidate?: boolean;
schema?: any;
validate?: (o: ObjectValue) => Promise<string | null | undefined>;
transform?: (
o: ObjectValue,
) =>
@@ -38,6 +39,12 @@ type TagDefinition = {
| null;
};
export class ObjectValidationError extends Error {
constructor(message: string, readonly object: ObjectValue) {
super(message);
}
}
export class ObjectIndex {
constructor(
private ds: DataStore,
@@ -285,6 +292,29 @@ export class ObjectIndex {
page: string,
objects: ObjectValue<T>[],
): Promise<void> {
const kvs = await this.processObjectsToKVs<T>(page, objects, false);
if (kvs.length > 0) {
return this.batchSet(page, kvs);
} else {
return Promise.resolve();
}
}
/**
* Validate and transform objects, throws a ValidationError when it fails
* @param page
* @param objects
* @throw ValidationError
*/
public async validateObjects<T>(page: string, objects: ObjectValue<T>[]) {
await this.processObjectsToKVs(page, objects, true);
}
private async processObjectsToKVs<T>(
page: string,
objects: ObjectValue<T>[],
throwOnValidationErrors: boolean,
): Promise<KV<T>[]> {
const kvs: KV<T>[] = [];
const tagDefinitions: Record<string, TagDefinition> = this.config.get(
"tags",
@@ -301,21 +331,54 @@ export class ObjectIndex {
const allTags = [obj.tag, ...obj.tags || []];
for (const tag of allTags) {
const tagDefinition = tagDefinitions[tag];
// Validate object if required
if (tagDefinition?.mustValidate && tagDefinition?.schema) {
// Validate object based on schema if required
if (
tagDefinition?.schema &&
(tagDefinition?.mustValidate || throwOnValidationErrors)
) {
const validationError = validateObject(tagDefinition?.schema, obj);
if (validationError) {
console.error(
`Object failed ${tag} validation so won't be indexed:`,
obj,
"Error:",
validationError,
);
continue;
if (!throwOnValidationErrors) {
console.warn(
`Object failed ${tag} validation so won't be indexed:`,
obj,
"Validation error:",
validationError,
);
continue;
} else {
throw new ObjectValidationError(validationError, obj);
}
}
}
// Validate object based on validate callback if required
if (
tagDefinition?.validate &&
(tagDefinition?.mustValidate || throwOnValidationErrors)
) {
const validationError = await tagDefinition.validate(obj);
if (validationError) {
if (!throwOnValidationErrors) {
console.warn(
`Object failed ${tag} validation so won't be indexed:`,
obj,
"Validation error:",
validationError,
);
continue;
} else {
throw new ObjectValidationError(validationError, obj);
}
}
}
// Transform object
if (tagDefinition?.transform) {
let newObjects = await tagDefinition.transform(obj);
let newObjects;
try {
newObjects = await tagDefinition.transform(obj);
} catch (e: any) {
throw new ObjectValidationError(e.message, obj);
}
if (!newObjects) {
// null value returned, just index as usual
@@ -367,11 +430,7 @@ export class ObjectIndex {
}
}
}
if (kvs.length > 0) {
return this.batchSet(page, kvs);
} else {
return Promise.resolve();
}
return kvs;
}
deleteObject(
client/plugos/syscalls/index.ts
+23 -1
View File
@@ -4,7 +4,10 @@ import type {
LuaQueryCollection,
} from "../../space_lua/query_collection.ts";
import type { ObjectIndex } from "../../data/object_index.ts";
import {
type ObjectIndex,
ObjectValidationError,
} from "../../data/object_index.ts";
import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index";
import type { Client } from "../../client.ts";
import type { LuaTable } from "../../space_lua/runtime.ts";
@@ -30,6 +33,25 @@ export function indexSyscalls(
): Promise<void> => {
return objectIndex.indexObjects(page, objects);
},
"index.validateObjects": async (
_ctx,
page: string,
objects: ObjectValue[],
): Promise<{ error: string; object: ObjectValue } | null> => {
try {
await objectIndex.validateObjects(page, objects);
return null;
} catch (e: any) {
if (e instanceof ObjectValidationError) {
return {
error: e.message,
object: e.object,
};
} else {
throw e;
}
}
},
"index.getObjectByRef": (
_ctx,
page: string,
libraries/Library/Std/Config.md
+2
View File
@@ -333,6 +333,8 @@ config.define("tags", {
schema = schema.schema(),
-- Whether or not an object HAS to validate to be indexed (defaults to false), has a performance penalty
mustValidate = schema.boolean(),
-- Additional custom validation logic
validate = schema.func(),
-- Invoked by the object indexer, takes a proposed object as input, returns an array of objects (can be empty table to skip indexing altogether)
transform = schema.func(),
},
libraries/Library/Std/Infrastructure/Builtin Tags.md
+1 -1
View File
@@ -135,7 +135,7 @@ tag.define {
name = readOnlyType("string"),
page = readOnlyType("string"),
parent = readOnlyType("string"),
pos = readOnlyType("string"),
pos = readOnlyType("number"),
text = readOnlyType("string"),
},
},
plug-api/syscalls/index.ts
+7
View File
@@ -20,6 +20,13 @@ export function indexObjects<T>(
return syscall("index.indexObjects", page, objects);
}
export function validateObjects<T>(
page: string,
objects: ObjectValue<T>[],
): Promise<{ error: string; object: ObjectValue } | null> {
return syscall("index.validateObjects", page, objects);
}
/**
* Queries objects using a Lua-based collection query
* @param tag - The tag to filter objects by
plug-api/types/index.ts
+1
View File
@@ -50,6 +50,7 @@ export type SyscallMeta = {
export type ObjectValue<T = any> = {
ref: string;
tag: string; // main tag
range?: [number, number];
tags?: string[];
itags?: string[]; // implicit or inherited tags (inherited from the page for instance)
} & T;
plugs/index/data.ts
+10 -4
View File
@@ -24,6 +24,7 @@ export function indexData(
frontmatter: FrontMatter,
tree: ParseTree,
) {
const separator = "---";
const dataObjects: ObjectValue<DataObject>[] = [];
const tagObjects: Map<string, ObjectValue<TagObject>> = new Map();
@@ -44,24 +45,29 @@ export function indexData(
const codeText = codeTextNode.children![0].text!;
const dataType = fenceType === "data" ? "data" : fenceType.substring(1);
try {
const docs = codeText.split("---");
const docs = codeText.split(separator);
let cursor = 0;
// We support multiple YAML documents in one block
for (let i = 0; i < docs.length; i++) {
const docStart = t.from! + cursor;
const docEnd = docStart + docs[i].length;
const doc = YAML.load(docs[i]);
if (!doc) {
cursor += docs[i].length;
continue;
}
const pos = t.from! + i;
const dataObj = {
ref: `${pageMeta.name}@${pos}`,
ref: `${pageMeta.name}@${docStart}`,
tag: dataType,
itags: ["data"],
pos: docStart,
range: [docStart, docEnd],
...doc,
pos,
page: pageMeta.name,
};
updateITags(dataObj, frontmatter);
dataObjects.push(dataObj);
cursor += docs[i].length;
}
tagObjects.set(dataType, {
ref: dataType,
plugs/index/frontmatter.ts
+18 -13
View File
@@ -13,7 +13,11 @@ import type { CompleteEvent } from "@silverbulletmd/silverbullet/type/client";
import { determineTags } from "./cheap_yaml.ts";
import { attributeCompletion } from "./complete.ts";
export type FrontMatter = { tags?: string[] } & Record<string, any>;
export type FrontMatter = {
tags?: string[];
// Location in the document where the frontmatter appears
range?: [number, number];
} & Record<string, any>;
export type FrontMatterExtractOptions = {
removeKeys?: string[];
@@ -31,7 +35,7 @@ export function extractFrontMatter(
options: FrontMatterExtractOptions = {},
): FrontMatter {
addParentPointers(tree);
let data: FrontMatter = {
let frontmatter: FrontMatter = {
tags: [],
};
const tags: string[] = [];
@@ -76,22 +80,23 @@ export function extractFrontMatter(
if (t.type === "FrontMatter") {
const yamlNode = t.children![1].children![0];
const yamlText = renderToText(yamlNode);
frontmatter.range = [t.from!, t.to!];
try {
const parsedData: any = cleanupJSON(YAML.load(yamlText));
// console.log("Parsed front matter", parsedData);
const newData = { ...parsedData };
data = { ...data, ...parsedData };
frontmatter = { ...frontmatter, ...parsedData };
// Make sure we have a tags array
if (!data.tags) {
data.tags = [];
if (!frontmatter.tags) {
frontmatter.tags = [];
}
// Normalize tags to an array
// support "tag1, tag2" as well as "tag1 tag2" as well as "#tag1 #tag2" notations
if (typeof data.tags === "string") {
tags.push(...(data.tags as string).split(/,\s*|\s+/));
if (typeof frontmatter.tags === "string") {
tags.push(...(frontmatter.tags as string).split(/,\s*|\s+/));
}
if (Array.isArray(data.tags)) {
tags.push(...data.tags);
if (Array.isArray(frontmatter.tags)) {
tags.push(...frontmatter.tags);
}
if (options.removeKeys && options.removeKeys.length > 0) {
@@ -127,7 +132,7 @@ export function extractFrontMatter(
const name = nameNode.children![0].text!;
const val = valueNode.children![0].text!;
try {
data[name] = cleanupJSON(YAML.load(val));
frontmatter[name] = cleanupJSON(YAML.load(val));
} catch (e: any) {
console.error("Error parsing attribute value as YAML", val, e);
}
@@ -139,7 +144,7 @@ export function extractFrontMatter(
});
try {
data.tags = [
frontmatter.tags = [
...new Set([...tags.map((t) => {
// Always treat tags as strings
const tagAsString = String(t);
@@ -152,9 +157,9 @@ export function extractFrontMatter(
}
// Expand property names (e.g. "foo.bar" => { foo: { bar: true } })
data = cleanupJSON(data);
frontmatter = cleanupJSON(frontmatter);
return data;
return frontmatter;
}
const attributeRegex = /^[\w\-_]+$/;
plugs/index/header.ts
+1
View File
@@ -59,6 +59,7 @@ export function indexHeaders(
text,
page: pageMeta.name,
pos: n.from!,
range: [n.from!, n.to!],
...attributes,
});
}
plugs/index/index.plug.yaml
+7 -2
View File
@@ -8,7 +8,7 @@ functions:
path: api.ts:patchFrontmatter
syscall: index.patchFrontmatter
text:
path: index.ts:indexMarkdown
path: indexer.ts:indexMarkdown
syscall: index.markdown
queryLuaObjects:
# DEPRECATED: use index.queryLuaObjects syscall instead
@@ -44,7 +44,7 @@ functions:
- document:index
indexPage:
path: index.ts:indexPage
path: indexer.ts:indexPage
events:
- page:index
@@ -114,6 +114,11 @@ functions:
events:
- editor:lint
lintObjects:
path: lint.ts:lintObjects
events:
- editor:lint
# Tasks
updateTaskState:
path: task.ts:updateTaskState
plugs/index/index.ts → plugs/index/indexer.ts
+1 -1
View File
@@ -24,7 +24,7 @@ export type IndexerFunction = (
text: string,
) => Promise<ObjectValue<any>[]>;
const allIndexers: IndexerFunction[] = [
export const allIndexers: IndexerFunction[] = [
pageIndexPage,
indexData,
indexItems,
plugs/index/item.ts
+2 -3
View File
@@ -21,7 +21,6 @@ export type ItemObject = ObjectValue<
name: string;
text: string;
pos: number;
toPos: number;
parent?: string;
links?: string[];
ilinks?: string[];
@@ -96,11 +95,11 @@ export function extractItemFromNode(
const item: ItemObject | TaskObject = {
ref: `${name}@${itemNode.from}`,
tag: "item",
pos: itemNode.from!,
range: [itemNode.from!, itemNode.to!],
name: "", // to be replaced
text: "", // to be replaced
page: name,
pos: itemNode.from!,
toPos: itemNode.to!,
};
// This will only be valid for items, not task
plugs/index/link.ts
+15 -6
View File
@@ -84,11 +84,12 @@ export async function indexLinks(
const url = wikiLinkPage.children![0].text!;
const pos = wikiLinkPage.from!;
const link: any = {
const link: Partial<LinkObject> = {
ref: `${name}@${pos}`,
tag: "link",
snippet: extractSnippet(name, pageText, pos),
pos,
range: [n.from!, n.to!],
page: name,
pageLastModified: pageMeta.lastModified,
};
@@ -106,7 +107,7 @@ export async function indexLinks(
if (wikiLinkAlias) {
link.alias = wikiLinkAlias.children![0].text!;
}
updateITags(link, frontmatter);
updateITags(link as LinkObject, frontmatter);
objects.push(link);
return true;
}
@@ -131,12 +132,14 @@ export async function indexLinks(
const pos = n.from!;
url = resolveMarkdownLink(name, decodeURI(url));
const link: any = {
const link: Partial<LinkObject> = {
ref: `${name}@${pos}`,
tag: "link",
snippet: extractSnippet(name, pageText, pos),
pos,
range: [n.from!, n.to!],
page: name,
pageLastModified: pageMeta.lastModified,
};
const ref = parseToRef(url);
@@ -152,7 +155,7 @@ export async function indexLinks(
if (alias) {
link.alias = alias;
}
updateITags(link, frontmatter);
updateITags(link as LinkObject, frontmatter);
objects.push(link);
return true;
}
@@ -171,12 +174,17 @@ export async function indexLinks(
if (match && match.groups && match[0] === trimmed) {
const { leadingTrivia, stringRef, alias } = match.groups;
const pos = textNode.from! + match.index! + leadingTrivia.length;
const link: any = {
const link: Partial<LinkObject> = {
ref: `${name}@${pos}`,
tag: "link",
page: name,
snippet: extractSnippet(name, pageText, pos),
pos: pos,
range: [
textNode.from! + match.index!,
textNode.from! + match.index! + match[0].length,
],
pageLastModified: pageMeta.lastModified,
};
const ref = parseToRef(stringRef);
@@ -192,7 +200,7 @@ export async function indexLinks(
if (alias) {
link.alias = alias;
}
updateITags(link, frontmatter);
updateITags(link as LinkObject, frontmatter);
objects.push(link);
}
}
@@ -209,6 +217,7 @@ export async function indexLinks(
tag: "aspiring-page",
page: name,
pos: link.pos,
range: link.range,
name: link.toPage,
} as AspiringPageObject);
console.info(
plugs/index/lint.ts
+48 -29
View File
@@ -1,7 +1,8 @@
import { config, jsonschema, lua } from "@silverbulletmd/silverbullet/syscalls";
import { index, lua } from "@silverbulletmd/silverbullet/syscalls";
import {
findNodeOfType,
renderToText,
traverseTree,
traverseTreeAsync,
} from "@silverbulletmd/silverbullet/lib/tree";
import type {
@@ -11,44 +12,26 @@ import type {
import YAML from "js-yaml";
import { extractFrontMatter } from "./frontmatter.ts";
import { allIndexers } from "./indexer.ts";
export async function lintYAML(
/**
* Lint YAML syntax in frontmatter and fenced code blocks
*/
export function lintYAML(
{ tree, name }: LintEvent,
): Promise<LintDiagnostic[]> {
): LintDiagnostic[] {
const diagnostics: LintDiagnostic[] = [];
const frontmatter = extractFrontMatter(tree);
await traverseTreeAsync(tree, async (node) => {
traverseTree(tree, (node) => {
if (node.type === "FrontMatterCode") {
const yamlText = renderToText(node);
const lintResult = lintYaml(
const lintResult = lintYamlBlock(
yamlText,
node.from!,
name,
);
if (lintResult) {
diagnostics.push(lintResult);
} else {
const parsed = YAML.load(yamlText);
// Parses as valid YAML, now let's see if we need to do schema validation
for (const tag of frontmatter.tags || []) {
const schema = await config.get(["tags", tag, "schema"], undefined);
if (schema) {
const validationError = await jsonschema.validateObject(
schema,
parsed,
);
if (validationError) {
diagnostics.push({
message: `${tag} validation failed: ${validationError}`,
severity: "error",
from: node.from!,
to: node.to!,
});
}
}
}
}
return true;
}
@@ -67,7 +50,7 @@ export async function lintYAML(
return true;
}
const yamlCode = renderToText(codeText);
const lintResult = lintYaml(
const lintResult = lintYamlBlock(
yamlCode,
codeText.from!,
);
@@ -84,7 +67,14 @@ export async function lintYAML(
const errorRegex = /\((\d+):(\d+)\)/;
function lintYaml(
/**
* Lint a YAML block
* @param yamlText - The YAML text to lint
* @param startPos - The start position of the YAML block
* @param pageName - The page name to check against
* @returns A LintDiagnostic if there is an error, undefined otherwise
*/
function lintYamlBlock(
yamlText: string,
startPos: number,
pageName?: string,
@@ -121,6 +111,10 @@ function lintYaml(
}
}
/**
* Lint Lua code in fenced code blocks
* @returns A list of LintDiagnostics for any errors found
*/
export async function lintLua({ tree }: LintEvent): Promise<LintDiagnostic[]> {
const diagnostics: LintDiagnostic[] = [];
await traverseTreeAsync(tree, async (node) => {
@@ -166,3 +160,28 @@ export async function lintLua({ tree }: LintEvent): Promise<LintDiagnostic[]> {
});
return diagnostics;
}
/**
* Lint objects in the page
*/
export async function lintObjects(
{ tree, pageMeta: meta, text, name }: LintEvent,
): Promise<LintDiagnostic[]> {
const frontmatter = extractFrontMatter(tree);
// Index the page
const allObjects = (await Promise.all(allIndexers.map((indexer) => {
return indexer(meta, frontmatter, tree, text);
}))).flat();
const result = await index.validateObjects(name, allObjects);
// If validation failed, return the error
if (result?.object?.range) {
return [{
from: result.object.range[0],
to: result.object.range[1],
severity: "error",
message: result.error,
}];
}
return [];
}
plugs/index/paragraph.ts
+1
View File
@@ -76,6 +76,7 @@ export async function indexParagraphs(
text: fullText,
page: pageMeta.name,
pos,
range: [p.from!, p.to!],
...attrs,
};
if (tags.length > 0) {
plugs/index/space_lua.ts
+1
View File
@@ -41,6 +41,7 @@ export function indexSpaceLua(
allSpaceLuas.push({
ref: `${pageMeta.name}@${t.from!}`,
tag: "space-lua",
range: [t.from!, t.to!],
script: codeText,
priority: priority !== undefined ? +priority : undefined,
});
plugs/index/space_style.ts
+1
View File
@@ -46,6 +46,7 @@ export function indexSpaceStyle(
allStyles.push({
ref: `${pageMeta.name}@${t.from!}`,
tag: "space-style",
range: [t.from!, t.to!],
style: codeText,
priority: priority !== undefined ? +priority : undefined,
});
plugs/index/table.ts
+1
View File
@@ -84,6 +84,7 @@ export function indexTables(
tags: [...tags],
page: pageMeta.name,
pos: row.from!,
range: [row.from!, row.to!],
};
cells.forEach((c, i) => {
replaceNodesMatching(c, (tree) => {
website/API/tag.md
+24 -3
View File
@@ -12,9 +12,10 @@ Defines a tag explicitly.
`spec` is a table that can contain:
* `name` (required) the name of the tag
* `metatable` to set a custom Lua metatable for objects with this tag.
* `mustValidate` a boolean defining whether or not schema validation must pass for the object to be indexed
* `schema` [[Schema]] to validate against
* `metatable` to set a custom Lua metatable for objects with this tag.
* `validate` callback function invoked when an objects needs to be validated, returns `nil` or an error message.
* `transform` callback function invoked when an object with tag `name` has been indexed. Allows you to make changes to it, skip indexing altogether or generate additional objects.
When `tag.define` is called multiple times with the same `name`, the specs will be merged. This e.g. enables overriding the schema of a built-in tag, or augment it with a custom `transform` callback.
@@ -24,7 +25,7 @@ This is a very powerful API, with a wide range of potential use cases.
# Use cases
## Schema validation
To define a [[Schema]] for validating an object (e.g. a page) tagged with `#person` ensuring that the `age` attribute is always a number, you can do the following:
```space-lua
```lua
tag.define {
name = "person",
-- mustValidate = true,
@@ -73,15 +74,35 @@ Let’s say that you’d like the encode a deadline attribute for tasks without
* [ ] Hello 📅 2026-12-31
In addition:
We’d like to highlight tasks that use a 📅 but then don’t follow the correct date format, like here:
* [ ] Hello task 📅 31-12-2026
And we’d like the `name` attribute to be cleaned from the deadline syntax.
This can be implemented by defining a custom `transform` for tasks:
```lua
local deadlinePattern = "📅%s*(%d%d%d%d%-%d%d%-%d%d)"
tag.define {
name = "task",
validate = function(o)
if o.name:find("📅") then
if not o.name:match(deadlinePattern) then
return "Found 📅, but did not match YYYY-mm-dd format"
end
end
end,
transform = function(o)
-- Use a regular expression to find a deadline
local date = o.name:match("📅%s*(%d%d%d%d%-%d%d%-%d%d)")
local date = o.name:match(deadlinePattern)
if date then
-- Remove the deadline from the name
o.name = o.name:gsub(deadlinePattern, "")
-- And put it in as attribute
o.deadline = date
end
return o
website/CHANGELOG.md
+2 -1
View File
@@ -5,7 +5,8 @@ Whenever a commit is pushed to the `main` branch, within ~10 minutes, it will be
* New _experimental_ API: [[API/tag#tag.define(spec)]], see linked page for docs and example uses. Brings back ability to define 📅 deadlines for tasks (see example). Another part of this is [[Schema]] support for [[Tag|tags]]. When a schema is defined for a tag, you get:
* [[Frontmatter]] **attribute completion and linting** (in-editor error indicators) for attributes defined as part of the tag’s schema.
* [[Space Lua/Lua Integrated Query]] **attribute code completion** _if_ you use the `from v = index.tag("bla")` style syntax (so explicitly bind your iterator variable)
* [[Space Lua/Lua Integrated Query]] **attribute code completion** _if_ you use the `from v = index.tag("bla")` style syntax (so explicitly bind your iterator variable).
* Item-level linting (highlights the object in-line in case of validation errors)
* Changed keyboard bindings (sorry!). CodeMirror no longer directly allows `Alt-<letter>` and `Alt-<special-character>` [[Keyboard Shortcuts]], meaning I had to **remap a few key bindings**. It’s basically a mission impossible to pick great ones, but here are the new defaults:
* `Quick note` is now bound to both `Ctrl-q q` (type `Ctrl-q` first, then hit `q` again) and `Ctrl-q Ctrl-q` (hit `Ctrl-q` twice)
* `Navigate: Home` is now bound to `Ctrl-g h`
website/CONFIG.md
+15 -1
View File
@@ -65,6 +65,7 @@ config.set {
```
# Custom tag definitions
(further detailed in [[API/tag#Use cases]])
```space-lua
tag.define {
name = "person",
@@ -74,11 +75,24 @@ tag.define {
end
}
local deadlinePattern = "📅%s*(%d%d%d%d%-%d%d%-%d%d)"
tag.define {
name = "task",
validate = function(o)
if o.name:find("📅") then
if not o.name:match(deadlinePattern) then
return "Found 📅, but did not match YYYY-mm-dd format"
end
end
end,
transform = function(o)
local date = o.name:match("📅%s*(%d%d%d%d%-%d%d%-%d%d)")
-- Use a regular expression to find a deadline
local date = o.name:match(deadlinePattern)
if date then
-- Remove the deadline from the name
o.name = o.name:gsub(deadlinePattern, "")
-- And put it in as attribute
o.deadline = date
end
return o