diff --git a/client/client_system.ts b/client/client_system.ts index f4cba888..2139eb6e 100644 --- a/client/client_system.ts +++ b/client/client_system.ts @@ -46,6 +46,8 @@ import type { LuaCollectionQuery } from "./space_lua/query_collection.ts"; import type { Command } from "./types/command.ts"; import { SpaceLuaEnvironment } from "./space_lua.ts"; import { builtinPlugPaths } from "../plugs/builtin_plugs.ts"; +import { ServiceRegistry } from "./service_registry.ts"; +import { serviceRegistrySyscalls } from "./plugos/syscalls/service_registry.ts"; const indexVersionKey = ["$indexVersion"]; const indexQueuedKey = ["$indexQueued"]; @@ -66,6 +68,9 @@ export class ClientSystem { namespaceHook!: PlugNamespaceHook; codeWidgetHook!: CodeWidgetHook; documentEditorHook!: DocumentEditorHook; + mqHook!: MQHook; + + serviceRegistry!: ServiceRegistry; // Space Lua spaceLuaEnv: SpaceLuaEnvironment; @@ -91,6 +96,7 @@ export class ClientSystem { }); this.spaceLuaEnv = new SpaceLuaEnvironment(this.system); + this.serviceRegistry = new ServiceRegistry(this.eventHook, client.config); setInterval(() => { mq.requeueTimeouts(mqTimeout, mqTimeoutRetry, true).catch(console.error); @@ -133,7 +139,8 @@ export class ClientSystem { this.slashCommandHook = new SlashCommandHook(this.client); // MQ hook - this.system.addHook(new MQHook(this.system, this.mq)); + this.mqHook = new MQHook(this.system, this.mq, this.client.config); + this.system.addHook(this.mqHook); // Syscall hook this.system.addHook(new SyscallHook()); @@ -162,6 +169,7 @@ export class ClientSystem { //commandSyscalls(client), luaSyscalls(this), mqSyscalls(this.mq), + serviceRegistrySyscalls(this.serviceRegistry), dataStoreReadSyscalls(this.ds, this), dataStoreWriteSyscalls(this.ds), syncSyscalls(this.client), @@ -219,6 +227,7 @@ export class ClientSystem { // Make scripted (slash) commands available this.commandHook.throttledBuildAllCommandsAndEmit(); this.slashCommandHook.throttledBuildAllCommands(); + this.mqHook.throttledReloadQueues(); this.scriptsLoaded = true; } diff --git a/client/plugos/hooks/event.ts b/client/plugos/hooks/event.ts index 13e3bd9d..ce531fa6 100644 --- a/client/plugos/hooks/event.ts +++ b/client/plugos/hooks/event.ts @@ -96,17 +96,18 @@ export class EventHook implements EventHookI { } // Local listeners - const localListeners = this.localListeners.get(eventName); - if (localListeners) { - for (const localListener of localListeners) { - // Queue the promise - promises.push((async () => { - return await Promise.resolve(localListener(...args)); - })()); + for (const [name, localListeners] of this.localListeners) { + if (eventNameToRegex(name).test(eventName)) { + for (const localListener of localListeners) { + // Queue the promise + promises.push((async () => { + return await Promise.resolve(localListener(...args)); + })()); + } } } - // Space script listeners + // Space Lua listeners if (this.config) { const configListeners: Record = this.config.get( "eventListeners", diff --git a/client/plugos/hooks/mq.ts b/client/plugos/hooks/mq.ts index c3d9377a..3f24259c 100644 --- a/client/plugos/hooks/mq.ts +++ b/client/plugos/hooks/mq.ts @@ -1,9 +1,22 @@ +// deno-lint-ignore-file ban-types import type { Hook, Manifest } from "../types.ts"; import type { System } from "../system.ts"; import { throttle } from "@silverbulletmd/silverbullet/lib/async"; import type { MQHookT } from "@silverbulletmd/silverbullet/type/manifest"; import type { DataStoreMQ, QueueWorker } from "../../data/mq.datastore.ts"; -import type { MQMessage } from "@silverbulletmd/silverbullet/type/datastore"; +import type { + MQMessage, + MQSubscribeOptions, +} from "@silverbulletmd/silverbullet/type/datastore"; +import type { Config } from "../../config.ts"; + +export type MQListenerSpec = + & MQSubscribeOptions + & { + queue: string; + autoAck?: boolean; + run: Function; + }; export class MQHook implements Hook { subscriptions: QueueWorker[] = []; @@ -11,7 +24,11 @@ export class MQHook implements Hook { this.reloadQueues(); }, 1000); - constructor(private system: System, readonly mq: DataStoreMQ) { + constructor( + private system: System, + readonly mq: DataStoreMQ, + readonly config: Config, + ) { } apply(system: System): void { @@ -35,6 +52,7 @@ export class MQHook implements Hook { reloadQueues() { this.stop(); + // Plug based subscriptions for (const plug of this.system.loadedPlugs.values()) { if (!plug.manifest) { continue; @@ -81,6 +99,42 @@ export class MQHook implements Hook { } } } + // Space Lua based subscriptions + const configListeners: Record = this.config.get( + "mqSubscriptions", + {}, + ); + for (const [queue, listeners] of Object.entries(configListeners)) { + for (const listener of listeners) { + // console.log("Subscribing to", queue, listener); + this.subscriptions.push( + this.mq.subscribe( + queue, + { + batchSize: listener.batchSize, + pollInterval: listener.pollInterval, + }, + async (messages: MQMessage[]) => { + try { + await listener.run(messages); + if (listener.autoAck) { + await this.mq.batchAck(queue, messages.map((m) => m.id)); + } + } catch (e: any) { + console.error( + "Execution of mqSubscription for queue", + queue, + "with messages", + messages, + "failed:", + e, + ); + } + }, + ), + ); + } + } } validateManifest(manifest: Manifest): string[] { diff --git a/client/plugos/syscalls/event.ts b/client/plugos/syscalls/event.ts index 86be4366..9330ceda 100644 --- a/client/plugos/syscalls/event.ts +++ b/client/plugos/syscalls/event.ts @@ -23,18 +23,14 @@ export function eventSyscalls( def: EventSubscription, ) => { // console.log("Registering Lua event listener: ", def.name); - // deno-lint-ignore ban-types - const listeners = client.config.get([ + client.config.insert([ "eventListeners", def.name, - ], []); - listeners.push((...args: any[]) => { + ], async (...args: any[]) => { // Convert return value to JS - return def.run(...args).then((val) => - luaValueToJS(val, LuaStackFrame.lostFrame) - ); + const val = await def.run(...args); + return luaValueToJS(val, LuaStackFrame.lostFrame); }); - client.config.set(["eventListeners", def.name], listeners); }, }; } diff --git a/client/plugos/syscalls/mq.ts b/client/plugos/syscalls/mq.ts index 8292673f..9472249a 100644 --- a/client/plugos/syscalls/mq.ts +++ b/client/plugos/syscalls/mq.ts @@ -1,10 +1,29 @@ import type { DataStoreMQ } from "../../data/mq.datastore.ts"; +import type { MQListenerSpec } from "../hooks/mq.ts"; import type { SysCallMapping } from "../system.ts"; +export type EventSubscription = MQListenerSpec & { + run: (...args: any[]) => Promise; +}; + export function mqSyscalls( mq: DataStoreMQ, ): SysCallMapping { return { + /** + * Define a Lua event listener + */ + "mq.subscribe": ( + _ctx, + def: MQListenerSpec, + ) => { + def.autoAck = def.autoAck != false; + // console.log("Registering Lua event listener: ", def.name); + client.config.insert([ + "mqSubscriptions", + def.queue, + ], def); + }, "mq.send": (_ctx, queue: string, body: any) => { return mq.send(queue, body); }, diff --git a/client/plugos/syscalls/service_registry.ts b/client/plugos/syscalls/service_registry.ts new file mode 100644 index 00000000..b92b5d0a --- /dev/null +++ b/client/plugos/syscalls/service_registry.ts @@ -0,0 +1,43 @@ +import type { + ServiceMatch, + ServiceRegistry, + ServiceSpec, +} from "../../service_registry.ts"; +import type { SysCallMapping } from "../system.ts"; + +export function serviceRegistrySyscalls( + serviceRegistry: ServiceRegistry, +): SysCallMapping { + return { + /** + * Define a Lua event listener + */ + "service.define": ( + _ctx, + def: ServiceSpec, + ) => { + return serviceRegistry.define(def); + }, + "service.discover": ( + _ctx, + selector: string, + data: any, + ): Promise => { + return serviceRegistry.discover(selector, data); + }, + "service.invoke": ( + _ctx, + service: ServiceMatch, + data: any, + ): Promise => { + return serviceRegistry.invoke(service, data); + }, + "service.invokeBestMatch": ( + _ctx, + selector: string, + data: any, + ): Promise => { + return serviceRegistry.invokeBestMatch(selector, data); + }, + }; +} diff --git a/client/service_registry.test.ts b/client/service_registry.test.ts new file mode 100644 index 00000000..0ab4b7ea --- /dev/null +++ b/client/service_registry.test.ts @@ -0,0 +1,78 @@ +import type { EventHookT } from "@silverbulletmd/silverbullet/type/manifest"; +import { EventHook } from "./plugos/hooks/event.ts"; +import { System } from "./plugos/system.ts"; +import { ServiceRegistry } from "./service_registry.ts"; +import { assertEquals } from "@std/assert"; +import { Config } from "./config.ts"; + +Deno.test("Test services", async () => { + const system = new System(); + const config = new Config(); + const eventHook = new EventHook(config); + system.addHook(eventHook); + + const registry = new ServiceRegistry(eventHook, config); + + registry.define({ + selector: "greeter", + match: { + priority: 1, + }, + run: (name: string) => { + return Promise.resolve(`Hello ${name}!`); + }, + }); + + registry.define({ + selector: "greeter-multi", + match: () => { + return Promise.resolve({ + priority: 1, + }); + }, + run: (name: string) => { + return Promise.resolve(`Hello 1 ${name}!`); + }, + }); + + registry.define({ + selector: "greeter-multi", + match: { + priority: 2, + }, + run: (name: string) => { + return Promise.resolve(`Hello 2 ${name}!`); + }, + }); + + registry.define({ + selector: "greeter-multi", + match: () => { + return Promise.resolve(null); + }, + run: () => { + throw new Error("I should never be run"); + }, + }); + + const matches = await registry.discover("greeter", {}); + assertEquals(matches.length, 1); + assertEquals(matches[0].priority, 1); + + const matchesMulti = await registry.discover("greeter-multi", {}); + assertEquals(matchesMulti.length, 2); + // higher prio should come first + assertEquals(matchesMulti[0].priority, 2); + assertEquals(matchesMulti[1].priority, 1); + + assertEquals(await registry.invoke(matches[0], "Pete"), "Hello Pete!"); + assertEquals( + await registry.invokeBestMatch("greeter", "Pete"), + "Hello Pete!", + ); + + assertEquals( + await registry.invokeBestMatch("greeter-multi", "Pete"), + "Hello 2 Pete!", + ); +}); diff --git a/client/service_registry.ts b/client/service_registry.ts new file mode 100644 index 00000000..402e58a0 --- /dev/null +++ b/client/service_registry.ts @@ -0,0 +1,69 @@ +import type { Config } from "./config.ts"; +import type { EventHook } from "./plugos/hooks/event.ts"; + +export type ServiceMatch = { + id?: string; // uuid, set automatically + priority?: number; +} & Record; + +export type ServiceSpec = { + selector: string; + match: + | ServiceMatch + | ((data: any) => Promise); + run: (data: any) => Promise; +}; + +export class ServiceRegistry { + constructor(private eventHook: EventHook, private config: Config) { + } + + public define(spec: ServiceSpec): void { + const id = crypto.randomUUID(); + // Register with discover:* event + this.config.insert([ + "eventListeners", + `discover:${spec.selector}`, + ], async (e: any) => { + const matchResult = typeof spec.match === "function" + ? await spec.match(e.data) + : spec.match; + if (matchResult) { + return { + ...matchResult, + id, + }; + } + }); + // Register callback when invoked + this.config.insert(["eventListeners", `service:${id}`], (e: any) => { + return spec.run(e.data); + }); + } + + public async discover(selector: string, opts: any): Promise { + const discoveryResults: ServiceMatch[] = await this.eventHook.dispatchEvent( + `discover:${selector}`, + opts, + ); + discoveryResults.sort((a, b) => (b.priority || 0) - (a.priority || 0)); + return discoveryResults; + } + + public async invoke(match: ServiceMatch, data: any): Promise { + const results = await this.eventHook.dispatchEvent( + `service:${match.id}`, + data, + ); + // Note: results may be an empty array in case no service actually returned a result (void) case, which is implicitly passed on here + return results[0]; + } + + public async invokeBestMatch(selector: string, data: any): Promise { + const results = await this.discover(selector, data); + if (results.length === 0) { + throw new Error(`No services matching: ${selector}`); + } + return this.invoke(results[0], data); + } +} diff --git a/client/space_lua/stdlib.ts b/client/space_lua/stdlib.ts index 224bfa1b..24ae9208 100644 --- a/client/space_lua/stdlib.ts +++ b/client/space_lua/stdlib.ts @@ -24,6 +24,8 @@ import { evalStatement } from "./eval.ts"; import { encodingApi } from "./stdlib/encoding.ts"; import { luaToNumber } from "./tonumber.ts"; import { luaLoad } from "./stdlib/load.ts"; +import { cryptoApi } from "./stdlib/crypto.ts"; +import { netApi } from "./stdlib/net.ts"; const printFunction = new LuaBuiltinFunction(async (_sf, ...args) => { console.log( @@ -379,6 +381,8 @@ export function luaBuildStandardEnv() { env.set("each", eachFunction); env.set("spacelua", spaceluaApi); env.set("encoding", encodingApi); + env.set("crypto", cryptoApi); + env.set("net", netApi); env.set("some", someFunction); return env; } diff --git a/plugs/index/queue.ts b/plugs/index/queue.ts index 8ecd3057..4ad78729 100644 --- a/plugs/index/queue.ts +++ b/plugs/index/queue.ts @@ -64,9 +64,6 @@ async function updateIndexProgressInUI() { export async function processIndexQueue(messages: MQMessage[]) { for (const message of messages) { let name: string = message.body; - if (name.startsWith("_plug/")) { - continue; - } console.log(`Indexing file ${name}`); if (name.endsWith(".md")) { name = name.slice(0, -3); diff --git a/website/API/service.md b/website/API/service.md new file mode 100644 index 00000000..8ec6938f --- /dev/null +++ b/website/API/service.md @@ -0,0 +1,57 @@ +# Service registry API +Exposes a simple service registry API leveraged by various parts of SilverBullet, see [[Services]]. + +## Example +```space-lua +service.define { + selector = "greeter-service", + name = "generic-greeter", + match = function(name) + -- fallback option, priority undefined (so: low) + return {} + end, + run = function(name) + return "Hello " .. name + end +} + +service.define { + selector = "greeter-service", + name = "pete-greeter", + match = function(name) + if name == "Pete" then + -- takes precendence with an exact match + return {priority=10} + else + return nil + end + end, + run = function(name) + return "Hello Pete, so happy to see you!" + end +} +``` + +To invoke: ${service.invokeBestMatch("greeter-service", "Pete")} and ${service.invokeBestMatch("greeter-service", "Hank")} + +## API +### service.define(spec) +Defines a service matching a selector. + +Spec arguments: +* `selector`: service selector, can contain wildcards (e.g. `hello:*`) +* `name`: globally unique service name +* `name` (gid for invocation) +* `match(data)`: callback (`data` argument), returns a match table if this service is a match or `nil` otherwise. A match table is service specific, but at least contains a `priority` key to set the service priority (used for sorting by `service.discover` and `service.invokeBestMatch`). +* `run(data)`: callback to be invoked when the service is invoked + +### service.discover(selector, data) +Discovers previously defined service matching a given selector. Return value is a list of tables (can be empty) with keys: +* `name`: name of the service +* `priority`: priority of service match (list will already be pre-sorted based on this key if present) + +### service.invoke(name, data) +Invokes a service by name (as returned by [[#service.discover(selector, options)]]). + +### service.invokeBest(selector, options) +Performs a `discover` based on the selector, then immediately performs and `invoke` on the best match (based on priority). \ No newline at end of file diff --git a/website/Services.md b/website/Services.md new file mode 100644 index 00000000..e900cf5c --- /dev/null +++ b/website/Services.md @@ -0,0 +1,14 @@ +The SilverBullet service bus is an extension mechanism built into SilverBullet. + +# Concepts +* Discovery: the process of discovering available services based on a _selector_. +* Selector: a string (possibly with a wildcard `*`) used to advertise as well as discover services. +* Service: defined via [[API/service#service.define(spec)]]), implements a service advertised via a selector. +* Invocation: running a discovered service. + +# Architecture +Services are an abstraction built on top of [[Events]]. When a new service is defined, it registers itself listening to the `discover:<>` event (to be discovered), and under `service:<>` to be invoked. + +Service discovery happens by broadcasting an event on the event bus with the given selector, returning a list of services that are a match, ordered by (self assigned) priority. One (or all) of these services can then be invoked based on their returned ID. + +See [[API/service]] for API details. \ No newline at end of file