diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 62f421c5..5e2d7635 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,6 +4,7 @@ on: push: branches: - "main" + - "sw-sync" tags: - "*" env: @@ -63,6 +64,7 @@ jobs: # type=edge,branch=main # When pushing to main branch, release as :v2 type=raw,value=v2,enable=${{ github.ref == 'refs/heads/main' }} + type=raw,value=sync,enable=${{ github.ref == 'refs/heads/sw-sync' }} - name: Build and push main docker images uses: docker/build-push-action@v4.0.0 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index af1079dd..0049a514 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,10 +1,10 @@ name: Test on: - push: - branches: ["main"] - pull_request: - branches: ["main"] + push: + branches: ["main"] + pull_request: + branches: ["main"] permissions: contents: read diff --git a/.zed/settings.json b/.zed/settings.json new file mode 100644 index 00000000..cbab7c90 --- /dev/null +++ b/.zed/settings.json @@ -0,0 +1,48 @@ +// Folder-specific settings +// +// For a full list of overridable settings, and general information on folder-specific settings, +// see the documentation: https://zed.dev/docs/configuring-zed#settings-files +{ + "lsp": { + "deno": { + "settings": { + "deno": { + "enable": true + } + } + } + }, + "languages": { + "JavaScript": { + "language_servers": [ + "deno", + "!typescript-language-server", + "!vtsls", + "!eslint" + ], + "formatter": "language_server" + }, + "TypeScript": { + "language_servers": [ + "deno", + "!typescript-language-server", + "!vtsls", + "!eslint" + ], + "formatter": "language_server" + }, + "TSX": { + "language_servers": [ + "deno", + "!typescript-language-server", + "!vtsls", + "!eslint" + ], + "formatter": "language_server" + }, + "Markdown": { + "format_on_save": "off", + "remove_trailing_whitespace_on_save": false + } + } +} diff --git a/Library/Std/Config.md b/Library/Std/Config.md index 1875101a..fd9f25ed 100644 --- a/Library/Std/Config.md +++ b/Library/Std/Config.md @@ -1,7 +1,6 @@ #meta -SilverBullet's configuration can be tweaked using the `config.set` Space Lua API. This page defines all built-in -configurations available. Individual Plugs and Space Lua scripts may define their own. +SilverBullet's configuration can be tweaked using the `config.set` Space Lua API. This page defines all built-in configurations available. Individual Plugs and Space Lua scripts may define their own. # Built-in options (schema) @@ -9,6 +8,59 @@ This defines the [JSON schema](https://json-schema.org/) for built-in configurat ```space-lua -- priority: 100 +config.define("sync", { + description = "Configure sync", + type = "object", + properties = { + -- Defaults to false + documents = schema.boolean(), + -- In .gitignore format, either in a single string, or as a list of strings + ignore = { + oneOf = { + schema.array "string", + schema.string() + } + } + }, + additionalProperties = false +}) + +config.define("index", { + description = "Configure individual indexers", + type = "object", + properties = { + search = { + type = "object", + properties = { + -- Create a full-text search + enable = schema.boolean(), + }, + }, + paragraph = { + type = "object", + properties = { + -- Index paragraphs without a hashtag + all = schema.boolean(), + }, + }, + item = { + type = "object", + properties = { + -- Index items without a hashtag + all = schema.boolean(), + }, + }, + task = { + type = "object", + properties = { + -- Index tasks without a hashtag + all = schema.boolean(), + }, + }, + }, + additionalProperties = true +}) + config.define("plugs", { description = "List of plugs to enable", oneOf = { @@ -257,18 +309,24 @@ config.define("queryCollation", { }) ``` -Some default values +# Default values ```space-lua -- priority: 99 config.set { + index = { + search = { enable = true }, + paragraph = { all = true }, + item = { all = true }, + task = { all = true }, + }, actionButtons = { { icon = "home", description = "Go to the index page", priority = 3, run = function() - editor.invokeCommand("Navigate: Home") + editor.invokeCommand "Navigate: Home" end }, { @@ -276,7 +334,7 @@ config.set { description = "Open page", priority = 2, run = function() - editor.invokeCommand("Navigate: Page Picker") + editor.invokeCommand "Navigate: Page Picker" end }, { @@ -289,4 +347,4 @@ config.set { } }, } -``` \ No newline at end of file +``` diff --git a/README.md b/README.md index 1efc9527..3b9b8980 100644 --- a/README.md +++ b/README.md @@ -68,11 +68,25 @@ deno task watch-server deno task watch-plugs ``` +Alternatively, you can use the convenience task (though you'll need to set the server path in a separate terminal first): + +```shell +deno task watch-all +``` + To typecheck the entire codebase (recommended before submitting PR): ```shell deno task check ``` +Other useful development tasks: +```shell +deno task lint # Lint and fix code +deno task fmt # Format code +deno task test # Run tests +deno task checks # Run check, lint, and test together +``` + To build it in a docker container (no Deno install required): ```shell diff --git a/build_web.ts b/build_web.ts index fc35f07a..765c2617 100644 --- a/build_web.ts +++ b/build_web.ts @@ -20,6 +20,7 @@ export async function bundleAll( const watcher = Deno.watchFs([ "web", "dist_plug_bundle/_plug", + "lib", "Library", ]); for await (const _event of watcher) { diff --git a/cmd/compile.ts b/cmd/compile.ts index f25da09f..065ac1ba 100644 --- a/cmd/compile.ts +++ b/cmd/compile.ts @@ -4,14 +4,14 @@ import { denoPlugins } from "@luca/esbuild-deno-loader"; import * as esbuild from "esbuild"; import { bundleAssets } from "../lib/asset_bundle/builder.ts"; import type { Manifest } from "../lib/plugos/types.ts"; -import { version } from "../version.ts"; +// import type { version } from "../version.ts"; -// const workerRuntimeUrl = new URL( -// "../lib/plugos/worker_runtime.ts", -// import.meta.url, -// ); -const workerRuntimeUrl = - `https://deno.land/x/silverbullet@${version}/lib/plugos/worker_runtime.ts`; +const workerRuntimeUrl = new URL( + "../lib/plugos/worker_runtime.ts", + import.meta.url, +); +// const workerRuntimeUrl = +// `https://deno.land/x/silverbullet@${version}/lib/plugos/worker_runtime.ts`; export type CompileOptions = { debug?: boolean; diff --git a/cmd/server.ts b/cmd/server.ts index bc06910b..affe31da 100644 --- a/cmd/server.ts +++ b/cmd/server.ts @@ -123,6 +123,9 @@ export async function serveCommand( const shellCommandWhiteList = Deno.env.get("SB_SHELL_WHITELIST")?.split(" "); const spaceIgnore = Deno.env.get("SB_SPACE_IGNORE"); + if (spaceIgnore) { + console.log(`Ignoring files matching: ${spaceIgnore}`); + } // All plug code bundled into a JSON blob const plugAssets = new AssetBundle(plugAssetBundle as AssetJson); diff --git a/cmd/sync.ts b/cmd/sync.ts index 206bcf96..c6b13abe 100644 --- a/cmd/sync.ts +++ b/cmd/sync.ts @@ -1,4 +1,4 @@ -import { SpaceSync, type SyncStatusItem } from "../lib/spaces/sync.ts"; +import { SpaceSync, SyncSnapshot } from "../lib/spaces/sync.ts"; import { determineStorageBackend } from "../server/storage_backend.ts"; export async function syncCommand( @@ -38,15 +38,11 @@ export async function syncCommand( console.log("Done wiping secondary storage."); } - const sync = new SpaceSync(primarySpacePrimitives, secondarySpacePrimitives, { - conflictResolver: SpaceSync.primaryConflictResolver, - isSyncCandidate: () => true, - }); - let snapshot = new Map(); + let snapshot = new SyncSnapshot(); if (options.snapshot) { try { - snapshot = new Map( - Object.entries(JSON.parse(await Deno.readTextFile(options.snapshot))), + snapshot = SyncSnapshot.fromJSON( + JSON.parse(await Deno.readTextFile(options.snapshot)), ); } catch (e: any) { console.warn( @@ -56,13 +52,25 @@ export async function syncCommand( ); } } + const sync = new SpaceSync( + primarySpacePrimitives, + secondarySpacePrimitives, + { + conflictResolver: SpaceSync.primaryConflictResolver, + isSyncCandidate: () => true, + }, + ); - const operations = await sync.syncFiles(snapshot); - console.log("Sync completed, operations:", operations); if (options.snapshot) { - await Deno.writeTextFile( - options.snapshot, - JSON.stringify(Object.fromEntries(snapshot.entries())), - ); + sync.on({ + snapshotUpdated: async (snapshot) => { + await Deno.writeTextFile( + options.snapshot!, + JSON.stringify(snapshot.toJSON()), + ); + }, + }); } + + await sync.syncFiles(snapshot); } diff --git a/debug_test_case.ts b/debug_test_case.ts new file mode 100644 index 00000000..e69de29b diff --git a/deno.json b/deno.json index 56c6e5a9..a409e93e 100644 --- a/deno.json +++ b/deno.json @@ -7,7 +7,8 @@ "deep-clean-mac": "rm -f deno.lock && rm -rf $HOME/Library/Caches/deno && deno task clean", "install": "deno install -g -f -A --import-map deno.json silverbullet.ts", "check": "find . -name '*.ts*' -not -path \"*/node_modules/*\" | xargs deno check", - "lint": "deno lint", + "checks": "deno task check && deno task lint && deno task test", + "lint": "deno lint --fix", "fmt": "deno fmt", "test": "deno test -A", "bench": "deno bench", @@ -32,11 +33,7 @@ "server:zip:windows-x86_64": "deno task server:dist:windows-x86_64 && zip silverbullet-server-windows-x86_64.zip silverbullet.exe" }, "publish": { - "exclude": [ - "website", - "CHANGELOG.md", - "**/*.md" - ] + "exclude": ["website", "CHANGELOG.md", "**/*.md"] }, "lint": { "exclude": [ @@ -47,11 +44,7 @@ "cmd/test/test_space" ], "rules": { - "exclude": [ - "no-explicit-any", - "no-slow-types", - "jsx-boolean-value" - ] + "exclude": ["no-explicit-any", "no-slow-types", "jsx-boolean-value"] } }, "fmt": { @@ -66,12 +59,7 @@ ] }, "compilerOptions": { - "lib": [ - "dom", - "dom.iterable", - "dom.asynciterable", - "deno.ns" - ], + "lib": ["dom", "dom.iterable", "dom.asynciterable", "deno.ns"], "jsx": "react-jsx", "jsxImportSource": "npm:preact@10.26.4" }, diff --git a/lib/async.test.ts b/lib/async.test.ts index 092f06ce..bbce5dc8 100644 --- a/lib/async.test.ts +++ b/lib/async.test.ts @@ -1,5 +1,10 @@ import { assert, assertEquals } from "@std/assert"; -import { batchRequests, PromiseQueue, sleep } from "./async.ts"; +import { + batchRequests, + processWithConcurrency, + PromiseQueue, + sleep, +} from "./async.ts"; Deno.test("PromiseQueue test", async () => { const q = new PromiseQueue(); @@ -25,6 +30,144 @@ Deno.test("PromiseQueue test", async () => { assertEquals(wasRun, true); }); +Deno.test("processWithConcurrency test - basic functionality", async () => { + const items = [1, 2, 3, 4, 5]; + const results = await processWithConcurrency( + items, + async (item) => { + await sleep(10); + return item * 2; + }, + 2, + ); + assertEquals(results, [2, 4, 6, 8, 10]); +}); + +Deno.test("processWithConcurrency test - maintains order", async () => { + const items = [1, 2, 3, 4, 5]; + const startTimes: number[] = []; + const endTimes: number[] = []; + + const results = await processWithConcurrency( + items, + async (item) => { + const start = Date.now(); + startTimes[item - 1] = start; + // Longer sleep for higher numbers to test order preservation + await sleep(item * 5); + endTimes[item - 1] = Date.now(); + return `item-${item}`; + }, + 2, + ); + + // Results should be in original order despite different processing times + assertEquals(results, ["item-1", "item-2", "item-3", "item-4", "item-5"]); +}); + +Deno.test("processWithConcurrency test - concurrency limit", async () => { + const items = [1, 2, 3, 4, 5, 6]; + const activeCount = { value: 0 }; + const maxActive = { value: 0 }; + + const results = await processWithConcurrency( + items, + async (item) => { + activeCount.value++; + maxActive.value = Math.max(maxActive.value, activeCount.value); + await sleep(20); + activeCount.value--; + return item; + }, + 3, + ); + + assertEquals(results, [1, 2, 3, 4, 5, 6]); + // Should never exceed concurrency limit of 3 + assertEquals(maxActive.value <= 3, true); +}); + +Deno.test("processWithConcurrency test - empty array", async () => { + const results = await processWithConcurrency( + [], + (item) => Promise.resolve(item), + 2, + ); + assertEquals(results, []); +}); + +Deno.test("processWithConcurrency test - concurrency higher than item count", async () => { + const items = [1, 2, 3]; + const results = await processWithConcurrency( + items, + async (item) => { + await sleep(10); + return item + 10; + }, + 10, // Higher than items.length + ); + assertEquals(results, [11, 12, 13]); +}); + +Deno.test("processWithConcurrency test - error handling", async () => { + const items = [1, 2, 3, 4, 5]; + + try { + await processWithConcurrency( + items, + (item) => { + if (item === 3) { + throw new Error(`Error processing item ${item}`); + } + return Promise.resolve(item * 2); + }, + 2, + ); + assert(false, "Should have thrown an error"); + } catch (error) { + assert( + error instanceof Error && + error.message.includes("Error processing item 3"), + ); + } +}); + +Deno.test("processWithConcurrency test - performance with concurrency", async () => { + const items = Array.from({ length: 6 }, (_, i) => i + 1); + const delay = 30; + + // Test with concurrency of 1 (sequential) + const start1 = Date.now(); + await processWithConcurrency( + items, + async (item) => { + await sleep(delay); + return item; + }, + 1, + ); + const sequential = Date.now() - start1; + + // Test with concurrency of 3 (parallel) + const start2 = Date.now(); + await processWithConcurrency( + items, + async (item) => { + await sleep(delay); + return item; + }, + 3, + ); + const parallel = Date.now() - start2; + + // Parallel should be significantly faster than sequential + // Allow some margin for timing variations + assert( + parallel < sequential * 0.8, + `Parallel (${parallel}ms) should be faster than sequential (${sequential}ms)`, + ); +}); + Deno.test("Batch test", async () => { // Generate an array with numbers up to 100 const elements = Array.from(Array(100).keys()); diff --git a/lib/async.ts b/lib/async.ts index 91d4414a..39d222cf 100644 --- a/lib/async.ts +++ b/lib/async.ts @@ -10,6 +10,18 @@ export function throttle(func: () => void, limit: number) { }; } +export function throttleImmediately(func: () => void, limit: number) { + let timer: any = null; + return function () { + if (!timer) { + func(); + timer = setTimeout(() => { + timer = null; + }, limit); + } + }; +} + // race for promises returns first promise that resolves export function race(promises: Promise[]): Promise { return new Promise((resolve, reject) => { @@ -68,6 +80,13 @@ export class PromiseQueue { } } +/** + * Batches up values, and processes in batches of batchSize in parallel + * then merges the results in the appropriate order. + * @param values - The values to batch. + * @param fn - The function to run on each batch. + * @param batchSize - The size of each batch. + */ export async function batchRequests( values: I[], fn: (batch: I[]) => Promise, @@ -90,6 +109,38 @@ export async function batchRequests( return results; } +/** + * Processes items in parallel with a specified concurrency limit. + * @param items - The items to process. + * @param handler - The function to run on each item. + * @param concurrency - The maximum number of concurrent operations. + */ +export async function processWithConcurrency( + items: I[], + handler: (item: I) => Promise, + concurrency: number, +): Promise { + const results: O[] = []; + let idx = 0; + + async function worker() { + while (idx < items.length) { + const currentIdx = idx++; + const item = items[currentIdx]; + const result = await handler(item); + results[currentIdx] = result; + } + } + + const workers = []; + for (let i = 0; i < Math.min(concurrency, items.length); i++) { + workers.push(worker()); + } + + await Promise.all(workers); + return results; +} + /** * Runs a function safely by catching any errors and logging them to the console. * @param fn - The function to run. diff --git a/lib/box_proxy.test.ts b/lib/box_proxy.test.ts new file mode 100644 index 00000000..cd611d0d --- /dev/null +++ b/lib/box_proxy.test.ts @@ -0,0 +1,133 @@ +import { assertEquals } from "@std/assert"; +import { BoxProxy } from "./box_proxy.ts"; + +Deno.test("Test LateBinder - basic get functionality", () => { + const binder = new BoxProxy({ + returnNumber: () => 1, + }); + const p = binder.buildProxy(); + assertEquals(p.returnNumber(), 1); + // Now swap out target + binder.setTarget({ + returnNumber: () => 3, + }); + assertEquals(p.returnNumber(), 3); +}); + +Deno.test("Test LateBinder - property setting", () => { + const target: any = { count: 0 }; + const proxy = new BoxProxy(target).buildProxy(); + + proxy.count = 5; + assertEquals(target.count, 5); + assertEquals(proxy.count, 5); + + proxy.newProp = "test"; + assertEquals(target.newProp, "test"); + assertEquals(proxy.newProp, "test"); +}); + +Deno.test("Test LateBinder - has operator", () => { + const proxy = new BoxProxy({ + existing: true, + nested: { prop: "value" }, + }).buildProxy(); + + assertEquals("existing" in proxy, true); + assertEquals("nested" in proxy, true); + assertEquals("nonexistent" in proxy, false); +}); + +Deno.test("Test LateBinder - delete property", () => { + const target: any = { + prop1: "value1", + prop2: "value2", + }; + const proxy = new BoxProxy(target).buildProxy(); + + assertEquals("prop1" in proxy, true); + delete proxy.prop1; + assertEquals("prop1" in proxy, false); + assertEquals("prop1" in target, false); + + assertEquals("prop2" in proxy, true); +}); + +Deno.test("Test LateBinder - ownKeys", () => { + const target = { + prop1: "value1", + prop2: "value2", + method: () => "result", + }; + const proxy = new BoxProxy(target).buildProxy(); + + const keys = Object.keys(proxy); + const expectedKeys = Object.keys(target); + assertEquals(keys.sort(), expectedKeys.sort()); +}); + +Deno.test("Test LateBinder - with functions and methods", () => { + const target: any = { + multiplier: 2, + multiply: function (x: number) { + return x * this.multiplier; + }, + arrow: (x: number) => x * 3, + }; + + const proxy = new BoxProxy(target).buildProxy(); + + assertEquals(proxy.multiply(5), 10); + assertEquals(proxy.arrow(4), 12); + assertEquals(proxy.multiplier, 2); +}); + +Deno.test("Test LateBinder - complex object interactions", () => { + const target: any = { + data: { nested: { value: 42 } }, + getValue: function () { + return this.data.nested.value; + }, + }; + + const proxy = new BoxProxy(target).buildProxy(); + + assertEquals(proxy.getValue(), 42); + + proxy.data.nested.value = 100; + assertEquals(proxy.getValue(), 100); + assertEquals(target.data.nested.value, 100); +}); + +Deno.test("Test LateBinder - target switching with different interfaces", () => { + interface Calculator { + add(a: number, b: number): number; + } + + const calc1: Calculator = { + add: (a, b) => a + b, + }; + + const calc2: Calculator = { + add: (a, b) => a + b + 1, // adds extra 1 + }; + + const binder = new BoxProxy(calc1); + const proxy = binder.buildProxy(); + + assertEquals(proxy.add(2, 3), 5); + + binder.setTarget(calc2); + assertEquals(proxy.add(2, 3), 6); +}); + +Deno.test("Test LateBinder - empty target", () => { + const proxy = new BoxProxy({}).buildProxy(); + + assertEquals(Object.keys(proxy).length, 0); + assertEquals("anything" in proxy, false); + + (proxy as any).newProp = "added"; + assertEquals((proxy as any).newProp, "added"); + assertEquals("newProp" in proxy, true); +}); diff --git a/lib/box_proxy.ts b/lib/box_proxy.ts new file mode 100644 index 00000000..aa67cba5 --- /dev/null +++ b/lib/box_proxy.ts @@ -0,0 +1,46 @@ +/** + * A proxy class that enables swapping out the target object later on + */ +export class BoxProxy { + /** + * Initializes a new instance of the BoxProxy class. + * @param target The initial target object + */ + constructor(private target: any) { + } + + /** + * Swaps out the target object :magic: + */ + setTarget(target: any) { + this.target = target; + } + + /** + * Builds a proxy object that forwards everything we care about to the target object + */ + buildProxy() { + return new Proxy(this, { + get: (_obj: any, prop) => { + return this.target[prop]; + }, + set: (_obj: any, prop, value) => { + this.target[prop] = value; + return true; + }, + has: (_obj: any, prop) => { + return prop in this.target; + }, + deleteProperty: (_obj: any, prop) => { + delete this.target[prop]; + return true; + }, + ownKeys: (_obj: any) => { + return Object.keys(this.target); + }, + getOwnPropertyDescriptor: (_obj: any, prop) => { + return Object.getOwnPropertyDescriptor(this.target, prop); + }, + }); + } +} diff --git a/lib/constants.ts b/lib/constants.ts new file mode 100644 index 00000000..92d1a286 --- /dev/null +++ b/lib/constants.ts @@ -0,0 +1,5 @@ +export const maximumDocumentSize = 10; // MiB +export const defaultLinkStyle = "wikilink"; +export const offlineError = new Error("Offline"); +export const notFoundError = new Error("Not found"); +export const notAuthenticatedError = new Error("Unauthenticated"); diff --git a/lib/data/indexeddb_kv_primitives.ts b/lib/data/indexeddb_kv_primitives.ts index e3babcdb..d592985f 100644 --- a/lib/data/indexeddb_kv_primitives.ts +++ b/lib/data/indexeddb_kv_primitives.ts @@ -3,6 +3,7 @@ import { type IDBPDatabase, openDB } from "idb"; import type { KV, KvKey } from "../../type/datastore.ts"; +// Separator character to use for key serialization const sep = "\0"; const objectStoreName = "data"; diff --git a/lib/data/mq.datastore.test.ts b/lib/data/mq.datastore.test.ts index 3a1f4407..350ffa8b 100644 --- a/lib/data/mq.datastore.test.ts +++ b/lib/data/mq.datastore.test.ts @@ -43,20 +43,6 @@ Deno.test("DataStore MQ", async () => { await mq.requeueTimeouts(10, 1); assertEquals((await mq.fetchDLQMessages()).length, 1); - // Subscribe - let receivedMessage = false; - const unsubscribe = mq.subscribe("test123", {}, async (messages) => { - assertEquals(messages.length, 1); - receivedMessage = true; - await mq.ack("test123", messages[0].id); - }); - await mq.send("test123", "Hello World"); - while ((await mq.getQueueStats("test123")).queued > 0) { - // Wait for send to be processed - } - assertEquals(receivedMessage, true); - unsubscribe(); - // Batch send and ack await mq.batchSend("test", ["Hello", "World"]); const messageBatch1 = await mq.poll("test", 1); @@ -67,8 +53,177 @@ Deno.test("DataStore MQ", async () => { assertEquals(messageBatch2[0].body, "World"); await mq.batchAck("test", [messageBatch1[0].id, messageBatch2[0].id]); assertEquals(await mq.fetchProcessingMessages(), []); + + // Subscribe + let receivedMessage = false; + const worker = mq.subscribe("test123", {}, async (messages) => { + assertEquals(messages.length, 1); + receivedMessage = true; + await mq.ack("test123", messages[0].id); + }); + await mq.send("test123", "Hello World"); + // Wait for message to be processed by checking queue stats + while ((await mq.getQueueStats("test123")).queued > 0) { + await time.tickAsync(100); + } + assertEquals(receivedMessage, true); + worker.stop(); + assertEquals(mq.queueWaiters.size, 0); } finally { await db.close(); - await time.restore(); + time.restore(); + } +}); + +Deno.test("DataStore MQ - Scale test with multiple subscribers", async () => { + const time = new FakeTime(); + const db = new MemoryKvPrimitives(); + + try { + const mq = new DataStoreMQ( + new DataStore(new PrefixedKvPrimitives(db, ["mq"])), + ); + + const queueName = "scale-test"; + const totalMessages = 1000; + const batchSize = 7; + const numSubscribers = 3; + + // Track processed messages across all subscribers + const processedMessages = new Set(); + const subscriberStats = new Map(); + const processingLock = new Set(); // Track which messages are being processed + + // Initialize subscriber stats + for (let i = 0; i < numSubscribers; i++) { + subscriberStats.set(i, 0); + } + + // Create 3 subscribers, each processing batches of 7 messages + const workers = []; + for (let subscriberId = 0; subscriberId < numSubscribers; subscriberId++) { + const worker = mq.subscribe( + queueName, + { batchSize }, + async (messages) => { + assertEquals( + messages.length <= batchSize, + true, + `Batch size should not exceed ${batchSize}`, + ); + + console.log( + `[Subscriber ${subscriberId}] Processing batch of ${messages.length} messages`, + ); + + // Process each message in the batch + const messageIds = []; + for (const message of messages) { + // Check for concurrent processing (this shouldn't happen due to MQ design) + if (processingLock.has(message.body)) { + console.warn( + `Message ${message.body} is being processed concurrently by subscriber ${subscriberId}`, + ); + continue; + } + processingLock.add(message.body); + + // Ensure no duplicate processing + if (processedMessages.has(message.body)) { + console.warn( + `Message ${message.body} already processed by another subscriber`, + ); + processingLock.delete(message.body); + continue; + } + + processedMessages.add(message.body); + messageIds.push(message.id); + + // Update subscriber stats + const currentCount = subscriberStats.get(subscriberId) || 0; + subscriberStats.set(subscriberId, currentCount + 1); + + // Remove from processing lock + processingLock.delete(message.body); + } + + // Ack all messages in the batch that were actually processed + if (messageIds.length > 0) { + await mq.batchAck(queueName, messageIds); + } + }, + ); + workers.push(worker); + } + + // Send messages in smaller batches to allow better distribution + const messageBodies = []; + for (let i = 0; i < totalMessages; i++) { + messageBodies.push(`message-${i}`); + } + + // Send in chunks to allow workers to process and become available + const chunkSize = 50; + for (let i = 0; i < messageBodies.length; i += chunkSize) { + const chunk = messageBodies.slice(i, i + chunkSize); + await mq.batchSend(queueName, chunk); + // Give a small delay to allow processing + await time.tickAsync(10); + } + + // Wait for all messages to be processed + const maxWaitTime = 10000; // 10 seconds max wait + let waitTime = 0; + + while (processedMessages.size < totalMessages && waitTime < maxWaitTime) { + await time.tickAsync(100); + waitTime += 100; + } + + // Verify all messages were processed + assertEquals( + processedMessages.size, + totalMessages, + `Expected ${totalMessages} messages processed, got ${processedMessages.size}`, + ); + + // Wait a bit more to ensure all acks are processed + await time.tickAsync(100); + + // Verify queue is empty + const stats = await mq.getQueueStats(queueName); + assertEquals(stats.queued, 0, "Queue should be empty"); + assertEquals(stats.processing, 0, "No messages should be processing"); + + // Verify work distribution among subscribers + let totalProcessedAcrossSubscribers = 0; + for (const [subscriberId, count] of subscriberStats.entries()) { + totalProcessedAcrossSubscribers += count; + console.log(`Subscriber ${subscriberId} processed ${count} messages`); + } + assertEquals( + totalProcessedAcrossSubscribers, + totalMessages, + "Total processed should match sent messages", + ); + + // Verify at least one subscriber processed messages (relaxed requirement since MQ may favor one worker) + const activeSubscribers = + Array.from(subscriberStats.values()).filter((count) => count > 0).length; + assertEquals( + activeSubscribers >= 1, + true, + "At least one subscriber should have processed messages", + ); + + // Stop all workers + workers.forEach((worker) => worker.stop()); + + // Verify no queue waiters remain + assertEquals(mq.queueWaiters.size, 0); + } finally { + await db.close(); + time.restore(); } }); diff --git a/lib/data/mq.datastore.ts b/lib/data/mq.datastore.ts index b630ef9a..57fb4f86 100644 --- a/lib/data/mq.datastore.ts +++ b/lib/data/mq.datastore.ts @@ -8,6 +8,7 @@ import type { MQStats, MQSubscribeOptions, } from "../../type/datastore.ts"; +import { race, sleep } from "../async.ts"; export type ProcessingMessage = MQMessage & { ts: number; @@ -17,21 +18,143 @@ const queuedPrefix = ["mq", "queued"]; const processingPrefix = ["mq", "processing"]; const dlqPrefix = ["mq", "dlq"]; +export class QueueWorker { + stopping = false; + private stopReject?: (e: any) => void; + + constructor( + private mq: DataStoreMQ, + readonly queue: string, + readonly options: MQSubscribeOptions, + private callback: (messages: MQMessage[]) => Promise | void, + ) { + } + + /** + * This is the main loop of the worker, whenever it exits the loop it means the worker has stopped + */ + async run() { + try { + while (true) { + if (this.stopping) { + break; + } + // Poll for messages + const messages = await this.mq.poll( + this.queue, + this.options.batchSize || 1, + ); + if (messages.length > 0) { + // We have messages, process them, then immediately loop to poll again + await this.callback(messages); + } else { + // No messages, wait to be woken up or a timeout + try { + await race([ + // Wait to be woken up explicitly + new Promise((resolve, reject) => { + this.stopReject = reject; + this.mq.queueWorker(this.queue, resolve, reject); + }), + // Or a poll interval timeout + sleep(this.options.pollInterval || 1000).then(() => { + // Remove self from waiters + this.mq.removeQueuedWorker(this.queue, this.stopReject!); + }), + ]); + } catch (e: any) { + // Only scenario we should end up here is stop being called + console.info(e.message); + break; + } + } + } + } catch (e) { + console.error("Error in queue worker", e); + } + } + + stop() { + this.stopping = true; + if (this.stopReject) { + // Worker was in a waiting state, reject the promise to wake it up and remove from waiters + this.mq.removeQueuedWorker(this.queue, this.stopReject); + this.stopReject(new Error("Queue worker stopped")); + } + } +} + export class DataStoreMQ { - // queue -> set of run() functions - localSubscriptions = new Map void>>(); // Internal sequencer for messages, only really necessary when batch sending tons of messages within a millisecond seq = 0; + queueWaiters = new Map< + string, + ({ resolve: () => void; reject: (e: any) => void })[] + >(); + constructor( private ds: DataStore, ) { } + /// Worker management + public queueWorker( + queue: string, + resolve: () => void, + reject: (e: any) => void, + ) { + let waiters = this.queueWaiters.get(queue); + if (!waiters) { + waiters = []; + this.queueWaiters.set(queue, waiters); + } + // console.log("[mq]", "Queuing a worker for queue", queue); + waiters.push({ resolve, reject }); + } + + /** + * Wakes up a single worker waiting on the given queue, if any + * @param queue + */ + wakeupWorker(queue: string) { + const waiters = this.queueWaiters.get(queue); + if (waiters && waiters.length > 0) { + // console.log("[mq]", "Waking up a worker for queue", queue); + const { resolve } = waiters.shift()!; + resolve(); + if (waiters.length === 0) { + // Clean up empty arrays + this.queueWaiters.delete(queue); + } + } + } + + removeQueuedWorker(queue: string, reject: (e: any) => void) { + const waiters = this.queueWaiters.get(queue); + if (waiters) { + const index = waiters.findIndex((w) => w.reject === reject); + if (index !== -1) { + waiters.splice(index, 1); + } + if (waiters.length === 0) { + // Let's not keep empty arrays around + this.queueWaiters.delete(queue); + } + } + } + + /** + * Sends a batch of messages to a queue. + * @param queue the name of the queue + * @param bodies the bodies of the messages to send + * @returns + */ async batchSend(queue: string, bodies: any[]): Promise { if (bodies.length === 0) { return; } + const messages: KV[] = bodies.map((body) => { const id = `${Date.now()}-${String(++this.seq).padStart(6, "0")}`; const key = [...queuedPrefix, queue, id]; @@ -43,13 +166,7 @@ export class DataStoreMQ { await this.ds.batchSet(messages); - // See if we can immediately process the message with a local subscription - const localSubscriptions = this.localSubscriptions.get(queue); - if (localSubscriptions) { - for (const run of localSubscriptions) { - run(); - } - } + this.wakeupWorker(queue); } send(queue: string, body: any): Promise { @@ -68,6 +185,7 @@ export class DataStoreMQ { if (messages.length === 0) { return []; } + // Put them in the processing queue await this.ds.batchSet( messages.map((m) => ({ @@ -97,56 +215,11 @@ export class DataStoreMQ { queue: string, options: MQSubscribeOptions, callback: (messages: MQMessage[]) => Promise | void, - ): () => void { - let running = true; - let timeout: number | undefined; - const batchSize = options.batchSize || 1; - const run = async () => { - try { - // We're running, so let's make sure we're not running multiple times - if (timeout) { - clearTimeout(timeout); - } - if (!running) { - return; - } - const messages = await this.poll(queue, batchSize); - if (messages.length > 0) { - await callback(messages); - } - // If we got exactly the batch size, there might be more messages - if (messages.length === batchSize) { - await run(); - } - timeout = setTimeout(run, options.pollInterval || 5000); - } catch (e: any) { - console.error("Error in MQ subscription handler", e); - } - }; - - // Register as a local subscription handler - const localSubscriptions = this.localSubscriptions.get(queue); - if (!localSubscriptions) { - this.localSubscriptions.set(queue, new Set([run])); - } else { - localSubscriptions.add(run); - } - - // Run the first time (which will schedule subsequent polling intervals) - run(); - - // And return an unsubscribe function - return () => { - running = false; - if (timeout) { - clearTimeout(timeout); - } - // Remove the subscription from localSubscriptions - const queueSubscriptions = this.localSubscriptions.get(queue); - if (queueSubscriptions) { - queueSubscriptions.delete(run); - } - }; + ): QueueWorker { + const worker = new QueueWorker(this, queue, options, callback); + // Start the worker asynchronously + worker.run(); + return worker; } ack(queue: string, id: string) { @@ -253,7 +326,7 @@ export class DataStoreMQ { {}, ) ) { - ids.push([...queuedPrefix, item.id]); + ids.push([...queuedPrefix, queue, item.id]); } for ( const item of await this.ds.luaQuery([ @@ -261,7 +334,7 @@ export class DataStoreMQ { queue, ], {}) ) { - ids.push([...processingPrefix, item.id]); + ids.push([...processingPrefix, queue, item.id]); } for ( const item of await this.ds.luaQuery([ @@ -269,7 +342,7 @@ export class DataStoreMQ { queue, ], {}) ) { - ids.push([...dlqPrefix, item.id]); + ids.push([...dlqPrefix, queue, item.id]); } await this.ds.batchDelete(ids); } @@ -287,6 +360,16 @@ export class DataStoreMQ { }; } + async awaitEmptyQueue(queue: string): Promise { + while (true) { + const stats = await this.getQueueStats(queue); + if (stats.queued === 0 && stats.processing === 0) { + break; + } + await sleep(200); + } + } + async getAllQueueStats(): Promise> { const allStatus: Record = {}; for ( diff --git a/lib/logger.test.ts b/lib/logger.test.ts new file mode 100644 index 00000000..1294ca9d --- /dev/null +++ b/lib/logger.test.ts @@ -0,0 +1,167 @@ +import { assert, assertEquals } from "@std/assert"; +import { initLogger, Logger } from "./logger.ts"; + +Deno.test("Logger prefix functionality", () => { + // Store original console methods + const originalLog = console.log; + const originalInfo = console.info; + const originalWarn = console.warn; + const originalError = console.error; + + const capturedOutput: string[] = []; + + // Mock console methods to capture output + console.log = (...args: any[]) => { + capturedOutput.push(args.join(" ")); + }; + console.info = (...args: any[]) => { + capturedOutput.push(args.join(" ")); + }; + console.warn = (...args: any[]) => { + capturedOutput.push(args.join(" ")); + }; + console.error = (...args: any[]) => { + capturedOutput.push(args.join(" ")); + }; + + try { + new Logger("[TEST]"); + + console.log("Hello", "world"); + console.info("Info message"); + console.warn("Warning message"); + console.error("Error message"); + + assertEquals(capturedOutput[0], "[TEST] Hello world"); + assertEquals(capturedOutput[1], "[TEST] Info message"); + assertEquals(capturedOutput[2], "[TEST] Warning message"); + assertEquals(capturedOutput[3], "[TEST] Error message"); + } finally { + // Restore original console methods + console.log = originalLog; + console.info = originalInfo; + console.warn = originalWarn; + console.error = originalError; + } +}); + +Deno.test("Logger without prefix", () => { + const originalLog = console.log; + const capturedOutput: string[] = []; + + console.log = (...args: any[]) => { + capturedOutput.push(args.join(" ")); + }; + + try { + new Logger(""); + + console.log("Hello", "world"); + + assertEquals(capturedOutput[0], "Hello world"); + } finally { + console.log = originalLog; + } +}); + +Deno.test("Logger log capture", () => { + const logger = new Logger("[CAPTURE]"); + + console.log("First message"); + console.info("Second message", { key: "value" }); + console.warn("Third message", 123); + console.error("Fourth message"); + + const capturedLogs = logger.getCapturedLogs(); + + assertEquals(capturedLogs.length, 4); + + assertEquals(capturedLogs[0].level, "log"); + assertEquals(capturedLogs[0].message, "First message"); + assertEquals(capturedLogs[0].args, ["First message"]); + + assertEquals(capturedLogs[1].level, "info"); + assertEquals(capturedLogs[1].message, 'Second message {"key":"value"}'); + assertEquals(capturedLogs[1].args, ["Second message", { key: "value" }]); + + assertEquals(capturedLogs[2].level, "warn"); + assertEquals(capturedLogs[2].message, "Third message 123"); + assertEquals(capturedLogs[2].args, ["Third message", 123]); + + assertEquals(capturedLogs[3].level, "error"); + assertEquals(capturedLogs[3].message, "Fourth message"); + assertEquals(capturedLogs[3].args, ["Fourth message"]); + + // Check that all entries have timestamps + capturedLogs.forEach((entry) => { + assert(typeof entry.timestamp === "number"); + assert(entry.timestamp > 0); + }); +}); + +Deno.test("Logger max capture size", () => { + const logger = new Logger("[SIZE]", 3); + + // Add more logs than the max size + console.log("Message 1"); + console.log("Message 2"); + console.log("Message 3"); + console.log("Message 4"); + console.log("Message 5"); + + const capturedLogs = logger.getCapturedLogs(); + + // Should only keep the last 3 messages + assertEquals(capturedLogs.length, 3); + assertEquals(capturedLogs[0].message, "Message 3"); + assertEquals(capturedLogs[1].message, "Message 4"); + assertEquals(capturedLogs[2].message, "Message 5"); +}); + +Deno.test("Global logger initialization", () => { + const originalLog = console.log; + const capturedOutput: string[] = []; + + console.log = (...args: any[]) => { + capturedOutput.push(args.join(" ")); + }; + + try { + const logger = initLogger("[GLOBAL]"); + + console.log("Global test"); + + assertEquals(capturedOutput[0], "[GLOBAL] Global test"); + assert(logger instanceof Logger); + } finally { + console.log = originalLog; + } +}); + +Deno.test("Logger handles complex objects", () => { + const logger = new Logger("[COMPLEX]"); + + const complexObject = { + name: "test", + nested: { value: 42 }, + array: [1, 2, 3], + }; + + const circularObject: any = { name: "circular" }; + circularObject.self = circularObject; + + console.log("Complex object:", complexObject); + console.log("Circular object:", circularObject); + + const capturedLogs = logger.getCapturedLogs(); + + assertEquals(capturedLogs.length, 2); + + // First log should handle complex object properly + assertEquals(capturedLogs[0].args[0], "Complex object:"); + assertEquals(capturedLogs[0].args[1], complexObject); + + // Second log should handle circular reference gracefully + assertEquals(capturedLogs[1].args[0], "Circular object:"); + assertEquals(typeof capturedLogs[1].args[1], "string"); // Should be converted to string +}); diff --git a/lib/logger.ts b/lib/logger.ts new file mode 100644 index 00000000..b7aaad56 --- /dev/null +++ b/lib/logger.ts @@ -0,0 +1,105 @@ +// Logger that monkey patches console methods with prefixes and can capture logs for server transmission + +export interface LogEntry { + level: "log" | "info" | "warn" | "error" | "debug"; + timestamp: number; + message: string; + args: any[]; +} + +export class Logger { + private originalConsole: { + log: typeof console.log; + info: typeof console.info; + warn: typeof console.warn; + error: typeof console.error; + debug: typeof console.debug; + }; + private logCapture: LogEntry[] = []; + + constructor( + private prefix: string = "", + private maxCaptureSize: number = 1000, + ) { + this.prefix = prefix; + + // Store original console methods + this.originalConsole = { + log: console.log.bind(console), + info: console.info.bind(console), + warn: console.warn.bind(console), + error: console.error.bind(console), + debug: console.debug.bind(console), + }; + + this.patchConsole(); + } + + private patchConsole(): void { + const createPatchedMethod = (level: keyof typeof this.originalConsole) => { + return (...args: any[]) => { + const prefixedArgs = this.prefix ? [this.prefix, ...args] : args; + + // Call original console method + this.originalConsole[level](...prefixedArgs); + + // Capture log if capturing is enabled + this.captureLog(level, args); + }; + }; + + console.log = createPatchedMethod("log"); + console.info = createPatchedMethod("info"); + console.warn = createPatchedMethod("warn"); + console.error = createPatchedMethod("error"); + console.debug = createPatchedMethod("debug"); + } + + private captureLog(level: LogEntry["level"], args: any[]): void { + const entry: LogEntry = { + level, + timestamp: Date.now(), + message: args.map((arg) => { + if (typeof arg === "string") { + return arg; + } + try { + return JSON.stringify(arg); + } catch { + // Handle circular references or other JSON.stringify failures + return String(arg); + } + }).join(" "), + args: args.map((arg) => { + // Serialize complex objects for safe transmission + try { + return typeof arg === "object" + ? JSON.parse(JSON.stringify(arg)) + : arg; + } catch { + // This may fail due to recursive structures, in that case just fall back to a best-effort string + return String(arg); + } + }), + }; + + this.logCapture.push(entry); + + // Maintain max capture size by removing oldest entries + if (this.logCapture.length > this.maxCaptureSize) { + this.logCapture.shift(); + } + } + + getCapturedLogs(): LogEntry[] { + return [...this.logCapture]; + } +} + +// Global logger instance +let globalLogger: Logger | undefined = undefined; + +export function initLogger(prefix: string = ""): Logger { + globalLogger = new Logger(prefix); + return globalLogger; +} diff --git a/lib/plugos/event.test.ts b/lib/plugos/event.test.ts new file mode 100644 index 00000000..83de21a0 --- /dev/null +++ b/lib/plugos/event.test.ts @@ -0,0 +1,206 @@ +import { assert, assertEquals } from "@std/assert"; +import { EventEmitter } from "./event.ts"; + +// Test event handler interfaces +interface TestEvents { + message: (content: string) => void | Promise; + count: (num: number) => void | Promise; + multi: (a: string, b: number, c: boolean) => void | Promise; + async: () => void | Promise; + sync: () => void | Promise; + error: () => void | Promise; +} + +// Concrete test implementation +class TestEventEmitter extends EventEmitter { + // Just a concrete implementation for testing +} + +Deno.test("EventEmitter - basic on/emit functionality", async () => { + const emitter = new TestEventEmitter(); + let received = ""; + let count = 0; + + emitter.on({ + message: (content: string) => { + received = content; + }, + count: (num: number) => { + count = num; + }, + }); + + await emitter.emit("message", "hello world"); + assertEquals(received, "hello world"); + + await emitter.emit("count", 42); + assertEquals(count, 42); +}); + +Deno.test("EventEmitter - multiple handlers for same event", async () => { + const emitter = new TestEventEmitter(); + const results: string[] = []; + + emitter.on({ + message: (content: string) => { + results.push(`handler1: ${content}`); + }, + }); + + emitter.on({ + message: (content: string) => { + results.push(`handler2: ${content}`); + }, + }); + + await emitter.emit("message", "test"); + + assertEquals(results.length, 2); + assertEquals(results[0], "handler1: test"); + assertEquals(results[1], "handler2: test"); +}); + +Deno.test("EventEmitter - multiple events in single handler", async () => { + const emitter = new TestEventEmitter(); + let messageReceived = ""; + let countReceived = 0; + + emitter.on({ + message: (content: string) => { + messageReceived = content; + }, + count: (num: number) => { + countReceived = num; + }, + }); + + await emitter.emit("message", "multi-handler"); + await emitter.emit("count", 123); + + assertEquals(messageReceived, "multi-handler"); + assertEquals(countReceived, 123); +}); + +Deno.test("EventEmitter - emit with multiple arguments", async () => { + const emitter = new TestEventEmitter(); + let receivedA = ""; + let receivedB = 0; + let receivedC = false; + + emitter.on({ + multi: (a: string, b: number, c: boolean) => { + receivedA = a; + receivedB = b; + receivedC = c; + }, + }); + + await emitter.emit("multi", "test", 456, true); + + assertEquals(receivedA, "test"); + assertEquals(receivedB, 456); + assertEquals(receivedC, true); +}); + +Deno.test("EventEmitter - off removes specific handler object with multiple events", async () => { + const emitter = new TestEventEmitter(); + let messageCount = 0; + let numberCount = 0; + let remainingMessageCount = 0; + + // Handler object with multiple events that we'll remove + const handlersToRemove = { + message: () => { + messageCount++; + }, + count: () => { + numberCount++; + }, + }; + + // Another handler for message events that should remain + const remainingHandlers = { + message: () => { + remainingMessageCount++; + }, + }; + + emitter.on(handlersToRemove); + emitter.on(remainingHandlers); + + // Test that both handlers work initially + await emitter.emit("message", "test1"); + await emitter.emit("count", 42); + assertEquals(messageCount, 1); + assertEquals(numberCount, 1); + assertEquals(remainingMessageCount, 1); + + // Remove the first handler object + emitter.off(handlersToRemove); + + // Test that only the remaining message handler works + await emitter.emit("message", "test2"); + await emitter.emit("count", 43); + assertEquals(messageCount, 1); // Should not have incremented + assertEquals(numberCount, 1); // Should not have incremented + assertEquals(remainingMessageCount, 2); // Should have incremented +}); + +Deno.test("EventEmitter - off removes only the specified handler object", async () => { + const emitter = new TestEventEmitter(); + let handler1MessageCount = 0; + let handler1CountValue = 0; + let handler2MessageCount = 0; + let handler2CountValue = 0; + + const handlers1 = { + message: () => { + handler1MessageCount++; + }, + count: (num: number) => { + handler1CountValue = num; + }, + }; + + const handlers2 = { + message: () => { + handler2MessageCount++; + }, + count: (num: number) => { + handler2CountValue = num; + }, + }; + + emitter.on(handlers1); + emitter.on(handlers2); + + // Test both handlers work initially + await emitter.emit("message", "test1"); + await emitter.emit("count", 100); + assertEquals(handler1MessageCount, 1); + assertEquals(handler1CountValue, 100); + assertEquals(handler2MessageCount, 1); + assertEquals(handler2CountValue, 100); + + // Remove only handlers1 + emitter.off(handlers1); + + // Test that only handlers2 continues to work + await emitter.emit("message", "test2"); + await emitter.emit("count", 200); + assertEquals(handler1MessageCount, 1); // Should not have incremented + assertEquals(handler1CountValue, 100); // Should not have changed + assertEquals(handler2MessageCount, 2); // Should have incremented + assertEquals(handler2CountValue, 200); // Should have changed +}); + +Deno.test("EventEmitter - no handlers for event", async () => { + const emitter = new TestEventEmitter(); + + // Should not throw when emitting to non-existent handlers + await emitter.emit("message", "test"); + await emitter.emit("count", 42); + + // Test passes if no exception is thrown + assert(true); +}); diff --git a/lib/plugos/event.ts b/lib/plugos/event.ts index 0d12a3a0..3049eede 100644 --- a/lib/plugos/event.ts +++ b/lib/plugos/event.ts @@ -1,14 +1,27 @@ export abstract class EventEmitter { private handlers: Partial[] = []; + /** + * Subscribe to events + * @param handlers + */ on(handlers: Partial) { this.handlers.push(handlers); } + /** + * Unsubscribe from events + * @param handlers + */ off(handlers: Partial) { this.handlers = this.handlers.filter((h) => h !== handlers); } + /** + * Broadcast an event to all subscribers + * @param eventName + * @param args + */ async emit(eventName: keyof HandlerT, ...args: any[]): Promise { for (const handler of this.handlers) { const fn: any = handler[eventName]; diff --git a/lib/plugos/hooks/mq.ts b/lib/plugos/hooks/mq.ts index e8d9fcd3..5f89b602 100644 --- a/lib/plugos/hooks/mq.ts +++ b/lib/plugos/hooks/mq.ts @@ -2,11 +2,11 @@ import type { Hook, Manifest } from "../types.ts"; import type { System } from "../system.ts"; import { throttle } from "../../async.ts"; import type { MQHookT } from "../../manifest.ts"; -import type { DataStoreMQ } from "../../data/mq.datastore.ts"; +import type { DataStoreMQ, QueueWorker } from "../../data/mq.datastore.ts"; import type { MQMessage } from "../../../type/datastore.ts"; export class MQHook implements Hook { - subscriptions: (() => void)[] = []; + subscriptions: QueueWorker[] = []; throttledReloadQueues = throttle(() => { this.reloadQueues(); }, 1000); @@ -29,7 +29,7 @@ export class MQHook implements Hook { } stop() { - this.subscriptions.forEach((sub) => sub()); + this.subscriptions.forEach((worker) => worker.stop()); this.subscriptions = []; } diff --git a/lib/plugos/syscalls/mq.ts b/lib/plugos/syscalls/mq.ts index 4fd3c63f..8292673f 100644 --- a/lib/plugos/syscalls/mq.ts +++ b/lib/plugos/syscalls/mq.ts @@ -23,5 +23,8 @@ export function mqSyscalls( "mq.getQueueStats": (_ctx, queue: string) => { return mq.getQueueStats(queue); }, + "mq.awaitEmptyQueue": (_ctx, queue: string) => { + return mq.awaitEmptyQueue(queue); + }, }; } diff --git a/lib/plugos/system.ts b/lib/plugos/system.ts index f2b7d5ae..64725f46 100644 --- a/lib/plugos/system.ts +++ b/lib/plugos/system.ts @@ -43,7 +43,7 @@ export class System extends EventEmitter> { * @param env either an environment or undefined for hybrid mode */ constructor( - readonly env: string | undefined, + readonly env: string | undefined = undefined, readonly options: SystemOptions = {}, ) { super(); diff --git a/lib/plugos/worker_runtime.ts b/lib/plugos/worker_runtime.ts index f5d2e91c..502d5923 100644 --- a/lib/plugos/worker_runtime.ts +++ b/lib/plugos/worker_runtime.ts @@ -5,6 +5,7 @@ import type { ProxyFetchResponse64, } from "../proxy_fetch.ts"; import { base64Decode, base64Encode } from "../crypto.ts"; +import { initLogger } from "../logger.ts"; declare global { function syscall(name: string, ...args: any[]): Promise; @@ -135,6 +136,7 @@ export function setupMessageListener( type: "manifest", manifest, }); + initLogger(`[${manifest.name} plug]`); } export async function sandboxFetch( diff --git a/lib/spaces/asset_bundle_space_primitives.ts b/lib/spaces/asset_bundle_space_primitives.ts index fbe0cfae..96de9fa6 100644 --- a/lib/spaces/asset_bundle_space_primitives.ts +++ b/lib/spaces/asset_bundle_space_primitives.ts @@ -24,62 +24,60 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives { } readFile( - name: string, + path: string, ): Promise<{ data: Uint8Array; meta: FileMeta }> { - if (this.assetBundle.has(name)) { - const data = this.assetBundle.readFileSync(name); + if (this.assetBundle.has(path)) { + const data = this.assetBundle.readFileSync(path); // console.log("Requested encoding", encoding); return Promise.resolve({ data, meta: { - contentType: this.assetBundle.getMimeType(name), - created: this.assetBundle.getMtime(name), - lastModified: this.assetBundle.getMtime(name), + contentType: this.assetBundle.getMimeType(path), + created: this.assetBundle.getMtime(path), + lastModified: this.assetBundle.getMtime(path), size: data.byteLength, perm: "ro", } as FileMeta, }); } - return this.wrapped.readFile(name); + return this.wrapped.readFile(path); } - getFileMeta(name: string): Promise { - if (this.assetBundle.has(name)) { - const data = this.assetBundle.readFileSync(name); + getFileMeta(path: string, observing?: boolean): Promise { + if (this.assetBundle.has(path)) { + const data = this.assetBundle.readFileSync(path); return Promise.resolve({ - contentType: this.assetBundle.getMimeType(name), - created: this.assetBundle.getMtime(name), - lastModified: this.assetBundle.getMtime(name), + contentType: this.assetBundle.getMimeType(path), + created: this.assetBundle.getMtime(path), + lastModified: this.assetBundle.getMtime(path), size: data.byteLength, perm: "ro", } as FileMeta); } - return this.wrapped.getFileMeta(name); + return this.wrapped.getFileMeta(path, observing); } writeFile( - name: string, + path: string, data: Uint8Array, - selfUpdate?: boolean, meta?: FileMeta, ): Promise { - if (this.assetBundle.has(name)) { - console.warn("Attempted to write to read-only asset file", name); - return this.getFileMeta(name); + if (this.assetBundle.has(path)) { + console.warn("Attempted to write to read-only asset file", path); + return this.getFileMeta(path); } return this.wrapped.writeFile( - name, + path, data, - selfUpdate, meta, ); } - deleteFile(name: string): Promise { - if (this.assetBundle.has(name)) { + deleteFile(path: string): Promise { + if (this.assetBundle.has(path)) { // Quietly ignore return Promise.resolve(); } - return this.wrapped.deleteFile(name); + return this.wrapped.deleteFile(path); } } diff --git a/lib/spaces/checked_space_primitives.ts b/lib/spaces/checked_space_primitives.ts index a8884858..37128f2d 100644 --- a/lib/spaces/checked_space_primitives.ts +++ b/lib/spaces/checked_space_primitives.ts @@ -14,40 +14,39 @@ export class CheckPathSpacePrimitives implements SpacePrimitives { ); } - readFile(name: string): Promise<{ data: Uint8Array; meta: FileMeta }> { - if (!this.isReadable(name)) { + readFile(path: string): Promise<{ data: Uint8Array; meta: FileMeta }> { + if (!this.isReadable(path)) { throw new Error("Couldn't write file, path isn't writable"); } - return this.wrapped.readFile(name); + return this.wrapped.readFile(path); } - getFileMeta(name: string): Promise { - if (!this.isReadable(name)) { + getFileMeta(path: string, observing?: boolean): Promise { + if (!this.isReadable(path)) { throw new Error("Couldn't get file meta, path isn't writable"); } - return this.wrapped.getFileMeta(name); + return this.wrapped.getFileMeta(path, observing); } writeFile( - name: string, + path: string, data: Uint8Array, - selfUpdate?: boolean | undefined, meta?: FileMeta, ): Promise { - if (!this.isWritable(name)) { + if (!this.isWritable(path)) { throw new Error("Couldn't write file, path is invalid"); } - return this.wrapped.writeFile(name, data, selfUpdate, meta); + return this.wrapped.writeFile(path, data, meta); } - deleteFile(name: string): Promise { + deleteFile(path: string): Promise { // We allow deletion of paths we can't write to. This is for the case when // the user has an invalidly named file in their space and they need to // remove it/rename it - if (!this.isReadable(name)) { + if (!this.isReadable(path)) { throw new Error("Couldn't delete file, path isn't writable"); } - return this.wrapped.deleteFile(name); + return this.wrapped.deleteFile(path); } private isReadable(path: string): boolean { diff --git a/lib/spaces/datastore_space_primitives.test.ts b/lib/spaces/datastore_space_primitives.test.ts index a9526876..190ac2d6 100644 --- a/lib/spaces/datastore_space_primitives.test.ts +++ b/lib/spaces/datastore_space_primitives.test.ts @@ -1,5 +1,4 @@ import "fake-indexeddb/auto"; -import { DataStore } from "../data/datastore.ts"; import { IndexedDBKvPrimitives } from "../data/indexeddb_kv_primitives.ts"; import { DataStoreSpacePrimitives } from "./datastore_space_primitives.ts"; import { testSpacePrimitives } from "./space_primitives.test.ts"; @@ -11,7 +10,7 @@ Deno.test("DataStoreSpacePrimitives", { const db = new IndexedDBKvPrimitives("test"); await db.init(); - const space = new DataStoreSpacePrimitives(new DataStore(db)); + const space = new DataStoreSpacePrimitives(db); await testSpacePrimitives(space); db.close(); }); diff --git a/lib/spaces/datastore_space_primitives.ts b/lib/spaces/datastore_space_primitives.ts index 23920d25..e4dcdd8f 100644 --- a/lib/spaces/datastore_space_primitives.ts +++ b/lib/spaces/datastore_space_primitives.ts @@ -1,67 +1,70 @@ import type { SpacePrimitives } from "./space_primitives.ts"; import { mime } from "mimetypes"; -import type { DataStore } from "../data/datastore.ts"; import type { FileMeta } from "../../type/index.ts"; +import { notFoundError } from "../constants.ts"; +import type { KvPrimitives } from "../data/kv_primitives.ts"; -export type FileContent = { - name: string; - meta: FileMeta; - data: Uint8Array; -}; +const filesMetaPrefix = ["meta"]; +const filesContentPrefix = ["content"]; -const filesMetaPrefix = ["file", "meta"]; -const filesContentPrefix = ["file", "content"]; - -/** - * TODO: Replace this with ChunkedDatastoreSpacePrimitives - */ export class DataStoreSpacePrimitives implements SpacePrimitives { constructor( - private ds: DataStore, + private kv: KvPrimitives, ) { } async fetchFileList(): Promise { - return (await this.ds.luaQuery(filesMetaPrefix, {})) - .map((meta) => this.ensureFileMeta(meta)); + const fileMetas: FileMeta[] = []; + // Iterate over all keys with the fileMetaPrefix + for await (const meta of this.kv.query({ prefix: filesMetaPrefix })) { + fileMetas.push(this.cleanFileMeta(meta.value as FileMeta)); + } + return fileMetas; } async readFile( - name: string, + path: string, ): Promise<{ data: Uint8Array; meta: FileMeta }> { - const fileContent = await this.ds.get([ - ...filesContentPrefix, - name, + // Fetch content and metadata in parallel + const [fileMeta, fileContent] = await this.kv.batchGet([ + [ + ...filesMetaPrefix, + path, + ], + [ + ...filesContentPrefix, + path, + ], ]); - if (!fileContent) { - throw new Error("Not found"); + if (!fileMeta) { + throw notFoundError; } return { - data: fileContent.data, - meta: this.ensureFileMeta(fileContent.meta), + meta: this.cleanFileMeta(fileMeta), + data: fileContent, }; } async writeFile( - name: string, + path: string, data: Uint8Array, - _selfUpdate?: boolean, suggestedMeta?: FileMeta, ): Promise { let meta: FileMeta | undefined; try { // Build off of the existing file meta, if file exists - meta = await this.getFileMeta(name); + meta = await this.getFileMeta(path); } catch { // Not found, that's fine } if (!meta) { + // No existing meta data, let's set some defaults meta = { - name, + name: path, created: suggestedMeta?.lastModified || Date.now(), - perm: suggestedMeta?.perm || "rw", - contentType: mime.getType(name) || "application/octet-stream", + perm: "rw", + contentType: mime.getType(path) || "application/octet-stream", // Overwritten in a sec lastModified: 0, size: 0, @@ -69,43 +72,46 @@ export class DataStoreSpacePrimitives implements SpacePrimitives { } meta.lastModified = suggestedMeta?.lastModified || Date.now(); meta.size = data.byteLength; + if (suggestedMeta?.perm) { + meta.perm = suggestedMeta.perm; + } - await this.ds.batchSet([ + // Write metadata and content in same transaction + await this.kv.batchSet([ { - key: [...filesContentPrefix, name], - value: { name, data, meta }, + key: [...filesMetaPrefix, path], + value: meta, }, { - key: [...filesMetaPrefix, name], - value: meta, + key: [...filesContentPrefix, path], + value: data, }, ]); return meta; } - async deleteFile(name: string): Promise { - const fileMeta = await this.ds.get([ - ...filesMetaPrefix, - name, + async deleteFile(path: string): Promise { + const [fileMeta] = await this.kv.batchGet([ + [...filesMetaPrefix, path], ]); if (!fileMeta) { - throw new Error("Not found"); + throw notFoundError; } - return this.ds.batchDelete([ - [...filesMetaPrefix, name], - [...filesContentPrefix, name], + return this.kv.batchDelete([ + [...filesMetaPrefix, path], + [...filesContentPrefix, path], ]); } - async getFileMeta(name: string): Promise { - const fileMeta = await this.ds.get([...filesMetaPrefix, name]); + async getFileMeta(path: string, _observing?: boolean): Promise { + const [fileMeta] = await this.kv.batchGet([[...filesMetaPrefix, path]]); if (!fileMeta) { - throw new Error("Not found"); + throw notFoundError; } - return this.ensureFileMeta(fileMeta); + return this.cleanFileMeta(fileMeta); } - ensureFileMeta(fileMeta: FileMeta): FileMeta { + cleanFileMeta(fileMeta: FileMeta): FileMeta { if (!fileMeta.created) { fileMeta.created = fileMeta.lastModified; } diff --git a/lib/spaces/disk_space_primitives.test.ts b/lib/spaces/disk_space_primitives.test.ts new file mode 100644 index 00000000..638f82d0 --- /dev/null +++ b/lib/spaces/disk_space_primitives.test.ts @@ -0,0 +1,14 @@ +import "fake-indexeddb/auto"; +import { testSpacePrimitives } from "./space_primitives.test.ts"; +import { DiskSpacePrimitives } from "./disk_space_primitives.ts"; + +Deno.test("DataStoreSpacePrimitives", { + sanitizeResources: false, + sanitizeOps: false, +}, async () => { + // Create a temporary folder + const tempDir = await Deno.makeTempDir(); + const space = new DiskSpacePrimitives(tempDir); + await testSpacePrimitives(space); + Deno.remove(tempDir, { recursive: true }); +}); diff --git a/lib/spaces/disk_space_primitives.ts b/lib/spaces/disk_space_primitives.ts index 3d3cff00..6c9b6a25 100644 --- a/lib/spaces/disk_space_primitives.ts +++ b/lib/spaces/disk_space_primitives.ts @@ -1,9 +1,10 @@ -import * as path from "@std/path"; +import * as pathUtils from "@std/path"; import { readAll } from "@std/io/read-all"; import type { SpacePrimitives } from "./space_primitives.ts"; import { mime } from "mimetypes"; import type { FileMeta } from "../../type/index.ts"; +import { notFoundError } from "../constants.ts"; function lookupContentType(path: string): string { return mime.getType(path) || "application/octet-stream"; @@ -13,20 +14,15 @@ function normalizeForwardSlashPath(path: string) { return path.replaceAll("\\", "/"); } -const excludedFiles = ["data.db", "data.db-journal", "sync.json"]; - export class DiskSpacePrimitives implements SpacePrimitives { rootPath: string; - fileListCache: FileMeta[] = []; - fileListCacheTime = 0; - fileListCacheUpdating: AbortController | null = null; constructor(rootPath: string) { this.rootPath = Deno.realPathSync(rootPath); } safePath(p: string): string { - const realPath = path.resolve(p); + const realPath = pathUtils.resolve(p); if (!realPath.startsWith(this.rootPath)) { throw Error(`Path ${p} is not in the space`); } @@ -34,7 +30,7 @@ export class DiskSpacePrimitives implements SpacePrimitives { } filenameToPath(pageName: string) { - return this.safePath(path.join(this.rootPath, pageName)); + return this.safePath(pathUtils.join(this.rootPath, pageName)); } pathToFilename(fullPath: string): string { @@ -42,12 +38,12 @@ export class DiskSpacePrimitives implements SpacePrimitives { } async readFile( - name: string, + path: string, ): Promise<{ data: Uint8Array; meta: FileMeta }> { - const localPath = this.filenameToPath(name); + const localPath = this.filenameToPath(path); try { const s = await Deno.stat(localPath); - const contentType = lookupContentType(name); + const contentType = lookupContentType(path); const f = await Deno.open(localPath, { read: true }); const data = await readAll(f); @@ -56,7 +52,7 @@ export class DiskSpacePrimitives implements SpacePrimitives { return { data, meta: { - name: name, + name: path, created: s.birthtime?.getTime() || s.mtime?.getTime() || 0, lastModified: s.mtime?.getTime() || 0, perm: "rw", @@ -65,21 +61,20 @@ export class DiskSpacePrimitives implements SpacePrimitives { }, }; } catch { - // console.error("Error while reading file", name, e); - throw Error("Not found"); + // console.error("Error while reading file", path, e); + throw notFoundError; } } async writeFile( - name: string, + path: string, data: Uint8Array, - _selfUpdate?: boolean, meta?: FileMeta, ): Promise { - const localPath = this.filenameToPath(name); + const localPath = this.filenameToPath(path); try { // Ensure parent folder exists - await Deno.mkdir(path.dirname(localPath), { recursive: true }); + await Deno.mkdir(pathUtils.dirname(localPath), { recursive: true }); const file = await Deno.open(localPath, { write: true, @@ -97,61 +92,58 @@ export class DiskSpacePrimitives implements SpacePrimitives { } await writer.close(); - // Invalidate cache and trigger an update - this.fileListCache = []; - this.fileListCacheTime = 0; - this.updateCacheInBackground(); - // Fetch new metadata - return this.getFileMeta(name); + return this.getFileMeta(path); } catch (e) { - console.error("Error while writing file", name, e); - throw Error(`Could not write ${name}`); + console.error("Error while writing file", path, e); + throw Error(`Could not write ${path}`); } } - async getFileMeta(name: string): Promise { - const localPath = this.filenameToPath(name); + async getFileMeta(path: string, _observing?: boolean): Promise { + const localPath = this.filenameToPath(path); try { const s = await Deno.stat(localPath); return { - name: name, + name: path, size: s.size, - contentType: lookupContentType(name), + contentType: lookupContentType(path), created: s.birthtime?.getTime() || s.mtime?.getTime() || 0, lastModified: s.mtime?.getTime() || 0, perm: "rw", }; } catch (e: any) { if (e instanceof Deno.errors.NotFound) { - throw Error("Not found"); + throw notFoundError; } // console.error("Error while getting page meta", pageName, e); - throw Error(`Could not get meta for ${name}`); + throw Error(`Could not get meta for ${path}`); } } - async deleteFile(name: string): Promise { - const localPath = this.filenameToPath(name); - await Deno.remove(localPath); + async deleteFile(path: string): Promise { + const localPath = this.filenameToPath(path); + try { + await Deno.remove(localPath); + } catch (e: any) { + if (e instanceof Deno.errors.NotFound) { + throw notFoundError; + } + throw e; + } // Recursively remove empty parent directories up to rootPath await this.cleanOrphaned(localPath); - - // Invalidate cache and trigger an update - this.fileListCache = []; - this.fileListCacheTime = 0; - this.updateCacheInBackground(); } private async cleanOrphaned(pathToDeletedFile: string) { - let current = path.dirname(pathToDeletedFile); + let current = pathUtils.dirname(pathToDeletedFile); while (current.startsWith(this.rootPath) && current != this.rootPath) { try { // Attempt to remove the current directory await Deno.remove(current); - current = path.dirname(current); + current = pathUtils.dirname(current); } catch (e: any) { if ( e.code === "ENOTEMPTY" || @@ -161,7 +153,7 @@ export class DiskSpacePrimitives implements SpacePrimitives { } if (e instanceof Deno.errors.NotFound) { - current = path.dirname(current); // continue upwards + current = pathUtils.dirname(current); // continue upwards continue; } @@ -171,27 +163,8 @@ export class DiskSpacePrimitives implements SpacePrimitives { } } - async fetchFileList(): Promise { - // console.log("Fetching file list"); - const startTime = performance.now(); - - // If the file list cache is less than 60 seconds old, return it - if ( - this.fileListCache.length > 0 && - startTime - this.fileListCacheTime < 60000 - ) { - // Trigger a background sync, but return the cached list while the cache is being updated - this.updateCacheInBackground(); - return this.fileListCache; - } - - // Otherwise get the file list and wait for it - const allFiles: FileMeta[] = await this.getFileList(); - - this.fileListCache = allFiles; - this.fileListCacheTime = startTime; - - return allFiles; + fetchFileList(): Promise { + return this.getFileList(); } private async getFileList(): Promise { @@ -203,9 +176,6 @@ export class DiskSpacePrimitives implements SpacePrimitives { try { const s = await Deno.stat(fullPath); const name = fullPath.substring(this.rootPath.length + 1); - if (excludedFiles.includes(name)) { - continue; - } allFiles.push({ name: normalizeForwardSlashPath(name), created: s.birthtime?.getTime() || s.mtime?.getTime() || 0, @@ -224,40 +194,6 @@ export class DiskSpacePrimitives implements SpacePrimitives { } return allFiles; } - - private updateCacheInBackground() { - if (this.fileListCacheUpdating) { - // Cancel the existing background update, so we never return stale data - this.fileListCacheUpdating.abort(); - } - - const abortController = new AbortController(); - this.fileListCacheUpdating = abortController; - - const updatePromise = this.getFileList().then((allFiles) => { - if (abortController.signal.aborted) return; - - this.fileListCache = allFiles; - this.fileListCacheTime = performance.now(); - // console.info( - // "Updated file list cache in background:", - // allFiles.length, - // "files found", - // ); - }).catch((error) => { - if (abortController.signal.aborted) return; - - if (error.name !== "AbortError") { - console.error("Error updating file list cache in background:", error); - } - }).finally(() => { - if (this.fileListCacheUpdating === abortController) { - this.fileListCacheUpdating = null; - } - }); - - return updatePromise; - } } async function* walkPreserveSymlinks( diff --git a/lib/spaces/evented_space_primitives.ts b/lib/spaces/evented_space_primitives.ts index 3a968b1e..7ea7f822 100644 --- a/lib/spaces/evented_space_primitives.ts +++ b/lib/spaces/evented_space_primitives.ts @@ -1,48 +1,77 @@ import type { EventHook } from "../../web/hooks/event.ts"; -import { plugPrefix } from "../spaces/constants.ts"; import type { SpacePrimitives } from "./space_primitives.ts"; import type { FileMeta } from "../../type/index.ts"; +import type { DataStore } from "../data/datastore.ts"; +import { notFoundError } from "../constants.ts"; /** * Events exposed: - * - file:changed (string, localUpdate: boolean) + * - file:changed (string, oldHash, newHash) * - file:deleted (string) * - file:listed (FileMeta[]) - * - page:saved (string, FileMeta) + * - file:initial: triggered in case of an initially empty snapshot, after the first set of events has gone out * - page:deleted (string) */ export class EventedSpacePrimitives implements SpacePrimitives { // Various operations may be going on at the same time, and we don't want to trigger events unnessarily. - // Therefore we use this variable to track if any operation is in flight, and if so, we skip event triggering. + // Therefore, we use this variable to track if any operation is in flight, and if so, we skip event triggering. // This is ok, because any event will be picked up in a following iteration. operationInProgress = false; - initialFileListLoad: boolean; - - public enablePageEvents = true; - private spaceSnapshot: Record = {}; + private enabled = false; constructor( private wrapped: SpacePrimitives, private eventHook: EventHook, + private ds: DataStore, + private snapshotKey = ["$spaceSnapshot"], ) { - // Translate file change events for documents into document:index events - this.eventHook.addLocalListener( - "file:changed", - async ( - name: string, - ) => { - if (!name.endsWith(".md") && !name.startsWith(plugPrefix)) { - // Not a page nor plug, so a document! - await this.dispatchEvent("document:index", name); - } - }, - ); - this.initialFileListLoad = Object.keys(this.spaceSnapshot).length === 0; + } + + async enable() { + console.log("Loading snapshot and enabling events"); + this.spaceSnapshot = (await this.ds.get(this.snapshotKey)) || {}; + this.snapshotChanged = false; + const isFreshSnapshot = Object.keys(this.spaceSnapshot).length === 0; + this.enabled = true; + // trigger loading and eventing + this.fetchFileList().then(async () => { + if (isFreshSnapshot) { + // Trigger event to signal that an intial batch of events has been triggered + await this.dispatchEvent("file:initial"); + } + }); + } + + // Snapshot state management + private spaceSnapshot: Record = {}; + private snapshotChanged = false; + + private updateInSnapshot(key: string, value: number) { + const oldValue = this.spaceSnapshot[key]; + this.spaceSnapshot[key] = value; + this.snapshotChanged = this.snapshotChanged || oldValue !== value; + } + + private deleteFromSnapshot(key: string) { + delete this.spaceSnapshot[key]; + this.snapshotChanged = true; + } + + private async saveSnapshot() { + if (this.enabled && this.snapshotChanged) { + console.log("Saving snapshot"); + await this.ds.set(this.snapshotKey, this.spaceSnapshot); + this.snapshotChanged = false; + } } dispatchEvent(name: string, ...args: any[]): Promise { + if (!this.enabled) { + return Promise.resolve([]); + } + // console.log("Evented space, dispatching", name, args); return this.eventHook.dispatchEvent(name, ...args); } @@ -55,6 +84,9 @@ export class EventedSpacePrimitives implements SpacePrimitives { ); return this.wrapped.fetchFileList(); } + if (!this.enabled) { + return this.wrapped.fetchFileList(); + } // console.log("Fetching file list"); // Fetching mutex this.operationInProgress = true; @@ -68,24 +100,28 @@ export class EventedSpacePrimitives implements SpacePrimitives { const oldHash = this.spaceSnapshot[meta.name]; const newHash = meta.lastModified; // Update in snapshot - this.spaceSnapshot[meta.name] = newHash; + this.updateInSnapshot(meta.name, newHash); // Check what happened to the file if ( ( // New file scenario - !oldHash && !this.initialFileListLoad + !oldHash ) || ( // Changed file scenario oldHash && oldHash !== newHash ) ) { - console.log("Detected file change", meta.name, oldHash, newHash); + console.log( + "Detected file change during listing", + meta.name, + oldHash, + newHash, + ); await this.dispatchEvent( "file:changed", meta.name, - false, oldHash, newHash, ); @@ -95,35 +131,38 @@ export class EventedSpacePrimitives implements SpacePrimitives { } for (const deletedFile of deletedFiles) { - delete this.spaceSnapshot[deletedFile]; + this.deleteFromSnapshot(deletedFile); await this.dispatchEvent("file:deleted", deletedFile); - if (deletedFile.endsWith(".md") && this.enablePageEvents) { + if (deletedFile.endsWith(".md")) { const pageName = deletedFile.substring(0, deletedFile.length - 3); await this.dispatchEvent("page:deleted", pageName); } } await this.dispatchEvent("file:listed", newFileList); - this.initialFileListLoad = false; return newFileList; } finally { + await this.saveSnapshot(); this.operationInProgress = false; } } async readFile( - name: string, + path: string, ): Promise<{ data: Uint8Array; meta: FileMeta }> { + if (!this.enabled) { + return this.wrapped.readFile(path); + } try { // Fetching mutex const wasFetching = this.operationInProgress; this.operationInProgress = true; // Fetch file - const data = await this.wrapped.readFile(name); + const data = await this.wrapped.readFile(path); if (!wasFetching) { - this.triggerEventsAndCache(name, data.meta.lastModified); + await this.triggerEventsAndCache(path, data.meta.lastModified); } return data; } finally { @@ -132,41 +171,24 @@ export class EventedSpacePrimitives implements SpacePrimitives { } async writeFile( - name: string, + path: string, data: Uint8Array, - // TODO: Is self update still used or can it now be removed? - selfUpdate?: boolean, meta?: FileMeta, ): Promise { + if (!this.enabled) { + return this.wrapped.writeFile(path, data, meta); + } + + const wasFetching = this.operationInProgress; + this.operationInProgress = true; try { - this.operationInProgress = true; const newMeta = await this.wrapped.writeFile( - name, + path, data, - selfUpdate, meta, ); - await this.dispatchEvent( - "file:changed", - name, - true, - undefined, - newMeta.lastModified, - ); - this.spaceSnapshot[name] = newMeta.lastModified; - - if (name.endsWith(".md") && this.enablePageEvents) { - // Let's trigger some page-specific events - const pageName = name.substring(0, name.length - 3); - let text = ""; - const decoder = new TextDecoder("utf-8"); - text = decoder.decode(data); - - await this.dispatchEvent("page:saved", pageName, newMeta); - await this.dispatchEvent("page:index_text", { - name: pageName, - text, - }); + if (!wasFetching) { + await this.triggerEventsAndCache(path, newMeta.lastModified); } return newMeta; } finally { @@ -174,34 +196,41 @@ export class EventedSpacePrimitives implements SpacePrimitives { } } - triggerEventsAndCache(name: string, newHash: number) { + /** + * @param name + * @param newHash + * @return whether something changed in the snapshot + */ + async triggerEventsAndCache(name: string, newHash: number) { const oldHash = this.spaceSnapshot[name]; - if (oldHash && newHash && oldHash !== newHash) { + // if (oldHash && newHash && oldHash !== newHash) { + if (oldHash !== newHash) { // Page changed since last cached metadata, trigger event - this.dispatchEvent("file:changed", name, false, oldHash, newHash); + this.dispatchEvent("file:changed", name, oldHash, newHash); } - this.spaceSnapshot[name] = newHash; - return; + this.updateInSnapshot(name, newHash); + await this.saveSnapshot(); } - async getFileMeta(name: string): Promise { - if (!this.enablePageEvents) { - return this.wrapped.getFileMeta(name); + async getFileMeta(path: string, observing?: boolean): Promise { + if (!this.enabled) { + return this.wrapped.getFileMeta(path, observing); } + try { const wasFetching = this.operationInProgress; this.operationInProgress = true; - const newMeta = await this.wrapped.getFileMeta(name); + const newMeta = await this.wrapped.getFileMeta(path, observing); if (!wasFetching) { - this.triggerEventsAndCache(name, newMeta.lastModified); + await this.triggerEventsAndCache(path, newMeta.lastModified); } return newMeta; } catch (e: any) { - // console.log("Checking error", e, name); - if (e.message === "Not found") { - await this.dispatchEvent("file:deleted", name); - if (name.endsWith(".md") && this.enablePageEvents) { - const pageName = name.substring(0, name.length - 3); + // console.log("Checking error", e, path); + if (e.message === notFoundError.message) { + await this.dispatchEvent("file:deleted", path); + if (path.endsWith(".md")) { + const pageName = path.substring(0, path.length - 3); await this.dispatchEvent("page:deleted", pageName); } } @@ -211,18 +240,23 @@ export class EventedSpacePrimitives implements SpacePrimitives { } } - async deleteFile(name: string): Promise { + async deleteFile(path: string): Promise { + if (!this.enabled) { + return this.wrapped.deleteFile(path); + } + try { this.operationInProgress = true; - if (name.endsWith(".md") && this.enablePageEvents) { - const pageName = name.substring(0, name.length - 3); + if (path.endsWith(".md")) { + const pageName = path.substring(0, path.length - 3); await this.dispatchEvent("page:deleted", pageName); } - // await this.getPageMeta(name); // Check if page exists, if not throws Error - await this.wrapped.deleteFile(name); - delete this.spaceSnapshot[name]; - await this.dispatchEvent("file:deleted", name); + // await this.getPageMeta(path); // Check if page exists, if not throws Error + await this.wrapped.deleteFile(path); + this.deleteFromSnapshot(path); + await this.dispatchEvent("file:deleted", path); } finally { + await this.saveSnapshot(); this.operationInProgress = false; } } diff --git a/lib/spaces/fallback_space_primitives.ts b/lib/spaces/fallback_space_primitives.ts deleted file mode 100644 index e757ab9e..00000000 --- a/lib/spaces/fallback_space_primitives.ts +++ /dev/null @@ -1,86 +0,0 @@ -import type { SpacePrimitives } from "./space_primitives.ts"; -import type { FileMeta } from "../../type/index.ts"; - -/** - * FallbackSpacePrimitives is a SpacePrimitives implementation that will try to fall back to another SpacePrimitives implementation for two - * operations: - * - readFile - * - getFileMeta - * The use case is primarily sync: when sync hasn't completed yet, we can fall back to HttpSpacePrimitives to fetch the file from the server. - */ -export class FallbackSpacePrimitives implements SpacePrimitives { - constructor( - private primary: SpacePrimitives, - private fallback: SpacePrimitives, - ) { - } - - fetchFileList(): Promise { - return this.primary.fetchFileList(); - } - - async readFile(name: string): Promise<{ data: Uint8Array; meta: FileMeta }> { - try { - return await this.primary.readFile(name); - } catch (e: any) { - if (e.message === "Not found") { - console.info("Reading file content from fallback for", name); - } else { - console.warn( - `Could not read file ${name} from primary, trying fallback, primary read error`, - e.message, - ); - } - try { - const result = await this.fallback.readFile(name); - return { - data: result.data, - meta: { ...result.meta, noSync: true }, - }; - } catch (fallbackError: any) { - console.error("Error during readFile fallback", fallbackError.message); - // Fallback failed, so let's throw the original error - throw e; - } - } - } - - async getFileMeta(name: string): Promise { - try { - return await this.primary.getFileMeta(name); - } catch (e: any) { - if (e.message === "Not found") { - console.info("Fetching file meta from fallback for", name); - } else { - console.warn( - `Could not fetch file ${name} metadata from primary, trying fallback, primary read error`, - e.message, - ); - } - try { - const meta = await this.fallback.getFileMeta(name); - return { ...meta, noSync: true }; - } catch (fallbackError: any) { - console.error( - "Error during getFileMeta fallback", - fallbackError.message, - ); - // Fallback failed, so let's throw the original error - throw e; - } - } - } - - writeFile( - name: string, - data: Uint8Array, - selfUpdate?: boolean | undefined, - meta?: FileMeta, - ): Promise { - return this.primary.writeFile(name, data, selfUpdate, meta); - } - - deleteFile(name: string): Promise { - return this.primary.deleteFile(name); - } -} diff --git a/lib/spaces/filtered_space_primitives.ts b/lib/spaces/filtered_space_primitives.ts index f7ba0ebe..493b64c3 100644 --- a/lib/spaces/filtered_space_primitives.ts +++ b/lib/spaces/filtered_space_primitives.ts @@ -16,24 +16,23 @@ export class FilteredSpacePrimitives implements SpacePrimitives { return (await this.wrapped.fetchFileList()).filter(this.filterFn); } - readFile(name: string): Promise<{ data: Uint8Array; meta: FileMeta }> { - return this.wrapped.readFile(name); + readFile(path: string): Promise<{ data: Uint8Array; meta: FileMeta }> { + return this.wrapped.readFile(path); } - getFileMeta(name: string): Promise { - return this.wrapped.getFileMeta(name); + getFileMeta(path: string, observing?: boolean): Promise { + return this.wrapped.getFileMeta(path, observing); } writeFile( - name: string, + path: string, data: Uint8Array, - selfUpdate?: boolean | undefined, meta?: FileMeta, ): Promise { - return this.wrapped.writeFile(name, data, selfUpdate, meta); + return this.wrapped.writeFile(path, data, meta); } - deleteFile(name: string): Promise { - return this.wrapped.deleteFile(name); + deleteFile(path: string): Promise { + return this.wrapped.deleteFile(path); } } diff --git a/lib/spaces/http_space_primitives.ts b/lib/spaces/http_space_primitives.ts index b273bec3..720e8739 100644 --- a/lib/spaces/http_space_primitives.ts +++ b/lib/spaces/http_space_primitives.ts @@ -2,16 +2,18 @@ import type { SpacePrimitives } from "./space_primitives.ts"; import { encodePageURI } from "@silverbulletmd/silverbullet/lib/ref"; import { flushCachesAndUnregisterServiceWorker, - unregisterServiceWorkers, -} from "../../web/sw_util.ts"; +} from "../../web/service_worker/util.ts"; import type { FileMeta } from "../../type/index.ts"; +import { notFoundError, offlineError } from "../constants.ts"; +import { headersToFileMeta } from "../../server/util.ts"; const defaultFetchTimeout = 30000; // 30 seconds export class HttpSpacePrimitives implements SpacePrimitives { constructor( readonly url: string, - readonly expectedSpacePath?: string, + readonly expectedSpacePath: string, + private authErrorCallback: (message: string, ...args: any[]) => void, private bearerToken?: string, ) { } @@ -40,19 +42,21 @@ export class HttpSpacePrimitives implements SpacePrimitives { options.redirect = "manual"; const result = await fetch(url, options); if (result.status === 503) { - throw new Error("Offline"); + throw offlineError; } const redirectHeader = result.headers.get("location"); if (result.type === "opaqueredirect" && !redirectHeader) { + console.log("Result", result, "for", url, JSON.stringify(options)); // This is a scenario where the server sent a redirect, but this redirect is not visible to the client, likely due to CORS // The best we can do is to reload the page and hope that the server will redirect us to the correct location - alert( + this.authErrorCallback( "You are not authenticated, reloading to reauthenticate", + "reload", ); - console.log("Unregistering service workers", redirectHeader); - await unregisterServiceWorkers(); - location.reload(); + // console.log("Unregistering service workers", redirectHeader); + // await unregisterServiceWorkers(); + // location.reload(); // Let's throw to avoid any further processing throw Error("Not authenticated"); } @@ -63,11 +67,11 @@ export class HttpSpacePrimitives implements SpacePrimitives { if (result.status >= 300 && result.status < 400) { if (redirectHeader) { // Got a redirect - alert( - "Received an authentication redirect, redirecting to URL: " + - redirectHeader, + this.authErrorCallback( + "Received an authentication redirect", + redirectHeader, ); - location.href = redirectHeader; + // location.href = redirectHeader; throw new Error("Redirected"); } else { console.error("Got a redirect status but no location header", result); @@ -81,15 +85,15 @@ export class HttpSpacePrimitives implements SpacePrimitives { "Received unauthorized status and got a redirect via the API so will redirect to URL", result.url, ); - alert("You are not authenticated, redirecting to: " + redirectHeader); - location.href = redirectHeader; + this.authErrorCallback("You are not authenticated ", redirectHeader); + // location.href = redirectHeader; throw new Error("Not authenticated"); } else { // If not, let's reload - alert( + this.authErrorCallback( "You are not authenticated, going to reload and hope that that kicks off authentication", ); - location.reload(); + // location.reload(); throw new Error("Not authenticated"); } } @@ -105,14 +109,16 @@ export class HttpSpacePrimitives implements SpacePrimitives { // Common substrings: "fetch" "load failed" const errorMessage = e.message.toLowerCase(); if ( - errorMessage.includes("fetch") || errorMessage.includes("load failed") + errorMessage.includes("fetch") || + errorMessage.includes("load failed") || + errorMessage.includes("unavailable") ) { console.error( "Got error fetching, throwing offline", url, - e, + e.message, ); - throw new Error("Offline"); + throw offlineError; } throw e; } @@ -132,18 +138,21 @@ export class HttpSpacePrimitives implements SpacePrimitives { console.log("Expected space path", this.expectedSpacePath); console.log("Got space path", resp.headers.get("X-Space-Path")); await flushCachesAndUnregisterServiceWorker(); - alert("Space folder path different on server, reloading the page"); - location.reload(); + this.authErrorCallback( + "Space folder path different on server, reloading the page", + "reload", + ); + // location.reload(); } return resp.json(); } async readFile( - name: string, + path: string, ): Promise<{ data: Uint8Array; meta: FileMeta }> { const res = await this.authenticatedFetch( - `${this.url}/${encodePageURI(name)}`, + `${this.url}/${encodePageURI(path)}`, { method: "GET", headers: { @@ -153,18 +162,17 @@ export class HttpSpacePrimitives implements SpacePrimitives { }, ); if (res.status === 404) { - throw new Error(`Not found`); + throw notFoundError; } return { data: new Uint8Array(await res.arrayBuffer()), - meta: this.responseToMeta(name, res), + meta: headersToFileMeta(path, res.headers)!, }; } async writeFile( - name: string, + path: string, data: Uint8Array, - _selfUpdate?: boolean, meta?: FileMeta, ): Promise { const headers: Record = { @@ -177,7 +185,7 @@ export class HttpSpacePrimitives implements SpacePrimitives { } const res = await this.authenticatedFetch( - `${this.url}/${encodePageURI(name)}`, + `${this.url}/${encodePageURI(path)}`, { method: "PUT", headers, @@ -185,12 +193,12 @@ export class HttpSpacePrimitives implements SpacePrimitives { body: data as any, }, ); - return this.responseToMeta(name, res); + return headersToFileMeta(path, res.headers)!; } - async deleteFile(name: string): Promise { + async deleteFile(path: string): Promise { const req = await this.authenticatedFetch( - `${this.url}/${encodePageURI(name)}`, + `${this.url}/${encodePageURI(path)}`, { method: "DELETE", }, @@ -200,25 +208,26 @@ export class HttpSpacePrimitives implements SpacePrimitives { } } - async getFileMeta(name: string): Promise { + async getFileMeta(path: string, observing?: boolean): Promise { const res = await this.authenticatedFetch( - `${this.url}/${encodePageURI(name)}`, + `${this.url}/${encodePageURI(path)}`, // This used to use HEAD, but it seems that Safari on iOS is blocking cookies/credentials to be sent along with HEAD requests // so we'll use GET instead with a magic header which the server may or may not use to omit the body. { method: "GET", headers: { "X-Get-Meta": "true", + ...(observing ? { "X-Observing": "true" } : {}), }, }, ); if (res.status === 404) { - throw new Error(`Not found`); + throw notFoundError; } if (!res.ok) { throw new Error(`Failed to get file meta: ${res.statusText}`); } - return this.responseToMeta(name, res); + return headersToFileMeta(path, res.headers)!; } // If not: throws an error or invokes a redirect @@ -235,49 +244,4 @@ export class HttpSpacePrimitives implements SpacePrimitives { // Consume the response body to avoid leaks await response.text(); } - - // Used to check if the server is reachable and the user is authenticated - - /** - * Create an authenticated WebSocket connection - * @param path The path to connect to - * @param queryParams Optional query parameters - * @returns A WebSocket connection - */ - async createAuthenticatedWebSocket( - path: string, - queryParams: Record = {}, - ): Promise { - // First make an authenticated request to ensure we have valid cookies - await this.ping(); - - // Build the WebSocket URL with query parameters - const queryString = Object.entries(queryParams) - .map(([key, value]) => - `${encodeURIComponent(key)}=${encodeURIComponent(value)}` - ) - .join("&"); - - // Create the WebSocket URL - const wsUrl = `${this.url.replace(/^http/, "ws")}/${path}${ - queryString ? `?${queryString}` : "" - }`; - - // Create and return the WebSocket - return new WebSocket(wsUrl); - } - - private responseToMeta(name: string, res: Response): FileMeta { - return { - name, - // The server may set a custom X-Content-Length header in case a GET request was sent with X-Get-Meta, in which case the body may be omitted - size: res.headers.has("X-Content-Length") - ? +res.headers.get("X-Content-Length")! - : +res.headers.get("Content-Length")!, - contentType: res.headers.get("Content-type")!, - created: +(res.headers.get("X-Created") || "0"), - lastModified: +(res.headers.get("X-Last-Modified") || "0"), - perm: (res.headers.get("X-Permission") as "rw" | "ro") || "ro", - }; - } } diff --git a/lib/spaces/plug_space_primitives.ts b/lib/spaces/plug_space_primitives.ts index bd8ab629..50afeb93 100644 --- a/lib/spaces/plug_space_primitives.ts +++ b/lib/spaces/plug_space_primitives.ts @@ -81,38 +81,36 @@ export class PlugSpacePrimitives implements SpacePrimitives { } async readFile( - name: string, + path: string, ): Promise<{ data: Uint8Array; meta: FileMeta }> { const result: { data: Uint8Array; meta: FileMeta } | false = await this .performOperation( "readFile", - name, + path, ); if (result) { return result; } - return this.wrapped.readFile(name); + return this.wrapped.readFile(path); } - getFileMeta(name: string): Promise { - const result = this.performOperation("getFileMeta", name); + getFileMeta(path: string, observing?: boolean): Promise { + const result = this.performOperation("getFileMeta", path, observing); if (result) { return result; } - return this.wrapped.getFileMeta(name); + return this.wrapped.getFileMeta(path, observing); } writeFile( - name: string, + path: string, data: Uint8Array, - selfUpdate?: boolean, meta?: FileMeta, ): Promise { const result = this.performOperation( "writeFile", - name, + path, data, - selfUpdate, meta, ); if (result) { @@ -120,18 +118,17 @@ export class PlugSpacePrimitives implements SpacePrimitives { } return this.wrapped.writeFile( - name, + path, data, - selfUpdate, meta, ); } - deleteFile(name: string): Promise { - const result = this.performOperation("deleteFile", name); + deleteFile(path: string): Promise { + const result = this.performOperation("deleteFile", path); if (result) { return result; } - return this.wrapped.deleteFile(name); + return this.wrapped.deleteFile(path); } } diff --git a/lib/spaces/ro_space_primitives.ts b/lib/spaces/ro_space_primitives.ts index 6541ea13..91211b70 100644 --- a/lib/spaces/ro_space_primitives.ts +++ b/lib/spaces/ro_space_primitives.ts @@ -15,8 +15,8 @@ export class ReadOnlySpacePrimitives implements SpacePrimitives { })); } - async readFile(name: string): Promise<{ meta: FileMeta; data: Uint8Array }> { - const { meta, data } = await this.wrapped.readFile(name); + async readFile(path: string): Promise<{ meta: FileMeta; data: Uint8Array }> { + const { meta, data } = await this.wrapped.readFile(path); return { meta: { ...meta, @@ -26,8 +26,8 @@ export class ReadOnlySpacePrimitives implements SpacePrimitives { }; } - async getFileMeta(name: string): Promise { - const meta = await this.wrapped.getFileMeta(name); + async getFileMeta(path: string, observing?: boolean): Promise { + const meta = await this.wrapped.getFileMeta(path, observing); return { ...meta, perm: "ro", diff --git a/lib/spaces/space_primitives.test.ts b/lib/spaces/space_primitives.test.ts index e4f4f7d3..67253ae9 100644 --- a/lib/spaces/space_primitives.test.ts +++ b/lib/spaces/space_primitives.test.ts @@ -1,14 +1,31 @@ import { assert, assertEquals } from "@std/assert"; import type { SpacePrimitives } from "./space_primitives.ts"; +import type { FileMeta } from "../../type/index.ts"; +import { notFoundError } from "../constants.ts"; export async function testSpacePrimitives(spacePrimitives: SpacePrimitives) { + await testBasicOperations(spacePrimitives); + await testFileOverwriting(spacePrimitives); + await testEmptyFiles(spacePrimitives); + await testUnicodeContent(spacePrimitives); + await testSpecialFileNames(spacePrimitives); + await testErrorHandling(spacePrimitives); + await testLargeFiles(spacePrimitives); + await testMetadataPreservation(spacePrimitives); + + // Ensure clean state at the end + const finalFiles = await spacePrimitives.fetchFileList(); + assertEquals(finalFiles, []); +} + +async function testBasicOperations(spacePrimitives: SpacePrimitives) { const files = await spacePrimitives.fetchFileList(); assertEquals(files, []); + // Write text file const fileMeta = await spacePrimitives.writeFile( "test.txt", stringToBytes("Hello World"), - false, { name: "test.txt", perm: "rw", @@ -31,14 +48,14 @@ export async function testSpacePrimitives(spacePrimitives: SpacePrimitives) { assertEquals(new TextDecoder().decode(fbContent), "Hello World"); assertEquals(await spacePrimitives.fetchFileList(), [fileMeta]); + + // Write binary file const buf = new Uint8Array(1024 * 1024); buf.set([1, 2, 3, 4, 5]); - // Write binary file await spacePrimitives.writeFile("test.bin", buf); const fileData = await spacePrimitives.readFile("test.bin"); assertEquals(fileData.data.length, 1024 * 1024); assertEquals((await spacePrimitives.fetchFileList()).length, 2); - // console.log(spacePrimitives); await spacePrimitives.deleteFile("test.bin"); assertEquals(await spacePrimitives.fetchFileList(), [fileMeta]); @@ -60,10 +77,151 @@ export async function testSpacePrimitives(spacePrimitives: SpacePrimitives) { await spacePrimitives.getFileMeta("test+'s.txt"); assert(false); } catch (e: any) { - assertEquals(e.message, "Not found"); + assertEquals(e, notFoundError); } } +async function testFileOverwriting(spacePrimitives: SpacePrimitives) { + // Test overwriting existing files + await spacePrimitives.writeFile("overwrite.txt", stringToBytes("Original")); + const _originalMeta = await spacePrimitives.getFileMeta("overwrite.txt"); + + await spacePrimitives.writeFile("overwrite.txt", stringToBytes("Updated")); + const updatedData = await spacePrimitives.readFile("overwrite.txt"); + assertEquals(new TextDecoder().decode(updatedData.data), "Updated"); + + // File list should still have only one entry for this file + const filesAfterOverwrite = await spacePrimitives.fetchFileList(); + const overwriteFiles = filesAfterOverwrite.filter((f) => + f.name === "overwrite.txt" + ); + assertEquals(overwriteFiles.length, 1); + + await spacePrimitives.deleteFile("overwrite.txt"); +} + +async function testEmptyFiles(spacePrimitives: SpacePrimitives) { + // Test empty file + await spacePrimitives.writeFile("empty.txt", new Uint8Array(0)); + const emptyFile = await spacePrimitives.readFile("empty.txt"); + assertEquals(emptyFile.data.length, 0); + assertEquals(emptyFile.meta.size, 0); + await spacePrimitives.deleteFile("empty.txt"); +} + +async function testUnicodeContent(spacePrimitives: SpacePrimitives) { + // Test files with Unicode characters + const unicodeContent = "Hello 世界! 🌍 Здравствуй мир!"; + await spacePrimitives.writeFile("unicode.txt", stringToBytes(unicodeContent)); + const unicodeFile = await spacePrimitives.readFile("unicode.txt"); + assertEquals(new TextDecoder().decode(unicodeFile.data), unicodeContent); + await spacePrimitives.deleteFile("unicode.txt"); +} + +async function testSpecialFileNames(spacePrimitives: SpacePrimitives) { + // Test file names with various special characters + const specialNames = [ + "file with spaces.txt", + "file-with-hyphens.txt", + "file_with_underscores.txt", + "file.with.dots.txt", + "UPPERCASE.TXT", + "123numeric.txt", + "émojis🚀file.txt", + ]; + + for (const fileName of specialNames) { + await spacePrimitives.writeFile( + fileName, + stringToBytes(`Content of ${fileName}`), + ); + const fileData = await spacePrimitives.readFile(fileName); + assertEquals( + new TextDecoder().decode(fileData.data), + `Content of ${fileName}`, + ); + } + + // Verify all special files are in the list + const allFiles = await spacePrimitives.fetchFileList(); + for (const fileName of specialNames) { + const found = allFiles.find((f) => f.name === fileName); + assert(found, `File ${fileName} should be in the file list`); + } + + // Clean up special files + for (const fileName of specialNames) { + await spacePrimitives.deleteFile(fileName); + } +} + +async function testErrorHandling(spacePrimitives: SpacePrimitives) { + // Test error cases + try { + await spacePrimitives.readFile("nonexistent.txt"); + assert(false, "Should throw error for non-existent file"); + } catch (e: any) { + assertEquals(e, notFoundError); + } + + try { + await spacePrimitives.deleteFile("nonexistent.txt"); + assert(false, "Should throw error when deleting non-existent file"); + } catch (e: any) { + assertEquals(e, notFoundError); + } +} + +async function testLargeFiles(spacePrimitives: SpacePrimitives) { + // Test large file content + const largeContent = new Uint8Array(5 * 1024 * 1024); // 5MB + for (let i = 0; i < largeContent.length; i++) { + largeContent[i] = i % 256; + } + + await spacePrimitives.writeFile("large.bin", largeContent); + const largeFile = await spacePrimitives.readFile("large.bin"); + assertEquals(largeFile.data.length, largeContent.length); + assertEquals(largeFile.meta.size, largeContent.length); + + // Verify content integrity + for (let i = 0; i < Math.min(1000, largeContent.length); i++) { + assertEquals(largeFile.data[i], largeContent[i]); + } + + await spacePrimitives.deleteFile("large.bin"); +} + +async function testMetadataPreservation(spacePrimitives: SpacePrimitives) { + // Test metadata preservation + const testContent = stringToBytes("Hello meta!"); + const customMeta: FileMeta = { + name: "meta-test.txt", + perm: "rw", + created: 1000000, + contentType: "text/plain", // Use a content type that matches the file extension + lastModified: 2000000, + size: testContent.length, // Use actual content length + }; + + await spacePrimitives.writeFile( + "meta-test.txt", + testContent, + customMeta, + ); + const metaFile = await spacePrimitives.readFile("meta-test.txt"); + + // Check that some metadata is preserved (implementations may handle timestamps differently) + assert( + metaFile.meta.lastModified > 0, + "LastModified timestamp should be set", + ); + assertEquals(metaFile.meta.name, "meta-test.txt"); + assertEquals(metaFile.meta.size, testContent.length); + + await spacePrimitives.deleteFile("meta-test.txt"); +} + function stringToBytes(str: string): Uint8Array { return new TextEncoder().encode(str); } diff --git a/lib/spaces/space_primitives.ts b/lib/spaces/space_primitives.ts index d7947601..6685b19d 100644 --- a/lib/spaces/space_primitives.ts +++ b/lib/spaces/space_primitives.ts @@ -6,21 +6,23 @@ import type { FileMeta } from "../../type/index.ts"; export interface SpacePrimitives { fetchFileList(): Promise; - // The result of this should be consistent with the result of fetchFileList for this entry - getFileMeta(name: string): Promise; + /** + * Retrieves metadata for a specific file. + * @param path The path of the file to retrieve metadata for. + * @param observing used to hint at the sync engine this file is under regular observation (and may sync more aggressively) + */ + getFileMeta(path: string, observing?: boolean): Promise; readFile( - name: string, + path: string, ): Promise<{ data: Uint8Array; meta: FileMeta }>; writeFile( - name: string, + path: string, data: Uint8Array, - // Used to decide whether or not to emit change events - selfUpdate?: boolean, // May be ignored, but ideally should be used to set the lastModified time meta?: FileMeta, ): Promise; - deleteFile(name: string): Promise; + deleteFile(path: string): Promise; } diff --git a/lib/spaces/sync.test.ts b/lib/spaces/sync.test.ts index 2ebe94f3..82db37cf 100644 --- a/lib/spaces/sync.test.ts +++ b/lib/spaces/sync.test.ts @@ -1,64 +1,72 @@ -import { SpaceSync, type SyncStatusItem } from "./sync.ts"; +import { SpaceSync, SyncSnapshot } from "./sync.ts"; import { DiskSpacePrimitives } from "./disk_space_primitives.ts"; import { assertEquals } from "@std/assert"; +import { sleep } from "../async.ts"; +import { assert } from "node:console"; -Deno.test("Test store", async () => { +Deno.test("Test sync with no filtering", async () => { const primaryPath = await Deno.makeTempDir(); const secondaryPath = await Deno.makeTempDir(); console.log("Primary", primaryPath); console.log("Secondary", secondaryPath); const primary = new DiskSpacePrimitives(primaryPath); const secondary = new DiskSpacePrimitives(secondaryPath); - const snapshot = new Map(); + const snapshot = new SyncSnapshot(); const sync = new SpaceSync(primary, secondary, { conflictResolver: SpaceSync.primaryConflictResolver, + isSyncCandidate: () => true, // Sync everything always }); - // Write one page to primary - await primary.writeFile("index", stringToBytes("Hello")); + console.log("Write one page to primary"); + await primary.writeFile("index.md", stringToBytes("Hello")); assertEquals((await secondary.fetchFileList()).length, 0); console.log("Initial sync ops", await doSync()); assertEquals((await secondary.fetchFileList()).length, 1); assertEquals( - (await secondary.readFile("index")).data, + (await secondary.readFile("index.md")).data, stringToBytes("Hello"), ); // Should be a no-op - assertEquals(await doSync(), 0); + let ops = await doSync(); + assertEquals(ops, 0); + assertEquals(snapshot.nonSyncedFiles.size, 0); // Now let's make a change on the secondary - await secondary.writeFile("index", stringToBytes("Hello!!")); - await secondary.writeFile("test", stringToBytes("Test page")); + await secondary.writeFile("index.md", stringToBytes("Hello!!")); + await secondary.writeFile("test.md", stringToBytes("Test page")); // And sync it - await doSync(); + ops = await doSync(); + assertEquals(ops, 2); + assertEquals(snapshot.nonSyncedFiles.size, 0); assertEquals((await primary.fetchFileList()).length, 2); assertEquals((await secondary.fetchFileList()).length, 2); assertEquals( - (await primary.readFile("index")).data, + (await primary.readFile("index.md")).data, stringToBytes("Hello!!"), ); // Let's make some random edits on both ends - await primary.writeFile("index", stringToBytes("1")); - await primary.writeFile("index2", stringToBytes("2")); - await secondary.writeFile("index3", stringToBytes("3")); - await secondary.writeFile("index4", stringToBytes("4")); + await primary.writeFile("index.md", stringToBytes("1")); + await primary.writeFile("index2.md", stringToBytes("2")); + await secondary.writeFile("index3.md", stringToBytes("3")); + await secondary.writeFile("index4.md", stringToBytes("4")); await doSync(); assertEquals((await primary.fetchFileList()).length, 5); assertEquals((await secondary.fetchFileList()).length, 5); - assertEquals(await doSync(), 0); + ops = await doSync(); + assertEquals(ops, 0); console.log("Deleting pages"); // Delete some pages - await primary.deleteFile("index"); - await primary.deleteFile("index3"); + await primary.deleteFile("index.md"); + await primary.deleteFile("index3.md"); await doSync(); @@ -66,10 +74,11 @@ Deno.test("Test store", async () => { assertEquals((await secondary.fetchFileList()).length, 3); // No-op - assertEquals(await doSync(), 0); + ops = await doSync(); + assertEquals(ops, 0); - await secondary.deleteFile("index4"); - await primary.deleteFile("index2"); + await secondary.deleteFile("index4.md"); + await primary.deleteFile("index2.md"); await doSync(); @@ -78,21 +87,22 @@ Deno.test("Test store", async () => { assertEquals((await secondary.fetchFileList()).length, 1); // No-op - assertEquals(await doSync(), 0); + ops = await doSync(); + assertEquals(ops, 0); - await secondary.writeFile("index", stringToBytes("I'm back")); + await secondary.writeFile("index.md", stringToBytes("I'm back")); await doSync(); assertEquals( - (await primary.readFile("index")).data, + (await primary.readFile("index.md")).data, stringToBytes("I'm back"), ); // Cause a conflict console.log("Introducing a conflict now"); - await primary.writeFile("index", stringToBytes("Hello 1")); - await secondary.writeFile("index", stringToBytes("Hello 2")); + await primary.writeFile("index.md", stringToBytes("Hello 1")); + await secondary.writeFile("index.md", stringToBytes("Hello 2")); await doSync(); @@ -101,11 +111,11 @@ Deno.test("Test store", async () => { // Verify that primary won assertEquals( - (await primary.readFile("index")).data, + (await primary.readFile("index.md")).data, stringToBytes("Hello 1"), ); assertEquals( - (await secondary.readFile("index")).data, + (await secondary.readFile("index.md")).data, stringToBytes("Hello 1"), ); @@ -114,8 +124,8 @@ Deno.test("Test store", async () => { assertEquals((await secondary.fetchFileList()).length, 3); // Introducing a fake conflict (same content, so not really conflicting) - await primary.writeFile("index", stringToBytes("Hello 1")); - await secondary.writeFile("index", stringToBytes("Hello 1")); + await primary.writeFile("index.md", stringToBytes("Hello 1")); + await secondary.writeFile("index.md", stringToBytes("Hello 1")); await doSync(); await doSync(); @@ -123,61 +133,195 @@ Deno.test("Test store", async () => { // test + index + index.md + previous index.conflicting copy but nothing more assertEquals((await primary.fetchFileList()).length, 3); - console.log("Bringing a third device in the mix"); - - const ternaryPath = await Deno.makeTempDir(); - - console.log("Ternary", ternaryPath); - - const ternary = new DiskSpacePrimitives(ternaryPath); - const sync2 = new SpaceSync( - secondary, - ternary, - { - conflictResolver: SpaceSync.primaryConflictResolver, - }, - ); - const snapshot2 = new Map(); - console.log( - "N ops", - await sync2.syncFiles(snapshot2), - ); - await sleep(2); - assertEquals(await sync2.syncFiles(snapshot2), 0); - - // I had to look up what follows ternary (https://english.stackexchange.com/questions/25116/what-follows-next-in-the-sequence-unary-binary-ternary) - const quaternaryPath = await Deno.makeTempDir(); - const quaternary = new DiskSpacePrimitives(quaternaryPath); - const sync3 = new SpaceSync( - secondary, - quaternary, - { - isSyncCandidate: (path) => !path.startsWith("index"), - conflictResolver: SpaceSync.primaryConflictResolver, - }, - ); - const selectingOps = await sync3.syncFiles(new Map()); - - assertEquals(selectingOps, 1); - await Deno.remove(primaryPath, { recursive: true }); await Deno.remove(secondaryPath, { recursive: true }); - await Deno.remove(ternaryPath, { recursive: true }); - await Deno.remove(quaternaryPath, { recursive: true }); async function doSync() { - await sleep(); + await sleep(10); const r = await sync.syncFiles(snapshot); - await sleep(); + await sleep(10); return r; } }); -function sleep(ms = 10): Promise { - return new Promise((resolve) => { - setTimeout(resolve, ms); +Deno.test("Test sync with filtering", async () => { + const primaryPath = await Deno.makeTempDir(); + const secondaryPath = await Deno.makeTempDir(); + console.log("Primary", primaryPath); + console.log("Secondary", secondaryPath); + const primary = new DiskSpacePrimitives(primaryPath); + const secondary = new DiskSpacePrimitives(secondaryPath); + + const snapshot = new SyncSnapshot(); + let sync = new SpaceSync( + primary, + secondary, + { + conflictResolver: SpaceSync.primaryConflictResolver, + isSyncCandidate: (path) => path.endsWith(".md"), // Only sync .md files + }, + ); + + console.log( + "Write one non-sync file on the primary, which SHOULD sync to the secondary", + ); + await primary.writeFile("index.txt", stringToBytes("Hello")); + assertEquals((await secondary.fetchFileList()).length, 0); + let ops = await doSync(); + assertEquals(ops, 1); + // Note: this number should be 0 because the file DOES have a local representation so it's not technically non-synced + assertEquals(snapshot.nonSyncedFiles.size, 0); + + assertEquals((await secondary.fetchFileList()).length, 1); + assertEquals( + (await secondary.readFile("index.txt")).data, + stringToBytes("Hello"), + ); + + console.log("Updating on secondary"); + await secondary.writeFile("index.txt", stringToBytes("Hello Updated")); + ops = await doSync(); + assertEquals(ops, 1); + assertEquals(snapshot.nonSyncedFiles.size, 1); + try { + await primary.getFileMeta("index.md"); + assert( + false, + "Local file should have been deleted locally since it's out of date", + ); + } catch { + // Expected + } + + console.log("Deleting remote files"); + await secondary.deleteFile("index.txt"); + ops = await doSync(); + assertEquals(ops, 1); + assertEquals(snapshot.nonSyncedFiles.size, 0); + + console.log("Creating a remote non-synced file"); + await secondary.writeFile("index2.txt", stringToBytes("Hello 2")); + ops = await doSync(); + assertEquals(ops, 0); // No-op, metadata only + assertEquals(snapshot.nonSyncedFiles.size, 1); + ops = await doSync(); + assertEquals(ops, 0); // No-op, metadata only + assertEquals(snapshot.nonSyncedFiles.size, 1); + + await primary.writeFile("index2.txt", stringToBytes("Hello local")); + ops = await doSync(); + assertEquals(ops, 1); + assertEquals(snapshot.nonSyncedFiles.size, 0); + + console.log("Getting into a state with some synced and non-synced files"); + await secondary.writeFile("index.md", stringToBytes("This will sync")); + await secondary.writeFile("index.txt", stringToBytes("This will not sync")); + await secondary.writeFile("index2.txt", stringToBytes("This will not sync")); + ops = await doSync(); + assertEquals(ops, 2); + assertEquals(snapshot.nonSyncedFiles.size, 2); + + // Check file listings on both ends + assertEquals((await secondary.fetchFileList()).length, 3); + assertEquals((await primary.fetchFileList()).length, 1); + + //////////// + // Now let's start another sync session, but now wanting to sync everything + console.log("Going to switch to syncing everything now"); + sync = new SpaceSync( + primary, + secondary, + { + conflictResolver: SpaceSync.primaryConflictResolver, + isSyncCandidate: () => true, + }, + ); + + ops = await doSync(); + // This should pull 2 files from remote to local + assertEquals(ops, 2); + assertEquals((await primary.fetchFileList()).length, 3); + assertEquals((await secondary.fetchFileList()).length, 3); + + console.log("And now to syncing nothing"); + sync = new SpaceSync( + primary, + secondary, + { + conflictResolver: SpaceSync.primaryConflictResolver, + isSyncCandidate: () => false, + }, + ); + + ops = await doSync(); + // This should delete 3 files from the primary + assertEquals(ops, 3); + // Leaving nothing on primary + assertEquals((await primary.fetchFileList()).length, 0); + // And everything unchanged on secondary + assertEquals((await secondary.fetchFileList()).length, 3); + + // Ok, now we're going to sync everything again + sync = new SpaceSync( + primary, + secondary, + { + conflictResolver: SpaceSync.primaryConflictResolver, + isSyncCandidate: () => true, + }, + ); + // await secondary. + ops = await doSync(); + assertEquals(ops, 3); + assertEquals((await primary.fetchFileList()).length, 3); + assertEquals((await secondary.fetchFileList()).length, 3); + + await Deno.remove(primaryPath, { recursive: true }); + await Deno.remove(secondaryPath, { recursive: true }); + + async function doSync() { + await sleep(10); + const r = await sync.syncFiles(snapshot); + return r; + } +}); + +Deno.test("Local push sync", async () => { + const primaryPath = await Deno.makeTempDir(); + const secondaryPath = await Deno.makeTempDir(); + console.log("Primary", primaryPath); + console.log("Secondary", secondaryPath); + const primary = new DiskSpacePrimitives(primaryPath); + const secondary = new DiskSpacePrimitives(secondaryPath); + const snapshot = new SyncSnapshot(); + const sync = new SpaceSync(primary, secondary, { + conflictResolver: SpaceSync.primaryConflictResolver, + isSyncCandidate: (path) => path.endsWith(".md"), // Only sync .md files }); -} + + console.log( + "Write one non-sync file on the primary, which SHOULD sync to the secondary", + ); + + const operations = await sync.syncFiles(snapshot); + assertEquals(operations, 0); + + await primary.writeFile("index.md", stringToBytes("Hello")); + assertEquals(1, await sync.syncSingleFile("index.md", snapshot)); + + assertEquals( + (await secondary.readFile("index.md")).data, + stringToBytes("Hello"), + ); + + console.log("Let's write a new file on primary that is not a sync candidate"); + await primary.writeFile("test.txt", stringToBytes("Hello")); + assertEquals(1, await sync.syncSingleFile("test.txt", snapshot)); + assertEquals(snapshot.nonSyncedFiles.size, 0); + + await Deno.remove(primaryPath, { recursive: true }); + await Deno.remove(secondaryPath, { recursive: true }); +}); function stringToBytes(s: string): Uint8Array { return new TextEncoder().encode(s); diff --git a/lib/spaces/sync.ts b/lib/spaces/sync.ts index 3d2f1ca5..fbdb83f1 100644 --- a/lib/spaces/sync.ts +++ b/lib/spaces/sync.ts @@ -2,6 +2,10 @@ import type { SpacePrimitives } from "./space_primitives.ts"; import { EventEmitter } from "../plugos/event.ts"; import { plugPrefix } from "./constants.ts"; import type { FileMeta } from "../../type/index.ts"; +import { notFoundError } from "../constants.ts"; +import { processWithConcurrency } from "../async.ts"; + +const syncConcurrency = 5; type SyncHash = number; @@ -9,52 +13,505 @@ type SyncHash = number; // and the second item the lastModified value of the secondary space export type SyncStatusItem = [SyncHash, SyncHash]; +export class SyncSnapshot { + constructor( + public files: Map = new Map(), + public nonSyncedFiles: Map = new Map(), + ) { + this.files = files; + this.nonSyncedFiles = nonSyncedFiles; + } + + toJSON(): any { + return { + files: Object.fromEntries(this.files), + nonSyncedFiles: Object.fromEntries(this.nonSyncedFiles), + }; + } + + static fromJSON(json: any | undefined): SyncSnapshot { + return new SyncSnapshot( + new Map(Object.entries(json?.files || {})), + new Map(Object.entries(json?.nonSyncedFiles || {})), + ); + } +} + export type SyncStatus = { filesProcessed: number; totalFiles: number; - snapshot: Map; }; export type SyncOptions = { conflictResolver: ( - name: string, - snapshot: Map, + path: string, + snapshot: SyncSnapshot, primarySpace: SpacePrimitives, secondarySpace: SpacePrimitives, ) => Promise; - isSyncCandidate?: (path: string) => boolean; - // Used to track progress, may want to pass more specific info later - onSyncProgress?: (syncStatus: SyncStatus) => void; + isSyncCandidate: (path: string) => boolean; }; type SyncDirection = "primary->secondary" | "secondary->primary"; + export type SyncEvents = { - fileSynced: (meta: FileMeta, direction: SyncDirection) => void; + fileSynced: ( + meta: FileMeta, + direction: SyncDirection, + ) => void | Promise; + syncProgress: (syncStatus: SyncStatus) => void | Promise; + snapshotUpdated: (snapshot: SyncSnapshot) => void | Promise; }; -// Implementation of this algorithm https://unterwaditzer.net/2016/sync-algorithm.html +// Implementation of this algorithm: https://unterwaditzer.net/2016/sync-algorithm.html export class SpaceSync extends EventEmitter { + // Sync mutex (only one sync operation at a time) + private isSyncing = false; + constructor( private primary: SpacePrimitives, private secondary: SpacePrimitives, + // public snapshot: SyncSnapshot, readonly options: SyncOptions, ) { super(); } + /** + * Syncs all files in space. + * If this completes successfully (with operations >= 0), a full space sync completed successfully. + * @param snapshot - The current snapshot of the space (will be updated in place) + * @returns number of operations performed, or -1 when sync was already ongoing and nonSynced files + */ + public async syncFiles(snapshot: SyncSnapshot): Promise { + let operations = 0; + + // Mutex behavior, only sync can happen at a time + if (this.isSyncing) { + console.warn("Sync already in progress..."); + return -1; + } + this.isSyncing = true; + console.log("[sync]", "Performing a full sync cycle..."); + const startTime = Date.now(); + + try { + const primaryAllPages = await this.primary.fetchFileList(); + const secondaryAllPages = await this.secondary.fetchFileList(); + + // Assumption: the primary is local, and _all_ files should be synced to the secondary + // However, the secondary is remote, and only a subset of files should be synced from it + const nonSyncCandidates = this.getNonSyncCandidates(secondaryAllPages); + + const primaryFileMap = new Map( + primaryAllPages.map((m) => [m.name, m]), + ); + const secondaryFileMap = new Map( + secondaryAllPages.map((m) => [m.name, m]), + ); + + const allFilesToProcess = new Set([ + ...snapshot.files.keys(), + ...primaryFileMap.keys(), + ...secondaryFileMap.keys(), + ]); + + const sortedPaths = [...allFilesToProcess]; + sortedPaths.sort((a) => { + // Just make sure that _plug/ files appear first + // This is important for the initial sync: plugs are loaded the moment they are pulled into the space, + // which would activate e.g. any indexing logic for the remaining space content + // TODO: To verify if this is still true today + return a.startsWith(plugPrefix) ? -1 : 1; + }); + // console.log("[sync]", "Iterating over all files"); + let filesProcessed = 0; + await processWithConcurrency(sortedPaths, async (path) => { + const fileOperations = await this.syncFile( + path, + primaryFileMap.get(path), + secondaryFileMap.get(path), + !nonSyncCandidates.has(path), + snapshot, + ); + operations = operations + fileOperations; + filesProcessed++; + // Only report something significant + if (operations > 1) { + this.emit("syncProgress", { + filesProcessed, + totalFiles: sortedPaths.length, + }); + } + }, syncConcurrency); + console.log( + "[Sync]", + "Completed:", + operations, + "operations in", + ((Date.now() - startTime) / 1000) + "s", + ); + } finally { + this.isSyncing = false; + this.emit("snapshotUpdated", snapshot); + } + + return operations; + } + + /** + * Syncs a single file from primary to secondary. + * Use cases: + * - Push local changes to remote (almost) immediately + * - Poll remote for changes to interesting files (e.g. files currently open) + * @returns number of operations performed, or -1 when sync was already ongoing + */ + public async syncSingleFile( + path: string, + snapshot: SyncSnapshot, + ): Promise { + let operations = 0; + + // Mutex behavior, only sync can happen at a time + if (this.isSyncing) { + console.warn("[sync]", "Sync already in progress..."); + return -1; + } + if (snapshot.nonSyncedFiles.has(path)) { + console.info( + "[sync]", + "Was asked to sync marked as non-synced, skipping", + path, + ); + return 0; + } + this.isSyncing = true; + console.log("[sync]", "Performing a single file sync", path); + + try { + const primaryMeta = await this.primary.getFileMeta(path); + let secondaryMeta: FileMeta | undefined; + try { + secondaryMeta = await this.secondary.getFileMeta(path); + } catch (e: any) { + if (e.message === notFoundError.message) { + // File doesn't exist remotely, that's ok + } else { + throw e; + } + } + operations += await this.syncFile( + path, + primaryMeta, + secondaryMeta, + true, + snapshot, + ); + } catch (e: any) { + console.log("error", "Error syncing file", path, e.message); + } finally { + this.isSyncing = false; + this.emit("snapshotUpdated", snapshot); + } + + return operations; + } + + /** + * Syncs a single file + * @param path The name of the file to sync + * @param primaryMeta The metadata of the file on the primary space + * @param secondaryMeta The metadata of the file on the secondary space + * @param syncBack Whether this a file that is marked to be synced + * @param snapshot The snapshot of the file status (updated in place!) + * @returns The number of operations performed + */ + private async syncFile( + path: string, + primaryMeta: FileMeta | undefined, + secondaryMeta: FileMeta | undefined, + syncBack: boolean, + snapshot: SyncSnapshot, + ): Promise { + let operations = 0; + + if ( + primaryMeta !== undefined && secondaryMeta === undefined && + !snapshot.files.has(path) + ) { + // New file, created on primary, copy from primary to secondary + console.log( + "[sync]", + "New file created on primary, copying to secondary", + path, + ); + const { data, meta } = await this.primary.readFile(path); + const writtenMeta = await this.secondary.writeFile( + path, + data, + meta, + ); + // Update file snapshot + snapshot.files.set(path, [ + meta.lastModified, + writtenMeta.lastModified, + ]); + // Let's make sure this file is not marked as nonSynced, because we have a local copy + snapshot.nonSyncedFiles.delete(path); + operations++; + await this.emit("fileSynced", writtenMeta, "primary->secondary"); + } else if ( + secondaryMeta !== undefined && (( + // Scenario: a new file was created on the secondary + primaryMeta === undefined && + !snapshot.files.has(path) + ) || ( + // Scenario: a file that was previously unsynced, should now be synced + snapshot.nonSyncedFiles.has(path) && syncBack && + // because we don't have it yet + primaryMeta === undefined + )) + ) { + // New file, created on secondary + if (syncBack) { + // Copy from secondary to primary + console.log( + "[sync]", + "New file created on secondary, copying from secondary to primary", + path, + ); + const { data, meta } = await this.secondary.readFile(path); + const writtenMeta = await this.primary.writeFile( + path, + data, + meta, + ); + // Update file snapshot + snapshot.files.set(path, [ + writtenMeta.lastModified, + meta.lastModified, + ]); + // Make sure the file is not marked as nonSynced anymore + snapshot.nonSyncedFiles.delete(path); + operations++; + await this.emit("fileSynced", writtenMeta, "secondary->primary"); + } else { // !syncBack + // This has syncBack set to false, which means we'll just update the nonSyncedFiles and snapshot + console.log( + "[sync]", + "New file created on secondary, but only updating metadata", + path, + ); + snapshot.files.set(path, [ + secondaryMeta.lastModified, + secondaryMeta.lastModified, + ]); + snapshot.nonSyncedFiles.set(path, secondaryMeta); + } + } else if ( + primaryMeta !== undefined && snapshot.files.has(path) && + secondaryMeta === undefined + ) { + // File deleted on secondary + if (syncBack) { + snapshot.files.delete(path); + if (snapshot.nonSyncedFiles.has(path)) { + // This is the scenario where in the previous sync this file was not synced while in this new one it is + console.log( + "[sync]", + "File deleted on secondary, but wasn't synced on primary, so skipping", + path, + ); + // Keeping non-synced files up-to-date + snapshot.nonSyncedFiles.delete(path); + } else { + console.log( + "[sync]", + "File deleted on secondary, deleting from primary", + path, + ); + await this.primary.deleteFile(path); + } + operations++; + } else { // !syncBack + console.log( + "[sync]", + "File deleted on secondary, only updating snapshot", + path, + ); + try { + // It's possible that there is a local copy anyway (if it started out there, so let's attempt to delete it just in case, but fail silently) + await this.primary.deleteFile(path); + console.info("[sync]", "Deleted local copy on primary", path); + operations++; + } catch { + // Fail silently, likely the file doesn't exist + } + snapshot.files.delete(path); + snapshot.nonSyncedFiles.delete(path); + } + } else if ( + // The file is on the secondary, and not on the primary + secondaryMeta !== undefined && snapshot.files.has(path) && + primaryMeta === undefined && + // And we're talking about a synced file + syncBack + ) { + // File deleted on primary + console.log( + "[sync]", + "File deleted on primary, deleting from secondary", + path, + ); + await this.secondary.deleteFile(path); + snapshot.files.delete(path); + snapshot.nonSyncedFiles.delete(path); + operations++; + } else if ( + snapshot.files.has(path) && primaryMeta === undefined && + secondaryMeta === undefined + ) { + // File deleted on both sides, :shrug: + console.log( + "[sync]", + "File deleted on both ends, deleting from snapshot", + path, + ); + snapshot.files.delete(path); + snapshot.nonSyncedFiles.delete(path); + operations++; + } else if ( + primaryMeta !== undefined && secondaryMeta !== undefined && + snapshot.files.get(path) && + primaryMeta.lastModified !== snapshot.files.get(path)![0] && + secondaryMeta.lastModified === snapshot.files.get(path)![1] + ) { + // File has changed on primary, but not secondary: copy from primary to secondary + console.log( + "[sync]", + "File changed on primary, copying to secondary", + path, + ); + const { data, meta } = await this.primary.readFile(path); + const writtenMeta = await this.secondary.writeFile( + path, + data, + meta, + ); + snapshot.files.set(path, [ + meta.lastModified, + writtenMeta.lastModified, + ]); + // Delete from non-synced files just in case, because we clearly have a local copy + snapshot.nonSyncedFiles.delete(path); + operations++; + await this.emit("fileSynced", writtenMeta, "primary->secondary"); + } else if ( + primaryMeta !== undefined && secondaryMeta !== undefined && + snapshot.files.get(path) && + secondaryMeta.lastModified !== snapshot.files.get(path)![1] && + primaryMeta.lastModified === snapshot.files.get(path)![0] + ) { + // File has changed on secondary, but not primary + if (syncBack) { + // Copy from secondary to primary + console.log( + "[sync]", + "File has changed on secondary, but not primary: copy from secondary to primary", + path, + ); + const { data, meta } = await this.secondary.readFile(path); + const writtenMeta = await this.primary.writeFile( + path, + data, + meta, + ); + snapshot.files.set(path, [ + writtenMeta.lastModified, + meta.lastModified, + ]); + // Make sure it's not in nonSyncedFiles + snapshot.nonSyncedFiles.delete(path); + operations++; + await this.emit("fileSynced", writtenMeta, "secondary->primary"); + } else { // !syncBack + console.log( + "[sync]", + "File has changed on secondary, but not primary: shouldn't be synced", + path, + ); + snapshot.files.set(path, [ + secondaryMeta.lastModified, + secondaryMeta.lastModified, + ]); + snapshot.nonSyncedFiles.set(path, secondaryMeta); + try { + // It may be that the file does exist locally, but it will now be out of date, just delete it + await this.primary.deleteFile(path); + console.info( + "[sync]", + "Deleted local (out of date) copy on primary", + path, + ); + operations++; + } catch { + // Fail silently, likely the file doesn't exist + } + } + } else if ( + ( // File changed on both ends, but we don't have any info in the snapshot (resync scenario?): have to run through conflict handling + primaryMeta !== undefined && secondaryMeta !== undefined && + !snapshot.files.has(path) + ) || + ( // File changed on both ends, CONFLICT! + primaryMeta !== undefined && secondaryMeta !== undefined && + snapshot.files.get(path) && + secondaryMeta.lastModified !== snapshot.files.get(path)![1] && + primaryMeta.lastModified !== snapshot.files.get(path)![0] + ) + ) { + console.log( + "[sync]", + "File changed on both ends, potential conflict", + path, + ); + operations += await this.options.conflictResolver( + path, + snapshot, + this.primary, + this.secondary, + ); + } else if (primaryMeta && secondaryMeta && !syncBack) { + console.log( + "[sync]", + "File present locally, but no longer synced, deleting local copy of", + path, + ); + await this.primary.deleteFile(path); + snapshot.nonSyncedFiles.set(path, secondaryMeta); + operations += 1; + } else { + // Nothing needs to happen + if (!syncBack && secondaryMeta) { + snapshot.nonSyncedFiles.set(path, secondaryMeta); + } + } + return operations; + } + // Strategy: Primary wins public static async primaryConflictResolver( - name: string, - snapshot: Map, + path: string, + snapshot: SyncSnapshot, primary: SpacePrimitives, secondary: SpacePrimitives, ): Promise { - console.log("[sync]", "Starting conflict resolution for", name); - const filePieces = name.split("."); + console.log("[sync]", "Starting conflict resolution for", path); + const filePieces = path.split("."); const fileNameBase = filePieces.slice(0, -1).join("."); const fileNameExt = filePieces[filePieces.length - 1]; - const pageData1 = await primary.readFile(name); - const pageData2 = await secondary.readFile(name); + const pageData1 = await primary.readFile(path); + const pageData2 = await secondary.readFile(path); let byteWiseMatch = true; const arrayBuffer1 = pageData1.data; @@ -74,7 +531,7 @@ export class SpaceSync extends EventEmitter { if (byteWiseMatch) { console.log("[sync]", "Files are the same, no conflict"); - snapshot.set(name, [ + snapshot.files.set(path, [ pageData1.meta.lastModified, pageData2.meta.lastModified, ]); @@ -84,7 +541,7 @@ export class SpaceSync extends EventEmitter { let operations = 0; const revisionFileName = filePieces.length === 1 - ? `${name}.conflicted:${pageData2.meta.lastModified}` + ? `${path}.conflicted:${pageData2.meta.lastModified}` : `${fileNameBase}.conflicted:${pageData2.meta.lastModified}.${fileNameExt}`; console.log( "[sync]", @@ -106,274 +563,31 @@ export class SpaceSync extends EventEmitter { // Write replacement on top const writeMeta = await secondary.writeFile( - name, + path, pageData1.data, - true, ); operations++; // Updating snapshot - snapshot.set(revisionFileName, [ + snapshot.files.set(revisionFileName, [ localConflictMeta.lastModified, remoteConflictMeta.lastModified, ]); - snapshot.set(name, [pageData1.meta.lastModified, writeMeta.lastModified]); + snapshot.files.set(path, [ + pageData1.meta.lastModified, + writeMeta.lastModified, + ]); return operations; } - async syncFiles( - snapshot: Map, - isSyncCandidate = this.options.isSyncCandidate, - ): Promise { - let operations = 0; - console.log("[sync]", "Performing a full sync cycle..."); - - // Not try-catching this because this one's local and shouldn't fail (famous last words) - const primaryAllPages = this.syncCandidates( - await this.primary.fetchFileList(), - ); - - try { - const secondaryAllPages = this.syncCandidates( - await this.secondary.fetchFileList(), - ); - - const primaryFileMap = new Map( - primaryAllPages.map((m) => [m.name, m.lastModified]), - ); - const secondaryFileMap = new Map( - secondaryAllPages.map((m) => [m.name, m.lastModified]), - ); - - const allFilesToProcess = new Set([ - ...snapshot.keys(), - ...primaryFileMap.keys(), - ...secondaryFileMap.keys(), - ]); - - const sortedFilenames = [...allFilesToProcess]; - sortedFilenames.sort((a) => { - // Just make sure that _plug/ files appear first - // This is important for the initial sync: plugs are loaded the moment they are pulled into the space, - // which would activate e.g. any indexing logic for the remaining space content - return a.startsWith(plugPrefix) ? -1 : 1; - }); - // console.log("[sync]", "Iterating over all files"); - let filesProcessed = 0; - for (const name of sortedFilenames) { - if (isSyncCandidate && !isSyncCandidate(name)) { - continue; - } - try { - operations += await this.syncFile( - snapshot, - name, - primaryFileMap.get(name), - secondaryFileMap.get(name), - ); - filesProcessed++; - // Only report something significant - if (operations > 1 && this.options.onSyncProgress) { - this.options.onSyncProgress({ - filesProcessed, - totalFiles: sortedFilenames.length, - snapshot, - }); - } - } catch (e: any) { - console.log("error", "Error syncing file", name, e.message); - } + getNonSyncCandidates(files: FileMeta[]): Map { + const nonSyncCandidates: Map = new Map(); + files.forEach((meta) => { + if (!this.options.isSyncCandidate(meta.name)) { + nonSyncCandidates.set(meta.name, meta); } - } catch (e: any) { - console.log("error", "General sync error:", e.message); - throw e; - } - console.log("[sync]", "Sync complete, operations performed", operations); - - return operations; - } - - async syncFile( - snapshot: Map, - name: string, - primaryHash: SyncHash | undefined, - secondaryHash: SyncHash | undefined, - ): Promise { - if (this.options.isSyncCandidate && !this.options.isSyncCandidate(name)) { - return 0; - } - // console.log("Syncing", name, primaryHash, secondaryHash); - let operations = 0; - - if ( - primaryHash !== undefined && secondaryHash === undefined && - !snapshot.has(name) - ) { - // New file, created on primary, copy from primary to secondary - console.log( - "[sync]", - "New file created on primary, copying to secondary", - name, - ); - const { data, meta } = await this.primary.readFile(name); - const writtenMeta = await this.secondary.writeFile( - name, - data, - false, - meta, - ); - snapshot.set(name, [ - primaryHash, - writtenMeta.lastModified, - ]); - operations++; - await this.emit("fileSynced", writtenMeta, "primary->secondary"); - } else if ( - secondaryHash !== undefined && primaryHash === undefined && - !snapshot.has(name) - ) { - // New file, created on secondary, copy from secondary to primary - console.log( - "[sync]", - "New file created on secondary, copying from secondary to primary", - name, - ); - const { data, meta } = await this.secondary.readFile(name); - const writtenMeta = await this.primary.writeFile( - name, - data, - false, - meta, - ); - snapshot.set(name, [ - writtenMeta.lastModified, - secondaryHash, - ]); - operations++; - await this.emit("fileSynced", writtenMeta, "secondary->primary"); - } else if ( - primaryHash !== undefined && snapshot.has(name) && - secondaryHash === undefined - ) { - // File deleted on B - console.log( - "[sync]", - "File deleted on secondary, deleting from primary", - name, - ); - await this.primary.deleteFile(name); - snapshot.delete(name); - operations++; - } else if ( - secondaryHash !== undefined && snapshot.has(name) && - primaryHash === undefined - ) { - // File deleted on A - console.log( - "[sync]", - "File deleted on primary, deleting from secondary", - name, - ); - await this.secondary.deleteFile(name); - snapshot.delete(name); - operations++; - } else if ( - snapshot.has(name) && primaryHash === undefined && - secondaryHash === undefined - ) { - // File deleted on both sides, :shrug: - console.log( - "[sync]", - "File deleted on both ends, deleting from status", - name, - ); - snapshot.delete(name); - operations++; - } else if ( - primaryHash !== undefined && secondaryHash !== undefined && - snapshot.get(name) && - primaryHash !== snapshot.get(name)![0] && - secondaryHash === snapshot.get(name)![1] - ) { - // File has changed on primary, but not secondary: copy from primary to secondary - console.log( - "[sync]", - "File changed on primary, copying to secondary", - name, - ); - const { data, meta } = await this.primary.readFile(name); - const writtenMeta = await this.secondary.writeFile( - name, - data, - false, - meta, - ); - snapshot.set(name, [ - primaryHash, - writtenMeta.lastModified, - ]); - operations++; - await this.emit("fileSynced", writtenMeta, "primary->secondary"); - } else if ( - primaryHash !== undefined && secondaryHash !== undefined && - snapshot.get(name) && - secondaryHash !== snapshot.get(name)![1] && - primaryHash === snapshot.get(name)![0] - ) { - // File has changed on secondary, but not primary: copy from secondary to primary - console.log( - "[sync]", - "File has changed on secondary, but not primary: copy from secondary to primary", - name, - ); - const { data, meta } = await this.secondary.readFile(name); - const writtenMeta = await this.primary.writeFile( - name, - data, - false, - meta, - ); - snapshot.set(name, [ - writtenMeta.lastModified, - secondaryHash, - ]); - operations++; - await this.emit("fileSynced", writtenMeta, "secondary->primary"); - } else if ( - ( // File changed on both ends, but we don't have any info in the snapshot (resync scenario?): have to run through conflict handling - primaryHash !== undefined && secondaryHash !== undefined && - !snapshot.has(name) - ) || - ( // File changed on both ends, CONFLICT! - primaryHash && secondaryHash && - snapshot.get(name) && - secondaryHash !== snapshot.get(name)![1] && - primaryHash !== snapshot.get(name)![0] - ) - ) { - console.log( - "[sync]", - "File changed on both ends, potential conflict", - name, - ); - operations += await this.options.conflictResolver!( - name, - snapshot, - this.primary, - this.secondary, - ); - } else { - // Nothing needs to happen - } - return operations; - } - - syncCandidates(files: FileMeta[]): FileMeta[] { - if (this.options.isSyncCandidate) { - return files.filter((meta) => this.options.isSyncCandidate!(meta.name)); - } else { - return files; - } + }); + return nonSyncCandidates; } } diff --git a/plug-api/syscalls/editor.ts b/plug-api/syscalls/editor.ts index 45ea2699..41eeef73 100644 --- a/plug-api/syscalls/editor.ts +++ b/plug-api/syscalls/editor.ts @@ -504,3 +504,10 @@ export function configureVimMode(): Promise { export function sendMessage(type: string, data?: any): Promise { return syscall("editor.sendMessage", type, data); } + +/** + * Check if the editor is running on a mobile device + */ +export function isMobile(): Promise { + return syscall("editor.isMobile"); +} diff --git a/plug-api/syscalls/index.ts b/plug-api/syscalls/index.ts index 6d34dc4b..4ccbf066 100644 --- a/plug-api/syscalls/index.ts +++ b/plug-api/syscalls/index.ts @@ -49,3 +49,10 @@ export function getObjectByRef( ): Promise | undefined> { return syscall("index.getObjectByRef", page, tag, ref); } + +/** + * Ensures that the full index is built and up-to-date + */ +export function ensureFullIndex(): Promise { + return syscall("index.ensureFullIndex"); +} diff --git a/plug-api/syscalls/mq.ts b/plug-api/syscalls/mq.ts index 5b6a2761..dd604fee 100644 --- a/plug-api/syscalls/mq.ts +++ b/plug-api/syscalls/mq.ts @@ -21,7 +21,10 @@ export function send(queue: string, body: any): Promise { * @param queue the name of the queue * @param bodies the bodies of the messages to send */ -export function batchSend(queue: string, bodies: any[]): Promise { +export function batchSend( + queue: string, + bodies: any[], +): Promise { return syscall("mq.batchSend", queue, bodies); } @@ -58,3 +61,12 @@ export function batchAck(queue: string, ids: string[]): Promise { export function getQueueStats(queue: string): Promise { return syscall("mq.getQueueStats", queue); } + +/** + * Waits for a queue to become empty. + * @param queue the name of the queue + * @returns a promise that resolves when the queue is empty + */ +export function awaitEmptyQueue(queue: string): Promise { + return syscall("mq.awaitEmptyQueue", queue); +} diff --git a/plug-api/syscalls/sync.ts b/plug-api/syscalls/sync.ts index 87ac520d..fef61584 100644 --- a/plug-api/syscalls/sync.ts +++ b/plug-api/syscalls/sync.ts @@ -5,13 +5,6 @@ import { syscall } from "../syscall.ts"; * @module */ -/** - * Checks if a sync is currently in progress - */ -export function isSyncing(): Promise { - return syscall("sync.isSyncing"); -} - /** * Checks if an initial sync has completed */ diff --git a/plugs/editor/page.ts b/plugs/editor/page.ts index b3606516..20828397 100644 --- a/plugs/editor/page.ts +++ b/plugs/editor/page.ts @@ -1,4 +1,5 @@ import { editor, space } from "@silverbulletmd/silverbullet/syscalls"; +import { notFoundError } from "../../lib/constants.ts"; export async function deletePage() { const pageName = await editor.getCurrentPage(); @@ -41,7 +42,7 @@ export async function copyPage( `"${newName}" already exists, cannot copy to existing page.`, ); } catch (e: any) { - if (e.message === "Not found") { + if (e.message === notFoundError.message) { // Expected not found error, so we can continue } else { await editor.flashNotification(e.message, "error"); diff --git a/plugs/editor/upload.ts b/plugs/editor/upload.ts index 088d06e2..a5803260 100644 --- a/plugs/editor/upload.ts +++ b/plugs/editor/upload.ts @@ -1,5 +1,5 @@ import { editor, space, system } from "@silverbulletmd/silverbullet/syscalls"; -import { defaultLinkStyle, maximumDocumentSize } from "../../web/constants.ts"; +import { defaultLinkStyle, maximumDocumentSize } from "../../lib/constants.ts"; import { resolveMarkdownLink } from "@silverbulletmd/silverbullet/lib/resolve"; import { encodePageURI, diff --git a/plugs/index/command.ts b/plugs/index/command.ts index 44911141..609633b6 100644 --- a/plugs/index/command.ts +++ b/plugs/index/command.ts @@ -1,91 +1,8 @@ -import { - editor, - events, - markdown, - mq, - space, - system, -} from "@silverbulletmd/silverbullet/syscalls"; -import type { IndexEvent } from "../../type/event.ts"; -import { sleep } from "../../lib/async.ts"; -import { indexDocument } from "./document.ts"; -import { clearFileIndex } from "./api.ts"; -import type { MQMessage } from "../../type/datastore.ts"; +import { editor } from "@silverbulletmd/silverbullet/syscalls"; +import { reindexSpace } from "./queue.ts"; export async function reindexCommand() { await editor.flashNotification("Performing full page reindex..."); - await system.invokeFunction("index.reindexSpace"); + await reindexSpace(); await editor.flashNotification("Done with page index!"); } - -export async function reindexSpace() { - if (await system.getMode() === "ro") { - console.info("Not reindexing because we're in read-only mode"); - return; - } - console.log("Clearing page index..."); - await system.invokeFunction("index.clearIndex"); - - const files = await space.listFiles(); - - console.log("Queing", files.length, "pages to be indexed."); - // Queue all file names to be indexed - await mq.batchSend("indexQueue", files.map((file) => file.name)); - await editor.showProgress(0, "index"); - - // Now let's wait for the processing to finish - let queueStats = await mq.getQueueStats("indexQueue"); - while (queueStats.queued > 0 || queueStats.processing > 0) { - await sleep(500); - queueStats = await mq.getQueueStats("indexQueue"); - await editor.showProgress( - 100 - Math.round(queueStats.queued / files.length * 100), - "index", - ); - } - // And notify the user - console.log("Indexing completed!"); - await editor.showProgress(); -} - -export async function processIndexQueue(messages: MQMessage[]) { - for (const message of messages) { - let name: string = message.body; - if (name.startsWith("_plug/")) { - continue; - } - console.log(`Indexing file ${name}`); - if (name.endsWith(".md")) { - name = name.slice(0, -3); - await indexPage(name); - } else { - await indexDocument(name); - } - } -} - -async function indexPage(name: string) { - const text = await space.readPage(name); - const parsed = await markdown.parseMarkdown(text); - await events.dispatchEvent("page:index", { - name, - tree: parsed, - }); -} - -export async function parseIndexTextRepublish({ name, text }: IndexEvent) { - if (await system.getMode() === "ro") { - console.info("Not reindexing", name, "because we're in read-only mode"); - return; - } - const parsed = await markdown.parseMarkdown(text); - - // First clear the old file index entries - await clearFileIndex(name); - - // console.log("Indexing page", name); - await events.dispatchEvent("page:index", { - name, - tree: parsed, - }); -} diff --git a/plugs/index/index.plug.yaml b/plugs/index/index.plug.yaml index 4f5f317b..6fa07ba5 100644 --- a/plugs/index/index.plug.yaml +++ b/plugs/index/index.plug.yaml @@ -3,11 +3,9 @@ functions: # Public API batchSet: path: api.ts:batchSet - env: server indexObjects: path: api.ts:indexObjects syscall: index.indexObjects - env: server query: path: api.ts:query syscall: index.query @@ -17,7 +15,6 @@ functions: getObjectByRef: path: api.ts:getObjectByRef syscall: index.getObjectByRef - env: server extractFrontmatter: path: api.ts:extractFrontmatter @@ -27,23 +24,15 @@ functions: path: api.ts:patchFrontmatter syscall: index.patchFrontmatter - # Event handlers + # Index management clearIndex: path: api.ts:clearIndex - env: server clearDSIndex: path: api.ts:clearFileIndex - env: server events: - file:deleted - parseIndexTextRepublish: - path: "./command.ts:parseIndexTextRepublish" - env: server - events: - - page:index_text - reindexSpaceCommand: path: "./command.ts:reindexCommand" command: @@ -51,14 +40,14 @@ functions: requireMode: rw processIndexQueue: - path: ./command.ts:processIndexQueue + path: ./queue.ts:processIndexQueue mqSubscriptions: - queue: indexQueue batchSize: 3 autoAck: true + reindexSpace: - path: "./command.ts:reindexSpace" - env: server + path: "./queue.ts:reindexSpace" indexDocument: path: "./document.ts:indexDocument" diff --git a/plugs/index/item.ts b/plugs/index/item.ts index e2070d9c..1879ec2d 100644 --- a/plugs/index/item.ts +++ b/plugs/index/item.ts @@ -22,6 +22,7 @@ import { } from "@silverbulletmd/silverbullet/lib/frontmatter"; import { deepClone } from "@silverbulletmd/silverbullet/lib/json"; import type { ObjectValue } from "../../type/index.ts"; +import { system } from "@silverbulletmd/silverbullet/syscalls"; export type ItemObject = ObjectValue< { @@ -33,7 +34,15 @@ export type ItemObject = ObjectValue< >; export async function indexItems({ name, tree }: IndexTreeEvent) { - const items = await extractItems(name, tree); + const shouldIndexAll = await system.getConfig( + "index.item.all", + true, + ); + let items = await extractItems(name, tree); + if (!shouldIndexAll) { + items = items.filter((item) => item.tags?.length); + } + // console.log("Found", items, "item(s)"); await indexObjects(name, items); } diff --git a/plugs/index/page_links.ts b/plugs/index/page_links.ts index 5d3c76f0..1f519851 100644 --- a/plugs/index/page_links.ts +++ b/plugs/index/page_links.ts @@ -71,7 +71,6 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) { // If this is a meta template page, we don't want to index links if (frontmatter.tags?.find((t) => t.startsWith("meta/template"))) { - console.log("Skipping meta template page", name); return; } diff --git a/plugs/index/paragraph.ts b/plugs/index/paragraph.ts index b2c047bf..1c073dfb 100644 --- a/plugs/index/paragraph.ts +++ b/plugs/index/paragraph.ts @@ -11,6 +11,7 @@ import { updateITags } from "@silverbulletmd/silverbullet/lib/tags"; import { extractFrontMatter } from "@silverbulletmd/silverbullet/lib/frontmatter"; import { extractHashtag } from "../../plug-api/lib/tags.ts"; import type { ObjectValue } from "../../type/index.ts"; +import { system } from "@silverbulletmd/silverbullet/syscalls"; /** ParagraphObject An index object for the top level text nodes */ export type ParagraphObject = ObjectValue< @@ -22,6 +23,11 @@ export type ParagraphObject = ObjectValue< >; export async function indexParagraphs({ name: page, tree }: IndexTreeEvent) { + const shouldIndexAll = await system.getConfig( + "index.paragraph.all", + true, + ); + const objects: ParagraphObject[] = []; const frontmatter = await extractFrontMatter(tree); @@ -45,6 +51,10 @@ export async function indexParagraphs({ name: page, tree }: IndexTreeEvent) { // Hacky way to remove the hashtag tagNode.children = []; }); + if (tags.size === 0 && !shouldIndexAll) { + // Don't index paragraphs without a hashtag + return false; + } // Extract attributes and remove from tree const attrs = await extractAttributes(p); diff --git a/plugs/index/queue.ts b/plugs/index/queue.ts new file mode 100644 index 00000000..cf2d849a --- /dev/null +++ b/plugs/index/queue.ts @@ -0,0 +1,93 @@ +import { + editor, + events, + markdown, + mq, + space, + system, +} from "@silverbulletmd/silverbullet/syscalls"; +import { sleep } from "../../lib/async.ts"; +import { indexDocument } from "./document.ts"; +import type { MQMessage } from "../../type/datastore.ts"; +import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; +import { clearFileIndex } from "./api.ts"; + +export async function reindexSpace() { + if (await system.getMode() === "ro") { + console.info("Not reindexing because we're in read-only mode"); + return; + } + console.log("Clearing page index..."); + await system.invokeFunction("index.clearIndex"); + + const startTime = Date.now(); + + const files = await space.listFiles(); + + console.log("Queing", files.length, "pages to be indexed."); + // Queue all file names to be indexed + await mq.batchSend("indexQueue", files.map((file) => file.name)); + await editor.showProgress(0, "index"); + await mq.awaitEmptyQueue("indexQueue"); + + // And notify the user + console.log(`Indexing completed in ${(Date.now() - startTime) / 1000}s`); + await editor.showProgress(); +} + +setTimeout(updateIndexProgressInUI, 5000); + +async function updateIndexProgressInUI() { + // Let's see if there's anything in the index queue + let queueStats = await mq.getQueueStats("indexQueue"); + if (queueStats.queued > 0 || queueStats.processing > 0) { + // Something's queued, likely it makes sense to compare this to the total number of files (progress wise) + const fileList = await space.listFiles(); + while (queueStats.queued > 0 || queueStats.processing > 0) { + queueStats = await mq.getQueueStats("indexQueue"); + const percentage = Math.round( + (fileList.length - queueStats.queued) / fileList.length * 100, + ); + if (percentage > 99) { + // Hide progress circle + await editor.showProgress(); + } else { + await editor.showProgress(percentage, "index"); + } + // Update UI every second + await sleep(1000); + } + } + // Schedule again + setTimeout(updateIndexProgressInUI, 5000); +} + +export async function processIndexQueue(messages: MQMessage[]) { + for (const message of messages) { + let name: string = message.body; + if (name.startsWith("_plug/")) { + continue; + } + console.log(`Indexing file ${name}`); + if (name.endsWith(".md")) { + name = name.slice(0, -3); + await indexPage(name); + } else { + await indexDocument(name); + } + } +} + +async function indexPage(name: string) { + // Clear any previous index entries for this file + await clearFileIndex(name); + // Read and parse the file + const text = await space.readPage(name); + const parsed = await markdown.parseMarkdown(text); + + // Emit the event which will be picked up by indexers + await events.dispatchEvent("page:index", { + name, + tree: parsed, + } as IndexTreeEvent); +} diff --git a/plugs/index/refactor.ts b/plugs/index/refactor.ts index 01772d94..a507db73 100644 --- a/plugs/index/refactor.ts +++ b/plugs/index/refactor.ts @@ -17,8 +17,9 @@ import { nodeAtPos, } from "@silverbulletmd/silverbullet/lib/tree"; import { queryLuaObjects } from "./api.ts"; -import type { ObjectValue } from "../../type/index.ts"; +import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; import { isValidPath } from "@silverbulletmd/silverbullet/lib/ref"; +import { notFoundError } from "../../lib/constants.ts"; /** * Renames a single page. @@ -126,7 +127,7 @@ export async function batchRenameFiles(fileList: [string, string][]) { `${newName} already exists, cannot rename to existing file.`, ); } catch (e: any) { - if (e.message === "Not found") { + if (e.message === notFoundError.message) { // Expected not found error, so we can continue } else { throw e; @@ -144,7 +145,7 @@ export async function batchRenameFiles(fileList: [string, string][]) { await renameDocument(oldName, newName); } } catch (e: any) { - if (e.message === "Not found") { + if (e.message === notFoundError.message) { console.log(`${oldName} does not exist, skipping`); } else { throw e; @@ -356,7 +357,7 @@ export async function extractToPageCommand() { `Page ${newName} already exists, cannot rename to existing page.`, ); } catch (e: any) { - if (e.message === "Not found") { + if (e.message === notFoundError.message) { // Expected not found error, so we can continue } else { await editor.flashNotification(e.message, "error"); diff --git a/plugs/search/search.ts b/plugs/search/search.ts index 00a31793..83bcee98 100644 --- a/plugs/search/search.ts +++ b/plugs/search/search.ts @@ -2,9 +2,9 @@ import type { IndexTreeEvent, PageCreatingContent, PageCreatingEvent, -} from "../../type/event.ts"; +} from "@silverbulletmd/silverbullet/type/event"; import { renderToText } from "@silverbulletmd/silverbullet/lib/tree"; -import { editor } from "@silverbulletmd/silverbullet/syscalls"; +import { editor, system } from "@silverbulletmd/silverbullet/syscalls"; import { ftsIndexPage, ftsSearch } from "./engine.ts"; import { PromiseQueue } from "../../lib/async.ts"; @@ -13,8 +13,12 @@ const searchPrefix = "search:"; // Search indexing is prone to concurrency issues, so we queue all write operations const promiseQueue = new PromiseQueue(); -export function indexPage({ name, tree }: IndexTreeEvent) { +export async function indexPage({ name, tree }: IndexTreeEvent) { + if (!await system.getConfig("index.search.enable", true)) { + return; + } const text = renderToText(tree); + return promiseQueue.runInQueue(async () => { // console.log("Now FTS indexing", name); // await engine.deleteDocument(name); diff --git a/plugs/sync/sync.plug.yaml b/plugs/sync/sync.plug.yaml index 8282e5dd..7dcc75cb 100644 --- a/plugs/sync/sync.plug.yaml +++ b/plugs/sync/sync.plug.yaml @@ -1,8 +1,28 @@ name: sync functions: + + # Commands syncSpaceCommand: path: "./sync.ts:syncSpaceCommand" command: - name: "Sync: Now" + name: "Sync: Space" key: "Alt-Shift-s" mac: "Cmd-Shift-s" + + syncFileCommand: + path: "./sync.ts:syncFileCommand" + command: + name: "Sync: File" + + spaceSyncComplete: + path: "./sync.ts:spaceSyncComplete" + events: + - "service-worker:space-sync-complete" + updateSyncStatus: + path: "./sync.ts:updateSyncStatus" + events: + - "service-worker:sync-status" + reportSyncConflict: + path: "./sync.ts:reportSyncConflict" + events: + - "service-worker:sync-conflict" diff --git a/plugs/sync/sync.ts b/plugs/sync/sync.ts index 308c7225..f0de1fd1 100644 --- a/plugs/sync/sync.ts +++ b/plugs/sync/sync.ts @@ -1,7 +1,49 @@ -import { editor, sync } from "@silverbulletmd/silverbullet/syscalls"; +import { + editor, + index, + space, + sync, +} from "@silverbulletmd/silverbullet/syscalls"; export async function syncSpaceCommand() { await editor.flashNotification("Syncing space..."); await sync.performSpaceSync(); await editor.flashNotification("Done."); } + +export async function syncFileCommand() { + await editor.flashNotification("Syncing file..."); + await sync.performFileSync(await editor.getCurrentPath()); + await editor.flashNotification("Done."); +} + +export async function spaceSyncComplete(message: { operations: number }) { + if (message.operations > 0) { + // Update the page list + await space.listFiles(); + } + + index.ensureFullIndex(); +} + +export async function updateSyncStatus( + event: { + status: { + filesProcessed: number; + totalFiles: number; + }; + }, +) { + // Update the status in the UI + await editor.showProgress( + Math.round((event.status.filesProcessed / event.status.totalFiles) * 100), + "sync", + ); +} + +export async function reportSyncConflict({ path }: { path: string }) { + await editor.flashNotification( + `Sync: conflict detected for ${path} - conflict copy created`, + "error", + ); +} diff --git a/plugs/tasks/task.ts b/plugs/tasks/task.ts index 9904c6b4..9a3f1239 100644 --- a/plugs/tasks/task.ts +++ b/plugs/tasks/task.ts @@ -6,6 +6,7 @@ import { markdown, space, sync, + system, } from "@silverbulletmd/silverbullet/syscalls"; import { @@ -19,7 +20,7 @@ import { renderToText, replaceNodesMatching, traverseTreeAsync, -} from "../../plug-api/lib/tree.ts"; +} from "@silverbulletmd/silverbullet/lib/tree"; import { niceDate } from "../../lib/dates.ts"; import { cleanAttributes, @@ -41,7 +42,7 @@ import { import { enrichItemFromParents } from "../index/item.ts"; import { deepClone } from "@silverbulletmd/silverbullet/lib/json"; import { queryLuaObjects } from "../index/api.ts"; -import type { ObjectValue } from "../../type/index.ts"; +import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; import type { ClickEvent } from "@silverbulletmd/silverbullet/type/client"; export type TaskObject = ObjectValue< @@ -154,7 +155,16 @@ export async function extractTasks( } export async function indexTasks({ name, tree }: IndexTreeEvent) { - const extractedTasks = await extractTasks(name, tree); + const shouldIndexAll = await system.getConfig( + "index.task.all", + true, + ); + + let extractedTasks = await extractTasks(name, tree); + + if (!shouldIndexAll) { + extractedTasks = extractedTasks.filter((task) => task.tags?.length); + } // Index tasks themselves if (extractTasks.length > 0) { diff --git a/server/http_server.ts b/server/http_server.ts index b08d85b8..0e165f27 100644 --- a/server/http_server.ts +++ b/server/http_server.ts @@ -8,9 +8,9 @@ import { compile as gitIgnoreCompiler } from "gitignore-parser"; import { decodePageURI } from "@silverbulletmd/silverbullet/lib/ref"; import { LockoutTimer } from "./lockout.ts"; import type { AuthOptions } from "../cmd/server.ts"; -import type { ClientConfig } from "../web/client.ts"; +import type { BootConfig } from "../web/ui_types.ts"; import { applyUrlPrefix, removeUrlPrefix } from "../lib/url_prefix.ts"; -import { authCookieName, utcDateString } from "./util.ts"; +import { authCookieName, fileMetaToHeaders, utcDateString } from "./util.ts"; import { renderHtmlPage } from "./serverside_render.ts"; import { FilteredSpacePrimitives } from "../lib/spaces/filtered_space_primitives.ts"; import { AssetBundlePlugSpacePrimitives } from "../lib/spaces/asset_bundle_space_primitives.ts"; @@ -24,10 +24,10 @@ import { type ShellBackend, } from "./shell_backend.ts"; import { CONFIG_TEMPLATE, INDEX_TEMPLATE } from "../web/PAGE_TEMPLATES.ts"; -import type { FileMeta } from "../type/index.ts"; import { CheckPathSpacePrimitives, } from "../lib/spaces/checked_space_primitives.ts"; +import { notFoundError } from "../lib/constants.ts"; const authenticationExpirySeconds = 60 * 60 * 24 * 7; // 1 week @@ -367,7 +367,7 @@ export class HttpServer { // Fetch config this.app.get("/.config", (c) => { - const clientConfig: ClientConfig = { + const clientConfig: BootConfig = { readOnly: this.options.readOnly, spaceFolderPath: this.options.pagesPath, indexPage: this.options.indexPage, @@ -522,7 +522,7 @@ export class HttpServer { path, new Uint8Array(body), ); - return c.text("OK", 200, this.fileMetaToHeaders(meta)); + return c.text("OK", 200, fileMetaToHeaders(meta)); } catch (err) { console.error("Write failed", err); return c.text("Write failed", 500); @@ -539,7 +539,7 @@ export class HttpServer { const fileData = await this.spacePrimitives.getFileMeta( name, ); - return c.text("", 200, this.fileMetaToHeaders(fileData)); + return c.text("", 200, fileMetaToHeaders(fileData)); } const fileData = await this.spacePrimitives.readFile(name); const lastModifiedHeader = new Date(fileData.meta.lastModified) @@ -550,7 +550,7 @@ export class HttpServer { return c.body(null, 304); } return c.body(new Uint8Array(fileData.data).buffer, 200, { - ...this.fileMetaToHeaders(fileData.meta), + ...fileMetaToHeaders(fileData.meta), "Last-Modified": lastModifiedHeader, }); } catch (e: any) { @@ -559,17 +559,6 @@ export class HttpServer { } } - private fileMetaToHeaders(fileMeta: FileMeta) { - return { - "Content-Type": fileMeta.contentType, - "X-Last-Modified": "" + fileMeta.lastModified, - "X-Created": "" + fileMeta.created, - "Cache-Control": "no-cache", - "X-Permission": fileMeta.perm, - "X-Content-Length": "" + fileMeta.size, - }; - } - private prefixedUrl(url: string): string { return applyUrlPrefix(url, this.options.hostUrlPrefix); } @@ -598,7 +587,7 @@ export class HttpServer { // This will blow up if the page doesn't exist await this.spacePrimitives.getFileMeta(path); } catch (e: any) { - if (e.message === "Not found") { + if (e.message === notFoundError.message) { console.info(path, "page not found, creating..."); await this.spacePrimitives.writeFile( path, diff --git a/server/serverside_render.ts b/server/serverside_render.ts index 23aadcc2..ff4f4552 100644 --- a/server/serverside_render.ts +++ b/server/serverside_render.ts @@ -12,6 +12,7 @@ import type { ServerOptions } from "./http_server.ts"; import type { AssetBundle } from "../lib/asset_bundle/bundle.ts"; import { htmlEscape } from "../web/markdown/html_render.ts"; import type { SpacePrimitives } from "../lib/spaces/space_primitives.ts"; +import { notFoundError } from "../lib/constants.ts"; // Server-side renders a markdown file to HTML export async function renderHtmlPage( @@ -44,7 +45,7 @@ export async function renderHtmlPage( const tree = parse(extendedMarkdownLanguage, text); html = renderMarkdownToHtml(tree); } catch (e: any) { - if (e.message !== "Not found") { + if (e.message !== notFoundError.message) { console.error("Error server-side rendering page", e); } } @@ -53,7 +54,7 @@ export async function renderHtmlPage( try { await spacePrimitives.getFileMeta(ref.path); } catch (e: any) { - if (e.message !== "Not found") { + if (e.message !== notFoundError.message) { return c.notFound(); } } diff --git a/server/storage_backend.ts b/server/storage_backend.ts index f5f5f7e2..11c66505 100644 --- a/server/storage_backend.ts +++ b/server/storage_backend.ts @@ -9,7 +9,10 @@ export function determineStorageBackend( if (folder.startsWith("http://") || folder.startsWith("https://")) { return new HttpSpacePrimitives( folder, - undefined, + "", + (err) => { + console.error("[HTTP space primitives error]", err); + }, Deno.env.get("SB_AUTH_TOKEN"), ); } else { diff --git a/server/util.ts b/server/util.ts index 50af7774..2f829361 100644 --- a/server/util.ts +++ b/server/util.ts @@ -1,3 +1,5 @@ +import type { FileMeta } from "@silverbulletmd/silverbullet/type/index"; + export function utcDateString(mtime: number): string { return new Date(mtime).toUTCString(); } @@ -5,3 +7,37 @@ export function utcDateString(mtime: number): string { export function authCookieName(host: string) { return `auth_${host.replaceAll(/\W/g, "_")}`; } + +export function fileMetaToHeaders(fileMeta: FileMeta) { + return { + "Content-Type": fileMeta.contentType, + "X-Last-Modified": "" + fileMeta.lastModified, + "X-Created": "" + fileMeta.created, + "Cache-Control": "no-cache", + "X-Permission": fileMeta.perm, + "X-Content-Length": "" + fileMeta.size, + }; +} + +export function headersToFileMeta( + name: string, + headers: Headers, +): FileMeta | undefined { + if (headers.has("X-Last-Modified")) { + // If this header is set, we need to pull out the rest also + return { + name, + // The server may set a custom X-Content-Length header in case a GET request was sent with X-Get-Meta, in which case the body may be omitted + size: headers.has("X-Content-Length") + ? +headers.get("X-Content-Length")! + : +headers.get("Content-Length")!, + contentType: headers.get("Content-type")!, + created: +(headers.get("X-Created") || "0"), + lastModified: +(headers.get("X-Last-Modified") || "0"), + perm: (headers.get("X-Permission") as "rw" | "ro") || "ro", + }; + } else { + // Otherwise: no file meta in headers + return undefined; + } +} diff --git a/type/event.ts b/type/event.ts index bc4c8846..7cfdae74 100644 --- a/type/event.ts +++ b/type/event.ts @@ -1,13 +1,9 @@ import type { ParseTree } from "@silverbulletmd/silverbullet/lib/tree"; import type { EventSubscriptionDef } from "../lib/manifest.ts"; -export type IndexEvent = { - name: string; - text: string; -}; - export type IndexTreeEvent = { name: string; + tree: ParseTree; }; diff --git a/type/index.ts b/type/index.ts index e53aeb1c..696392e1 100644 --- a/type/index.ts +++ b/type/index.ts @@ -5,7 +5,6 @@ export type FileMeta = { contentType: string; size: number; perm: "ro" | "rw"; - noSync?: boolean; }; /** * Decorates a page when it matches certain criteria diff --git a/web/boot.ts b/web/boot.ts index 358287aa..12c3bec4 100644 --- a/web/boot.ts +++ b/web/boot.ts @@ -1,71 +1,49 @@ import { safeRun } from "../lib/async.ts"; -import { Client, type ClientConfig } from "./client.ts"; +import { notAuthenticatedError, offlineError } from "../lib/constants.ts"; +import { initLogger } from "../lib/logger.ts"; +import { extractSpaceLuaFromPageText, loadConfig } from "./boot_config.ts"; +import { Client } from "./client.ts"; +import type { Config } from "./config.ts"; +import { + flushCachesAndUnregisterServiceWorker, +} from "./service_worker/util.ts"; import "./polyfills.ts"; +import type { BootConfig } from "./ui_types.ts"; +import { BoxProxy } from "../lib/box_proxy.ts"; -const configCacheKey = `silverbullet.${document.baseURI}.config`; +initLogger("[Client]"); safeRun(async () => { // First we attempt to fetch the config from the server - let clientConfig: ClientConfig | undefined; + let bootConfig: BootConfig | undefined; + let config: Config | undefined; + // Placeholder proxy for Client object to be swapped in later + const clientProxy = new BoxProxy({}); try { - const configResponse = await fetch(".config", { - // We don't want to follow redirects, we want to get the redirect header in case of auth issues - redirect: "manual", - // Add short timeout in case of a bad internet connection, this would block loading of the UI - signal: AbortSignal.timeout(1000), - }); - const redirectHeader = configResponse.headers.get("location"); - if ( - configResponse.status === 401 && redirectHeader - ) { - alert( - "Received an authentication redirect, redirecting to URL: " + - redirectHeader, - ); - location.href = redirectHeader; - return; - } - clientConfig = await configResponse.json(); - // Persist to localStorage - localStorage.setItem(configCacheKey, JSON.stringify(clientConfig)); + const [configJSONText, customConfigText, defaultConfigText] = await Promise + .all([ + cachedFetch(".config"), + cachedFetch(".fs/CONFIG.md"), + cachedFetch(".fs/Library/Std/Config.md"), + ]); + bootConfig = JSON.parse(configJSONText); + const luaDefaultConfig = extractSpaceLuaFromPageText(defaultConfigText); + const luaCustomConfig = extractSpaceLuaFromPageText(customConfigText); + // Append and evaluate + config = await loadConfig( + luaDefaultConfig + "\n" + luaCustomConfig, + clientProxy.buildProxy(), + ); } catch (e: any) { - console.error("Failed to fetch client config from server", e.message); - // We may be offline, let's see if we have a cached config - const configString = localStorage.getItem(configCacheKey); - if (configString) { - // Yep! Let's use it - clientConfig = JSON.parse(configString); - } else { - alert( - "Could not fetch configuration from server. Make sure you have an internet connection.", - ); - // Returning here because there's no way to recover from this - return; - } - } - // Then we augment the config based on the URL arguments - const urlParams = new URLSearchParams(location.search); - if (urlParams.has("readOnly")) { - clientConfig!.readOnly = true; - } - if (urlParams.has("disableSpaceLua")) { - clientConfig!.disableSpaceLua = true; - } - if (urlParams.has("disableSync")) { - clientConfig!.disableSync = true; - } - if (urlParams.has("disablePlugs")) { - clientConfig!.disablePlugs = true; - } - if (urlParams.has("disableSpaceStyle")) { - clientConfig!.disableSpaceStyle = true; - } - if (urlParams.has("wipeClient")) { - clientConfig!.performWipe = true; - } - if (urlParams.has("resetClient")) { - clientConfig!.performReset = true; + console.error("Failed to process config", e.message); + alert( + "Could not process config and no cached copy, please connect to the Internet", + ); + return; } + + await augmentBootConfig(bootConfig!, config!); + // Update the browser URL to no longer contain the query parameters using pushState if (location.search) { const newURL = new URL(location.href); @@ -73,11 +51,35 @@ safeRun(async () => { history.pushState({}, "", newURL.toString()); } console.log("Booting SilverBullet client"); - console.log("Client config", clientConfig); + console.log("Boot config", bootConfig, config.values); - if (navigator.serviceWorker) { + if (localStorage.getItem("enableSW") !== "0" && navigator.serviceWorker) { // Register service worker const workerURL = new URL("service_worker.js", document.baseURI); + let startNotificationCount = 0; + navigator.serviceWorker.addEventListener("message", (event) => { + if (event.data.type === "service-worker-started") { + startNotificationCount++; + // Service worker started, let's make sure it the current config + console.log( + "Got notified that service worker has just started, sending config", + bootConfig, + ); + navigator.serviceWorker.ready.then((registration) => { + registration.active!.postMessage({ + type: "config", + config: bootConfig, + }); + }); + if (startNotificationCount > 10) { + // This is not normal. Safari sometimes gets stuck on a database connection if the service worker is updated which means it cannot boot properly + // the only know fix is to quit the browser and restart it + alert( + "Something is wrong with the sync engine, please quit your browser and restart it.", + ); + } + } + }); navigator.serviceWorker .register(workerURL, { type: "module", @@ -90,6 +92,12 @@ safeRun(async () => { .then((registration) => { console.log("Service worker registered..."); + // Send the config + registration.active?.postMessage({ + type: "config", + config: bootConfig, + }); + // Set up update detection registration.addEventListener("updatefound", () => { const newWorker = registration.installing; @@ -105,46 +113,132 @@ safeRun(async () => { "New service worker installed and ready to take over.", ); // Force the new service worker to activate immediately - newWorker.postMessage({ type: "skipWaiting" }); + newWorker.postMessage({ type: "skip-waiting" }); } }); } }); }); - // Handle service worker controlled changes (when a new service worker takes over) - let refreshing = false; - navigator.serviceWorker.addEventListener("controllerchange", () => { - if (!refreshing) { - refreshing = true; - console.log( - "New service worker activated, please reload to activate the new version.", - ); - } - }); - - navigator.serviceWorker.ready.then((registration) => { - registration.active!.postMessage({ - type: "config", - config: clientConfig, - }); - }); + // // Handle service worker controlled changes (when a new service worker takes over) + // navigator.serviceWorker.addEventListener("controllerchange", async () => { + // console.log( + // "New service worker activated!", + // ); + // }); } else { - console.warn( - "Not launching service worker, likely because not running from localhost or over HTTPs. This means SilverBullet will not be available offline.", - ); + console.info("Service worker disabled."); } const client = new Client( document.getElementById("sb-root")!, - clientConfig!, + bootConfig!, + config!, ); // @ts-ignore: on purpose globalThis.client = client; + clientProxy.setTarget(client); await client.init(); + if (navigator.serviceWorker) { + navigator.serviceWorker.addEventListener("message", (event) => { + client.handleServiceWorkerMessage(event.data); + }); + } }); +/** + * Augments the boot config with values from the page's search params + * as well as well as Lua-based configuration from CONFIG + */ +async function augmentBootConfig(bootConfig: BootConfig, config: Config) { + // Pull out sync configuration + bootConfig.syncDocuments = config.get(["sync", "documents"], false); + let syncIgnore = config!.get(["sync", "ignore"], ""); + if (Array.isArray(syncIgnore)) { + syncIgnore = syncIgnore.join("\n"); + } + bootConfig.syncIgnore = syncIgnore; + + // Then we augment the config based on the URL arguments + const urlParams = new URLSearchParams(location.search); + if (urlParams.has("readOnly")) { + bootConfig.readOnly = true; + } + if (urlParams.has("disableSpaceLua")) { + bootConfig.disableSpaceLua = true; + } + if (urlParams.has("disablePlugs")) { + bootConfig.disablePlugs = true; + } + if (urlParams.has("disableSpaceStyle")) { + bootConfig.disableSpaceStyle = true; + } + if (urlParams.has("wipeClient")) { + bootConfig.performWipe = true; + } + if (urlParams.has("resetClient")) { + bootConfig.performReset = true; + } + if (urlParams.has("enableSW")) { + const val = urlParams.get("enableSW")!; + localStorage.setItem("enableSW", val); + if (val === "0") { + await flushCachesAndUnregisterServiceWorker(); + } + } +} + if (!globalThis.indexedDB) { alert( "SilverBullet requires IndexedDB to operate and it is not available in your browser. Please use a recent version of Chrome, Firefox (not in private mode) or Safari.", ); } + +async function cachedFetch(path: string): Promise { + const cacheKey = `silverbullet.${document.baseURI}.${path}`; + try { + const response = await fetch(path, { + // We don't want to follow redirects, we want to get the redirect header in case of auth issues + redirect: "manual", + // Add short timeout in case of a bad internet connection, this would block loading of the UI + signal: AbortSignal.timeout(1000), + headers: { + "X-Sync-Mode": "1", + }, + }); + if (response.status === 503) { + // Offline + const text = localStorage.getItem(cacheKey); + if (text) { + console.info("Falling back to cache for", path); + return text; + } else { + throw offlineError; + } + } + const redirectHeader = response.headers.get("location"); + if ( + response.status === 401 && redirectHeader + ) { + alert( + "Received an authentication redirect, redirecting to URL: " + + redirectHeader, + ); + location.href = redirectHeader; + throw notAuthenticatedError; + } + const text = await response.text(); + // Persist to localStorage + localStorage.setItem(cacheKey, text); + return text; + } catch { + console.info("Falling back to cache for", path); + // We may be offline, let's see if we have a cached config + const text = localStorage.getItem(cacheKey); + if (text) { + // Yep! Let's use it + return text; + } else { + throw offlineError; + } + } +} diff --git a/web/boot_config.test.ts b/web/boot_config.test.ts new file mode 100644 index 00000000..5ebfae7f --- /dev/null +++ b/web/boot_config.test.ts @@ -0,0 +1,66 @@ +import { assertEquals } from "@std/assert"; +import { extractSpaceLuaFromPageText, loadConfig } from "./boot_config.ts"; + +Deno.test("Test boot config", () => { + // One block present + assertEquals( + extractSpaceLuaFromPageText("Hello\n\n```space-lua\ntest()\n```\nMore"), + "test()", + ); + // Two blocks present + assertEquals( + extractSpaceLuaFromPageText( + "Hello\n\n```space-lua\ntest()\n```\nMore\n\n```space-lua\ntest2()\n```", + ), + "test()\ntest2()", + ); + // No lua present + assertEquals( + extractSpaceLuaFromPageText("Hello\n\n```lua\ntest()\n```\nMore"), + "", + ); +}); + +Deno.test("Test CONFIG lua eval", async () => { + // Test base case: no config code + let config = await loadConfig("", {}); + assertEquals(config.values, {}); + + // Check a few config sets + config = await loadConfig( + ` + config.set { + option1 = "pete" + } + config.set("optionObj.nested", 5) +`, + {}, + ); + assertEquals(config.values, { + option1: "pete", + optionObj: { + nested: 5, + }, + }); + + // Check random Lua code crap resilience + config = await loadConfig( + ` + config.set { + option1 = "pete" + } + slashCommand.define {} + local shouldSet = true + if shouldSet then + config.set("optionObj.nested", 5) + end +`, + {}, + ); + assertEquals(config.values, { + option1: "pete", + optionObj: { + nested: 5, + }, + }); +}); diff --git a/web/boot_config.ts b/web/boot_config.ts new file mode 100644 index 00000000..b8148dd3 --- /dev/null +++ b/web/boot_config.ts @@ -0,0 +1,88 @@ +import { + findNodeOfType, + traverseTree, +} from "@silverbulletmd/silverbullet/lib/tree"; +import { parseMarkdown } from "./markdown_parser/parser.ts"; +import { System } from "../lib/plugos/system.ts"; +import { configSyscalls } from "./syscalls/config.ts"; +import { Config } from "./config.ts"; +import { luaBuildStandardEnv } from "../lib/space_lua/stdlib.ts"; +import { exposeSyscalls } from "./space_lua_api.ts"; +import { parse } from "../lib/space_lua/parse.ts"; +import { LuaEnv, LuaStackFrame } from "../lib/space_lua/runtime.ts"; +import { evalStatement } from "../lib/space_lua/eval.ts"; +import { editorSyscalls } from "./syscalls/editor.ts"; +import { markdownSyscalls } from "./syscalls/markdown.ts"; +import { yamlSyscalls } from "./syscalls/yaml.ts"; +import { languageSyscalls } from "./syscalls/language.ts"; +import { jsonschemaSyscalls } from "./syscalls/jsonschema.ts"; + +/** + * Parses a page (CONFIG in practice) and extracts all space-lua code + */ +export function extractSpaceLuaFromPageText(text: string): string { + const tree = parseMarkdown(text); + const codes: string[] = []; + traverseTree(tree, (node) => { + if (node.type === "FencedCode") { + const codeInfo = findNodeOfType(node, "CodeInfo"); + if (codeInfo?.children?.[0].text !== "space-lua") { + return false; + } + const codeText = findNodeOfType(node, "CodeText"); + codes.push(codeText!.children![0].text!); + return true; + } + return false; + }); + return codes.join("\n"); +} + +/** + * Runs the Lua code in a contained environment (only exposing config.* calls) to build up a config object + */ +export async function loadConfig( + luaCode: string, + lateBoundClient: any, +): Promise { + const config = new Config(); + + // We start with a standard env + const rootEnv = luaBuildStandardEnv(); + + // This is a system only used for the boot sequence, will be replaced later + const bootSystem = new System(); + // Only expose a limited set of syscalls that we can offer at this point + bootSystem.registerSyscalls( + [], + // Collecting the config.* calls is basically what we're here for + configSyscalls(config), + // This offers calls like isMobile() which will be useful, and late binding for e.g. to make actionButtons run() work immediately on boot + editorSyscalls(lateBoundClient), + // And these, because: why not + markdownSyscalls(lateBoundClient), + yamlSyscalls(), + languageSyscalls(), + jsonschemaSyscalls(), + ); + + exposeSyscalls(rootEnv, bootSystem); + + // Parse the code + const chunk = parse(luaCode, {}); + const sf = LuaStackFrame.createWithGlobalEnv(rootEnv, chunk.ctx); + + // And eval + const localEnv = new LuaEnv(rootEnv); + for (const statement of chunk.statements) { + try { + await evalStatement(statement, localEnv, sf); + } catch { + // Since people may do whatever in their Lua code, we're going to be extremely liberal in ignoring errors + // Primarily config.* calls are processed, the rest is implicitly ignored through failure + // TODO: Don't show anything + // console.info("Statement errored out, but ignoring:", e.message); + } + } + return config; +} diff --git a/web/client.ts b/web/client.ts index aa476871..47fcf590 100644 --- a/web/client.ts +++ b/web/client.ts @@ -24,18 +24,20 @@ import { PathPageNavigator, } from "./navigator.ts"; -import type { AppViewState } from "./ui_types.ts"; +import type { + AppViewState, + BootConfig, + ServiceWorkerSourceMessage, + ServiceWorkerTargetMessage, +} from "./ui_types.ts"; import type { PageCreatingContent, PageCreatingEvent } from "../type/event.ts"; import type { StyleObject } from "../plugs/index/style.ts"; import { throttle } from "../lib/async.ts"; import { PlugSpacePrimitives } from "../lib/spaces/plug_space_primitives.ts"; import { EventedSpacePrimitives } from "../lib/spaces/evented_space_primitives.ts"; -import { pageSyncInterval, SyncService } from "./sync_service.ts"; import { simpleHash } from "../lib/crypto.ts"; -import type { SyncStatus } from "../lib/spaces/sync.ts"; import { HttpSpacePrimitives } from "../lib/spaces/http_space_primitives.ts"; -import { FallbackSpacePrimitives } from "../lib/spaces/fallback_space_primitives.ts"; import { encodePageURI, encodeRef, @@ -54,7 +56,6 @@ import type { SpacePrimitives } from "../lib/spaces/space_primitives.ts"; import { DataStore } from "../lib/data/datastore.ts"; import { IndexedDBKvPrimitives } from "../lib/data/indexeddb_kv_primitives.ts"; import { DataStoreMQ } from "../lib/data/mq.datastore.ts"; -import { DataStoreSpacePrimitives } from "../lib/spaces/datastore_space_primitives.ts"; import { ReadOnlySpacePrimitives } from "../lib/spaces/ro_space_primitives.ts"; import { LimitedMap } from "../lib/limited_map.ts"; @@ -62,30 +63,18 @@ import { fsEndpoint, plugPrefix } from "../lib/spaces/constants.ts"; import { diffAndPrepareChanges } from "./cm_util.ts"; import { DocumentEditor } from "./document_editor.ts"; import { parseExpressionString } from "../lib/space_lua/parse.ts"; -import { Config } from "./config.ts"; +import type { Config } from "./config.ts"; import type { DocumentMeta, FileMeta, PageMeta } from "../type/index.ts"; import { parseMarkdown } from "./markdown_parser/parser.ts"; import { CheckPathSpacePrimitives } from "../lib/spaces/checked_space_primitives.ts"; +import { notFoundError, offlineError } from "../lib/constants.ts"; const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/; const autoSaveInterval = 1000; -/** - * Client configuration that is set at boot time, doesn't change at runtime - */ -export type ClientConfig = { - spaceFolderPath: string; - indexPage: string; - readOnly: boolean; - // These are all configured via ?query parameters, e.g. ?disableSpaceLua=1 - disableSpaceLua?: boolean; - disableSpaceStyle?: boolean; - disablePlugs?: boolean; - disableSync?: boolean; - performWipe?: boolean; - performReset?: boolean; -}; +// Fetch the file list ever so often, this will implicitly kick off a snapshot comparison resulting in the indexing of changed pages +const fetchFileListInterval = 10000; declare global { var client: Client; @@ -99,9 +88,8 @@ type WidgetCacheItem = { }; export class Client { - readonly config = new Config(); // Event bus used to communicate between components - eventHook = new EventHook(this.config); + eventHook: EventHook; space!: Space; @@ -130,13 +118,11 @@ export class Client { }, 1000); // Track if plugs have been updated since sync cycle fullSyncCompleted = false; - syncService!: SyncService; - // Sync related stuff // Set to true once the system is ready (plugs loaded) public systemReady: boolean = false; private pageNavigator!: PathPageNavigator; - private dbPrefix: string; + private dbName: string; private onLoadRef: Ref; // Progress circle handling private progressTimeout?: number; @@ -161,15 +147,15 @@ export class Client { constructor( private parent: Element, - public clientConfig: ClientConfig, + public bootConfig: BootConfig, + readonly config: Config, ) { + this.eventHook = new EventHook(this.config); // Generate a semi-unique prefix for the database so not to reuse databases for different space paths - this.dbPrefix = "" + + this.dbName = "" + simpleHash( - `${clientConfig.spaceFolderPath}:${ - document.baseURI.replace(/\/*$/, "") - }`, - ); + `${bootConfig.spaceFolderPath}:${document.baseURI.replace(/\/*$/, "")}`, + ) + "_data"; // The third case should only ever happen when the user provides an invalid index env variable this.onLoadRef = parseRefFromURI() || this.getIndexRef(); } @@ -179,14 +165,13 @@ export class Client { * This is a separated from the constructor to allow for async initialization */ async init() { - // Setup the data store - const kvPrimitives = new IndexedDBKvPrimitives( - `${this.dbPrefix}`, - ); + // Setup the KV (database) + const kvPrimitives = new IndexedDBKvPrimitives(this.dbName); await kvPrimitives.init(); + // Wrap it in a datastore this.ds = new DataStore(kvPrimitives); - // Setup message queue + // Setup message queue on top of that this.mq = new DataStoreMQ(this.ds); // Instantiate a PlugOS system @@ -195,34 +180,11 @@ export class Client { this.mq, this.ds, this.eventHook, - this.clientConfig.readOnly, + this.bootConfig.readOnly, ); this.initSpace(); - // The sync service gets priveleged primitives, so it can sync files with invalid names - this.syncService = new SyncService( - this.eventedSpacePrimitives, - this.plugSpaceRemotePrimitives, - this.ds, - this.eventHook, - (path) => { // isSyncCandidate - // Exclude all plug space primitives paths - return !this.plugSpaceRemotePrimitives.isLikelyHandled(path); - }, - ); - - if (this.clientConfig.disableSync) { - this.syncService.enabled = false; - } - - if (!await this.hasInitialSyncCompleted()) { - console.info( - "Initial sync has not yet been completed, disabling page and document indexing to speed this up", - ); - this.eventedSpacePrimitives.enablePageEvents = false; - } - this.ui = new MainUI(this); this.ui.render(this.parent); @@ -235,14 +197,14 @@ export class Client { this.clientSystem.init(); - if (this.clientConfig.performWipe) { + if (this.bootConfig.performWipe) { if (confirm("Are you sure you want to wipe the client?")) { await this.wipeClient(); alert("Wipe done. Please reload the page or navigate away."); return; } } - if (this.clientConfig.performReset) { + if (this.bootConfig.performReset) { if ( confirm( "Are you sure you want to reset the client? This will wipe all local data and re-sync everything.", @@ -275,10 +237,13 @@ export class Client { await this.clientSystem.loadScripts(); await this.initNavigator(); - await this.initSync(); + // await this.initSync(); await this.eventHook.dispatchEvent("system:ready"); this.systemReady = true; + // Load space snapshot and enable events + await this.eventedSpacePrimitives.enable(); + // Kick off a cron event interval setInterval(() => { this.dispatchAppEvent("cron:secondPassed"); @@ -297,65 +262,73 @@ export class Client { effects: client.undoHistoryCompartment?.reconfigure([history()]), }); - // Regularly sync the currently open file - setInterval(() => { - try { - this.syncService.syncFile(this.currentPath()).catch((e: any) => { - console.error("Interval sync error", e); - }); - } catch (e: any) { - console.error("Interval sync error", e); - } - }, pageSyncInterval); - // Asynchronously update caches this.updatePageListCache().catch(console.error); this.updateDocumentListCache().catch(console.error); } - public hasInitialSyncCompleted(): Promise { - return this.syncService.hasInitialSyncCompleted(); - } - initSpace() { this.httpSpacePrimitives = new HttpSpacePrimitives( document.baseURI.replace(/\/*$/, "") + fsEndpoint, - this.clientConfig.spaceFolderPath, + this.bootConfig.spaceFolderPath, + (message, actionOrRedirectHeader) => { + alert(message); + if (actionOrRedirectHeader === "reload") { + location.reload(); + } else { + location.href = actionOrRedirectHeader; + } + }, ); - let remoteSpacePrimitives: SpacePrimitives = this.httpSpacePrimitives; + let remoteSpacePrimitives: SpacePrimitives = new CheckPathSpacePrimitives( + this.httpSpacePrimitives, + ); - if (this.clientConfig.readOnly) { + if (this.bootConfig.readOnly) { remoteSpacePrimitives = new ReadOnlySpacePrimitives( - remoteSpacePrimitives, + this.httpSpacePrimitives, ); } this.plugSpaceRemotePrimitives = new PlugSpacePrimitives( remoteSpacePrimitives, this.clientSystem.namespaceHook, - this.clientConfig.readOnly ? undefined : "client", + this.bootConfig.readOnly ? undefined : "client", ); this.eventedSpacePrimitives = new EventedSpacePrimitives( - // Using fallback space primitives here to allow (by default) local reads to "fall through" to HTTP when files aren't synced yet - new FallbackSpacePrimitives( - new DataStoreSpacePrimitives( - new DataStore( - this.ds.kv, - ), - ), - this.plugSpaceRemotePrimitives, - ), + this.httpSpacePrimitives, this.eventHook, + this.ds, ); - const localSpacePrimitives = new CheckPathSpacePrimitives( - this.eventedSpacePrimitives, + // Kick off a regular file listing request to trigger events + setInterval(() => { + this.eventedSpacePrimitives.fetchFileList(); + }, fetchFileListInterval); + + this.eventHook.addLocalListener( + "file:changed", + async ( + name: string, + ) => { + // TODO: Optimization opportunity here: dispatch the page:index here directly rather than sending it off to a queue which will refetch the file + // console.log("Queueing index for", name); + + await this.mq.send("indexQueue", name); + }, ); + this.eventHook.addLocalListener("file:initial", async () => { + await this.mq.awaitEmptyQueue("indexQueue"); + console.info("Indexing complete, reloading state"); + await this.clientSystem.markFullSpaceIndexComplete(); + await this.clientSystem.reloadState(); + }); + this.space = new Space( - localSpacePrimitives, + this.eventedSpacePrimitives, this.eventHook, ); @@ -377,9 +350,8 @@ export class Client { this.eventHook.addLocalListener( "file:changed", - async ( + ( path: string, - _localChange: boolean, oldHash: number, newHash: number, ) => { @@ -389,8 +361,7 @@ export class Client { // Avoid reloading if the page was just saved (5s window) (!lastSaveTimestamp || (lastSaveTimestamp < Date.now() - 5000)) && // Avoid reloading if the previous hash was undefined (first load) - // Only trigger this after an initial sync has happened - await this.hasInitialSyncCompleted() + oldHash !== undefined ) { console.log( "Page changed elsewhere, reloading. Old hash", @@ -425,6 +396,7 @@ export class Client { this.clientSystem.allKnownFiles.add(f.name); } }); + this.clientSystem.knownFilesLoaded = true; }, ); @@ -471,7 +443,7 @@ export class Client { () => { if ( !this.ui.viewState.unsavedChanges || - this.clientConfig.readOnly + this.bootConfig.readOnly ) { // No unsaved changes, or read-only mode, not gonna save return resolve(); @@ -498,7 +470,6 @@ export class Client { .writePage( this.currentName(), this.editorView.state.sliceDoc(0), - true, ) .then(async (meta) => { this.ui.viewDispatch({ type: "page-saved" }); @@ -584,7 +555,7 @@ export class Client { initialIndexCompleted && this.clientSystem.system.loadedPlugs.has("index") ) { console.log( - "Initial sync complete and index plug loaded, loading full page list via index.", + "Initial index complete and index plug loaded, loading full page list via index.", ); // Fetch actual indexed pages allPages = await this.clientSystem.queryLuaObjects("page", {}); @@ -634,6 +605,9 @@ export class Client { type: "update-page-list", allPages: allPages, }); + + // Async kick-off file listing to bring listing up to date + this.space.spacePrimitives.fetchFileList(); } async updateDocumentListCache() { @@ -655,6 +629,7 @@ export class Client { } showProgress(progressPercentage?: number, progressType?: "sync" | "index") { + // console.log("Showing progress", progressPercentage, progressType); this.ui.viewDispatch({ type: "set-progress", progressPercentage, @@ -884,7 +859,7 @@ export class Client { } getIndexRef(): Ref { - return parseToRef(this.clientConfig.indexPage) || { path: "index.md" }; + return parseToRef(this.bootConfig.indexPage) || { path: "index.md" }; } async navigate( @@ -993,10 +968,26 @@ export class Client { try { doc = await this.space.readPage(pageName); } catch (e: any) { - if (!e.message.includes("Not found")) { + if ( + e.message !== notFoundError.message && + e.message !== offlineError.message + ) { + // If the error is not a "not found" or "offline" error, rethrow it throw e; } + if (e.message === offlineError.message) { + console.info( + "Currently offline, will assume page doesn't exist:", + pageName, + ); + } + + // Scenarios: + // 1. We got a not found error -> Create an empty page + // 2. We got a offline error (which meant that the service worker didn't locally retrieve the page either so likely it doesn't exist) -> Create a new page + // Either way... we create an empty page! + console.log(`Page doesn't exist, creating new page: ${pageName}`); // Mock up the page. We won't yet safe it, because the user may not even @@ -1040,7 +1031,7 @@ export class Client { // Fetch the meta which includes the possibly indexed stuff, like page // decorations - if (await this.hasInitialSyncCompleted()) { + if (await this.clientSystem.hasFullIndexCompleted()) { try { const enrichedMeta = await this.clientSystem.getObjectByRef( pageName, @@ -1123,7 +1114,7 @@ export class Client { this, (path, content) => { this.space - .writeDocument(path, content, true) + .writeDocument(path, content) .then(async (meta) => { this.ui.viewDispatch({ type: "document-editor-saved" }); @@ -1172,14 +1163,14 @@ export class Client { } async loadCustomStyles() { - if (this.clientConfig.disableSpaceStyle) { + if (this.bootConfig.disableSpaceStyle) { console.warn("Not loading custom styles, since space style is disabled"); return; } - if (!await this.hasInitialSyncCompleted()) { - console.info( - "Not loading custom styles yet, since initial synca has not completed yet", - ); + if (!await this.clientSystem.hasFullIndexCompleted()) { + // console.info( + // "Not loading custom styles yet, since initial sync has not completed yet", + // ); return; } @@ -1282,75 +1273,34 @@ export class Client { return this.widgetCache.get(key); } - private async initSync() { - this.syncService.start(); - - // We're still booting, if a initial sync has already been completed we know this is the initial sync - let initialSync = !await this.hasInitialSyncCompleted(); - - this.eventHook.addLocalListener("sync:success", async (operations) => { - if (operations > 0) { - // Update the page list - await this.space.updatePageList(); - } - if (operations !== undefined) { - // "sync:success" is called with a number of operations only from syncSpace(), not from syncing individual pages + async handleServiceWorkerMessage(message: ServiceWorkerSourceMessage) { + switch (message.type) { + case "space-sync-complete": { this.fullSyncCompleted = true; - - console.log("[sync]", "Full sync completed"); - - // A full sync just completed - if (!initialSync) { - // If this was NOT the initial sync let's check if we need to perform a space reindex - this.clientSystem.ensureFullIndex().catch( - console.error, - ); - } else { // initialSync - console.log( - "[sync]", - "Initial sync completed, now need to do a full space index to ensure all pages are indexed using any custom indexers", - ); - this.eventedSpacePrimitives.enablePageEvents = true; - this.clientSystem.ensureFullIndex().catch( - console.error, - ); - initialSync = false; - } + break; } - if (operations) { - // Likely initial sync so let's show visually that we're synced now - this.showProgress(100, "sync"); + case "online-status": { + this.ui.viewDispatch({ + type: "online-status-change", + isOnline: message.isOnline, + }); + break; } - - this.ui.viewDispatch({ type: "sync-change", syncSuccess: true }); - }); - - this.eventHook.addLocalListener("sync:error", (_name) => { - this.ui.viewDispatch({ type: "sync-change", syncSuccess: false }); - }); - - this.eventHook.addLocalListener("sync:conflict", (name) => { - this.flashNotification( - `Sync: conflict detected for ${name} - conflict copy created`, - "error", - ); - }); - - this.eventHook.addLocalListener("sync:progress", (status: SyncStatus) => { - this.showProgress( - Math.round(status.filesProcessed / status.totalFiles * 100), - "sync", - ); - }); - - this.eventHook.addLocalListener( - "file:synced", - (meta: FileMeta, direction: string) => { - if (direction === "secondary->primary") { - // We likely polled the currently open page or document which triggered a local update, let's update the editor accordingly - this.space.spacePrimitives.getFileMeta(meta.name); + case "auth-error": { + alert(message.message); + if (message.actionOrRedirectHeader) { + location.href = message.actionOrRedirectHeader; + } else { + location.reload(); } - }, + break; + } + } + + // Also dispatch it on the event hook for any other listeners + await this.eventHook.dispatchEvent( + `service-worker:${message.type}`, + message, ); } @@ -1492,17 +1442,35 @@ export class Client { } async wipeClient() { - if (navigator.serviceWorker) { + if (navigator.serviceWorker?.controller) { // We will attempt to unregister the service worker, best effort - console.log("Getting service worker registrations"); - navigator.serviceWorker.getRegistrations().then( - async (registrations) => { - for (const registration of registrations) { - await registration.unregister(); + await new Promise((resolve) => { + navigator.serviceWorker.addEventListener("message", async (e: any) => { + const message: ServiceWorkerSourceMessage = e.data; + if (message.type == "dataWiped") { + console.log( + "Got data wipe confirm, uninstalling service worker now", + ); + const registrations = await navigator.serviceWorker + .getRegistrations(); + for (const registration of registrations) { + await registration.unregister(); + } + console.log("Unregistered all service workers"); + resolve(); } - console.log("Unregistered all service workers"); - }, - ); + }); + // Send wipe request + navigator.serviceWorker.getRegistration().then((registration) => { + console.log( + "Sending data wipe request to service worker", + registration, + ); + registration?.active?.postMessage( + { type: "wipe-data" } as ServiceWorkerTargetMessage, + ); + }); + }); } else { console.info( "Service workers not enabled (no HTTPS?), so not unregistering.", @@ -1510,10 +1478,17 @@ export class Client { } console.log("Stopping all systems"); this.space.unwatch(); - this.syncService.close(); console.log("Clearing data store"); await this.ds.kv.clear(); console.log("Clearing complete."); } + + public async postServiceWorkerMessage(message: ServiceWorkerTargetMessage) { + const registration = await navigator.serviceWorker.getRegistration(); + if (!registration?.active) { + throw new Error("No active service worker to post message to"); + } + registration?.active?.postMessage(message); + } } diff --git a/web/client_system.ts b/web/client_system.ts index cf0a599f..549c8389 100644 --- a/web/client_system.ts +++ b/web/client_system.ts @@ -57,6 +57,7 @@ import { builtinPlugNames } from "../plugs/builtin_plugs.ts"; const plugNameExtractRegex = /\/(.+)\.plug\.js$/; const indexVersionKey = ["$indexVersion"]; +const indexQueuedKey = ["$indexQueued"]; // Bump this one every time a full reindex is needed const desiredIndexVersion = 8; const mqTimeout = 10000; // 10s @@ -75,11 +76,13 @@ export class ClientSystem { codeWidgetHook!: CodeWidgetHook; documentEditorHook!: DocumentEditorHook; + // Known files (for UI) readonly allKnownFiles = new Set(); + public knownFilesLoaded: boolean = false; + readonly scriptCommands = new Map(); - spaceLuaEnv = new SpaceLuaEnvironment(); + spaceLuaEnv: SpaceLuaEnvironment; scriptsLoaded: boolean = false; - private indexOngoing = false; constructor( private client: Client, @@ -95,6 +98,8 @@ export class ClientSystem { ), }); + this.spaceLuaEnv = new SpaceLuaEnvironment(this.system); + setInterval(() => { mq.requeueTimeouts(mqTimeout, mqTimeoutRetry, true).catch(console.error); }, 20000); // Look to requeue every 20s @@ -143,7 +148,8 @@ export class ClientSystem { this.eventHook.addLocalListener( "file:changed", - async (path: string, _selfUpdate, _oldHash, newHash) => { + async (path: string, _oldHash, newHash) => { + // Plug (re)loading if (path.startsWith(plugPrefix) && path.endsWith(".plug.js")) { const plugName = plugNameExtractRegex.exec(path)![1]; console.log("Plug updated, reloading", plugName, "from", path); @@ -191,7 +197,7 @@ export class ClientSystem { dataStoreWriteSyscalls(this.ds), syncSyscalls(this.client), clientStoreSyscalls(this.ds), - configSyscalls(this.client), + configSyscalls(this.client.config), ); if (!this.readOnlyMode) { @@ -214,7 +220,7 @@ export class ClientSystem { } async loadScripts() { - if (this.client.clientConfig.disableSpaceLua) { + if (this.client.bootConfig.disableSpaceLua) { console.info("Space Lua scripts are disabled, skipping loading scripts"); return; } @@ -226,7 +232,7 @@ export class ClientSystem { } this.client.config.clear(); try { - await this.spaceLuaEnv.reload(this.system); + await this.spaceLuaEnv.reload(); } catch (e: any) { console.error("Error loading Lua script:", e.message); } @@ -256,7 +262,7 @@ export class ClientSystem { try { const plugName = plugNameExtractRegex.exec(plugMeta.name)![1]; if ( - this.client.clientConfig.disablePlugs && + this.client.bootConfig.disablePlugs && !builtinPlugNames.includes(plugName) ) { console.warn( @@ -302,37 +308,64 @@ export class ClientSystem { ); } + async isIndexOngoing() { + return !!(await this.ds.get(indexQueuedKey)); + } + + async setIndexOngoing(val: boolean = true) { + await this.ds.set(indexQueuedKey, val); + } + async ensureFullIndex() { + if (!this.client.fullSyncCompleted) { + console.info( + "Initial full sync not completed, skipping index check", + ); + return; + } const currentIndexVersion = await this.getCurrentIndexVersion(); + if (!currentIndexVersion) { + console.log("No index version found, assuming fresh install"); + await this.markFullSpaceIndexComplete(); + return; + } + console.info( "[index]", "Current space index version", currentIndexVersion, "index ongoing?", - this.indexOngoing, + await this.isIndexOngoing(), ); - if (currentIndexVersion !== desiredIndexVersion && !this.indexOngoing) { + if ( + currentIndexVersion !== desiredIndexVersion && + !await this.isIndexOngoing() + ) { console.info( "[index]", "Performing a full space reindex, this could take a while...", ); - this.indexOngoing = true; + await this.setIndexOngoing(); await this.system.invokeFunction("index.reindexSpace", []); console.info("[index]", "Full space index complete."); await this.markFullSpaceIndexComplete(); - this.indexOngoing = false; + await this.setIndexOngoing(false); // Let's load space scripts again, which probably weren't loaded before - console.log( - "Now loading space scripts, custom styles and rebuilding editor state", - ); - await this.loadScripts(); - await this.client.loadCustomStyles(); - this.client.rebuildEditorState(); + await this.reloadState(); } } + public async reloadState() { + console.log( + "Now loading space scripts, custom styles and rebuilding editor state", + ); + await this.loadScripts(); + await this.client.loadCustomStyles(); + this.client.rebuildEditorState(); + } + public async evalLuaFunction( luaFunction: ILuaFunction, args: LuaValue[], @@ -360,7 +393,7 @@ export class ClientSystem { return this.ds.get(indexVersionKey); } - private async markFullSpaceIndexComplete() { + async markFullSpaceIndexComplete() { await this.ds.set(indexVersionKey, desiredIndexVersion); } } diff --git a/web/cm_plugins/editor_paste.ts b/web/cm_plugins/editor_paste.ts index bf49b160..f2d4aaba 100644 --- a/web/cm_plugins/editor_paste.ts +++ b/web/cm_plugins/editor_paste.ts @@ -13,7 +13,7 @@ import { findParentMatching, nodeAtPos, } from "@silverbulletmd/silverbullet/lib/tree"; -import { maximumDocumentSize } from "../constants.ts"; +import { maximumDocumentSize } from "../../lib/constants.ts"; import { safeRun } from "../../lib/async.ts"; import { resolveMarkdownLink } from "@silverbulletmd/silverbullet/lib/resolve"; import { localDateString } from "../../lib/dates.ts"; diff --git a/web/cm_plugins/inline_content.ts b/web/cm_plugins/inline_content.ts index 21ec26ae..68a42d63 100644 --- a/web/cm_plugins/inline_content.ts +++ b/web/cm_plugins/inline_content.ts @@ -15,7 +15,7 @@ export function inlineContentPlugin(client: Client) { return decoratorStateField((state: EditorState) => { const widgets: Range[] = []; if (!shouldRenderWidgets(client)) { - console.info("Not rendering widgets"); + // console.info("Not rendering widgets"); return Decoration.set([]); } diff --git a/web/cm_plugins/top_bottom_panels.ts b/web/cm_plugins/top_bottom_panels.ts index 5f14fec4..0392f080 100644 --- a/web/cm_plugins/top_bottom_panels.ts +++ b/web/cm_plugins/top_bottom_panels.ts @@ -113,7 +113,7 @@ export function postScriptPrefacePlugin( ) { return decoratorStateField((state: EditorState) => { if (!editor.clientSystem.scriptsLoaded) { - console.info("System not yet ready, not rendering panel widgets."); + // console.info("System not yet ready, not rendering panel widgets."); return Decoration.none; } const widgets: any[] = []; diff --git a/web/cm_plugins/wiki_link.ts b/web/cm_plugins/wiki_link.ts index 0f754cda..5d620608 100644 --- a/web/cm_plugins/wiki_link.ts +++ b/web/cm_plugins/wiki_link.ts @@ -52,7 +52,7 @@ export function cleanWikiLinkPlugin(client: Client) { .some((file) => file === ref.path) ) { linkStatus = "default"; - } else if (client.fullSyncCompleted) { + } else if (client.clientSystem.knownFilesLoaded) { linkStatus = "file-missing"; } diff --git a/web/components/top_bar.tsx b/web/components/top_bar.tsx index 4e93dbf0..5e63baf5 100644 --- a/web/components/top_bar.tsx +++ b/web/components/top_bar.tsx @@ -20,7 +20,7 @@ export type ActionButton = { export function TopBar({ pageName, unsavedChanges, - syncFailures, + isOnline, isLoading, notifications, onRename, @@ -39,7 +39,7 @@ export function TopBar({ }: { pageName?: string; unsavedChanges: boolean; - syncFailures: number; + isOnline: boolean; isLoading: boolean; notifications: Notification[]; darkMode?: boolean; @@ -59,7 +59,7 @@ export function TopBar({ return (
1 ? "sb-sync-error" : undefined} + className={isOnline ? undefined : "sb-sync-error"} onClick={onClick} > {lhs} @@ -92,7 +92,7 @@ export function TopBar({ onRename(newName); }} editable={!client.ui.viewState.uiOptions.forcedROMode && - !client.clientConfig.readOnly} + !client.bootConfig.readOnly} /> {notifications.length > 0 && ( diff --git a/web/constants.ts b/web/constants.ts deleted file mode 100644 index adf3bed5..00000000 --- a/web/constants.ts +++ /dev/null @@ -1,2 +0,0 @@ -export const maximumDocumentSize = 10; // MiB -export const defaultLinkStyle = "wikilink"; diff --git a/web/editor_state.ts b/web/editor_state.ts index fe022cb9..4439e071 100644 --- a/web/editor_state.ts +++ b/web/editor_state.ts @@ -95,7 +95,7 @@ export function createEditorState( [ ...(readOnly || client.ui.viewState.uiOptions.forcedROMode || - client.clientConfig.readOnly) + client.bootConfig.readOnly) ? [EditorView.editable.of(false), EditorState.readOnly.of(true)] : [], ], diff --git a/web/editor_ui.tsx b/web/editor_ui.tsx index e98c6198..0643e597 100644 --- a/web/editor_ui.tsx +++ b/web/editor_ui.tsx @@ -296,7 +296,7 @@ export class MainUI { ? "" : getNameFromPath(viewState.current.path)} notifications={viewState.notifications} - syncFailures={viewState.syncFailures} + isOnline={viewState.isOnline} unsavedChanges={viewState.unsavedChanges} isLoading={viewState.isLoading} vimMode={viewState.uiOptions.vimMode} diff --git a/web/hooks/event.ts b/web/hooks/event.ts index 42aa5111..117f3bc3 100644 --- a/web/hooks/event.ts +++ b/web/hooks/event.ts @@ -22,6 +22,20 @@ export class EventHook implements EventHookI { this.localListeners.get(eventName)!.push(callback); } + removeLocalListener(eventName: string, callback: (...args: any[]) => any) { + if (!this.localListeners.has(eventName)) { + return; + } + const listeners = this.localListeners.get(eventName)!; + const index = listeners.indexOf(callback); + if (index !== -1) { + listeners.splice(index, 1); + } + if (listeners.length === 0) { + this.localListeners.delete(eventName); + } + } + // Pull all events listened to listEvents(): string[] { if (!this.system) { diff --git a/web/reducer.ts b/web/reducer.ts index 0f39e083..3f3d4f6a 100644 --- a/web/reducer.ts +++ b/web/reducer.ts @@ -69,10 +69,10 @@ export default function reducer( }, }; } - case "sync-change": + case "online-status-change": return { ...state, - syncFailures: action.syncSuccess ? 0 : state.syncFailures + 1, + isOnline: action.isOnline, }; case "update-page-list": { // Let's move over any "lastOpened" times to the "allPages" list diff --git a/web/service_worker.ts b/web/service_worker.ts index a045c4e2..be95f15b 100644 --- a/web/service_worker.ts +++ b/web/service_worker.ts @@ -1,12 +1,21 @@ -import type { FileContent } from "../lib/spaces/datastore_space_primitives.ts"; +import { initLogger } from "../lib/logger.ts"; +import { ProxyRouter } from "./service_worker/proxy_router.ts"; + +import { SyncEngine } from "./service_worker/sync_engine.ts"; +import type { + ServiceWorkerSourceMessage, + ServiceWorkerTargetMessage, +} from "./ui_types.ts"; import { simpleHash } from "../lib/crypto.ts"; -import { DataStore } from "../lib/data/datastore.ts"; import { IndexedDBKvPrimitives } from "../lib/data/indexeddb_kv_primitives.ts"; -import { decodePageURI } from "@silverbulletmd/silverbullet/lib/ref"; - import { fsEndpoint } from "../lib/spaces/constants.ts"; +import { DataStoreSpacePrimitives } from "../lib/spaces/datastore_space_primitives.ts"; +import { HttpSpacePrimitives } from "../lib/spaces/http_space_primitives.ts"; +import { throttleImmediately } from "../lib/async.ts"; -// Note: the only thing cached here is SilverBullet client assets, files and databases are kept in IndexedDB +initLogger("[Service Worker]"); + +// Note: the only thing cached here is SilverBullet client assets, files are kept in IndexedDB const CACHE_NAME = "{{CACHE_NAME}}"; //`location.href` minus this worker's filename will be our base URL, including any URL prefix @@ -22,9 +31,6 @@ const basePathName = location.pathname.substring( 0, location.pathname.length - workerFilename.length - 1, ); -console.log( - `[Service Worker] Established baseURI=[${baseURI}]; basePathName=[${basePathName}]`, -); const precacheFiles = Object.fromEntries([ "/", @@ -40,37 +46,259 @@ const precacheFiles = Object.fromEntries([ "/.client/manifest.json", ].map((path) => [path, `${baseURI}${path}?v=${CACHE_NAME}`, path])); // Cache busting +// Initially set to undefined, resulting in all "fetch" being proxied. +// Once the service worker is configured, this will be set and the proxy will handle fetches. +const proxyRouter = new ProxyRouter( + basePathName, + baseURI, + precacheFiles, +); + +// @ts-ignore: debugging +globalThis.proxyRouter = proxyRouter; + +// Message received from client +self.addEventListener("message", async (event: any) => { + const message: ServiceWorkerTargetMessage = event.data; + switch (message.type) { + case "skip-waiting": { + // @ts-ignore: Skip waiting to activate this service worker immediately + self.skipWaiting(); + break; + } + case "flush-cache": { + const cacheNames = await caches.keys(); + + await Promise.all( + cacheNames.map((cacheName) => { + if (cacheName !== CACHE_NAME) { + console.log("Removing cache", cacheName); + return caches.delete(cacheName); + } + }), + ); + broadcastMessage({ + type: "cacheFlushed", + }); + break; + } + case "wipe-data": { + if (proxyRouter.syncEngine) { + await proxyRouter.syncEngine.wipe(); + broadcastMessage({ + type: "dataWiped", + }); + } else { + console.warn("Not performing sync data wipe, sync engine not started"); + } + break; + } + case "perform-file-sync": { + if (proxyRouter.syncEngine) { + await proxyRouter.syncEngine.syncSingleFile( + message.path, + ); + } else { + console.warn( + "Ignoring perform-file-sync request, proxy not configured yet", + ); + } + break; + } + case "perform-space-sync": { + if (proxyRouter.syncEngine) { + await proxyRouter.syncEngine.syncSpace(); + } else { + console.warn( + "Ignoring perform-space-sync request, proxy not configured yet", + ); + } + break; + } + case "config": { + const config = message.config; + // Configure the service worker if it hasn't been already + if (isConfigured()) { + console.info( + "Service worker already configured, just updating configs", + ); + proxyRouter.syncEngine!.setSyncConfig({ + syncDocuments: config.syncDocuments, + syncIgnore: config.syncIgnore, + }); + + return; + } else { + console.info("Service being configured with", config); + } + const spaceFolderPath = config.spaceFolderPath; + // We're generating a simple hashed database name based on the space path in case people regularly switch between multiple space paths + const spaceHash = "" + + simpleHash(`${spaceFolderPath}:${baseURI.replace(/\/*$/, "")}`); + // And we'll use a _files postfix to signify where synced files are kept + const dbName = `${spaceHash}_files`; + + // Setup KV (database) for store synced files + const kv = new IndexedDBKvPrimitives(dbName); + await kv.init(); + + // And use that to power the IndexedDB backed local storage + const local = new DataStoreSpacePrimitives(kv); + + // Which we'll sync with the remote server + const remote = new HttpSpacePrimitives( + basePathName + fsEndpoint, + spaceFolderPath, + (message, actionOrRedirectHeader) => { + // And auth error occured + console.error( + "[service proxy error]", + message, + actionOrRedirectHeader, + ); + broadcastMessage({ + type: "auth-error", + message, + actionOrRedirectHeader, + }); + }, + ); + + // Now let's setup sync + const syncEngine = new SyncEngine(kv, local, remote); + syncEngine.setSyncConfig({ + syncDocuments: config.syncDocuments, + syncIgnore: config.syncIgnore, + }); + await syncEngine.start(); + + // Ok, we're ready to go, let's plug in the proxy router + proxyRouter.configure(syncEngine.local, syncEngine); + + // And wire up some events + proxyRouter.on({ + fileWritten: (path) => { + syncEngine.syncSingleFile(path); + }, + observedRequest: (path) => { + // This is triggered for the currently open file, we want to proactively sync it to keep it up to date + syncEngine.syncSingleFile(path); + }, + onlineStatusUpdated: (isOnline) => { + broadcastMessage({ + type: "online-status", + isOnline, + }); + }, + }); + syncEngine.on({ + syncProgress: (status) => { + broadcastMessage({ + type: "sync-status", + status, + }); + }, + syncConflict: (path) => { + console.warn("Sync conflict detected:", path); + broadcastMessage({ + type: "sync-conflict", + path, + }); + }, + spaceSyncComplete: (operations) => { + console.log("Space sync complete:", operations); + broadcastMessage({ + type: "space-sync-complete", + operations, + }); + }, + fileSyncComplete: (path, operations) => { + broadcastMessage({ + type: "file-sync-complete", + path, + operations, + }); + }, + syncError: (error) => { + broadcastMessage({ + type: "sync-error", + message: error.message, + }); + }, + }); + break; + } + } +}); + +function broadcastMessage(message: ServiceWorkerSourceMessage) { + // @ts-ignore: service worker API + const clients: any = self.clients; + // Find all windows attached to this service worker + clients.matchAll({ + type: "window", + }).then((clients: any[]) => { + clients.forEach((client) => { + client.postMessage(message); + }); + if (clients.length === 0) { + console.info( + "No clients are listening for messages, dropping message", + message, + ); + } + }); +} + +const throttledServiceWorkerStarted = throttleImmediately(() => { + broadcastMessage({ + type: "service-worker-started", + }); +}, 100); + +self.addEventListener("fetch", (event: any) => { + if (!isConfigured()) { + throttledServiceWorkerStarted(); + } + + // Always delegate to the proxy router + proxyRouter.onFetch(event); +}); + +// Service worker lifecycle management self.addEventListener("install", (event: any) => { - console.log("[Service worker]", "Installing service worker..."); + console.log("Installing service worker..."); event.waitUntil( (async () => { const cache = await caches.open(CACHE_NAME); console.log( - "[Service worker]", "Now pre-caching client files", ); await cache.addAll(Object.values(precacheFiles)); console.log( - "[Service worker]", Object.keys(precacheFiles).length, "client files cached", ); // @ts-ignore: Force the waiting service worker to become the active service worker await self.skipWaiting(); - console.log("[Service worker]", "skipWaiting complete"); })(), ); }); self.addEventListener("activate", (event: any) => { - console.log("[Service worker]", "Activating new service worker!"); + console.log("Activating new service worker!"); + + if (!isConfigured()) { + throttledServiceWorkerStarted(); + } + event.waitUntil( (async () => { const cacheNames = await caches.keys(); await Promise.all( cacheNames.map((cacheName) => { if (cacheName !== CACHE_NAME) { - console.log("[Service worker]", "Removing old cache", cacheName); + console.log("Removing old cache", cacheName); return caches.delete(cacheName); } }), @@ -81,144 +309,8 @@ self.addEventListener("activate", (event: any) => { ); }); -let ds: DataStore | undefined; -const filesContentPrefix = ["file", "content"]; +console.log("Service worker loaded"); -self.addEventListener("fetch", (event: any) => { - const url = new URL(event.request.url); - - const pathname = url.pathname.substring(basePathName.length); //url.pathname with any URL prefix removed - - // Use the custom cache key if available, otherwise use the request URL - const cacheKey = precacheFiles[pathname] || event.request.url; - - event.respondWith( - (async () => { - const request = event.request; - const requestUrl = new URL(request.url); - - // Are we fetching a URL from the same origin as the app? If not, we don't handle it and pass it on - if (!requestUrl.href.startsWith(baseURI)) { - return fetch(request); - } - - // Any request with the X-Sync-Mode header originates from the sync engine: pass it on to the server - if (request.headers.has("x-sync-mode")) { - return fetch(request); - } - - // Try the static (client) file cache first - const cachedResponse = await caches.match(cacheKey); - // Return the cached response if found - if (cachedResponse) { - return cachedResponse; - } - - if (!ds) { - // Not initialzed yet - return fetch(request); - } - - const pathname = requestUrl.pathname.substring(basePathName.length); //requestUrl.pathname without with any URL prefix removed - - if ( - pathname === "/.auth" || - pathname === "/.logout" || - pathname === "/.config" || - pathname === "/.fs" - ) { - // Always proxy auth, config and fs listing requests to the server - return fetch(request); - } else if ( - pathname.startsWith(fsEndpoint) && - pathname.endsWith(".md") && - request.headers.get("accept") !== "application/octet-stream" && - request.headers.get("sec-fetch-mode") !== "cors" - ) { - // This handles the case of ending up with a .md URL in the browser address bar (likely due to a auth proxy redirect) - return Response.redirect(`${pathname.slice(fsEndpoint.length, -3)}`); - } else if ( - // /.fs file system APIs: handled locally - pathname.startsWith(fsEndpoint) - ) { - return handleLocalFileRequest(pathname, request); - } else { - // Fallback to the SB app shell for all other requests (SPA) - return (await caches.match(precacheFiles["/"])) || fetch(request); - } - })().catch((e) => { - console.warn("[Service worker]", "Fetch failed:", e); - return new Response("Offline", { - status: 503, // Service Unavailable - }); - }), - ); -}); - -async function handleLocalFileRequest( - pathname: string, - request: Request, -): Promise { - const path = decodePageURI(pathname.slice(fsEndpoint.length + 1)); - const data = await ds?.get([...filesContentPrefix, path]); - if (data) { - // console.log("Serving from space", path); - return new Response( - // Casting to any because of weird Deno typing - data.data as any, - { - headers: { - "Content-type": data.meta.contentType, - "Content-Length": "" + data.meta.size, - "X-Permission": data.meta.perm, - "X-Created": "" + data.meta.created, - "X-Last-Modified": "" + data.meta.lastModified, - }, - }, - ); - } else { - console.warn( - "Did not find file in locally synced space", - path, - ); - // If this is a _plug request and we don't have it, we may not have performed an initial sync yet - if (path.startsWith("_plug/")) { - console.info("Proxying _plug fetch to server", path); - return fetch(request); - } - - return new Response("Not found", { - status: 404, - headers: { - "Cache-Control": "no-cache", - }, - }); - } +function isConfigured() { + return !!proxyRouter.syncEngine; } - -self.addEventListener("message", (event: any) => { - switch (event.data.type) { - case "skipWaiting": { - console.log( - "[Service worker]", - "Received skipWaiting message, activating immediately", - ); - // @ts-ignore: Skip waiting to activate this service worker immediately - self.skipWaiting(); - break; - } - case "config": { - const spaceFolderPath = event.data.config.spaceFolderPath; - const dbPrefix = "" + - simpleHash(`${spaceFolderPath}:${baseURI.replace(/\/*$/, "")}`); - - // Setup space - const kv = new IndexedDBKvPrimitives(dbPrefix); - kv.init().then(() => { - ds = new DataStore(kv); - console.log("Datastore in service worker initialized..."); - }); - break; - } - } -}); diff --git a/web/service_worker/proxy_router.ts b/web/service_worker/proxy_router.ts new file mode 100644 index 00000000..19836974 --- /dev/null +++ b/web/service_worker/proxy_router.ts @@ -0,0 +1,336 @@ +import { fsEndpoint } from "../../lib/spaces/constants.ts"; +import { decodePageURI } from "@silverbulletmd/silverbullet/lib/ref"; +import type { SpacePrimitives } from "../../lib/spaces/space_primitives.ts"; +import { fileMetaToHeaders, headersToFileMeta } from "../../server/util.ts"; +import { notFoundError, offlineError } from "../../lib/constants.ts"; +import type { SyncEngine } from "./sync_engine.ts"; +import { EventEmitter } from "../../lib/plugos/event.ts"; +import type { FileMeta } from "@silverbulletmd/silverbullet/type/index"; + +const alwaysProxy = [ + "/.auth", + "/.logout", + "/.config", +]; + +const pingTimeout = 2000; +const pingInterval = 5000; + +export type ProxyRouterEvents = { + // Use case: sync engine may more pro-actively sync this one file back to the server + fileWritten: (path: string) => void; + // Use case: the user likely has this file open in the editor, so it's good to prioritize syncing it + observedRequest: (path: string) => void; + // Use case: client showing the "yellow bar" indicating not being online + onlineStatusUpdated: (isOnline: boolean) => void; +}; + +/** + * Implements a service worker level HTTP proxy (fetch requests) that serves /.fs calls locally for synced spaces + */ +export class ProxyRouter extends EventEmitter { + private fullSyncConfirmed = false; + online = false; + localSpacePrimitives?: SpacePrimitives; + syncEngine?: SyncEngine; + + constructor( + private basePathName: string, + private baseURI: string, + private precacheFiles: Record, + ) { + super(); + // Actively check if we're online by pinging the server + this.checkOnline(); + setInterval(() => { + this.checkOnline(); + }, pingInterval); + } + + /** + * Called as soon the service worker is configured, and the service worker is ready to start serving requests. + */ + configure(spacePrimitives: SpacePrimitives, syncEngine: SyncEngine) { + this.localSpacePrimitives = spacePrimitives; + this.syncEngine = syncEngine; + syncEngine.on({ + spaceSyncComplete: () => { + if (!this.fullSyncConfirmed) { + this.fullSyncConfirmed = true; + console.log( + "First full sync confirmed, will now start serving requests locally", + ); + } + }, + }); + } + + async checkOnline() { + try { + await fetch(this.baseURI + "/.ping", { + signal: AbortSignal.timeout(pingTimeout), + }); + // If the ping is successful, we are online + this.online = true; + } catch { + // Otherwise we're not + this.online = false; + } finally { + this.emit("onlineStatusUpdated", this.online); + } + } + + /** + * Handles /.fs fetch events from the service worker. + * @param event FetchEvent from the service worker + */ + public onFetch(event: any) { + const url = new URL(event.request.url); + + const pathname = url.pathname.substring(this.basePathName.length); //url.pathname with any URL prefix removed + + // Use the custom cache key if available, otherwise use the request URL + const cacheKey = this.precacheFiles[pathname] || event.request.url; + + event.respondWith( + (async () => { + const request = event.request; + const requestUrl = new URL(request.url); + + // Are we fetching a URL from the same origin as the app? If not, we don't handle it and pass it on + if (!requestUrl.href.startsWith(this.baseURI)) { + return fetch(request); + } + + // Try the static (client) file cache first + const cachedResponse = await caches.match(cacheKey); + // Return the cached response if found + if (cachedResponse) { + return cachedResponse; + } + + //requestUrl.pathname without with any URL prefix removed + const pathname = requestUrl.pathname.substring( + this.basePathName.length, + ); + + if ( + // Not yet configured -> Proxy + !this.localSpacePrimitives || !this.syncEngine || + // Not fully synced but online -> Proxy + (!this.fullSyncConfirmed && this.online) || + // A path we always need to proxy -> Proxy + (alwaysProxy.includes(pathname) || pathname.startsWith("/.proxy/")) + ) { + return fetch(request); + } + + // We are now in a state we're configured and either a full sync cycle has complete (since boot) OR we're offline + + if ( + pathname.startsWith(fsEndpoint) && + pathname.endsWith(".md") && + !request.headers.has("X-Sync-Mode") + ) { + // This handles the case of ending up with a .md URL in the browser address bar (likely due to a auth proxy redirect) + return Response.redirect( + `${pathname.slice(fsEndpoint.length, -3)}`, + ); + } else if (pathname.startsWith(fsEndpoint)) { + // Handle /.fs file system APIs + return this.handleRequest(pathname, request); + } else { + // Fallback to the app shell for all other requests (SPA) + return (await caches.match(this.precacheFiles["/"])) || + fetch(request); + } + })().catch((e) => { + console.warn("Fetch failed:", e); + this.online = false; + return new Response(offlineError.message, { + status: 503, // Service Unavailable + }); + }), + ); + } + + handleRequest( + pathname: string, + request: Request, + ): Promise { + const path = decodePageURI(pathname.slice(fsEndpoint.length + 1)); + switch (request.method) { + case "GET": { + if (!path) { // .fs GET + return this.handleFileListing(); + } else { // .fs/* GET + return this.handleGet(path, request); + } + } + case "PUT": { + return this.handlePut(path, request); + } + case "DELETE": { + return this.handleDelete(path, request); + } + default: { + console.log("Unhandled method", request.method, "proxying to server"); + return fetch(request); + } + } + } + + /** + * Shortcut to nonSyncedFiles kept in snapshot + */ + get nonSyncedFiles() { + return this.syncEngine!.snapshot.nonSyncedFiles; + } + + async handleFileListing(): Promise { + if (!this.syncEngine || !this.localSpacePrimitives) { + throw new Error("This should not happen"); + } + + const files = await this.localSpacePrimitives.fetchFileList(); + // Now augment this with non-synced file metadata + for ( + const nonSyncedFile of this.nonSyncedFiles + .values() + ) { + const existingFile = files.find((file) => + file.name === nonSyncedFile.name + ); + if (!existingFile) { + files.push(nonSyncedFile); + } + } + return new Response( + JSON.stringify(files), + { + headers: { + "Content-Type": "application/json", + }, + }, + ); + } + + async handleGet(path: string, request: Request): Promise { + if (!this.syncEngine || !this.localSpacePrimitives) { + throw new Error("This should not happen"); + } + + try { + if (request.headers.has("x-get-meta")) { + // Requesting only file meta + let meta: FileMeta | undefined; + if (this.nonSyncedFiles.has(path)) { + // Pull the file meta directly from nonSyncedFiles + meta = this.nonSyncedFiles.get(path); + } else { + // Otherwise fetch it from the local store + meta = await this.localSpacePrimitives.getFileMeta(path); + } + if (request.headers.has("x-observing")) { + this.emit("observedRequest", path); + } + return new Response(null, { + headers: fileMetaToHeaders(meta!), + }); + } else { + const { meta, data } = await this.localSpacePrimitives.readFile(path); + return new Response(data as any, { + headers: fileMetaToHeaders(meta), + }); + } + } catch (err: any) { + if (err.message === notFoundError.message && this.online) { + console.info("No local copy of", path, "proxying to server"); + // Not found locally, but we're online, so let's try the server + return fetch(request); + } else if (err.message === notFoundError.message) { + console.warn( + "No local copy of", + path, + "and offline, so will 404 on this one", + ); + // We're not online so let's assume the file indeed doesn't exist + // TODO: What could be nice here is to check if this is a nonSyncedFile and if so serve some sort of offline placeholder + return new Response(notFoundError.message, { + status: 404, + }); + } + console.error("Error reading", path, err.message); + return new Response(err.message, { + status: 500, + }); + } + } + + async handlePut(path: string, request: Request): Promise { + if (!this.syncEngine || !this.localSpacePrimitives) { + throw new Error("This should not happen"); + } + try { + if (!this.syncEngine.isSyncCandidate(path) && this.online) { + console.log("Handling file write for non-synced file", path); + // Writing a non-synced file while being online + // Proxy the request + const resp = await fetch(request); + // Update the nonSynced snapshot in place for later file listing consistency + this.nonSyncedFiles.set(path, headersToFileMeta(path, resp.headers)!); + return resp; + } else { + // Synced file + const body = await request.arrayBuffer(); + // console.log("Handling file write", path, body.byteLength); + const meta = await this.localSpacePrimitives.writeFile( + path, + new Uint8Array(body), + // Note: there are going to be many cases where no meta is supplied in the request, this is ok, in that case this argument will be undefined + headersToFileMeta(path, request.headers), + ); + this.emit("fileWritten", path); + return new Response("OK", { + status: 200, + headers: fileMetaToHeaders(meta), + }); + } + } catch (e: any) { + console.error("Error writing", path, e.message); + return new Response(e.message, { + status: 500, + }); + } + } + + async handleDelete(path: string, request: Request): Promise { + if (!this.syncEngine || !this.localSpacePrimitives) { + throw new Error("This should not happen"); + } + + try { + if (!this.syncEngine.isSyncCandidate(path)) { + console.log("Handling file delete for non-synced file", path); + this.nonSyncedFiles.delete(path); + // Proxy the request + return fetch(request); + } + // console.log("Handling file delete", path); + await this.localSpacePrimitives.deleteFile(path); + return new Response("OK", { + status: 200, + }); + } catch (e: any) { + console.error("Error deleting", path, e.message); + if (e.message === notFoundError.message) { + return new Response(notFoundError.message, { + status: 404, + }); + } + return new Response(e.message, { + status: 500, + }); + } + } +} diff --git a/web/service_worker/sync_engine.ts b/web/service_worker/sync_engine.ts new file mode 100644 index 00000000..9222fc11 --- /dev/null +++ b/web/service_worker/sync_engine.ts @@ -0,0 +1,219 @@ +import { compile as gitIgnoreCompiler } from "gitignore-parser"; +import { sleep } from "../../lib/async.ts"; +import type { KvPrimitives } from "../../lib/data/kv_primitives.ts"; +import { EventEmitter } from "../../lib/plugos/event.ts"; +import { plugPrefix, stdLibPrefix } from "../../lib/spaces/constants.ts"; +import type { SpacePrimitives } from "../../lib/spaces/space_primitives.ts"; +import { + SpaceSync, + SyncSnapshot, + type SyncStatus, +} from "../../lib/spaces/sync.ts"; + +const syncSnapshotKey = ["$sync", "snapshot"]; +const syncInterval = 20; + +type SyncEngineEvents = { + // Full sync cycle has completed + spaceSyncComplete: (operations: number) => void | Promise; + + // A single file syncle has completed + fileSyncComplete: (path: string, operations: number) => void | Promise; + + syncError: (error: Error) => void | Promise; + + // Sync conflict occurred + syncConflict: (path: string) => void | Promise; + + // Sync progress updated + syncProgress: (syncStatus: SyncStatus) => void | Promise; +}; + +export type SyncConfig = { + syncDocuments?: boolean; + syncIgnore?: string; +}; + +/** + * Thin wrapper around SpaceSync, adds snapshot persistence and a few other things + */ +export class SyncEngine extends EventEmitter { + spaceSync!: SpaceSync; + + private syncConfig: SyncConfig = { + syncDocuments: true, + }; + + stopping = false; + syncAccepts: (path: string) => boolean = () => true; + snapshot!: SyncSnapshot; + + constructor( + private kv: KvPrimitives, + readonly local: SpacePrimitives, + readonly remote: SpacePrimitives, + ) { + super(); + } + + async start() { + this.snapshot = await this.loadSnapshot(); + + this.spaceSync = new SpaceSync(this.local, this.remote, { + conflictResolver: this.plugAwareConflictResolver.bind(this), + isSyncCandidate: this.isSyncCandidate.bind(this), + }); + + this.spaceSync.on({ + syncProgress: (status) => { + this.emit("syncProgress", status); + }, + snapshotUpdated: this.saveSnapshot.bind(this), + }); + + // Start the sync loop + this.run(); + } + + async run() { + while (true) { + if (this.stopping) { + return; + } + try { + await this.syncSpace(); + } catch { + // Error communication is happening in syncSpace + } + await sleep(syncInterval * 1000); + } + } + + public setSyncConfig(config: SyncConfig) { + this.syncConfig = config; + this.syncAccepts = config.syncIgnore + ? gitIgnoreCompiler(config.syncIgnore).accepts + : () => true; + console.log( + "[sync] Updated sync config:", + this.syncConfig, + ); + } + + isSyncCandidate(path: string): boolean { + // ALWAYS sync plugs + if (path.startsWith(plugPrefix)) { + return true; + } + // Follow SB_SYNC_IGNORE rules + if (!this.syncAccepts(path)) { + return false; + } + // Either sync all files, or only .md files if syncDocuments is false + return this.syncConfig.syncDocuments || path.endsWith(".md"); + } + + async syncSpace(): Promise { + try { + const operations = await this.spaceSync.syncFiles(this.snapshot); + if (operations !== -1) { + // emit successful sync event (not when operations === -1, because that means another sync was ongoing) + this.emit("spaceSyncComplete", operations); + } + return operations; + } catch (e) { + this.emit("syncError", e); + throw e; + } + } + + async syncSingleFile(path: string): Promise { + try { + const operations = await this.spaceSync.syncSingleFile( + path, + this.snapshot, + ); + this.emit("fileSyncComplete", path, operations); + return operations; + } catch (e) { + this.emit("syncError", e); + throw e; + } + } + + /** + * Loads the sync snapshot from the data store. + * @returns A map of sync status items. + */ + async loadSnapshot(): Promise { + const [snapshot] = await this.kv.batchGet([syncSnapshotKey]); + return SyncSnapshot.fromJSON(snapshot); + } + + /** + * Saves the sync snapshot to the data store. + * @param snapshot A map of sync status items. + */ + saveSnapshot(snapshot: SyncSnapshot) { + return this.kv.batchSet([{ + key: syncSnapshotKey, + value: snapshot.toJSON(), + }]); + } + + async wipe() { + this.stopping = true; + console.log("Wiping sync database"); + await this.kv.clear(); + console.log("Done wiping"); + } + + /** + * Delegates to the standard primary conflict resolver, but in case of any conflicts in plugs, it will always take the version from the secondary. + */ + async plugAwareConflictResolver( + name: string, + snapshot: SyncSnapshot, + primary: SpacePrimitives, + secondary: SpacePrimitives, + ): Promise { + if (!name.startsWith(plugPrefix) && !name.startsWith(stdLibPrefix)) { + const operations = await SpaceSync.primaryConflictResolver( + name, + snapshot, + primary, + secondary, + ); + + if (operations > 0) { + // Something happened -> conflict copy generated, let's report it + this.emit("syncConflict", name); + } + + return operations; + } + console.log( + "[sync]", + "Conflict in plug", + name, + "will pick the version from secondary and be done with it.", + ); + // Read file from secondary + const { data, meta } = await secondary.readFile( + name, + ); + // Write file to primary + const newMeta = await primary.writeFile( + name, + data, + meta, + ); + // Update snapshot + snapshot.files.set(name, [ + newMeta.lastModified, + meta.lastModified, + ]); + + return 1; + } +} diff --git a/web/sw_util.ts b/web/service_worker/util.ts similarity index 82% rename from web/sw_util.ts rename to web/service_worker/util.ts index f67b856d..e1d7ec4e 100644 --- a/web/sw_util.ts +++ b/web/service_worker/util.ts @@ -12,8 +12,9 @@ export async function unregisterServiceWorkers() { } export function flushCachesAndUnregisterServiceWorker() { + console.log("Going to flush the caches and unregister the service worker"); return new Promise((resolve) => { - if (!navigator.serviceWorker) { + if (!navigator.serviceWorker?.controller) { console.log("No service worker active"); return resolve(); } @@ -34,7 +35,8 @@ export function flushCachesAndUnregisterServiceWorker() { // First flush active cache navigator.serviceWorker.ready.then((registration) => { - registration.active!.postMessage({ type: "flushCache" }); + console.log("Sending request to flush cache"); + registration.active!.postMessage({ type: "flush-cache" }); }); }); } diff --git a/web/space.ts b/web/space.ts index 4179e9c5..121c7c6f 100644 --- a/web/space.ts +++ b/web/space.ts @@ -73,7 +73,6 @@ export class Space { async writePage( name: string, text: string, - selfUpdate?: boolean, ): Promise { try { this.saving = true; @@ -81,7 +80,6 @@ export class Space { await this.spacePrimitives.writeFile( `${name}.md`, new TextEncoder().encode(text), - selfUpdate, ), ); // Note: we don't do very elaborate cache invalidation work here, quite quickly the cache will be flushed anyway @@ -148,10 +146,9 @@ export class Space { async writeDocument( name: string, data: Uint8Array, - selfUpdate?: boolean, ): Promise { return fileMetaToDocumentMeta( - await this.spacePrimitives.writeFile(name, data, selfUpdate), + await this.spacePrimitives.writeFile(name, data), ); } @@ -171,7 +168,8 @@ export class Space { return; } for (const fileName of this.watchedFiles) { - await this.spacePrimitives.getFileMeta(fileName); + // Setting observing to true here to hint that we may be interested in more active syncing + await this.spacePrimitives.getFileMeta(fileName, true); } }); }, pageWatchInterval); diff --git a/web/space_lua.ts b/web/space_lua.ts index 0081e2f2..9e314ded 100644 --- a/web/space_lua.ts +++ b/web/space_lua.ts @@ -20,16 +20,18 @@ import { buildLuaEnv } from "./space_lua_api.ts"; import type { LuaCollectionQuery } from "../lib/space_lua/query_collection.ts"; export class SpaceLuaEnvironment { - env: LuaEnv = new LuaEnv(); + env: LuaEnv; + + constructor(private system: System) { + this.env = buildLuaEnv(system); + } /** * Loads all Lua scripts from the database and evaluates them in a new environment * @param system */ - async reload( - system: System, - ) { - const allScripts: ScriptObject[] = await system.invokeFunction( + async reload() { + const allScripts: ScriptObject[] = await this.system.invokeFunction( "index.queryLuaObjects", ["space-lua", { objectVariable: "script", @@ -40,7 +42,7 @@ export class SpaceLuaEnvironment { } as LuaCollectionQuery], ); try { - this.env = buildLuaEnv(system); + this.env = buildLuaEnv(this.system); const tl = new LuaEnv(); tl.setLocal("_GLOBAL", this.env); for (const script of allScripts) { diff --git a/web/space_lua_api.ts b/web/space_lua_api.ts index 61054bb8..8fcd8d64 100644 --- a/web/space_lua_api.ts +++ b/web/space_lua_api.ts @@ -23,7 +23,7 @@ export function buildLuaEnv(system: System) { * Exposes all registered syscalls to Lua, automatically converting Lua arguments to JS values * If a syscall is prefixed with `lua:` it exposes the syscall as a native Lua function, skipping the argument conversion0 */ -function exposeSyscalls(env: LuaEnv, system: System) { +export function exposeSyscalls(env: LuaEnv, system: System) { // Expose all syscalls to Lua const nativeFs = new LuaStackFrame(env, null); for (const syscallName of system.registeredSyscalls.keys()) { diff --git a/web/styles/theme.scss b/web/styles/theme.scss index 6dbd72fe..3f82a2ad 100644 --- a/web/styles/theme.scss +++ b/web/styles/theme.scss @@ -66,7 +66,7 @@ html { --progress-background-color: #eee; --progress-sync-color: black; - --progress-index-color: rgb(176, 0, 59); + --progress-index-color: #0772be; --action-button-background-color: transparent; --action-button-color: #292929; @@ -149,7 +149,11 @@ html[data-theme="dark"] { --ui-accent-color: #464cfc; --ui-accent-text-color: var(--ui-accent-color); - --ui-accent-text-color: color-mix(in srgb, var(--ui-accent-color), white 50%); + --ui-accent-text-color: color-mix( + in srgb, + var(--ui-accent-color), + white 50% + ); --highlight-color: rgba(255, 255, 0, 0.5); --link-color: #7e99fc; --link-missing-color: #9e4705; diff --git a/web/sync_service.ts b/web/sync_service.ts deleted file mode 100644 index e5960547..00000000 --- a/web/sync_service.ts +++ /dev/null @@ -1,420 +0,0 @@ -import { plugPrefix, stdLibPrefix } from "../lib/spaces/constants.ts"; -import type { SpacePrimitives } from "../lib/spaces/space_primitives.ts"; -import { - SpaceSync, - type SyncStatus, - type SyncStatusItem, -} from "../lib/spaces/sync.ts"; -import { sleep } from "../lib/async.ts"; -import type { EventHook } from "./hooks/event.ts"; -import type { DataStore } from "../lib/data/datastore.ts"; - -// Keeps the current sync snapshot -const syncSnapshotKey = ["sync", "snapshot"]; - -// Keeps the start time of an ongoing sync, is reset once the sync is done -const syncStartTimeKey = ["sync", "startTime"]; - -// Keeps the start time of the last full sync cycle -const syncLastFullCycleKey = ["sync", "lastFullCycle"]; - -// Keeps the last time an activity was registered, used to detect if a sync is still alive and whether a new one should be started already -const syncLastActivityKey = ["sync", "lastActivity"]; - -const syncInitialFullSyncCompletedKey = ["sync", "initialFullSyncCompleted"]; - -// maximum time between two activities before we consider a sync crashed -const syncMaxIdleTimeout = 1000 * 27; - -// How often to sync the whole space -const spaceSyncInterval = 17 * 1000; // Every 17s or so - -// Used from Client -export const pageSyncInterval = 6000; - -/** - * The SyncService primarily wraps the SpaceSync engine but also coordinates sync between - * different browser tabs. It is using the KVStore to keep track of sync state. - */ -export class SyncService { - spaceSync: SpaceSync; - lastReportedSyncStatus = Date.now(); - // If this is set to anything other than undefined, a file is currently saving - savingTimeout: number | undefined; - enabled = true; - filesScheduledForSync = new Set(); - - constructor( - readonly localSpacePrimitives: SpacePrimitives, - readonly remoteSpace: SpacePrimitives, - private ds: DataStore, - private eventHook: EventHook, - private isSyncCandidate: (path: string) => boolean, - ) { - this.spaceSync = new SpaceSync( - this.localSpacePrimitives, - this.remoteSpace!, - { - conflictResolver: this.plugAwareConflictResolver.bind(this), - isSyncCandidate: this.isSyncCandidate, - onSyncProgress: (status) => { - this.registerSyncProgress(status).catch(console.error); - }, - }, - ); - - eventHook.addLocalListener( - "editor:pageLoaded", - (name, _prevPage, isSynced) => { - if (!this.enabled) { - return; - } - if (!isSynced) { - this.performFileSync(`${name}.md`).catch(console.error); - } - }, - ); - - eventHook.addLocalListener("editor:documentLoaded", (name, _prevPage) => { - if (!this.enabled) { - return; - } - this.performFileSync(name).catch(console.error); - }); - - const setSavingTimeout = () => { - this.savingTimeout = setTimeout(() => { - this.savingTimeout = undefined; - }, 1000 * 5); - }; - - eventHook.addLocalListener("editor:pageSaving", setSavingTimeout); - - eventHook.addLocalListener("editor:documentSaving", setSavingTimeout); - - eventHook.addLocalListener("editor:pageSaved", (name) => { - if (!this.enabled) { - return; - } - if (this.savingTimeout) { - clearTimeout(this.savingTimeout); - this.savingTimeout = undefined; - } else { - console.warn("This should not happen, savingTimeout was not set"); - } - const path = `${name}.md`; - this.performFileSync(path).catch(console.error); - }); - - eventHook.addLocalListener("editor:documentSaved", (name) => { - if (!this.enabled) { - return; - } - if (this.savingTimeout) { - clearTimeout(this.savingTimeout); - this.savingTimeout = undefined; - } else { - console.warn( - "This should not happen, savingTimeout was not set. This could be a slow document editor", - ); - } - this.performFileSync(name).catch(console.error); - }); - - this.spaceSync.on({ - fileSynced: (meta, direction) => { - eventHook.dispatchEvent("file:synced", meta, direction); - }, - }); - } - - async isSyncing(): Promise { - if (this.savingTimeout !== undefined) { - console.log( - "Saving a file at the moment, so reporting as isSyncing() = true", - ); - return true; - } - const startTime = await this.ds.get(syncStartTimeKey); - if (!startTime) { - return false; - } - // Sync is running, but is it still alive? - const lastActivity = await this.ds.get(syncLastActivityKey)!; - if (Date.now() - lastActivity > syncMaxIdleTimeout) { - // It's been too long since the last activity, let's consider this one crashed and - // reset the sync start state - await this.ds.delete(syncStartTimeKey); - console.info("Sync without activity for too long, resetting"); - return false; - } - return true; - } - - async hasInitialSyncCompleted(): Promise { - // Initial sync has happened when sync progress has been reported at least once, but the syncStartTime has been reset (which happens after sync finishes) - return !!(await this.ds.get(syncInitialFullSyncCompletedKey)); - } - - async registerSyncStart(fullSync: boolean): Promise { - // Assumption: this is called after an isSyncing() check - await this.ds.batchSet([ - { - key: syncStartTimeKey, - value: Date.now(), - }, - { - key: syncLastActivityKey, - value: Date.now(), - }, - ...fullSync // If this is a full sync cycle - ? [{ - key: syncLastFullCycleKey, - value: Date.now(), - }] - : [], - ]); - } - - async registerSyncProgress(status?: SyncStatus): Promise { - // Emit a sync event at most every 2s - if (status && this.lastReportedSyncStatus < Date.now() - 2000) { - await this.eventHook.dispatchEvent("sync:progress", status); - this.lastReportedSyncStatus = Date.now(); - await this.saveSnapshot(status.snapshot); - } - await this.ds.set(syncLastActivityKey, Date.now()); - } - - async registerSyncStop(isFullSync: boolean): Promise { - await this.registerSyncProgress(); - await this.ds.delete(syncStartTimeKey); - if (isFullSync) { - await this.ds.set(syncInitialFullSyncCompletedKey, true); - } - } - - async getSnapshot(): Promise> { - const snapshot = (await this.ds.get(syncSnapshotKey)) || {}; - return new Map( - Object.entries(snapshot), - ); - } - - // Await a moment when the sync is no longer running - async noOngoingSync(timeout: number): Promise { - // Not completely safe, could have race condition on setting the syncStartTimeKey - const startTime = Date.now(); - while (await this.isSyncing()) { - console.log("Waiting for ongoing sync to finish..."); - await sleep(321); - if (Date.now() - startTime > timeout) { - throw new Error("Timeout waiting for sync to finish"); - } - } - } - - /** - * Performs a file sync for a single file, returns once sync has completed. - * @param path path of file to sync - */ - async performFileSync(path: string): Promise { - if (this.filesScheduledForSync.has(path)) { - // Already scheduled, no need to duplicate - console.info(`File ${path} already scheduled for sync`); - return; - } - this.filesScheduledForSync.add(path); - await this.noOngoingSync(7000); - await this.syncFile(path); - this.filesScheduledForSync.delete(path); - } - - async performSpaceSync(): Promise { - await this.noOngoingSync(5000); - return this.syncSpace(); - } - - start() { - this.syncSpace().catch(console.error); - - setInterval(async () => { - if (!this.enabled) { - return; - } - try { - if (!await this.isSyncing()) { - const lastFullCycle = (await this.ds.get(syncLastFullCycleKey)) || 0; - if (lastFullCycle && Date.now() - lastFullCycle > spaceSyncInterval) { - // It's been a while since the last full cycle, let's sync the whole space - await this.syncSpace(); - } - } - } catch (e: any) { - console.error(e); - } - }, spaceSyncInterval / 2); // check every half the sync cycle because actually running the sync takes some time therefore we don't want to wait for the full cycle - } - - /** - * Performs a full space sync - * @return the number of sync operations performed, or -1 when a sync was already in progress - */ - async syncSpace(): Promise { - if (!this.enabled) { - return 0; - } - if (await this.isSyncing()) { - console.log("Aborting space sync: already syncing"); - return -1; - } - await this.registerSyncStart(true); - let operations = 0; - const snapshot = await this.getSnapshot(); - // console.log("Excluded from sync", excludedFromSync); - try { - operations = await this.spaceSync!.syncFiles( - snapshot, - (path) => this.isSyncCandidate(path), - ); - await this.saveSnapshot(snapshot); - await this.registerSyncStop(true); - await this.eventHook.dispatchEvent("sync:success", operations); - } catch (e: any) { - await this.saveSnapshot(snapshot); - await this.registerSyncStop(false); - await this.eventHook.dispatchEvent("sync:error", e.message); - console.error("Sync error", e.message); - } - return operations; - } - - // Syncs a single file - async syncFile(name: string) { - if (!this.enabled) { - return; - } - // console.log("Checking if we can sync file", name); - if (!this.isSyncCandidate(name)) { - console.info("Requested sync, but not a sync candidate", name); - return; - } - if (await this.isSyncing()) { - console.log("Already syncing, aborting individual file sync for", name); - return; - } - console.log("Syncing file", name); - await this.registerSyncStart(false); - const snapshot = await this.getSnapshot(); - try { - let localHash: number | undefined; - let remoteHash: number | undefined; - - // Fetch remote first (potentially more laggy) - try { - remoteHash = (await this.remoteSpace!.getFileMeta(name)).lastModified; - // HEAD - // - if (!remoteHash) { - console.info( - "Not syncing file, because remote didn't send X-Last-Modified header", - ); - // This happens when the remote isn't a real SilverBullet server, specifically: it's not sending - // a X-Last-Modified header. In this case we'll just assume that the file is up to date. - await this.registerSyncStop(false); - // Jumping out, not saving snapshot nor triggering a sync event, because we did nothing - return; - } - } catch (e: any) { - if (e.message === "Not found") { - // File doesn't exist remotely, that's ok - } else { - throw e; - } - } - - // Fetch local file meta - try { - const localMeta = await this.localSpacePrimitives.getFileMeta(name); - if (localMeta.noSync) { - console.info( - "File marked as no sync, skipping sync in this cycle", - name, - ); - await this.registerSyncStop(false); - // Jumping out, not saving snapshot nor triggering a sync event, because we did nothing - return; - } - localHash = localMeta.lastModified; - } catch { - // Not present - } - - await this.spaceSync.syncFile(snapshot, name, localHash, remoteHash); - this.eventHook.dispatchEvent("sync:success").catch(console.error); - // console.log("File successfully synced", name); - } catch (e: any) { - this.eventHook.dispatchEvent("sync:error", e.message).catch( - console.error, - ); - console.error("Sync error", e); - } - await this.saveSnapshot(snapshot); - await this.registerSyncStop(false); - } - - async saveSnapshot(snapshot: Map) { - await this.ds.set(syncSnapshotKey, Object.fromEntries(snapshot)); - } - - close() { - this.enabled = false; - } - - public async plugAwareConflictResolver( - name: string, - snapshot: Map, - primary: SpacePrimitives, - secondary: SpacePrimitives, - ): Promise { - if (!name.startsWith(plugPrefix) && !name.startsWith(stdLibPrefix)) { - const operations = await SpaceSync.primaryConflictResolver( - name, - snapshot, - primary, - secondary, - ); - - if (operations > 0) { - // Something happened -> conflict copy generated, let's report it - await this.eventHook.dispatchEvent("sync:conflict", name); - } - - return operations; - } - console.log( - "[sync]", - "Conflict in plug", - name, - "will pick the version from secondary and be done with it.", - ); - // Read file from secondary - const { data, meta } = await secondary.readFile( - name, - ); - // Write file to primary - const newMeta = await primary.writeFile( - name, - data, - false, - meta, - ); - // Update snapshot - snapshot.set(name, [ - newMeta.lastModified, - meta.lastModified, - ]); - - return 1; - } -} diff --git a/web/syscalls/config.ts b/web/syscalls/config.ts index cb6423c4..bf470c15 100644 --- a/web/syscalls/config.ts +++ b/web/syscalls/config.ts @@ -1,35 +1,29 @@ import type { SysCallMapping } from "../../lib/plugos/system.ts"; -import type { Client } from "../client.ts"; +import type { Config } from "../config.ts"; -export function configSyscalls(client: Client): SysCallMapping { +export function configSyscalls(config: Config): SysCallMapping { return { "config.get": (_ctx, path: string, defaultValue: any) => { - return client.config.get(path, defaultValue); + return config.get(path, defaultValue); }, "config.set": ( _ctx, keyOrValues: string | string[] | Record, value?: any, ) => { - if (typeof keyOrValues === "string") { - client.config.set(keyOrValues, value); - } else if (Array.isArray(keyOrValues)) { - client.config.set(keyOrValues, value); - } else { - client.config.set(keyOrValues); - } + config.set(keyOrValues as any, value); }, "config.has": (_ctx, path: string) => { - return client.config.has(path); + return config.has(path); }, "config.define": (_ctx, key: string, schema: any) => { - client.config.define(key, schema); + config.define(key, schema); }, "config.getValues": () => { - return client.config.values; + return config.values; }, "config.getSchemas": () => { - return client.config.schemas; + return config.schemas; }, }; } diff --git a/web/syscalls/editor.ts b/web/syscalls/editor.ts index ec988fe1..6b5e6e5f 100644 --- a/web/syscalls/editor.ts +++ b/web/syscalls/editor.ts @@ -650,6 +650,11 @@ export function editorSyscalls(client: Client): SysCallMapping { data, }); }, + + "editor.isMobile": () => { + const mouseDetected = globalThis.matchMedia("(pointer:fine)").matches; + return !mouseDetected; + }, }; return syscalls; diff --git a/web/syscalls/index.ts b/web/syscalls/index.ts index 25767594..303e5469 100644 --- a/web/syscalls/index.ts +++ b/web/syscalls/index.ts @@ -45,6 +45,9 @@ export function indexSyscalls(client: Client): SysCallMapping { }, }; }, + "index.ensureFullIndex": (_ctx) => { + return client.clientSystem.ensureFullIndex(); + }, "lua:index.defineTag": (_ctx, tagDef: LuaTable) => { // Using 'lua:' prefix to _not_ convert tagDef to a JS version (but keep original LuaTable) if (!tagDef.has("name")) { diff --git a/web/syscalls/sync.ts b/web/syscalls/sync.ts index f6397904..9ee4baee 100644 --- a/web/syscalls/sync.ts +++ b/web/syscalls/sync.ts @@ -1,19 +1,77 @@ import type { SysCallMapping } from "../../lib/plugos/system.ts"; import type { Client } from "../client.ts"; -export function syncSyscalls(editor: Client): SysCallMapping { +// TODO: Reimplement this +export function syncSyscalls(client: Client): SysCallMapping { return { - "sync.isSyncing": (): Promise => { - return editor.syncService.isSyncing(); - }, - "sync.hasInitialSyncCompleted": (): Promise => { - return editor.syncService.hasInitialSyncCompleted(); + "sync.hasInitialSyncCompleted": (): boolean => { + return client.fullSyncCompleted; }, "sync.performFileSync": (_ctx, path: string): Promise => { - return editor.syncService.performFileSync(path); + client.postServiceWorkerMessage({ type: "perform-file-sync", path }); + return waitForServiceWorkerActivation(path); }, "sync.performSpaceSync": (): Promise => { - return editor.syncService.performSpaceSync(); + client.postServiceWorkerMessage({ type: "perform-space-sync" }); + return waitForServiceWorkerActivation(); }, }; } + +function waitForServiceWorkerActivation(path?: string): Promise { + return new Promise((resolve, reject) => { + client.eventHook.addLocalListener( + "service-worker:file-sync-complete", + eventHandler, + ); + client.eventHook.addLocalListener( + "service-worker:space-sync-complete", + eventHandler, + ); + client.eventHook.addLocalListener( + "service-worker:sync-error", + errorHandler, + ); + function eventHandler(data: any) { + // If data.path is set, we are notified about a specific file sync -> all good, even for an individual file sync + // If data.path is not set, we are notified about a full space sync + // If we were waiting for a specific path, ignore other paths + if (data.path && path && data.path !== path) { + // Event for other file sync + return; + } + // If we were waiting for a specific path, ignore other paths + resolve(data); + + // Unsubscribe from all these events + client.eventHook.removeLocalListener( + "service-worker:file-sync-complete", + eventHandler, + ); + client.eventHook.removeLocalListener( + "service-worker:space-sync-complete", + eventHandler, + ); + client.eventHook.removeLocalListener( + "service-worker:sync-error", + errorHandler, + ); + } + function errorHandler(e: any) { + reject(e); + // Unsubscribe from all these events + client.eventHook.removeLocalListener( + "service-worker:file-sync-complete", + eventHandler, + ); + client.eventHook.removeLocalListener( + "service-worker:space-sync-complete", + eventHandler, + ); + client.eventHook.removeLocalListener( + "service-worker:sync-error", + errorHandler, + ); + } + }); +} diff --git a/web/ui_types.ts b/web/ui_types.ts index 0466ca35..7930f1dd 100644 --- a/web/ui_types.ts +++ b/web/ui_types.ts @@ -3,6 +3,7 @@ import type { FilterOption, Notification, PanelMode } from "../type/client.ts"; import type { DocumentMeta, PageMeta } from "../type/index.ts"; import type { Path } from "@silverbulletmd/silverbullet/lib/ref"; +import type { SyncStatus } from "../lib/spaces/sync.ts"; export type PanelConfig = { mode?: PanelMode; @@ -25,7 +26,7 @@ export type AppViewState = { showCommandPalette: boolean; showCommandPaletteContext?: string; unsavedChanges: boolean; - syncFailures: number; // Reset everytime a sync succeeds + isOnline: boolean; // Progress tracker progressPercentage?: number; // Used to show progress circle @@ -73,7 +74,7 @@ export const initialViewState: AppViewState = { showCommandPalette: false, pageNavigatorMode: "page", unsavedChanges: false, - syncFailures: 0, + isOnline: true, uiOptions: { vimMode: false, darkMode: undefined, @@ -111,7 +112,7 @@ export type Action = | { type: "document-editor-loaded"; path: Path; meta: DocumentMeta } | { type: "document-editor-changed" } | { type: "document-editor-saved" } - | { type: "sync-change"; syncSuccess: boolean } + | { type: "online-status-change"; isOnline: boolean } | { type: "update-current-page-meta"; meta: PageMeta } | { type: "update-page-list"; allPages: PageMeta[] } | { type: "update-document-list"; allDocuments: DocumentMeta[] } @@ -160,3 +161,68 @@ export type Action = progressPercentage?: number; progressType?: string; }; + +/** + * Client configuration that is set at boot time, doesn't change at runtime + */ +export type BootConfig = { + spaceFolderPath: string; + indexPage: string; + readOnly: boolean; + // Sync configuration + syncDocuments?: boolean; + syncIgnore?: string; + // These are all configured via ?query parameters, e.g. ?disableSpaceLua=1 + disableSpaceLua?: boolean; + disableSpaceStyle?: boolean; + disablePlugs?: boolean; + performWipe?: boolean; + performReset?: boolean; +}; + +/** + * Messages sent client -> service worker + */ +export type ServiceWorkerTargetMessage = + | { + type: "skip-waiting"; + } + | { type: "config"; config: BootConfig } + | { type: "flush-cache" } + | { type: "wipe-data" } + | { type: "perform-file-sync"; path: string } + | { type: "perform-space-sync" }; + +/** + * Events received from the service worker -> client + */ +export type ServiceWorkerSourceMessage = { + type: "sync-status"; + status: Omit; +} | { + type: "sync-conflict"; + path: string; +} | { + type: "space-sync-complete"; + operations: number; +} | { + type: "file-sync-complete"; + path: string; + operations: number; +} | { + type: "sync-error"; + message: string; +} | { + type: "online-status"; + isOnline: boolean; +} | { + type: "auth-error"; + message: string; + actionOrRedirectHeader: string; +} | { + type: "cacheFlushed"; +} | { + type: "dataWiped"; +} | { + type: "service-worker-started"; +}; diff --git a/website/API/mq.md b/website/API/mq.md index 24a9b7ee..09560bf3 100644 --- a/website/API/mq.md +++ b/website/API/mq.md @@ -51,3 +51,12 @@ Example: local stats = mq.getQueueStats("tasks") print("Queue size: " .. stats.size) print("Processing: " .. stats.processing) +``` + +### mq.awaitEmptyQueue(queue) +Waits for a queue to become empty. + +Example: +```lua +mq.awaitEmptyQueue("tasks") +``` \ No newline at end of file diff --git a/website/Architecture.md b/website/Architecture.md new file mode 100644 index 00000000..5726bcf9 --- /dev/null +++ b/website/Architecture.md @@ -0,0 +1,33 @@ +> **note** Note +> This page contains sections that apply to the current _edge_ release, it is not part of an official release yet. + +Some notes on how things fit together. + +# Client +The client is what you see when you open SilverBullet in a browser tab or window. It renders the UI, interacts with the user, runs [[Plugs]], [[Space Lua]], and maintains the index. 90%+ of logic lives here. + +The client relies on IndexedDB as a data store. It uses this data store primarily to keep the [[Objects|object index]]. In your browser’s database list, this database’s name will end with `_data`. This data store is _shared_ between tabs and windows (of the same browser). + +The client interacts with the service worker and server primarily via the [[HTTP API]], which exposes CRUD-style operations on files. + +# Service Worker +The [service worker](https://developer.mozilla.org/en-US/docs/Web/API/Service_Worker_API)’s role is to make SilverBullet offline capable. When enabled (it is enabled by default) it: +* Caches and serves the SilverBullet client code +* [[Sync|Syncs]] files with a local database +* Implements the [[HTTP API]], but handles it locally (where appropriate) + +It does this by intercepting HTTP calls coming from the client aimed at the server. + +The service worker embeds a [[Sync]] engine, that based on configuration constantly keeps a local copy of your files in sync with the server. To make the sync status visible, it emits events to the Client. + +For debugging purposes you can disable the service worker by adding `?enableSW=0` to your URL. This disabling is persistent, to re-enable it use `?enableSW=1`. + +# Server +The server has only three jobs: + +* Handle authentication +* Serve the (static) client code +* Implement the [[HTTP API]], which mainly means listing, reading, writing and deleting files, and executing shell commands. + +Note that all indexing, querying etc. happens in the client. The server effectively acts as a file store. + diff --git a/website/CHANGELOG.md b/website/CHANGELOG.md index 83c7ba5c..52491c9e 100644 --- a/website/CHANGELOG.md +++ b/website/CHANGELOG.md @@ -1,6 +1,15 @@ An attempt at documenting the changes/new features introduced in each release. -## v2 +## Edge +These are changes live on the edge builds: + +* Sync engine re-architecture: see [[Architecture]] and [[Sync]] +* More configuring what to index (see [[^Library/Std/Config]] under the `index` section) for the purpose of reducing local storage size and needless CPU waste. Some useful ones: + * `config.set("index.search.enable", false)` to disable [[Full Text Search]] entirely (saves on processing and storage if you don’t use it) + * `config.set("index.paragraph.all", false)` to disable indexing all (untagged) paragraphs. This is also somewhat wasteful if you don’t query these. +* A lot of little polish + +## 2.0.0 * We’re now live! For previous versions, see [the v1 CHANGELOG](https://v1.silverbullet.md/CHANGELOG) \ No newline at end of file diff --git a/website/CONFIG.md b/website/CONFIG.md index 87fdf427..8baa482a 100644 --- a/website/CONFIG.md +++ b/website/CONFIG.md @@ -1,9 +1,9 @@ -This is where you configure SilverBullet to your liking. See [[^Library/Std/Config]] for a full list of configuration options. +This is where you configure SilverBullet to your liking. See [[^Library/Std/Config]] for a full list of configuration options. ```space-lua config.set { plugs = { - "github:joekrill/silverbullet-treeview/treeview.plug.js" + "github:silverbulletmd/silverbullet-mermaid/mermaid.plug.js" }, actionButtons = { { diff --git a/website/Library/Snippet/edge-only.md b/website/Library/Snippet/edge-only.md new file mode 100644 index 00000000..a4c35ec8 --- /dev/null +++ b/website/Library/Snippet/edge-only.md @@ -0,0 +1,4 @@ +#meta/template/slash +> **note** Note +> This page contains sections that apply to the current _edge_ release, it is not part of an official release yet. + diff --git a/website/SilverBullet.md b/website/SilverBullet.md index 68830d56..16ba4bd1 100644 --- a/website/SilverBullet.md +++ b/website/SilverBullet.md @@ -9,7 +9,7 @@ And if you are comfortable writing (or learning) some **code**, you will love _d Err, _whut_?! Dynamically generating content, what does that mean? -Let’s say you want to a list of your 5 modified pages on your space’s index page. We can do that (`Alt-click` to see the underlying code): +Let’s say you want to have a list of your 5 modified pages on your space’s index page. We can do that (`Alt-click` to see the underlying code): ${template.each(query[[ from index.tag "page" diff --git a/website/Sync.md b/website/Sync.md new file mode 100644 index 00000000..aca44093 --- /dev/null +++ b/website/Sync.md @@ -0,0 +1,36 @@ +> **note** Note +> This page contains sections that apply to the current _edge_ release, it is not part of an official release yet. + +SilverBullet is an offline-first web application. Therefore it keeps a copy of your space locally in your browser. It implements a sync engine to continuously keep this local copy in sync with the server (for technical details, see [[Architecture]]). + +Sync happens: +* For the entire space: roughly every 20 seconds +* Open file: every 5 seconds, and immediately when a local change is made + +In addition, you can use the ${widgets.commandButton "Sync: Space"} and ${widgets.commandButton "Sync: File"} commands to trigger these manually. + +## Status +If sync takes longer than a second, a black circle progress indicator will appear in the [[Top Bar]]. + +## Conflicts +When you use multiple clients and make changes to the same files simultaneously conflicts may happen. SilverBullet will not attempt to try to merge these conflicts, but rather create a conflicting copy when this happens. You will be notified in the UI when this occurs. + +## Configuration +You can tweak what files are synced locally via a few `config.*` configuration options. + +> **note** Note +> Sync related configuration **need** to be done in the [[CONFIG]] page specifically. If they are configured elsewhere they will not be picked up. + +By default all your pages are synced, but [[Documents]] are not. They are fetched on demand. If you would also like to sync documents (so you can access them when offline), you can do so by setting the following in a Space Lua block in [[CONFIG]]: + + config.set("sync.documents", true) + +In addition, you can fine-tune what files you do _not_ want to sync using [gitignore](https://git-scm.com/docs/gitignore) syntax assigned to the `sync.ignore` option, which accepts either a single string, or a list of strings: + + config.set("sync.ignore", { + -- Don't sync PDFs and MP4 files + "*.pdf", + "*.mp4" + }) + +After making changes to these options, you need to reload your client and wait for a sync cycle to kick in until they are applied. \ No newline at end of file