diff --git a/.air.toml b/.air.toml new file mode 100644 index 00000000..35f3cf63 --- /dev/null +++ b/.air.toml @@ -0,0 +1,37 @@ +root = "." +tmp_dir = "tmp" + +[build] + args_bin = [] + bin = "silverbullet" + cmd = "deno task build && go build" + # delay = 10000 + exclude_dir = ["node_modules", "website", "scripts", "dist_client_bundle", "dist_base_fs_bundle", "dist"] + exclude_file = ["public_version.ts"] + exclude_regex = ["_test.go", ".test.ts"] + exclude_unchanged = false + follow_symlink = false + full_bin = "" + include_dir = [] + include_ext = ["go", "html", "ts", "scss"] + #include_file = [] + kill_delay = "0s" + log = "build-errors.log" + poll = false + poll_interval = 0 + post_cmd = [] + pre_cmd = [] + #rerun = false + #rerun_delay = 1000 + stop_on_error = true + +[color] + app = "" + build = "yellow" + main = "magenta" + runner = "green" + watcher = "cyan" + +[screen] + clear_on_rebuild = true + keep_scroll = true diff --git a/.github/workflows/edge.yml b/.github/workflows/edge.yml index e28debfe..ed3cf304 100644 --- a/.github/workflows/edge.yml +++ b/.github/workflows/edge.yml @@ -1,38 +1,59 @@ name: Build & Release Edge on: - push: - branches: - - "main" + push: + branches: + - "main" jobs: - release: - runs-on: ubuntu-latest - steps: - - name: Setup repo - uses: actions/checkout@v3 - - name: Setup Deno - uses: denoland/setup-deno@v2 - with: - deno-version: v2.5.0 - - name: Compile for all platforms - run: | - deno task build - deno task server:zip:linux-x86_64 - deno task server:zip:linux-aarch64 - deno task server:zip:darwin-x86_64 - deno task server:zip:darwin-aarch64 - deno task server:zip:windows-x86_64 - - name: Update edge release - uses: softprops/action-gh-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - draft: false - tag_name: edge - body: Automated build from commit ${{ github.sha }} - prerelease: true - files: | - website/CHANGELOG.md - dist/plug-compile.js - dist/silverbullet.js - silverbullet-*.zip + release: + runs-on: ubuntu-latest + steps: + - name: Setup repo + uses: actions/checkout@v3 + - name: Setup Deno + uses: denoland/setup-deno@v2 + with: + deno-version: v2.5.0 + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: "1.25.1" + - name: Compile frontend and plug-compile with Deno + run: | + deno task build-production + deno task build-plug-compile + - name: Build Go binaries for multiple platforms + run: | + # Linux ARM64 + GOOS=linux GOARCH=arm64 go build -o silverbullet . + zip silverbullet-server-linux-aarch64.zip silverbullet + + # Linux x86_64 + GOOS=linux GOARCH=amd64 go build -o silverbullet . + zip silverbullet-server-linux-x86_64.zip silverbullet + + # macOS ARM64 + GOOS=darwin GOARCH=arm64 go build -o silverbullet . + zip silverbullet-server-darwin-aarch64.zip silverbullet + + # macOS x86_64 + GOOS=darwin GOARCH=amd64 go build -o silverbullet . + zip silverbullet-server-darwin-x86_64.zip silverbullet + + # Windows x86_64 + GOOS=windows GOARCH=amd64 go build -o silverbullet.exe . + zip silverbullet-server-windows-x86_64.zip silverbullet.exe + + - name: Update edge release + uses: softprops/action-gh-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + draft: false + tag_name: edge + body: Automated build from commit ${{ github.sha }} + prerelease: true + files: | + website/CHANGELOG.md + dist/plug-compile.js + silverbullet-*.zip diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fbb45f1c..8640da1e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,37 +1,37 @@ name: Build & Release on: - push: - tags: - - "*" + push: + tags: + - "*" jobs: - release: - runs-on: ubuntu-latest - permissions: - contents: write - id-token: write # The OIDC ID token is used for authentication with JSR. - steps: - - name: Setup repo - uses: actions/checkout@v3 - with: - fetch-depth: 0 - fetch-tags: true - - name: Setup Deno - uses: denoland/setup-deno@v2 - with: - deno-version: v2.5.0 - - name: Run build - run: | - deno task build - - name: Publish to JSR - run: deno publish --allow-dirty - - name: Release - uses: softprops/action-gh-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - draft: false - files: | - website/CHANGELOG.md - dist/plug-compile.js - dist/silverbullet.js + release: + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write # The OIDC ID token is used for authentication with JSR. + steps: + - name: Setup repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 + fetch-tags: true + - name: Setup Deno + uses: denoland/setup-deno@v2 + with: + deno-version: v2.5.0 + - name: Run build + run: | + deno task build-production + deno task build-plug-compile + - name: Publish to JSR + run: deno publish --allow-dirty + - name: Release + uses: softprops/action-gh-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + draft: false + files: | + website/CHANGELOG.md + dist/plug-compile.js diff --git a/.github/workflows/server.yml b/.github/workflows/server.yml index 2fbc8cf8..216114d2 100644 --- a/.github/workflows/server.yml +++ b/.github/workflows/server.yml @@ -1,43 +1,54 @@ name: Server on: - push: - tags: - - "*" - + push: + tags: + - "*" jobs: - build: - runs-on: ubuntu-latest + release: + runs-on: ubuntu-latest + steps: + - name: Setup repo + uses: actions/checkout@v3 + - name: Setup Deno + uses: denoland/setup-deno@v2 + with: + deno-version: v2.5.0 + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: "1.25.1" + - name: Compile client and plug-compile with Deno + run: | + deno task build-production + deno task build-plug-compile + - name: Build Go binaries for multiple platforms + run: | + # Linux ARM64 + GOOS=linux GOARCH=arm64 go build -o silverbullet . + zip silverbullet-server-linux-aarch64.zip silverbullet - steps: - - name: Setup repo - uses: actions/checkout@v3 - with: - fetch-depth: 0 - fetch-tags: true + # Linux x86_64 + GOOS=linux GOARCH=amd64 go build -o silverbullet . + zip silverbullet-server-linux-x86_64.zip silverbullet - - name: Setup Deno - uses: denoland/setup-deno@v2 - with: - deno-version: v2.5.0 + # macOS ARM64 + GOOS=darwin GOARCH=arm64 go build -o silverbullet . + zip silverbullet-server-darwin-aarch64.zip silverbullet - - name: Build bundles - run: | - deno task build + # macOS x86_64 + GOOS=darwin GOARCH=amd64 go build -o silverbullet . + zip silverbullet-server-darwin-x86_64.zip silverbullet - - name: Compile for all platforms - run: | - deno task server:zip:linux-x86_64 - deno task server:zip:linux-aarch64 - deno task server:zip:darwin-x86_64 - deno task server:zip:darwin-aarch64 - deno task server:zip:windows-x86_64 - - - name: Release - uses: softprops/action-gh-release@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - draft: true - files: | - silverbullet-*.zip + # Windows x86_64 + GOOS=windows GOARCH=amd64 go build -o silverbullet.exe . + zip silverbullet-server-windows-x86_64.zip silverbullet.exe + - name: Release + uses: softprops/action-gh-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + draft: true + files: | + dist/plug-compile.js + silverbullet-*.zip diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 08434184..325b1ec8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: Test +name: Tests on: push: @@ -10,7 +10,7 @@ permissions: contents: read jobs: - test: + test-frontend: runs-on: ubuntu-latest steps: @@ -43,20 +43,11 @@ jobs: with: go-version: "1.25" - - name: Verify Go installation - run: go version + - name: Run Go tests + run: go test ./server/... -v - - name: Verify Go modules - run: go mod verify + - name: Run Go tests with coverage + run: go test ./server/... -coverprofile=coverage.out - - name: Download dependencies - run: go mod download - - #- name: Run Go tests - # run: go test ./server_go -v - - #- name: Run Go tests with coverage - # run: go test ./server_go -coverprofile=coverage.out - - #- name: Display test coverage - # run: go tool cover -func=coverage.out + - name: Display test coverage + run: go tool cover -func=coverage.out diff --git a/.gitignore b/.gitignore index f77a9fcc..07431d9e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ .env .DS_Store dist_client_bundle -dist_plug_bundle +dist_base_fs_bundle dist *.js.map website_build @@ -16,3 +16,5 @@ deploy.json *.generated tmp_playground /public_version.ts +tmp +*.ipynb diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 13566b81..00000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml deleted file mode 100644 index 1a0176ac..00000000 --- a/.idea/codeStyles/codeStyleConfig.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - diff --git a/.idea/deno.xml b/.idea/deno.xml deleted file mode 100644 index 7b6555a3..00000000 --- a/.idea/deno.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - diff --git a/.idea/dictionaries/project.xml b/.idea/dictionaries/project.xml deleted file mode 100644 index 5d0c4432..00000000 --- a/.idea/dictionaries/project.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - itags - - - diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index a5994fc5..00000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - diff --git a/.idea/silverbullet.iml b/.idea/silverbullet.iml deleted file mode 100644 index 4d59d886..00000000 --- a/.idea/silverbullet.iml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index dcb6b8c4..00000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/Dockerfile b/Dockerfile index c612c0de..b1c62c28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,22 @@ - # Stage 1: Build the silverbullet binary FROM denoland/deno:2.5.0 AS builder -RUN apt update && apt install -y git +RUN apt update && apt install -y git wget + +ARG TARGETARCH +ENV GO_VERSION=1.25.1 + +RUN wget -P /tmp "https://dl.google.com/go/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz" +RUN tar -C /usr/local -xzf "/tmp/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz" + +ENV GOPATH=/go +ENV PATH=$GOPATH/bin:/usr/local/go/bin:$PATH + WORKDIR /app ADD . /app # This will produce the `silverbullet` self-contained binary in /app/silverbullet -RUN deno task build +RUN deno task build-production +RUN go build # Stage 2: Create the runtime from the build FROM ubuntu:noble diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..4c46e865 --- /dev/null +++ b/Makefile @@ -0,0 +1,35 @@ +build: + # Build frontend + deno task build + # Build plug-compile + deno task build-plug-compile + # Build backend + go build + +clean: + deno task clean + rm -f silverbullet + +check: + # Frontend type check + deno task check + # Frontend lint + deno task lint + # Backend lint + go vet + +fmt: + # Reformat frontend + deno task fmt + # Reformat backend + go fmt + +test: + # Run frontend tests + deno task test + # Run backend tests + go test ./server/... + +generate: + # Regenerate the Lua parser from the the grammar + deno run -A npm:@lezer/generator@1.5.1 lib/space_lua/lua.grammar -o lib/space_lua/parse-lua.js diff --git a/README.md b/README.md index 3b9b8980..ca62a531 100644 --- a/README.md +++ b/README.md @@ -34,60 +34,74 @@ Check out the [instructions](https://silverbullet.md/Install). ## Developing SilverBullet -SilverBullet is written in [TypeScript](https://www.typescriptlang.org/) and -built on top of the excellent [CodeMirror 6](https://codemirror.net/) editor -component. Additional UI is built using [Preact](https://preactjs.com). -[ESBuild]([https://parceljs.org/](https://esbuild.github.io)) is used to build both the front-end and -back-end bundles. The server backend runs as a HTTP server on Deno using and is written using [Hono](https://hono.dev). +SilverBullet's frontend is written in [TypeScript](https://www.typescriptlang.org/) and built on top of the excellent [CodeMirror 6](https://codemirror.net/) editor component. Additional UI is built using [Preact](https://preactjs.com). [ESBuild]([https://parceljs.org/](https://esbuild.github.io)) is used to build both the front-end. -To prepare the initial web and plug build run: +The server backend is written in Go. + +## Code structure +* `client/`: The SilverBullet client, implemented with TypeScript +* `server/`: The SilverBullet server, written in Go +* `plugs`: Set of built-in plugs that are distributed with SilverBullet +* `libraries`: A set of libraries (space scripts, page templates, slash templates) distributed with SilverBullet +* `plug-api/`: Useful APIs for use in plugs + * `lib/`: Useful libraries to be used in plugs + * `syscalls/`: TypeScript wrappers around syscalls + * `types/`: Various (client) types that can be references from plugs +* `bin` + * `plug_compile.ts` the plug compiler +* `scripts/`: Useful scripts +* `website/`: silverbullet.md website content + +### Requirements +* [Deno](https://deno.com/): Used to build the frontend and plugs +* [Go](https://go.dev/): Used to build the backend + +It's convenient to also install [air](https://github.com/air-verse/air) for development, this will automatically rebuild both the frontend and backend when changes are made: + +```shell +go install github.com/air-verse/air@latest +``` +Make sure your `$GOPATH/bin` is in your $PATH. + +To build everything and run the server: + +```shell +air +``` + +Alternatively, to build just the frontend: ```shell deno task build ``` -To symlink `silverbullet` to your locally checked-out version, run: +To build the backend (note: this will bundle the frontend into the same binary, so be sure to build that first): ```shell -deno task install +go build ``` -You can then run the server in "watch mode" (automatically restarting when you -change source files) with: +To run the resulting server: ```shell -deno task watch-server +./silverbullet ``` -After this initial build, it's convenient to run three commands in parallel (in -separate terminals): +### Useful development tasks ```shell -deno task watch-web -deno task watch-server -deno task watch-plugs +# Clean all generated files +make clean +# Typecheck and lint all code +make check +# Format all code +make fmt +# Run all tests +make test ``` -Alternatively, you can use the convenience task (though you'll need to set the server path in a separate terminal first): - -```shell -deno task watch-all -``` - -To typecheck the entire codebase (recommended before submitting PR): -```shell -deno task check -``` - -Other useful development tasks: -```shell -deno task lint # Lint and fix code -deno task fmt # Format code -deno task test # Run tests -deno task checks # Run check, lint, and test together -``` - -To build it in a docker container (no Deno install required): +### Build a docker container +Note, you do not need Deno nor Go locally installed for this to work: ```shell docker build -t silverbullet . diff --git a/plug-compile.ts b/bin/plug-compile.ts similarity index 79% rename from plug-compile.ts rename to bin/plug-compile.ts index 5e7bef25..bbbda613 100755 --- a/plug-compile.ts +++ b/bin/plug-compile.ts @@ -1,10 +1,9 @@ import.meta.main = false; import { Command } from "@cliffy/command"; -import { version } from "./version.ts"; +import { version } from "../version.ts"; -import { versionCommand } from "./cmd/version.ts"; -import { plugCompileCommand } from "./cmd/plug_compile.ts"; +import { plugCompileCommand } from "../lib/plugos/plug_compile.ts"; await new Command() .name("plug-compile") @@ -29,10 +28,6 @@ await new Command() .option("-c, --config ", "Path to deno.json file to use") .option("--runtimeUrl ", "URL to worker_runtime.ts to use") .action(plugCompileCommand) - // version - .command("version") - .description("Get current version") - .action(versionCommand) .parse(Deno.args); Deno.exit(0); diff --git a/build_web.ts b/build_client.ts similarity index 52% rename from build_web.ts rename to build_client.ts index 765c2617..5ee1d033 100644 --- a/build_web.ts +++ b/build_client.ts @@ -2,70 +2,42 @@ import { copy } from "@std/fs"; import { fileURLToPath } from "node:url"; import sass from "denosass"; -import { bundleFolder } from "./lib/asset_bundle/builder.ts"; -import { parseArgs } from "@std/cli/parse-args"; -import { patchDenoLibJS } from "./cmd/compile.ts"; +import { patchDenoLibJS } from "./lib/plugos/plug_compile.ts"; import { denoPlugins } from "@luca/esbuild-deno-loader"; import * as esbuild from "esbuild"; -import { updateVersionFile } from "./cmd/update_version.ts"; -export async function bundleAll( - watch: boolean, -): Promise { - let building = false; +// This builds the client and puts it into dist_client_bundle/ + +export async function bundleAll(): Promise { await buildCopyBundleAssets(); - let timer; - if (watch) { - const watcher = Deno.watchFs([ - "web", - "dist_plug_bundle/_plug", - "lib", - "Library", - ]); - for await (const _event of watcher) { - if (timer) { - clearTimeout(timer); - } - timer = setTimeout(() => { - console.log("Change detected, rebuilding..."); - if (building) { - return; - } - building = true; - buildCopyBundleAssets().finally(() => { - building = false; - }); - }, 1000); - } - } } export async function copyAssets(dist: string) { await Deno.mkdir(dist, { recursive: true }); - await copy("web/fonts", `${dist}`, { overwrite: true }); - await copy("web/index.html", `${dist}/index.html`, { + await copy("client/fonts", `${dist}`, { overwrite: true }); + await copy("client/index.html", `${dist}/index.html`, { overwrite: true, }); - await copy("web/auth.html", `${dist}/auth.html`, { + await copy("client/auth.html", `${dist}/auth.html`, { overwrite: true, }); - await copy("web/images/favicon.png", `${dist}/favicon.png`, { + await copy("client/images/favicon.png", `${dist}/favicon.png`, { overwrite: true, }); - await copy("web/images/logo.png", `${dist}/logo.png`, { + await copy("client/images/logo.png", `${dist}/logo.png`, { overwrite: true, }); - await copy("web/images/logo-dock.png", `${dist}/logo-dock.png`, { + await copy("client/images/logo-dock.png", `${dist}/logo-dock.png`, { overwrite: true, }); - await copy("web/manifest.json", `${dist}/manifest.json`, { + await copy("client/manifest.json", `${dist}/manifest.json`, { overwrite: true, }); const compiler = sass( - Deno.readTextFileSync("web/styles/main.scss"), + Deno.readTextFileSync("client/styles/main.scss"), { - load_paths: ["web/styles"], + load_paths: ["client/styles"], }, ); await Deno.writeTextFile( @@ -81,28 +53,18 @@ export async function copyAssets(dist: string) { async function buildCopyBundleAssets() { await Deno.mkdir("dist_client_bundle", { recursive: true }); - await Deno.mkdir("dist_plug_bundle", { recursive: true }); - - // Copy Library files - await copy("Library", "dist_plug_bundle/Library", { - overwrite: true, - }); - - await bundleFolder( - "dist_plug_bundle", - "dist/plug_asset_bundle.json", - ); + await Deno.mkdir("dist_base_fs_bundle", { recursive: true }); console.log("Now ESBuilding the client and service workers..."); const result = await esbuild.build({ entryPoints: [ { - in: "web/boot.ts", + in: "client/boot.ts", out: ".client/client", }, { - in: "web/service_worker.ts", + in: "client/service_worker.ts", out: "service_worker", }, ], @@ -135,22 +97,10 @@ async function buildCopyBundleAssets() { await copyAssets("dist_client_bundle/.client"); - await bundleFolder("dist_client_bundle", "dist/client_asset_bundle.json"); - console.log("Built!"); } if (import.meta.main) { - const args = parseArgs(Deno.args, { - boolean: ["watch"], - alias: { w: "watch" }, - default: { - watch: false, - }, - }); - await updateVersionFile(); - await bundleAll(args.watch); - if (!args.watch) { - esbuild.stop(); - } + await bundleAll(); + esbuild.stop(); } diff --git a/build_bundle.ts b/build_plug_compile.ts similarity index 84% rename from build_bundle.ts rename to build_plug_compile.ts index 5719b19e..b9060eb8 100644 --- a/build_bundle.ts +++ b/build_plug_compile.ts @@ -1,14 +1,10 @@ import { denoPlugins } from "@luca/esbuild-deno-loader"; import * as esbuild from "esbuild"; -import { updateVersionFile } from "./cmd/update_version.ts"; - -await updateVersionFile(); await Deno.mkdir("dist", { recursive: true }); const result = await esbuild.build({ entryPoints: { - silverbullet: "silverbullet.ts", - "plug-compile": "plug-compile.ts", + "plug-compile": "./bin/plug-compile.ts", }, outdir: "dist", format: "esm", diff --git a/build_plugs.ts b/build_plugs.ts deleted file mode 100644 index 5694c905..00000000 --- a/build_plugs.ts +++ /dev/null @@ -1,39 +0,0 @@ -import * as path from "@std/path"; -import * as esbuild from "esbuild"; -import { compileManifests } from "./cmd/compile.ts"; -import { builtinPlugNames } from "./plugs/builtin_plugs.ts"; -import { parseArgs } from "@std/cli/parse-args"; -import { fileURLToPath } from "node:url"; -import { updateVersionFile } from "./cmd/update_version.ts"; - -await updateVersionFile(); - -if (import.meta.main) { - const args = parseArgs(Deno.args, { - boolean: ["debug", "watch", "reload", "info"], - alias: { w: "watch" }, - }); - - const manifests = builtinPlugNames.map((name) => - `./plugs/${name}/${name}.plug.yaml` - ); - - const plugBundlePath = "dist_plug_bundle"; - const targetDir = path.join(plugBundlePath, "_plug"); - Deno.mkdirSync(targetDir, { recursive: true }); - Deno.mkdirSync("dist", { recursive: true }); - - // Build the other plugs - await compileManifests( - manifests, - targetDir, - args.watch, - { - debug: args.debug, - reload: args.reload, - info: args.info, - configPath: fileURLToPath(new URL("deno.json", import.meta.url)), - }, - ); - esbuild.stop(); -} diff --git a/build_plugs_libraries.ts b/build_plugs_libraries.ts new file mode 100644 index 00000000..4607d7cb --- /dev/null +++ b/build_plugs_libraries.ts @@ -0,0 +1,68 @@ +import * as path from "@std/path"; +import * as esbuild from "esbuild"; +import { compileManifests } from "./lib/plugos/plug_compile.ts"; +import { builtinPlugNames } from "./plugs/builtin_plugs.ts"; +import { parseArgs } from "@std/cli/parse-args"; +import { fileURLToPath } from "node:url"; +import { copy } from "@std/fs"; +import { version } from "./version.ts"; + +// This builds all built-in plugs and libraries and puts them into dist_base_fs_bundle/ + +if (import.meta.main) { + await updateVersionFile(); + const args = parseArgs(Deno.args, { + boolean: ["debug", "reload", "info"], + alias: { w: "watch" }, + }); + + const manifests = builtinPlugNames.map((name) => + `./plugs/${name}/${name}.plug.yaml` + ); + + const plugBundlePath = "dist_base_fs_bundle"; + const targetDir = path.join(plugBundlePath, "_plug"); + Deno.mkdirSync(targetDir, { recursive: true }); + Deno.mkdirSync("dist", { recursive: true }); + + // Copy Library files + await copy("libraries/Library", "dist_base_fs_bundle/Library", { + overwrite: true, + }); + + // Build the plugs + await compileManifests( + manifests, + targetDir, + { + debug: args.debug, + info: args.info, + configPath: fileURLToPath(new URL("deno.json", import.meta.url)), + }, + ); + esbuild.stop(); +} + +export async function updateVersionFile() { + const command = new Deno.Command("git", { + args: ["describe", "--tags", "--long"], + stdout: "piped", + stderr: "piped", + }); + + const { stdout } = await command.output(); + let commitVersion = new TextDecoder().decode(stdout).trim(); + + if (!commitVersion) { + // Probably no valid .git repo, fallback to GITHUB_SHA env var (used in CI) + commitVersion = `${version}-${Deno.env.get("GITHUB_SHA") || "unknown"}`; + } + + const versionFilePath = "./public_version.ts"; + const versionContent = `export const publicVersion = "${commitVersion}";`; + + await Deno.writeTextFile(versionFilePath, versionContent); + console.log( + `Updated public_version.ts with version information: ${commitVersion}`, + ); +} diff --git a/web/PAGE_TEMPLATES.ts b/client/PAGE_TEMPLATES.ts similarity index 100% rename from web/PAGE_TEMPLATES.ts rename to client/PAGE_TEMPLATES.ts diff --git a/web/README.md b/client/README.md similarity index 100% rename from web/README.md rename to client/README.md diff --git a/client/auth.html b/client/auth.html new file mode 100644 index 00000000..6049572d --- /dev/null +++ b/client/auth.html @@ -0,0 +1,209 @@ + + + + + + + + Login to {{.SpaceName}} + + + + +
+
+

+ Login to + {{.SpaceName}} +

+
+
+
+ + +
+
+ + +
+
+ + +
+
+ +
+
+ +
+
+ + + + diff --git a/web/boot.ts b/client/boot.ts similarity index 100% rename from web/boot.ts rename to client/boot.ts diff --git a/web/boot_config.test.ts b/client/boot_config.test.ts similarity index 100% rename from web/boot_config.test.ts rename to client/boot_config.test.ts diff --git a/web/boot_config.ts b/client/boot_config.ts similarity index 100% rename from web/boot_config.ts rename to client/boot_config.ts diff --git a/web/change.test.ts b/client/change.test.ts similarity index 100% rename from web/change.test.ts rename to client/change.test.ts diff --git a/web/change.ts b/client/change.ts similarity index 100% rename from web/change.ts rename to client/change.ts diff --git a/web/client.ts b/client/client.ts similarity index 99% rename from web/client.ts rename to client/client.ts index f47a1a91..8efe4a4c 100644 --- a/web/client.ts +++ b/client/client.ts @@ -17,7 +17,7 @@ import type { SlashCompletions, } from "@silverbulletmd/silverbullet/type/client"; import { EventHook } from "./hooks/event.ts"; -import type { Command } from "../type/command.ts"; +import type { Command } from "./types/command.ts"; import { type LocationState, parseRefFromURI, @@ -31,7 +31,10 @@ import type { ServiceWorkerTargetMessage, } from "./ui_types.ts"; -import type { PageCreatingContent, PageCreatingEvent } from "../type/event.ts"; +import type { + PageCreatingContent, + PageCreatingEvent, +} from "@silverbulletmd/silverbullet/type/event"; import type { StyleObject } from "../plugs/index/style.ts"; import { jitter, throttle } from "../lib/async.ts"; import { PlugSpacePrimitives } from "../lib/spaces/plug_space_primitives.ts"; @@ -64,7 +67,11 @@ import { diffAndPrepareChanges } from "./cm_util.ts"; import { DocumentEditor } from "./document_editor.ts"; import { parseExpressionString } from "../lib/space_lua/parse.ts"; import type { Config } from "./config.ts"; -import type { DocumentMeta, FileMeta, PageMeta } from "../type/index.ts"; +import type { + DocumentMeta, + FileMeta, + PageMeta, +} from "@silverbulletmd/silverbullet/type/index"; import { parseMarkdown } from "./markdown_parser/parser.ts"; import { CheckPathSpacePrimitives } from "../lib/spaces/checked_space_primitives.ts"; import { notFoundError, offlineError } from "../lib/constants.ts"; diff --git a/web/client_system.ts b/client/client_system.ts similarity index 99% rename from web/client_system.ts rename to client/client_system.ts index 17c474f0..ae20ac18 100644 --- a/web/client_system.ts +++ b/client/client_system.ts @@ -42,7 +42,7 @@ import { configSyscalls } from "./syscalls/config.ts"; import { eventListenerSyscalls } from "./syscalls/event.ts"; import { DocumentEditorHook } from "./hooks/document_editor.ts"; import type { LuaCollectionQuery } from "../lib/space_lua/query_collection.ts"; -import type { Command } from "../type/command.ts"; +import type { Command } from "./types/command.ts"; import { SpaceLuaEnvironment } from "./space_lua.ts"; import { type ILuaFunction, diff --git a/web/cm_plugins/admonition.ts b/client/cm_plugins/admonition.ts similarity index 100% rename from web/cm_plugins/admonition.ts rename to client/cm_plugins/admonition.ts diff --git a/web/cm_plugins/block.ts b/client/cm_plugins/block.ts similarity index 100% rename from web/cm_plugins/block.ts rename to client/cm_plugins/block.ts diff --git a/web/cm_plugins/block_quote.ts b/client/cm_plugins/block_quote.ts similarity index 100% rename from web/cm_plugins/block_quote.ts rename to client/cm_plugins/block_quote.ts diff --git a/web/cm_plugins/clean.ts b/client/cm_plugins/clean.ts similarity index 98% rename from web/cm_plugins/clean.ts rename to client/cm_plugins/clean.ts index d62d7738..c51af581 100644 --- a/web/cm_plugins/clean.ts +++ b/client/cm_plugins/clean.ts @@ -21,7 +21,7 @@ export function cleanModePlugins(client: Client) { luaDirectivePlugin(client), cleanWikiLinkPlugin(client), hashtagPlugin(), - frontmatterPlugin(client), + frontmatterPlugin(client), ]; if (client.ui.viewState.uiOptions.markdownSyntaxRendering) { diff --git a/web/cm_plugins/code_copy.ts b/client/cm_plugins/code_copy.ts similarity index 100% rename from web/cm_plugins/code_copy.ts rename to client/cm_plugins/code_copy.ts diff --git a/web/cm_plugins/code_widget.ts b/client/cm_plugins/code_widget.ts similarity index 100% rename from web/cm_plugins/code_widget.ts rename to client/cm_plugins/code_widget.ts diff --git a/web/cm_plugins/editor_paste.ts b/client/cm_plugins/editor_paste.ts similarity index 100% rename from web/cm_plugins/editor_paste.ts rename to client/cm_plugins/editor_paste.ts diff --git a/web/cm_plugins/escapes.ts b/client/cm_plugins/escapes.ts similarity index 100% rename from web/cm_plugins/escapes.ts rename to client/cm_plugins/escapes.ts diff --git a/web/cm_plugins/fenced_code.ts b/client/cm_plugins/fenced_code.ts similarity index 100% rename from web/cm_plugins/fenced_code.ts rename to client/cm_plugins/fenced_code.ts diff --git a/web/cm_plugins/frontmatter.ts b/client/cm_plugins/frontmatter.ts similarity index 90% rename from web/cm_plugins/frontmatter.ts rename to client/cm_plugins/frontmatter.ts index 68e5c66d..9b8806bc 100644 --- a/web/cm_plugins/frontmatter.ts +++ b/client/cm_plugins/frontmatter.ts @@ -1,9 +1,17 @@ -import type {EditorState} from "@codemirror/state"; -import {foldedRanges, syntaxTree} from "@codemirror/language"; -import {Decoration} from "@codemirror/view"; -import {decoratorStateField, HtmlWidget, isCursorInRange, LinkWidget} from "./util.ts"; -import type {Client} from "../client.ts"; -import {frontmatterUrlRegex, frontmatterWikiLinkRegex} from "../markdown_parser/constants.ts"; +import type { EditorState } from "@codemirror/state"; +import { foldedRanges, syntaxTree } from "@codemirror/language"; +import { Decoration } from "@codemirror/view"; +import { + decoratorStateField, + HtmlWidget, + isCursorInRange, + LinkWidget, +} from "./util.ts"; +import type { Client } from "../client.ts"; +import { + frontmatterUrlRegex, + frontmatterWikiLinkRegex, +} from "../markdown_parser/constants.ts"; import { processWikiLink, type WikiLinkMatch } from "./wiki_link_processor.ts"; export function frontmatterPlugin(client: Client) { @@ -127,7 +135,9 @@ export function frontmatterPlugin(client: Client) { if (e.altKey) { // Move cursor into the link's content client.editorView.dispatch({ - selection: {anchor: mFrom + wikiLinkMatch.leadingTrivia.length}, + selection: { + anchor: mFrom + wikiLinkMatch.leadingTrivia.length, + }, }); client.focus(); return; @@ -148,4 +158,4 @@ export function frontmatterPlugin(client: Client) { return Decoration.set(widgets, true); }, ); -} \ No newline at end of file +} diff --git a/web/cm_plugins/hashtag.ts b/client/cm_plugins/hashtag.ts similarity index 100% rename from web/cm_plugins/hashtag.ts rename to client/cm_plugins/hashtag.ts diff --git a/web/cm_plugins/hide_mark.ts b/client/cm_plugins/hide_mark.ts similarity index 100% rename from web/cm_plugins/hide_mark.ts rename to client/cm_plugins/hide_mark.ts diff --git a/web/cm_plugins/iframe_widget.ts b/client/cm_plugins/iframe_widget.ts similarity index 100% rename from web/cm_plugins/iframe_widget.ts rename to client/cm_plugins/iframe_widget.ts diff --git a/web/cm_plugins/inline_content.ts b/client/cm_plugins/inline_content.ts similarity index 100% rename from web/cm_plugins/inline_content.ts rename to client/cm_plugins/inline_content.ts diff --git a/web/cm_plugins/line_wrapper.ts b/client/cm_plugins/line_wrapper.ts similarity index 100% rename from web/cm_plugins/line_wrapper.ts rename to client/cm_plugins/line_wrapper.ts diff --git a/web/cm_plugins/link.ts b/client/cm_plugins/link.ts similarity index 100% rename from web/cm_plugins/link.ts rename to client/cm_plugins/link.ts diff --git a/web/cm_plugins/lint.ts b/client/cm_plugins/lint.ts similarity index 100% rename from web/cm_plugins/lint.ts rename to client/cm_plugins/lint.ts diff --git a/web/cm_plugins/list.ts b/client/cm_plugins/list.ts similarity index 100% rename from web/cm_plugins/list.ts rename to client/cm_plugins/list.ts diff --git a/web/cm_plugins/lua_directive.ts b/client/cm_plugins/lua_directive.ts similarity index 98% rename from web/cm_plugins/lua_directive.ts rename to client/cm_plugins/lua_directive.ts index 364d955b..f3ff24f7 100644 --- a/web/cm_plugins/lua_directive.ts +++ b/client/cm_plugins/lua_directive.ts @@ -27,7 +27,7 @@ import { import { resolveASTReference } from "../space_lua.ts"; import { LuaWidget } from "./lua_widget.ts"; import YAML from "js-yaml"; -import type { PageMeta } from "../../type/index.ts"; +import type { PageMeta } from "../../plug-api/types/index.ts"; export function luaDirectivePlugin(client: Client) { return decoratorStateField((state: EditorState) => { diff --git a/web/cm_plugins/lua_widget.ts b/client/cm_plugins/lua_widget.ts similarity index 100% rename from web/cm_plugins/lua_widget.ts rename to client/cm_plugins/lua_widget.ts diff --git a/web/cm_plugins/smart_quotes.ts b/client/cm_plugins/smart_quotes.ts similarity index 97% rename from web/cm_plugins/smart_quotes.ts rename to client/cm_plugins/smart_quotes.ts index 3bec3ea7..2f1b76b7 100644 --- a/web/cm_plugins/smart_quotes.ts +++ b/client/cm_plugins/smart_quotes.ts @@ -3,7 +3,7 @@ import { syntaxTree } from "@codemirror/language"; import { EditorSelection } from "@codemirror/state"; import type { Client } from "../client.ts"; -import type { SmartQuotesConfig } from "../../type/config.ts"; +import type { SmartQuotesConfig } from "../../plug-api/types/config.ts"; const straightQuoteContexts = [ "CommentBlock", diff --git a/web/cm_plugins/spell_checking.ts b/client/cm_plugins/spell_checking.ts similarity index 100% rename from web/cm_plugins/spell_checking.ts rename to client/cm_plugins/spell_checking.ts diff --git a/web/cm_plugins/table.ts b/client/cm_plugins/table.ts similarity index 100% rename from web/cm_plugins/table.ts rename to client/cm_plugins/table.ts diff --git a/web/cm_plugins/task.ts b/client/cm_plugins/task.ts similarity index 100% rename from web/cm_plugins/task.ts rename to client/cm_plugins/task.ts diff --git a/web/cm_plugins/top_bottom_panels.ts b/client/cm_plugins/top_bottom_panels.ts similarity index 100% rename from web/cm_plugins/top_bottom_panels.ts rename to client/cm_plugins/top_bottom_panels.ts diff --git a/web/cm_plugins/util.ts b/client/cm_plugins/util.ts similarity index 98% rename from web/cm_plugins/util.ts rename to client/cm_plugins/util.ts index 8000a663..2ee3cbda 100644 --- a/web/cm_plugins/util.ts +++ b/client/cm_plugins/util.ts @@ -10,7 +10,7 @@ import type { DecorationSet } from "@codemirror/view"; import { Decoration, EditorView, WidgetType } from "@codemirror/view"; import type { Client } from "../client.ts"; -import type { PageMeta } from "../../type/index.ts"; +import type { PageMeta } from "../../plug-api/types/index.ts"; type LinkOptions = { text: string; diff --git a/web/cm_plugins/widget_util.ts b/client/cm_plugins/widget_util.ts similarity index 100% rename from web/cm_plugins/widget_util.ts rename to client/cm_plugins/widget_util.ts diff --git a/web/cm_plugins/wiki_link.ts b/client/cm_plugins/wiki_link.ts similarity index 94% rename from web/cm_plugins/wiki_link.ts rename to client/cm_plugins/wiki_link.ts index 79abf3c5..f26debdb 100644 --- a/web/cm_plugins/wiki_link.ts +++ b/client/cm_plugins/wiki_link.ts @@ -12,7 +12,7 @@ import { processWikiLink, type WikiLinkMatch } from "./wiki_link_processor.ts"; export function cleanWikiLinkPlugin(client: Client) { return decoratorStateField((state) => { const widgets: any[] = []; - + syntaxTree(state).iterate({ enter: ({ type, from, to }) => { if (type.name !== "WikiLink") { @@ -46,7 +46,9 @@ export function cleanWikiLinkPlugin(client: Client) { if (e.altKey) { // Move cursor into the link client.editorView.dispatch({ - selection: { anchor: from + wikiLinkMatch.leadingTrivia.length }, + selection: { + anchor: from + wikiLinkMatch.leadingTrivia.length, + }, }); client.focus(); return; diff --git a/web/cm_plugins/wiki_link_processor.ts b/client/cm_plugins/wiki_link_processor.ts similarity index 95% rename from web/cm_plugins/wiki_link_processor.ts rename to client/cm_plugins/wiki_link_processor.ts index eb47e56c..9d5eec20 100644 --- a/web/cm_plugins/wiki_link_processor.ts +++ b/client/cm_plugins/wiki_link_processor.ts @@ -1,8 +1,10 @@ - import type { EditorState } from "@codemirror/state"; import { Decoration } from "@codemirror/view"; import type { Client } from "../client.ts"; -import { fileName, isBuiltinPath } from "@silverbulletmd/silverbullet/lib/resolve"; +import { + fileName, + isBuiltinPath, +} from "@silverbulletmd/silverbullet/lib/resolve"; import { encodePageURI, encodeRef, @@ -30,9 +32,10 @@ export interface WikiLinkProcessorOptions { } export function processWikiLink(options: WikiLinkProcessorOptions): any[] { - const { from, to, match, matchFrom, matchTo, client, state, callback } = options; + const { from, to, match, matchFrom, matchTo, client, state, callback } = + options; const widgets: any[] = []; - + const { leadingTrivia, stringRef, alias, trailingTrivia } = match; const ref = parseToRef(stringRef); @@ -43,7 +46,9 @@ export function processWikiLink(options: WikiLinkProcessorOptions): any[] { } else if (ref.path === "" || isBuiltinPath(ref.path)) { linkStatus = "default"; } else if ( - Array.from(client.clientSystem.allKnownFiles).some((file) => file === ref.path) + Array.from(client.clientSystem.allKnownFiles).some((file) => + file === ref.path + ) ) { linkStatus = "default"; } else if (client.fullSyncCompleted || client.clientSystem.knownFilesLoaded) { @@ -57,7 +62,7 @@ export function processWikiLink(options: WikiLinkProcessorOptions): any[] { }[linkStatus]; const renderingSyntax = client.ui.viewState.uiOptions.markdownSyntaxRendering; - + if (isCursorInRange(state, [from, to]) || renderingSyntax) { // Only attach a CSS class, then get out if (linkStatus !== "default") { diff --git a/web/cm_util.ts b/client/cm_util.ts similarity index 100% rename from web/cm_util.ts rename to client/cm_util.ts diff --git a/web/commands.ts b/client/commands.ts similarity index 100% rename from web/commands.ts rename to client/commands.ts diff --git a/web/components/basic_modals.tsx b/client/components/basic_modals.tsx similarity index 100% rename from web/components/basic_modals.tsx rename to client/components/basic_modals.tsx diff --git a/web/components/command_palette.tsx b/client/components/command_palette.tsx similarity index 97% rename from web/components/command_palette.tsx rename to client/components/command_palette.tsx index 33728f8f..a28d9883 100644 --- a/web/components/command_palette.tsx +++ b/client/components/command_palette.tsx @@ -4,7 +4,7 @@ import type { CompletionResult, } from "@codemirror/autocomplete"; import { Terminal } from "preact-feather"; -import type { Command } from "../../type/command.ts"; +import type { Command } from "../types/command.ts"; import type { FilterOption } from "@silverbulletmd/silverbullet/type/client"; export function CommandPalette({ diff --git a/web/components/filter.tsx b/client/components/filter.tsx similarity index 100% rename from web/components/filter.tsx rename to client/components/filter.tsx diff --git a/web/components/mini_editor.tsx b/client/components/mini_editor.tsx similarity index 100% rename from web/components/mini_editor.tsx rename to client/components/mini_editor.tsx diff --git a/web/components/page_navigator.tsx b/client/components/page_navigator.tsx similarity index 98% rename from web/components/page_navigator.tsx rename to client/components/page_navigator.tsx index 6e1f776d..df5901b4 100644 --- a/web/components/page_navigator.tsx +++ b/client/components/page_navigator.tsx @@ -6,7 +6,10 @@ import type { } from "@codemirror/autocomplete"; import { tagRegex as mdTagRegex } from "../markdown_parser/constants.ts"; import { extractHashtag } from "@silverbulletmd/silverbullet/lib/tags"; -import type { DocumentMeta, PageMeta } from "../../type/index.ts"; +import type { + DocumentMeta, + PageMeta, +} from "@silverbulletmd/silverbullet/type/index"; import { getNameFromPath, parseToRef, diff --git a/web/components/panel.tsx b/client/components/panel.tsx similarity index 100% rename from web/components/panel.tsx rename to client/components/panel.tsx diff --git a/web/components/panel_html.ts b/client/components/panel_html.ts similarity index 100% rename from web/components/panel_html.ts rename to client/components/panel_html.ts diff --git a/web/components/top_bar.tsx b/client/components/top_bar.tsx similarity index 100% rename from web/components/top_bar.tsx rename to client/components/top_bar.tsx diff --git a/web/components/widget_sandbox_iframe.ts b/client/components/widget_sandbox_iframe.ts similarity index 100% rename from web/components/widget_sandbox_iframe.ts rename to client/components/widget_sandbox_iframe.ts diff --git a/web/config.test.ts b/client/config.test.ts similarity index 100% rename from web/config.test.ts rename to client/config.test.ts diff --git a/web/config.ts b/client/config.ts similarity index 100% rename from web/config.ts rename to client/config.ts diff --git a/web/document_editor.ts b/client/document_editor.ts similarity index 99% rename from web/document_editor.ts rename to client/document_editor.ts index 948fd8da..0d558b01 100644 --- a/web/document_editor.ts +++ b/client/document_editor.ts @@ -1,6 +1,6 @@ import type { Client } from "./client.ts"; import { timeout } from "../lib/async.ts"; -import type { DocumentMeta } from "../type/index.ts"; +import type { DocumentMeta } from "../plug-api/types/index.ts"; import type { Ref } from "@silverbulletmd/silverbullet/lib/ref"; type DocumentEditorIframeWindow = Window & { diff --git a/web/editor_state.ts b/client/editor_state.ts similarity index 100% rename from web/editor_state.ts rename to client/editor_state.ts diff --git a/web/editor_ui.tsx b/client/editor_ui.tsx similarity index 100% rename from web/editor_ui.tsx rename to client/editor_ui.tsx diff --git a/web/filtered_material_icons.ts b/client/filtered_material_icons.ts similarity index 100% rename from web/filtered_material_icons.ts rename to client/filtered_material_icons.ts diff --git a/web/fonts/LICENSE.md b/client/fonts/LICENSE.md similarity index 100% rename from web/fonts/LICENSE.md rename to client/fonts/LICENSE.md diff --git a/web/fonts/iAWriterMonoS-Bold.woff2 b/client/fonts/iAWriterMonoS-Bold.woff2 similarity index 100% rename from web/fonts/iAWriterMonoS-Bold.woff2 rename to client/fonts/iAWriterMonoS-Bold.woff2 diff --git a/web/fonts/iAWriterMonoS-BoldItalic.woff2 b/client/fonts/iAWriterMonoS-BoldItalic.woff2 similarity index 100% rename from web/fonts/iAWriterMonoS-BoldItalic.woff2 rename to client/fonts/iAWriterMonoS-BoldItalic.woff2 diff --git a/web/fonts/iAWriterMonoS-Italic.woff2 b/client/fonts/iAWriterMonoS-Italic.woff2 similarity index 100% rename from web/fonts/iAWriterMonoS-Italic.woff2 rename to client/fonts/iAWriterMonoS-Italic.woff2 diff --git a/web/fonts/iAWriterMonoS-Regular.woff2 b/client/fonts/iAWriterMonoS-Regular.woff2 similarity index 100% rename from web/fonts/iAWriterMonoS-Regular.woff2 rename to client/fonts/iAWriterMonoS-Regular.woff2 diff --git a/web/fuse_search.test.ts b/client/fuse_search.test.ts similarity index 100% rename from web/fuse_search.test.ts rename to client/fuse_search.test.ts diff --git a/web/fuse_search.ts b/client/fuse_search.ts similarity index 100% rename from web/fuse_search.ts rename to client/fuse_search.ts diff --git a/web/hooks/code_widget.ts b/client/hooks/code_widget.ts similarity index 100% rename from web/hooks/code_widget.ts rename to client/hooks/code_widget.ts diff --git a/web/hooks/command.ts b/client/hooks/command.ts similarity index 97% rename from web/hooks/command.ts rename to client/hooks/command.ts index 1f296fa3..d7a27292 100644 --- a/web/hooks/command.ts +++ b/client/hooks/command.ts @@ -2,7 +2,7 @@ import type { Hook, Manifest } from "../../lib/plugos/types.ts"; import type { System } from "../../lib/plugos/system.ts"; import { EventEmitter } from "../../lib/plugos/event.ts"; import { throttle } from "../../lib/async.ts"; -import type { Command, CommandHookEvents } from "../../type/command.ts"; +import type { Command, CommandHookEvents } from "../types/command.ts"; import type { CommandHookT } from "../../lib/manifest.ts"; export class CommandHook extends EventEmitter diff --git a/web/hooks/document_editor.ts b/client/hooks/document_editor.ts similarity index 100% rename from web/hooks/document_editor.ts rename to client/hooks/document_editor.ts diff --git a/web/hooks/event.ts b/client/hooks/event.ts similarity index 100% rename from web/hooks/event.ts rename to client/hooks/event.ts diff --git a/web/hooks/plug_namespace.ts b/client/hooks/plug_namespace.ts similarity index 100% rename from web/hooks/plug_namespace.ts rename to client/hooks/plug_namespace.ts diff --git a/web/hooks/slash_command.ts b/client/hooks/slash_command.ts similarity index 98% rename from web/hooks/slash_command.ts rename to client/hooks/slash_command.ts index bcb165bf..2ff3f1c2 100644 --- a/web/hooks/slash_command.ts +++ b/client/hooks/slash_command.ts @@ -9,7 +9,7 @@ import type { Client } from "../client.ts"; import { syntaxTree } from "@codemirror/language"; import { safeRun, throttle } from "../../lib/async.ts"; import type { SlashCommandHookT } from "../../lib/manifest.ts"; -import type { SlashCommand } from "../../type/command.ts"; +import type { SlashCommand } from "../types/command.ts"; import type { SlashCompletionOption, SlashCompletions, diff --git a/web/hooks/syscall.ts b/client/hooks/syscall.ts similarity index 100% rename from web/hooks/syscall.ts rename to client/hooks/syscall.ts diff --git a/web/images/favicon.png b/client/images/favicon.png similarity index 100% rename from web/images/favicon.png rename to client/images/favicon.png diff --git a/web/images/logo-dock.png b/client/images/logo-dock.png similarity index 100% rename from web/images/logo-dock.png rename to client/images/logo-dock.png diff --git a/web/images/logo-large.png b/client/images/logo-large.png similarity index 100% rename from web/images/logo-large.png rename to client/images/logo-large.png diff --git a/web/images/logo.ico b/client/images/logo.ico similarity index 100% rename from web/images/logo.ico rename to client/images/logo.ico diff --git a/web/images/logo.png b/client/images/logo.png similarity index 100% rename from web/images/logo.png rename to client/images/logo.png diff --git a/web/images/silver-bullet.sketch b/client/images/silver-bullet.sketch similarity index 100% rename from web/images/silver-bullet.sketch rename to client/images/silver-bullet.sketch diff --git a/client/index.html b/client/index.html new file mode 100644 index 00000000..8b285293 --- /dev/null +++ b/client/index.html @@ -0,0 +1,75 @@ + + + + + + + + + + + + {{.Title}} + + + + + + + + + + + +
+
+
+
+
{{.Content | markdown}}
+
+
+
+
+
+ + diff --git a/web/languages.ts b/client/languages.ts similarity index 100% rename from web/languages.ts rename to client/languages.ts diff --git a/web/manifest.json b/client/manifest.json similarity index 100% rename from web/manifest.json rename to client/manifest.json diff --git a/web/markdown/html_render.test.ts b/client/markdown/html_render.test.ts similarity index 100% rename from web/markdown/html_render.test.ts rename to client/markdown/html_render.test.ts diff --git a/web/markdown/html_render.ts b/client/markdown/html_render.ts similarity index 100% rename from web/markdown/html_render.ts rename to client/markdown/html_render.ts diff --git a/web/markdown/inline.ts b/client/markdown/inline.ts similarity index 100% rename from web/markdown/inline.ts rename to client/markdown/inline.ts diff --git a/web/markdown/justified_tables.ts b/client/markdown/justified_tables.ts similarity index 100% rename from web/markdown/justified_tables.ts rename to client/markdown/justified_tables.ts diff --git a/web/markdown/markdown_render.test.ts b/client/markdown/markdown_render.test.ts similarity index 100% rename from web/markdown/markdown_render.test.ts rename to client/markdown/markdown_render.test.ts diff --git a/web/markdown/markdown_render.ts b/client/markdown/markdown_render.ts similarity index 99% rename from web/markdown/markdown_render.ts rename to client/markdown/markdown_render.ts index d69562d9..30c2ef4e 100644 --- a/web/markdown/markdown_render.ts +++ b/client/markdown/markdown_render.ts @@ -12,7 +12,7 @@ import { Fragment, renderHtml, type Tag } from "./html_render.ts"; import * as TagConstants from "../../plugs/index/constants.ts"; import { extractHashtag } from "@silverbulletmd/silverbullet/lib/tags"; import { justifiedTableRender } from "./justified_tables.ts"; -import type { PageMeta } from "../../type/index.ts"; +import type { PageMeta } from "../../plug-api/types/index.ts"; import { inlineContentFromURL, parseTransclusion } from "./inline.ts"; export type MarkdownRenderOptions = { diff --git a/web/markdown/result_render.ts b/client/markdown/result_render.ts similarity index 100% rename from web/markdown/result_render.ts rename to client/markdown/result_render.ts diff --git a/web/markdown_parser/constants.ts b/client/markdown_parser/constants.ts similarity index 87% rename from web/markdown_parser/constants.ts rename to client/markdown_parser/constants.ts index 805a1cef..f1d5a0fa 100644 --- a/web/markdown_parser/constants.ts +++ b/client/markdown_parser/constants.ts @@ -3,10 +3,9 @@ export const wikiLinkRegex = export const mdLinkRegex = /!?\[(?[^\]]*)\]\((?<url>.+)\)/g; export const tagRegex = /#(?:(?:\d*[^\d\s!@#$%^&*(),.?":{}|<>\\][^\s!@#$%^&*(),.?":{}|<>\\]*)|(?:<[^>\n]+>))/; -export const nakedUrlRegex= +export const nakedUrlRegex = /(^https?:\/\/([-a-zA-Z0-9@:%_\+~#=]|(?:[.](?!(\s|$)))){1,256})(([-a-zA-Z0-9(@:%_\+~#?&=\/]|(?:[.,:;)](?!(\s|$))))*)/; -export const frontmatterUrlRegex= - /"(https?:\/\/[^\s"']+)"/g; +export const frontmatterUrlRegex = /"(https?:\/\/[^\s"']+)"/g; export const frontmatterWikiLinkRegex = /"(?<leadingTrivia>!?\[\[)(?<stringRef>.*?)(?:\|(?<alias>.*?))?(?<trailingTrivia>\]\])"/g; export const pWikiLinkRegex = new RegExp("^" + wikiLinkRegex.source); // Modified regex used only in parser diff --git a/web/markdown_parser/customtags.ts b/client/markdown_parser/customtags.ts similarity index 100% rename from web/markdown_parser/customtags.ts rename to client/markdown_parser/customtags.ts diff --git a/web/markdown_parser/extended_task.ts b/client/markdown_parser/extended_task.ts similarity index 100% rename from web/markdown_parser/extended_task.ts rename to client/markdown_parser/extended_task.ts diff --git a/web/markdown_parser/parse_tree.ts b/client/markdown_parser/parse_tree.ts similarity index 100% rename from web/markdown_parser/parse_tree.ts rename to client/markdown_parser/parse_tree.ts diff --git a/web/markdown_parser/parser.test.ts b/client/markdown_parser/parser.test.ts similarity index 100% rename from web/markdown_parser/parser.test.ts rename to client/markdown_parser/parser.test.ts diff --git a/web/markdown_parser/parser.ts b/client/markdown_parser/parser.ts similarity index 98% rename from web/markdown_parser/parser.ts rename to client/markdown_parser/parser.ts index 8dfc6731..bcf62503 100644 --- a/web/markdown_parser/parser.ts +++ b/client/markdown_parser/parser.ts @@ -13,7 +13,7 @@ import * as ct from "./customtags.ts"; import { NakedURLTag } from "./customtags.ts"; import { TaskList } from "./extended_task.ts"; import { Table } from "./table_parser.ts"; -import {nakedUrlRegex, pWikiLinkRegex, tagRegex} from "./constants.ts"; +import { nakedUrlRegex, pWikiLinkRegex, tagRegex } from "./constants.ts"; import { parse } from "./parse_tree.ts"; import type { ParseTree } from "@silverbulletmd/silverbullet/lib/tree"; import { luaLanguage } from "../../lib/space_lua/parse.ts"; @@ -291,7 +291,7 @@ function regexParser({ const NakedURL = regexParser( { firstCharCode: 104, // h - regex:new RegExp(`^${nakedUrlRegex.source}`), + regex: new RegExp(`^${nakedUrlRegex.source}`), nodeType: "NakedURL", className: "sb-naked-url", tag: NakedURLTag, diff --git a/web/markdown_parser/table_parser.ts b/client/markdown_parser/table_parser.ts similarity index 100% rename from web/markdown_parser/table_parser.ts rename to client/markdown_parser/table_parser.ts diff --git a/web/navigator.ts b/client/navigator.ts similarity index 100% rename from web/navigator.ts rename to client/navigator.ts diff --git a/web/polyfills.ts b/client/polyfills.ts similarity index 100% rename from web/polyfills.ts rename to client/polyfills.ts diff --git a/web/reducer.ts b/client/reducer.ts similarity index 99% rename from web/reducer.ts rename to client/reducer.ts index 3f3d4f6a..dc08e766 100644 --- a/web/reducer.ts +++ b/client/reducer.ts @@ -1,5 +1,5 @@ import type { Action, AppViewState } from "./ui_types.ts"; -import type { PageMeta } from "../type/index.ts"; +import type { PageMeta } from "../plug-api/types/index.ts"; import { isMarkdownPath, parseToRef, diff --git a/web/service_worker.ts b/client/service_worker.ts similarity index 64% rename from web/service_worker.ts rename to client/service_worker.ts index 3458df14..24bbc776 100644 --- a/web/service_worker.ts +++ b/client/service_worker.ts @@ -54,6 +54,9 @@ const proxyRouter = new ProxyRouter( precacheFiles, ); +// Configuration mutex +let configuring = false; + // @ts-ignore: debugging globalThis.proxyRouter = proxyRouter; @@ -103,6 +106,7 @@ self.addEventListener("message", async (event: any) => { "Ignoring perform-file-sync request, proxy not configured yet", ); } + break; } case "perform-space-sync": { @@ -131,98 +135,113 @@ self.addEventListener("message", async (event: any) => { } else { console.info("Service being configured with", config); } - const spaceFolderPath = config.spaceFolderPath; - // We're generating a simple hashed database name based on the space path in case people regularly switch between multiple space paths - const spaceHash = "" + - simpleHash(`${spaceFolderPath}:${baseURI.replace(/\/*$/, "")}`); - // And we'll use a _files postfix to signify where synced files are kept - const dbName = `${spaceHash}_files`; + if (configuring) { + console.info("Configuration already in progress, skipping"); + return; + } + // Lock configuration mutex + configuring = true; + // Put a timeout on it, just in case + setTimeout(() => { + configuring = false; + }, 5000); + try { + const spaceFolderPath = config.spaceFolderPath; + // We're generating a simple hashed database name based on the space path in case people regularly switch between multiple space paths + const spaceHash = "" + + simpleHash(`${spaceFolderPath}:${baseURI.replace(/\/*$/, "")}`); + // And we'll use a _files postfix to signify where synced files are kept + const dbName = `${spaceHash}_files`; - // Setup KV (database) for store synced files - const kv = new IndexedDBKvPrimitives(dbName); - await kv.init(); + // Setup KV (database) for store synced files + const kv = new IndexedDBKvPrimitives(dbName); + await kv.init(); - // And use that to power the IndexedDB backed local storage - const local = new DataStoreSpacePrimitives(kv); + // And use that to power the IndexedDB backed local storage + const local = new DataStoreSpacePrimitives(kv); - // Which we'll sync with the remote server - const remote = new HttpSpacePrimitives( - basePathName + fsEndpoint, - spaceFolderPath, - (message, actionOrRedirectHeader) => { - // And auth error occured - console.error( - "[service proxy error]", - message, - actionOrRedirectHeader, - ); - broadcastMessage({ - type: "auth-error", - message, - actionOrRedirectHeader, - }); - }, - ); + // Which we'll sync with the remote server + const remote = new HttpSpacePrimitives( + basePathName + fsEndpoint, + spaceFolderPath, + (message, actionOrRedirectHeader) => { + // And auth error occured + console.error( + "[service proxy error]", + message, + actionOrRedirectHeader, + ); + broadcastMessage({ + type: "auth-error", + message, + actionOrRedirectHeader, + }); + }, + ); - // Now let's setup sync - const syncEngine = new SyncEngine(kv, local, remote); - syncEngine.setSyncConfig({ - syncDocuments: config.syncDocuments, - syncIgnore: config.syncIgnore, - }); - await syncEngine.start(); + // Now let's setup sync + const syncEngine = new SyncEngine(kv, local, remote); + syncEngine.setSyncConfig({ + syncDocuments: config.syncDocuments, + syncIgnore: config.syncIgnore, + }); + await syncEngine.start(); - // Ok, we're ready to go, let's plug in the proxy router - proxyRouter.configure(syncEngine.local, syncEngine); + // Ok, we're ready to go, let's plug in the proxy router + proxyRouter.configure(syncEngine); - // And wire up some events - proxyRouter.on({ - observedRequest: (path) => { - // This is triggered for the currently open file, we want to proactively sync it to keep it up to date - syncEngine.syncSingleFile(path); - }, - onlineStatusUpdated: (isOnline) => { - broadcastMessage({ - type: "online-status", - isOnline, - }); - }, - }); - syncEngine.on({ - syncProgress: (status) => { - broadcastMessage({ - type: "sync-status", - status, - }); - }, - syncConflict: (path) => { - console.warn("Sync conflict detected:", path); - broadcastMessage({ - type: "sync-conflict", - path, - }); - }, - spaceSyncComplete: (operations) => { - console.log("Space sync complete:", operations); - broadcastMessage({ - type: "space-sync-complete", - operations, - }); - }, - fileSyncComplete: (path, operations) => { - broadcastMessage({ - type: "file-sync-complete", - path, - operations, - }); - }, - syncError: (error) => { - broadcastMessage({ - type: "sync-error", - message: error.message, - }); - }, - }); + // And wire up some events + proxyRouter.on({ + observedRequest: (path) => { + // This is triggered for the currently open file, we want to proactively sync it to keep it up to date + syncEngine.syncSingleFile(path); + }, + onlineStatusUpdated: (isOnline) => { + broadcastMessage({ + type: "online-status", + isOnline, + }); + }, + }); + syncEngine.on({ + syncProgress: (status) => { + broadcastMessage({ + type: "sync-status", + status, + }); + }, + syncConflict: (path) => { + console.warn("Sync conflict detected:", path); + broadcastMessage({ + type: "sync-conflict", + path, + }); + }, + spaceSyncComplete: (operations) => { + console.log("Space sync complete:", operations); + broadcastMessage({ + type: "space-sync-complete", + operations, + }); + }, + fileSyncComplete: (path, operations) => { + broadcastMessage({ + type: "file-sync-complete", + path, + operations, + }); + }, + syncError: (error) => { + broadcastMessage({ + type: "sync-error", + message: error.message, + }); + }, + }); + } finally { + // Unlock mutex + configuring = false; + } break; } } diff --git a/web/service_worker/proxy_router.ts b/client/service_worker/proxy_router.ts similarity index 78% rename from web/service_worker/proxy_router.ts rename to client/service_worker/proxy_router.ts index 2e414fbe..6bec2666 100644 --- a/web/service_worker/proxy_router.ts +++ b/client/service_worker/proxy_router.ts @@ -1,7 +1,7 @@ import { fsEndpoint } from "../../lib/spaces/constants.ts"; import { decodePageURI } from "@silverbulletmd/silverbullet/lib/ref"; import type { SpacePrimitives } from "../../lib/spaces/space_primitives.ts"; -import { fileMetaToHeaders, headersToFileMeta } from "../../server/util.ts"; +import { fileMetaToHeaders, headersToFileMeta } from "../../lib/util.ts"; import { notFoundError, offlineError } from "../../lib/constants.ts"; import type { SyncEngine } from "./sync_engine.ts"; import { EventEmitter } from "../../lib/plugos/event.ts"; @@ -49,8 +49,8 @@ export class ProxyRouter extends EventEmitter<ProxyRouterEvents> { /** * Called as soon the service worker is configured, and the service worker is ready to start serving requests. */ - configure(spacePrimitives: SpacePrimitives, syncEngine: SyncEngine) { - this.localSpacePrimitives = spacePrimitives; + configure(syncEngine: SyncEngine) { + this.localSpacePrimitives = syncEngine.local; this.syncEngine = syncEngine; syncEngine.on({ spaceSyncComplete: () => { @@ -95,61 +95,62 @@ export class ProxyRouter extends EventEmitter<ProxyRouterEvents> { (async () => { const request = event.request; const requestUrl = new URL(request.url); + try { + // Are we fetching a URL from the same origin as the app? If not, we don't handle it and pass it on + if (!requestUrl.href.startsWith(this.baseURI)) { + return fetch(request); + } - // Are we fetching a URL from the same origin as the app? If not, we don't handle it and pass it on - if (!requestUrl.href.startsWith(this.baseURI)) { - return fetch(request); - } + // Try the static (client) file cache first + const cachedResponse = await caches.match(cacheKey); + // Return the cached response if found + if (cachedResponse) { + return cachedResponse; + } - // Try the static (client) file cache first - const cachedResponse = await caches.match(cacheKey); - // Return the cached response if found - if (cachedResponse) { - return cachedResponse; - } - - //requestUrl.pathname without with any URL prefix removed - const pathname = requestUrl.pathname.substring( - this.basePathName.length, - ); - - if ( - // Not yet configured -> Proxy - !this.localSpacePrimitives || !this.syncEngine || - // Not fully synced but online -> Proxy - (!this.fullSyncConfirmed && this.online) || - // A path we always need to proxy -> Proxy - (alwaysProxy.includes(pathname) || pathname.startsWith("/.proxy/")) - ) { - return fetch(request); - } - - // We are now in a state we're configured and either a full sync cycle has complete (since boot) OR we're offline - - if ( - pathname.startsWith(fsEndpoint) && - pathname.endsWith(".md") && - !request.headers.has("X-Sync-Mode") - ) { - // This handles the case of ending up with a .md URL in the browser address bar (likely due to a auth proxy redirect) - return Response.redirect( - `${pathname.slice(fsEndpoint.length, -3)}`, + //requestUrl.pathname without with any URL prefix removed + const pathname = requestUrl.pathname.substring( + this.basePathName.length, ); - } else if (pathname.startsWith(fsEndpoint)) { - // Handle /.fs file system APIs - return this.handleRequest(pathname, request); - } else { - // Fallback to the app shell for all other requests (SPA) - return (await caches.match(this.precacheFiles["/"])) || - fetch(request); + + if ( + // Not yet configured -> Proxy + !this.localSpacePrimitives || !this.syncEngine || + // Not fully synced but online -> Proxy + (!this.fullSyncConfirmed && this.online) || + // A path we always need to proxy -> Proxy + (alwaysProxy.includes(pathname) || pathname.startsWith("/.proxy/")) + ) { + return fetch(request); + } + + // We are now in a state we're configured and either a full sync cycle has complete (since boot) OR we're offline + + if ( + pathname.startsWith(fsEndpoint) && + pathname.endsWith(".md") && + !request.headers.has("X-Sync-Mode") + ) { + // This handles the case of ending up with a .md URL in the browser address bar (likely due to a auth proxy redirect) + return Response.redirect( + `${pathname.slice(fsEndpoint.length, -3)}`, + ); + } else if (pathname.startsWith(fsEndpoint)) { + // Handle /.fs file system APIs + return this.handleRequest(pathname, request); + } else { + // Fallback to the app shell for all other requests (SPA) + return (await caches.match(this.precacheFiles["/"])) || + fetch(request); + } + } catch (e: any) { + console.warn("Fetch failed for", request.url, "error:", e.message); + this.online = false; + return new Response(offlineError.message, { + status: 503, // Service Unavailable + }); } - })().catch((e) => { - console.warn("Fetch failed:", e); - this.online = false; - return new Response(offlineError.message, { - status: 503, // Service Unavailable - }); - }), + })(), ); } diff --git a/web/service_worker/sync_engine.ts b/client/service_worker/sync_engine.ts similarity index 100% rename from web/service_worker/sync_engine.ts rename to client/service_worker/sync_engine.ts diff --git a/web/service_worker/util.ts b/client/service_worker/util.ts similarity index 100% rename from web/service_worker/util.ts rename to client/service_worker/util.ts diff --git a/web/space.ts b/client/space.ts similarity index 97% rename from web/space.ts rename to client/space.ts index c2ce21fb..e360ba4d 100644 --- a/web/space.ts +++ b/client/space.ts @@ -1,9 +1,13 @@ import { plugPrefix } from "../lib/spaces/constants.ts"; -import type { EventHook } from "../web/hooks/event.ts"; +import type { EventHook } from "../client/hooks/event.ts"; import { jitter, safeRun } from "../lib/async.ts"; import { localDateString } from "../lib/dates.ts"; -import type { DocumentMeta, FileMeta, PageMeta } from "../type/index.ts"; +import type { + DocumentMeta, + FileMeta, + PageMeta, +} from "../plug-api/types/index.ts"; import type { SpacePrimitives } from "../lib/spaces/space_primitives.ts"; import { getPathExtension, diff --git a/web/space_lua.ts b/client/space_lua.ts similarity index 100% rename from web/space_lua.ts rename to client/space_lua.ts diff --git a/web/space_lua_api.ts b/client/space_lua_api.ts similarity index 100% rename from web/space_lua_api.ts rename to client/space_lua_api.ts diff --git a/web/style.ts b/client/style.ts similarity index 100% rename from web/style.ts rename to client/style.ts diff --git a/web/styles/colors.scss b/client/styles/colors.scss similarity index 100% rename from web/styles/colors.scss rename to client/styles/colors.scss diff --git a/web/styles/editor.scss b/client/styles/editor.scss similarity index 100% rename from web/styles/editor.scss rename to client/styles/editor.scss diff --git a/web/styles/main.scss b/client/styles/main.scss similarity index 100% rename from web/styles/main.scss rename to client/styles/main.scss diff --git a/web/styles/modals.scss b/client/styles/modals.scss similarity index 100% rename from web/styles/modals.scss rename to client/styles/modals.scss diff --git a/web/styles/theme.scss b/client/styles/theme.scss similarity index 99% rename from web/styles/theme.scss rename to client/styles/theme.scss index 39e11dd8..1abd2c45 100644 --- a/web/styles/theme.scss +++ b/client/styles/theme.scss @@ -358,4 +358,4 @@ html[data-markdown-syntax-rendering="on"] { --admonition-icon: url('data:image/svg+xml,<svg width="16" height="16" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather feather-check-circle"><path d="M22 11.08V12a10 10 0 1 1-5.93-9.14"></path><polyline points="22 4 12 14.01 9 11.01"></polyline></svg>'); --admonition-color: #00c853; -} \ No newline at end of file +} diff --git a/web/styles/top.scss b/client/styles/top.scss similarity index 100% rename from web/styles/top.scss rename to client/styles/top.scss diff --git a/web/syscalls/clientStore.ts b/client/syscalls/clientStore.ts similarity index 90% rename from web/syscalls/clientStore.ts rename to client/syscalls/clientStore.ts index ede34fdd..12dcbe46 100644 --- a/web/syscalls/clientStore.ts +++ b/client/syscalls/clientStore.ts @@ -1,7 +1,7 @@ import type { SysCallMapping } from "../../lib/plugos/system.ts"; import type { DataStore } from "../../lib/data/datastore.ts"; -import type { KvKey } from "../../type/datastore.ts"; +import type { KvKey } from "../../plug-api/types/datastore.ts"; export function clientStoreSyscalls( ds: DataStore, diff --git a/web/syscalls/client_code_widget.ts b/client/syscalls/client_code_widget.ts similarity index 100% rename from web/syscalls/client_code_widget.ts rename to client/syscalls/client_code_widget.ts diff --git a/web/syscalls/code_widget.ts b/client/syscalls/code_widget.ts similarity index 100% rename from web/syscalls/code_widget.ts rename to client/syscalls/code_widget.ts diff --git a/web/syscalls/config.ts b/client/syscalls/config.ts similarity index 100% rename from web/syscalls/config.ts rename to client/syscalls/config.ts diff --git a/web/syscalls/datastore.ts b/client/syscalls/datastore.ts similarity index 97% rename from web/syscalls/datastore.ts rename to client/syscalls/datastore.ts index 4c9a5947..49d7e6e5 100644 --- a/web/syscalls/datastore.ts +++ b/client/syscalls/datastore.ts @@ -13,7 +13,7 @@ import { import type { KvQueryOptions } from "../../lib/data/kv_primitives.ts"; import type { ClientSystem } from "../client_system.ts"; -import type { KV, KvKey } from "../../type/datastore.ts"; +import type { KV, KvKey } from "../../plug-api/types/datastore.ts"; /** * Exposes the datastore API to plugs, but scoping everything to a prefix based on the plug's name diff --git a/web/syscalls/editor.ts b/client/syscalls/editor.ts similarity index 99% rename from web/syscalls/editor.ts rename to client/syscalls/editor.ts index 6b5e6e5f..1f64a82e 100644 --- a/web/syscalls/editor.ts +++ b/client/syscalls/editor.ts @@ -31,8 +31,8 @@ import { type Ref, } from "@silverbulletmd/silverbullet/lib/ref"; import { insertNewlineContinueMarkup } from "@codemirror/lang-markdown"; -import type { VimConfig } from "../../type/config.ts"; -import type { PageMeta } from "../../type/index.ts"; +import type { VimConfig } from "../../plug-api/types/config.ts"; +import type { PageMeta } from "../../plug-api/types/index.ts"; export function editorSyscalls(client: Client): SysCallMapping { const syscalls: SysCallMapping = { diff --git a/web/syscalls/event.ts b/client/syscalls/event.ts similarity index 91% rename from web/syscalls/event.ts rename to client/syscalls/event.ts index c0d8036f..a7ea47fe 100644 --- a/web/syscalls/event.ts +++ b/client/syscalls/event.ts @@ -3,7 +3,7 @@ import type { SysCallMapping } from "../../lib/plugos/system.ts"; import { LuaStackFrame, luaValueToJS } from "../../lib/space_lua/runtime.ts"; import type { Client } from "../client.ts"; -import type { EventSubscription } from "../../type/event.ts"; +import type { EventSubscription } from "@silverbulletmd/silverbullet/type/event"; export function eventListenerSyscalls( client: Client, diff --git a/web/syscalls/fetch.ts b/client/syscalls/fetch.ts similarity index 100% rename from web/syscalls/fetch.ts rename to client/syscalls/fetch.ts diff --git a/web/syscalls/index.ts b/client/syscalls/index.ts similarity index 100% rename from web/syscalls/index.ts rename to client/syscalls/index.ts diff --git a/web/syscalls/jsonschema.ts b/client/syscalls/jsonschema.ts similarity index 100% rename from web/syscalls/jsonschema.ts rename to client/syscalls/jsonschema.ts diff --git a/web/syscalls/language.ts b/client/syscalls/language.ts similarity index 100% rename from web/syscalls/language.ts rename to client/syscalls/language.ts diff --git a/web/syscalls/lua.ts b/client/syscalls/lua.ts similarity index 100% rename from web/syscalls/lua.ts rename to client/syscalls/lua.ts diff --git a/web/syscalls/markdown.ts b/client/syscalls/markdown.ts similarity index 100% rename from web/syscalls/markdown.ts rename to client/syscalls/markdown.ts diff --git a/web/syscalls/shell.ts b/client/syscalls/shell.ts similarity index 100% rename from web/syscalls/shell.ts rename to client/syscalls/shell.ts diff --git a/web/syscalls/space.ts b/client/syscalls/space.ts similarity index 97% rename from web/syscalls/space.ts rename to client/syscalls/space.ts index 70aaca9a..86ef98a8 100644 --- a/web/syscalls/space.ts +++ b/client/syscalls/space.ts @@ -1,7 +1,11 @@ import type { Client } from "../client.ts"; import type { SysCallMapping } from "../../lib/plugos/system.ts"; -import type { DocumentMeta, FileMeta, PageMeta } from "../../type/index.ts"; +import type { + DocumentMeta, + FileMeta, + PageMeta, +} from "../../plug-api/types/index.ts"; export function spaceReadSyscalls(editor: Client): SysCallMapping { return { diff --git a/web/syscalls/sync.ts b/client/syscalls/sync.ts similarity index 100% rename from web/syscalls/sync.ts rename to client/syscalls/sync.ts diff --git a/web/syscalls/system.ts b/client/syscalls/system.ts similarity index 98% rename from web/syscalls/system.ts rename to client/syscalls/system.ts index 65aff355..5be25e79 100644 --- a/web/syscalls/system.ts +++ b/client/syscalls/system.ts @@ -2,7 +2,7 @@ import type { SysCallMapping } from "../../lib/plugos/system.ts"; import type { Client } from "../client.ts"; import { publicVersion } from "../../public_version.ts"; import type { CommandDef } from "../../lib/manifest.ts"; -import type { SyscallMeta } from "../../type/index.ts"; +import type { SyscallMeta } from "../../plug-api/types/index.ts"; export function systemSyscalls( client: Client, diff --git a/web/syscalls/yaml.ts b/client/syscalls/yaml.ts similarity index 100% rename from web/syscalls/yaml.ts rename to client/syscalls/yaml.ts diff --git a/type/command.ts b/client/types/command.ts similarity index 83% rename from type/command.ts rename to client/types/command.ts index 550f4680..8464eebf 100644 --- a/type/command.ts +++ b/client/types/command.ts @@ -1,4 +1,4 @@ -import type { CommandDef, SlashCommandDef } from "../lib/manifest.ts"; +import type { CommandDef, SlashCommandDef } from "../../lib/manifest.ts"; import type { SlashCompletions } from "@silverbulletmd/silverbullet/type/client"; diff --git a/web/ui_types.ts b/client/ui_types.ts similarity index 96% rename from web/ui_types.ts rename to client/ui_types.ts index 7930f1dd..c8d24ef9 100644 --- a/web/ui_types.ts +++ b/client/ui_types.ts @@ -1,7 +1,11 @@ -import type { Command } from "../type/command.ts"; -import type { FilterOption, Notification, PanelMode } from "../type/client.ts"; +import type { Command } from "./types/command.ts"; +import type { + FilterOption, + Notification, + PanelMode, +} from "../plug-api/types/client.ts"; -import type { DocumentMeta, PageMeta } from "../type/index.ts"; +import type { DocumentMeta, PageMeta } from "../plug-api/types/index.ts"; import type { Path } from "@silverbulletmd/silverbullet/lib/ref"; import type { SyncStatus } from "../lib/spaces/sync.ts"; diff --git a/cmd/README.md b/cmd/README.md deleted file mode 100644 index e810f573..00000000 --- a/cmd/README.md +++ /dev/null @@ -1 +0,0 @@ -Code that implements `silverbullet` sub commands \ No newline at end of file diff --git a/cmd/plug_compile.ts b/cmd/plug_compile.ts deleted file mode 100644 index 52bb74b4..00000000 --- a/cmd/plug_compile.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { compileManifests } from "./compile.ts"; -import * as esbuild from "esbuild"; - -export async function plugCompileCommand( - { watch, dist, debug, info, importmap, config, runtimeUrl }: { - watch: boolean; - dist: string; - debug: boolean; - info: boolean; - importmap?: string; - config?: string; - runtimeUrl?: string; - }, - ...manifestPaths: string[] -) { - await compileManifests( - manifestPaths, - dist, - watch, - { - debug: debug, - info: info, - runtimeUrl, - importMap: importmap - ? new URL(importmap, `file://${Deno.cwd()}/`).toString() - : undefined, - configPath: config, - }, - ); - esbuild.stop(); - Deno.exit(0); -} diff --git a/cmd/server.ts b/cmd/server.ts deleted file mode 100644 index affe31da..00000000 --- a/cmd/server.ts +++ /dev/null @@ -1,199 +0,0 @@ -import { HttpServer } from "../server/http_server.ts"; - -// To ship silverbullet as a single binary, we're importing all assets (client and plugs) as JSON blobs -import clientAssetBundle from "../dist/client_asset_bundle.json" with { - type: "json", -}; -import plugAssetBundle from "../dist/plug_asset_bundle.json" with { - type: "json", -}; - -import { AssetBundle, type AssetJson } from "../lib/asset_bundle/bundle.ts"; - -import { resolve } from "@std/path"; -import { sleep } from "../lib/async.ts"; -import { MemoryKvPrimitives } from "../lib/data/memory_kv_primitives.ts"; -import { isValidName } from "@silverbulletmd/silverbullet/lib/ref"; - -export type AuthOptions = { - authToken?: string; - user: string; - pass: string; - lockoutTime: number; - lockoutLimit: number; -}; - -export async function serveCommand( - options: { - hostname?: string; - port?: number; - user?: string; - auth?: string; - }, - folder?: string, -) { - const hostname = options.hostname || Deno.env.get("SB_HOSTNAME") || - "127.0.0.1"; - const port = options.port || - (Deno.env.get("SB_PORT") && +Deno.env.get("SB_PORT")!) || 3000; - - const readOnly = !!Deno.env.get("SB_READ_ONLY"); - - if (readOnly) { - console.info("Starting in read-only mode."); - } - - const indexPage = Deno.env.get("SB_INDEX_PAGE") || "index"; - if (!isValidName(indexPage)) { - console.error( - "SB_INDEX_PAGE has to be a valid page name.", - ); - Deno.exit(1); - } - - folder = folder || Deno.env.get("SB_FOLDER"); - if (!folder) { - console.error( - "No folder specified. Please pass a folder as an argument or set SB_FOLDER environment variable.", - ); - Deno.exit(1); - } - - // This database is only used to store some auth based temporary values - let dbFile = Deno.env.get("SB_KV_DB") || ".silverbullet.db.json"; - dbFile = resolve(folder, dbFile); - - console.info( - `Storing database in ${dbFile}.`, - ); - - // The only use case of this KV is to store auth-related stuff (keys, hashes) - const baseKvPrimitives = new MemoryKvPrimitives(dbFile, { - throttleMs: 100, - }); - await baseKvPrimitives.init(); - - console.info("Starting SilverBullet binding to", `${hostname}:${port}`); - if (hostname === "127.0.0.1") { - console.info( - `SilverBullet will only be available locally, to allow outside connections, pass -L0.0.0.0 as a flag, and put a TLS terminator on top.`, - ); - } - - let hostUrlPrefix = Deno.env.get("SB_URL_PREFIX"); - if (hostUrlPrefix) { - // Normalize the host URL prefix (if set) - if (!hostUrlPrefix.startsWith("/")) { - hostUrlPrefix = "/" + hostUrlPrefix; - } - if (hostUrlPrefix.endsWith("/")) { - hostUrlPrefix = hostUrlPrefix.replace(/\/*$/, ""); - } - - if (hostUrlPrefix !== "") { - console.log(`Host URL Prefix: ${hostUrlPrefix}`); - } else { - hostUrlPrefix = undefined; - } - } - - // Built-in authentication setup - const userAuth = options.user || Deno.env.get("SB_USER"); - let userCredentials: AuthOptions | undefined; - if (userAuth) { - const [user, pass] = userAuth.split(":"); - userCredentials = { - user, - pass, - authToken: Deno.env.get("SB_AUTH_TOKEN"), - // Default lockout: 10 failed login attempts in 1 minute - lockoutLimit: Deno.env.get("SB_LOCKOUT_LIMIT") - ? Number(Deno.env.get("SB_LOCKOUT_LIMIT")) - : 10, - lockoutTime: Deno.env.get("SB_LOCKOUT_TIME") - ? Number(Deno.env.get("SB_LOCKOUT_TIME")) - : 60, - }; - console.log( - `User authentication enabled for user "${user}" with lockout limit ${userCredentials.lockoutLimit} and lockout time ${userCredentials.lockoutTime}s`, - ); - } - - const shellBackend = Deno.env.get("SB_SHELL_BACKEND") || "local"; - const shellCommandWhiteList = Deno.env.get("SB_SHELL_WHITELIST")?.split(" "); - - const spaceIgnore = Deno.env.get("SB_SPACE_IGNORE"); - if (spaceIgnore) { - console.log(`Ignoring files matching: ${spaceIgnore}`); - } - - // All plug code bundled into a JSON blob - const plugAssets = new AssetBundle(plugAssetBundle as AssetJson); - // All client files bundled into a JSON blob - const clientAssets = new AssetBundle(clientAssetBundle as AssetJson); - - // Manifest configuration - buildPWAManifest(hostUrlPrefix, clientAssets); - - // Build the web server - const httpServer = new HttpServer( - { - hostname, - port, - hostUrlPrefix, - indexPage, - spaceIgnore, - auth: userCredentials, - readOnly, - shellBackend, - pagesPath: folder, - shellCommandWhiteList, - }, - clientAssets, - plugAssets, - baseKvPrimitives, - ); - - // And kick it off - await httpServer.start(); - - // Wait in an infinite loop (to keep the HTTP server running, only cancelable via Ctrl+C or other signal) - while (true) { - await sleep(10000); - } -} - -/** - * Overrides SilverBullet's manifest.json dynamically based on some (optional) environment variables - */ -function buildPWAManifest( - hostUrlPrefix: string | undefined, - clientAssets: AssetBundle, -) { - const manifestName = Deno.env.get("SB_NAME"); - const manifestDescription = Deno.env.get("SB_DESCRIPTION"); - - // Parse the default manifest.json - const manifestData = JSON.parse( - clientAssets.readTextFileSync(".client/manifest.json"), - ); - if (manifestName) { - manifestData.name = manifestData.short_name = manifestName; - } - if (manifestDescription) { - manifestData.description = manifestDescription; - } - if (hostUrlPrefix) { - for (const icon of manifestData.icons) { - if (icon.src) icon.src = hostUrlPrefix + icon.src; - } - manifestData.start_url = hostUrlPrefix + manifestData.start_url; - manifestData.scope = hostUrlPrefix + manifestData.scope; - } - // Write it back into the asset bundle to be served by the HTTP server - clientAssets.writeTextFileSync( - ".client/manifest.json", - "application/json", - JSON.stringify(manifestData), - ); -} diff --git a/cmd/sync.ts b/cmd/sync.ts deleted file mode 100644 index c6b13abe..00000000 --- a/cmd/sync.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { SpaceSync, SyncSnapshot } from "../lib/spaces/sync.ts"; -import { determineStorageBackend } from "../server/storage_backend.ts"; - -export async function syncCommand( - options: { - snapshot?: string; - wipeSecondary?: boolean; - }, - primary: string, - secondary: string, -) { - console.log("Going to synchronize", primary, "and", secondary); - - const primarySpacePrimitives = determineStorageBackend( - primary, - ); - const secondarySpacePrimitives = determineStorageBackend( - secondary, - ); - - if (options.wipeSecondary) { - if ( - !confirm( - `About to wipe the secondary storage at ${secondary}, are you sure?`, - ) - ) { - return; - } - const allFiles = await secondarySpacePrimitives.fetchFileList(); - for (const file of allFiles) { - try { - console.log("Deleting", file.name); - await secondarySpacePrimitives.deleteFile(file.name); - } catch (e: any) { - console.warn("Failed to delete file", file.name, e.message); - } - } - console.log("Done wiping secondary storage."); - } - - let snapshot = new SyncSnapshot(); - if (options.snapshot) { - try { - snapshot = SyncSnapshot.fromJSON( - JSON.parse(await Deno.readTextFile(options.snapshot)), - ); - } catch (e: any) { - console.warn( - "Failed to read snapshot file", - e.message, - "using empty snapshot", - ); - } - } - const sync = new SpaceSync( - primarySpacePrimitives, - secondarySpacePrimitives, - { - conflictResolver: SpaceSync.primaryConflictResolver, - isSyncCandidate: () => true, - }, - ); - - if (options.snapshot) { - sync.on({ - snapshotUpdated: async (snapshot) => { - await Deno.writeTextFile( - options.snapshot!, - JSON.stringify(snapshot.toJSON()), - ); - }, - }); - } - - await sync.syncFiles(snapshot); -} diff --git a/cmd/test/plug_test.ts b/cmd/test/plug_test.ts deleted file mode 100644 index 8140774e..00000000 --- a/cmd/test/plug_test.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { datastore } from "@silverbulletmd/silverbullet/syscalls"; - -export async function run() { - console.log("Hello from plug_test.ts"); - await datastore.set(["plug_test"], "Hello"); - return "Hello"; -} diff --git a/cmd/test/runtime.test.ts b/cmd/test/runtime.test.ts deleted file mode 100644 index 80d07037..00000000 --- a/cmd/test/runtime.test.ts +++ /dev/null @@ -1,58 +0,0 @@ -import type { SysCallMapping } from "../../lib/plugos/system.ts"; -import { System } from "../../lib/plugos/system.ts"; -import { assertEquals } from "@std/assert"; -import { compileManifest } from "../compile.ts"; -import * as esbuild from "esbuild"; -import { fileURLToPath } from "node:url"; -import { WorkerSandbox } from "../../lib/plugos/sandboxes/worker_sandbox.ts"; - -Deno.test("Run a deno sandbox", { - sanitizeResources: false, - sanitizeOps: false, -}, async () => { - const system = new System("server"); - system.registerSyscalls([], { - addNumbers: (_ctx, a, b) => { - return a + b; - }, - failingSyscall: () => { - throw new Error("#fail"); - }, - } as SysCallMapping); - system.registerSyscalls(["restricted"], { - restrictedSyscall: () => { - return "restricted"; - }, - }); - system.registerSyscalls(["dangerous"], { - dangerousSyscall: () => { - return "yay"; - }, - }); - - const tempDir = await Deno.makeTempDir(); - - const workerPath = await compileManifest( - fileURLToPath(new URL("test_runtime.plug.yaml", import.meta.url)), - tempDir, - { - configPath: fileURLToPath(new URL("../../deno.json", import.meta.url)), - }, - ); - - const plug = await system.loadPlug( - (plug) => new WorkerSandbox(plug, new URL(`file://${workerPath}`)), - "test", - ); - - assertEquals({ - addedNumbers: 3, - jsonMessage: JSON.stringify({ hello: "world" }), - }, await plug.invoke("boot", [])); - - await system.unloadAll(); - - await Deno.remove(tempDir, { recursive: true }); - - esbuild.stop(); -}); diff --git a/cmd/test/test_func.test.ts b/cmd/test/test_func.test.ts deleted file mode 100644 index 64fb77ca..00000000 --- a/cmd/test/test_func.test.ts +++ /dev/null @@ -1,10 +0,0 @@ -// import * as YAML from "js-yaml"; - -export async function hello() { - // @ts-ignore: syscall is a global function - const numbers = await syscall("addNumbers", 1, 2); - return { - jsonMessage: JSON.stringify({ hello: "world" }), - addedNumbers: numbers, - }; -} diff --git a/cmd/test/test_plug_run.plug.yaml b/cmd/test/test_plug_run.plug.yaml deleted file mode 100644 index 696c0961..00000000 --- a/cmd/test/test_plug_run.plug.yaml +++ /dev/null @@ -1,6 +0,0 @@ -name: test -requiredPermissions: - - shell -functions: - run: - path: plug_test.ts:run diff --git a/cmd/test/test_runtime.plug.yaml b/cmd/test/test_runtime.plug.yaml deleted file mode 100644 index db3c5dca..00000000 --- a/cmd/test/test_runtime.plug.yaml +++ /dev/null @@ -1,4 +0,0 @@ -name: test -functions: - boot: - path: "./test_func.test.ts:hello" diff --git a/cmd/update_version.ts b/cmd/update_version.ts deleted file mode 100644 index 61e4ae7d..00000000 --- a/cmd/update_version.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { version } from "../version.ts"; - -export async function updateVersionFile() { - const command = new Deno.Command("git", { - args: ["describe", "--tags", "--long"], - stdout: "piped", - stderr: "piped", - }); - - const { stdout } = await command.output(); - let commitVersion = new TextDecoder().decode(stdout).trim(); - - if (!commitVersion) { - // Probably no valid .git repo, fallback to GITHUB_SHA env var (used in CI) - commitVersion = `${version}-${Deno.env.get("GITHUB_SHA") || "unknown"}`; - } - - const versionFilePath = "./public_version.ts"; - const versionContent = `export const publicVersion = "${commitVersion}";`; - - await Deno.writeTextFile(versionFilePath, versionContent); - console.log( - `Updated public_version.ts with version information: ${commitVersion}`, - ); -} diff --git a/cmd/upgrade.ts b/cmd/upgrade.ts deleted file mode 100644 index c71d1a99..00000000 --- a/cmd/upgrade.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { dirname } from "@std/path"; - -async function upgrade(urlPrefix: string) { - const installDir = dirname(Deno.execPath()); - - console.log("Install dir:", installDir); - - const tmpDir = Deno.makeTempDirSync(); - const zipUrl = urlPrefix + - `/silverbullet-server-${Deno.build.os}-${Deno.build.arch}.zip`; - const zipPath = `${tmpDir}/silverbullet.zip`; - console.log("Downloading from", zipUrl); - const command = new Deno.Command("curl", { - args: [ - "-L", - "-o", - zipPath, - zipUrl, - ], - }); - const curlOutput = await command.output(); - if (curlOutput.code !== 0) { - console.error( - "Download failed", - new TextDecoder().decode(curlOutput.stderr), - ); - return; - } - console.log( - "Now going to replace the existing silverbullet binary in", - installDir, - ); - - const unzipCommand = new Deno.Command("unzip", { - args: [ - "-o", - "-d", - installDir, - `${tmpDir}/silverbullet.zip`, - ], - }); - const zipOutput = await unzipCommand.output(); - if (zipOutput.code !== 0) { - console.error( - "Download failed", - new TextDecoder().decode(curlOutput.stderr), - ); - return; - } - await Deno.chmod(`${installDir}/silverbullet`, 0o755); - await Deno.remove(zipPath); - console.log("And done! Restart your server to get the latest and greatest!"); -} - -export async function upgradeCommand() { - console.log("Now going to attempt an upgrade..."); - await upgrade( - "https://github.com/silverbulletmd/silverbullet/releases/latest/download", - ); -} - -export async function upgradeEdgeCommand() { - console.log("Now going to attempt an edge upgrade..."); - await upgrade( - "https://github.com/silverbulletmd/silverbullet/releases/download/edge", - ); -} diff --git a/cmd/version.ts b/cmd/version.ts deleted file mode 100644 index fd6170e3..00000000 --- a/cmd/version.ts +++ /dev/null @@ -1,5 +0,0 @@ -import { publicVersion } from "../public_version.ts"; - -export function versionCommand() { - console.log(publicVersion); -} diff --git a/deno.json b/deno.json index a409e93e..2dbcb369 100644 --- a/deno.json +++ b/deno.json @@ -2,47 +2,23 @@ "name": "@silverbulletmd/silverbullet", "version": "2.0.0", "tasks": { - "build": "rm -rf dist dist_client_bundle dist_plug_bundle && deno run -A build_plugs.ts && deno run -A build_web.ts --production && deno task compile", - "clean": "rm -rf dist dist_client_bundle dist_plug_bundle", + "build": "deno task clean && deno run -A build_plugs_libraries.ts && deno run -A build_client.ts", + "build-production": "deno task build --production", + "build-plug-compile": "deno run -A build_plug_compile.ts", + "clean": "rm -rf dist_client_bundle dist_base_fs_bundle dist public_version.ts", "deep-clean-mac": "rm -f deno.lock && rm -rf $HOME/Library/Caches/deno && deno task clean", - "install": "deno install -g -f -A --import-map deno.json silverbullet.ts", "check": "find . -name '*.ts*' -not -path \"*/node_modules/*\" | xargs deno check", "checks": "deno task check && deno task lint && deno task test", "lint": "deno lint --fix", "fmt": "deno fmt", "test": "deno test -A", - "bench": "deno bench", - "plugs": "deno run -A build_plugs.ts", - "server": "deno run -A --check silverbullet.ts", - "generate": "./scripts/generate.sh", - "watch-web": "deno run -A --check build_web.ts --watch", - "watch-server": "deno run -A --check --watch silverbullet.ts", - "watch-plugs": "deno run -A --check build_plugs.ts -w", - "watch-all": "deno task watch-web & deno task watch-plugs & deno task watch-server", - "bundle": "deno run -A build_bundle.ts", - "compile": "deno task bundle && deno compile -A -o silverbullet dist/silverbullet.js", - "server:dist:linux-x86_64": "deno task bundle && deno compile -A --target x86_64-unknown-linux-gnu -o silverbullet dist/silverbullet.js", - "server:zip:linux-x86_64": "deno task server:dist:linux-x86_64 && zip silverbullet-server-linux-x86_64.zip silverbullet", - "server:dist:linux-aarch64": "deno task bundle && deno compile -A --target aarch64-unknown-linux-gnu -o silverbullet dist/silverbullet.js", - "server:zip:linux-aarch64": "deno task server:dist:linux-aarch64 && zip silverbullet-server-linux-aarch64.zip silverbullet", - "server:dist:darwin-x86_64": "deno task bundle && deno compile -A --target x86_64-apple-darwin -o silverbullet dist/silverbullet.js", - "server:zip:darwin-x86_64": "deno task server:dist:darwin-x86_64 && zip silverbullet-server-darwin-x86_64.zip silverbullet", - "server:dist:darwin-aarch64": "deno task bundle && deno compile -A --target aarch64-apple-darwin -o silverbullet dist/silverbullet.js", - "server:zip:darwin-aarch64": "deno task server:dist:darwin-aarch64 && zip silverbullet-server-darwin-aarch64.zip silverbullet", - "server:dist:windows-x86_64": "deno task bundle && deno compile -A --target x86_64-pc-windows-msvc -o silverbullet.exe dist/silverbullet.js", - "server:zip:windows-x86_64": "deno task server:dist:windows-x86_64 && zip silverbullet-server-windows-x86_64.zip silverbullet.exe" + "plugs": "deno run -A build_plugs_libraries.ts" }, "publish": { "exclude": ["website", "CHANGELOG.md", "**/*.md"] }, "lint": { - "exclude": [ - "dist", - "dist_client_bundle", - "dist_plug_bundle", - "cmd/test_space", - "cmd/test/test_space" - ], + "exclude": ["dist", "dist_client_bundle", "dist_base_fs_bundle"], "rules": { "exclude": ["no-explicit-any", "no-slow-types", "jsx-boolean-value"] } @@ -51,7 +27,6 @@ "exclude": [ "dist*", "website", - "test_space", "**/*.md", "**/*.html", "**/*.js", @@ -72,27 +47,21 @@ "./lib/ref": "./plug-api/lib/ref.ts", "./lib/resolve": "./plug-api/lib/resolve.ts", "./lib/frontmatter": "./plug-api/lib/frontmatter.ts", - "./type/client": "./type/client.ts", - "./type/command": "./type/command.ts", - "./type/config": "./type/config.ts", - "./type/datastore": "./type/datastore.ts", - "./type/event": "./type/event.ts", - "./type/index": "./type/index.ts", - "./type/rpc": "./type/rpc.ts", - "./lib/tags": "./plug-api/lib/tags.ts" + "./lib/tags": "./plug-api/lib/tags.ts", + "./type/client": "./plug-api/types/client.ts", + "./type/config": "./plug-api/types/config.ts", + "./type/datastore": "./plug-api/types/datastore.ts", + "./type/event": "./plug-api/types/event.ts", + "./type/index": "./plug-api/types/index.ts" }, "imports": { - "@cliffy/command": "jsr:@cliffy/command@^1.0.0-rc.7", - "@cliffy/prompt": "jsr:@cliffy/prompt@^1.0.0-rc.7", "@silverbulletmd/silverbullet/syscall": "./plug-api/syscall.ts", "@silverbulletmd/silverbullet/syscalls": "./plug-api/syscalls.ts", - "@silverbulletmd/silverbullet/type/client": "./type/client.ts", - "@silverbulletmd/silverbullet/type/command": "./type/command.ts", - "@silverbulletmd/silverbullet/type/config": "./type/config.ts", - "@silverbulletmd/silverbullet/type/datastore": "./type/datastore.ts", - "@silverbulletmd/silverbullet/type/event": "./type/event.ts", - "@silverbulletmd/silverbullet/type/index": "./type/index.ts", - "@silverbulletmd/silverbullet/type/rpc": "./type/rpc.ts", + "@silverbulletmd/silverbullet/type/client": "./plug-api/types/client.ts", + "@silverbulletmd/silverbullet/type/config": "./plug-api/types/config.ts", + "@silverbulletmd/silverbullet/type/datastore": "./plug-api/types/datastore.ts", + "@silverbulletmd/silverbullet/type/event": "./plug-api/types/event.ts", + "@silverbulletmd/silverbullet/type/index": "./plug-api/types/index.ts", "@silverbulletmd/silverbullet/lib/json": "./plug-api/lib/json.ts", "@silverbulletmd/silverbullet/lib/tree": "./plug-api/lib/tree.ts", "@silverbulletmd/silverbullet/lib/attribute": "./plug-api/lib/attribute.ts", @@ -126,6 +95,7 @@ "@replit/codemirror-lang-nix": "npm:@replit/codemirror-lang-nix@6.0.1", "@replit/codemirror-vim": "npm:@replit/codemirror-vim@6.3.0", "@std/assert": "jsr:@std/assert@^1.0.1", + "@cliffy/command": "jsr:@cliffy/command@^1.0.0-rc.7", "@std/cli": "jsr:@std/cli@^1.0.1", "@std/fs": "jsr:@std/fs@^1.0.18", "@std/io": "jsr:@std/io@^0.225.2", @@ -135,16 +105,12 @@ "@std/yaml": "jsr:@std/yaml@^1.0.8", "crelt": "npm:crelt@1.0.6", "denosass": "https://deno.land/x/denosass@1.0.6/mod.ts", - "djwt": "https://deno.land/x/djwt@v3.0.2/mod.ts", "esbuild": "https://deno.land/x/esbuild@v0.25.1/mod.js", "fake-indexeddb/auto": "npm:fake-indexeddb@6.0.1/auto", "fast-diff": "npm:fast-diff@1.3.0", "fuse": "https://deno.land/x/fuse@v6.4.1/dist/fuse.esm.min.js", "gitignore-parser": "npm:gitignore-parser@0.0.2", "idb": "npm:idb@8.0.3", - "hono": "jsr:@hono/hono@4.9.2", - "hono/cookie": "jsr:@hono/hono@4.9.2/cookie", - "hono/validator": "jsr:@hono/hono@4.9.2/validator", "js-yaml": "npm:js-yaml@4.1.0", "mimetypes": "https://deno.land/x/mimetypes@v1.0.0/mod.ts", "porter-stemmer": "npm:porter-stemmer@0.9.1", @@ -154,7 +120,6 @@ "react-icons/md": "https://deno.land/x/react_icons_md@1.0.9/mod.ts", "react-icons/md/": "https://deno.land/x/react_icons_md@1.0.9/ico/", "react-icons/types": "https://deno.land/x/react_icons@1.0.9/mod.ts", - "s3_lite_client/": "https://deno.land/x/s3_lite_client@0.4.0/", "style-mod": "npm:style-mod@4.1.2", "turndown": "npm:turndown@7.2.0", "ajv": "npm:ajv@8.17.1", diff --git a/deno.lock b/deno.lock index d53b5250..11d63600 100644 --- a/deno.lock +++ b/deno.lock @@ -8,7 +8,6 @@ "jsr:@cliffy/internal@1.0.0-rc.8": "1.0.0-rc.8", "jsr:@cliffy/table@1.0.0-rc.7": "1.0.0-rc.7", "jsr:@cliffy/table@1.0.0-rc.8": "1.0.0-rc.8", - "jsr:@hono/hono@4.9.2": "4.9.2", "jsr:@luca/esbuild-deno-loader@~0.11.1": "0.11.1", "jsr:@std/assert@^1.0.1": "1.0.13", "jsr:@std/async@^1.0.13": "1.0.13", @@ -45,6 +44,7 @@ "npm:@joplin/turndown-plugin-gfm@1.0.62": "1.0.62", "npm:@lezer/common@1.2.3": "1.2.3", "npm:@lezer/css@1.2.1": "1.2.1", + "npm:@lezer/generator@1.5.1": "1.5.1", "npm:@lezer/highlight@1.2.1": "1.2.1", "npm:@lezer/html@1.3.10": "1.3.10", "npm:@lezer/javascript@1.5.1": "1.5.1", @@ -119,9 +119,6 @@ "jsr:@std/fmt" ] }, - "@hono/hono@4.9.2": { - "integrity": "1a8ae7b0d9e4fecef440f991c604e032d5548dc313d1961073ac5d94809be52b" - }, "@luca/esbuild-deno-loader@0.11.1": { "integrity": "dc020d16d75b591f679f6b9288b10f38bdb4f24345edb2f5732affa1d9885267", "dependencies": [ @@ -334,6 +331,14 @@ "@lezer/lr" ] }, + "@lezer/generator@1.5.1": { + "integrity": "sha512-vodJv2JPwsFsiBBHE463yBhvUI9TmhIu5duF/8MH304xNS6FyWH/vTyG61pjhERm5f+VBP94co0eiN+afWcvXw==", + "dependencies": [ + "@lezer/common", + "@lezer/lr" + ], + "bin": true + }, "@lezer/highlight@1.2.1": { "integrity": "sha512-Z5duk4RN/3zuVO7Jq0pGLJ3qynpxUVsh7IbUbGj88+uV2ApSAn6kWg2au3iJb+0Zi7kKtqffIESgNcRXWZWmSA==", "dependencies": [ @@ -589,8 +594,6 @@ "workspace": { "dependencies": [ "jsr:@cliffy/command@^1.0.0-rc.7", - "jsr:@cliffy/prompt@^1.0.0-rc.7", - "jsr:@hono/hono@4.9.2", "jsr:@luca/esbuild-deno-loader@~0.11.1", "jsr:@std/assert@^1.0.1", "jsr:@std/cli@^1.0.1", diff --git a/go.mod b/go.mod index f6323a90..04eb5392 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,24 @@ module github.com/silverbulletmd/silverbullet go 1.25.1 + +require ( + github.com/Diogenesoftoronto/go-gitignore v1.1.0 + github.com/djherbis/times v1.6.0 + github.com/go-chi/chi/v5 v5.2.3 + github.com/go-chi/render v1.0.3 + github.com/golang-jwt/jwt/v5 v5.3.0 + github.com/gomarkdown/markdown v0.0.0-20250810172220-2e2c11897d1a + github.com/spf13/cobra v1.10.1 + github.com/stretchr/testify v1.11.1 +) + +require ( + github.com/ajg/form v1.5.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/spf13/pflag v1.0.9 // indirect + golang.org/x/sys v0.36.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..241f6061 --- /dev/null +++ b/go.sum @@ -0,0 +1,35 @@ +github.com/Diogenesoftoronto/go-gitignore v1.1.0 h1:G8jh5GxjmCidSTvotSiNeBMxMLma9zOtjb2wZ/QvLk8= +github.com/Diogenesoftoronto/go-gitignore v1.1.0/go.mod h1:mmVD39ljrmsXQECsd829osXfYy+nzbxqYEH2rCLE+XA= +github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= +github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= +github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= +github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE= +github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops= +github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4= +github.com/go-chi/render v1.0.3/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/gomarkdown/markdown v0.0.0-20250810172220-2e2c11897d1a h1:l7A0loSszR5zHd/qK53ZIHMO8b3bBSmENnQ6eKnUT0A= +github.com/gomarkdown/markdown v0.0.0-20250810172220-2e2c11897d1a/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= +github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/install-edge.sh b/install-edge.sh deleted file mode 100755 index 2ba5fabd..00000000 --- a/install-edge.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh - -set -e - -if ! command -v unzip >/dev/null; then - echo "Error: unzip is required to install SilverBullet." 1>&2 - exit 1 -fi - -case $(uname -sm) in -"Darwin x86_64") target="darwin-x86_64" ;; -"Darwin arm64") target="darwin-aarch64" ;; -"Linux arm64") target="linux-aarch64" ;; -*) target="linux-x86_64" ;; -esac - -echo "Installing for $target" - -sb_uri="https://github.com/silverbulletmd/silverbullet/releases/download/edge/silverbullet-server-${target}.zip" - -exe=silverbullet -bin_dir=. - -curl --fail --location --progress-bar --output "$exe.zip" "$sb_uri" -unzip -d "$bin_dir" -o "$exe.zip" -chmod +x "$exe" -rm "$exe.zip" - -echo "SilverBullet server was installed successfully to $bin_dir, run it directly via ./$exe or move it to a more convenient place." \ No newline at end of file diff --git a/lib/asset_bundle/builder.ts b/lib/asset_bundle/builder.ts index 456e9e30..54729119 100644 --- a/lib/asset_bundle/builder.ts +++ b/lib/asset_bundle/builder.ts @@ -1,5 +1,5 @@ // import { globToRegExp, mime, path, walk } from "../deps_server.ts"; -import { dirname, globToRegExp } from "@std/path"; +import { globToRegExp } from "@std/path"; import { AssetBundle } from "./bundle.ts"; import { walk } from "@std/fs"; import { mime } from "mimetypes"; @@ -35,29 +35,3 @@ export async function bundleAssets( } return bundle; } - -export async function bundleFolder( - rootPath: string, - bundlePath: string, -) { - const bundle = new AssetBundle(); - - await Deno.mkdir(dirname(bundlePath), { recursive: true }); - for await ( - const { path: filePath } of walk(rootPath, { includeDirs: false }) - ) { - console.log("Bundling", filePath); - const stat = await Deno.stat(filePath); - const cleanPath = filePath.substring(`${rootPath}/`.length); - bundle.writeFileSync( - cleanPath, - mime.getType(filePath) || "application/octet-stream", - await Deno.readFile(filePath), - stat.mtime?.getTime(), - ); - } - await Deno.writeTextFile( - bundlePath, - JSON.stringify(bundle.toJSON(), null, 2), - ); -} diff --git a/lib/data/datastore.ts b/lib/data/datastore.ts index 4c5632c5..6ada8fd0 100644 --- a/lib/data/datastore.ts +++ b/lib/data/datastore.ts @@ -5,7 +5,7 @@ import { import { LuaEnv, LuaStackFrame } from "../space_lua/runtime.ts"; import type { KvPrimitives, KvQueryOptions } from "./kv_primitives.ts"; -import type { KV, KvKey } from "../../type/datastore.ts"; +import type { KV, KvKey } from "../../plug-api/types/datastore.ts"; /** * This is the data store class you'll actually want to use, wrapping the primitives diff --git a/lib/data/indexeddb_kv_primitives.ts b/lib/data/indexeddb_kv_primitives.ts index d592985f..448c3b64 100644 --- a/lib/data/indexeddb_kv_primitives.ts +++ b/lib/data/indexeddb_kv_primitives.ts @@ -1,7 +1,7 @@ import type { KvPrimitives, KvQueryOptions } from "./kv_primitives.ts"; import { type IDBPDatabase, openDB } from "idb"; -import type { KV, KvKey } from "../../type/datastore.ts"; +import type { KV, KvKey } from "../../plug-api/types/datastore.ts"; // Separator character to use for key serialization const sep = "\0"; diff --git a/lib/data/kv_primitives.test.ts b/lib/data/kv_primitives.test.ts index 8cf08652..5dfb317c 100644 --- a/lib/data/kv_primitives.test.ts +++ b/lib/data/kv_primitives.test.ts @@ -1,7 +1,7 @@ import type { KvPrimitives } from "./kv_primitives.ts"; import { assertEquals } from "@std/assert"; -import type { KV } from "../../type/datastore.ts"; +import type { KV } from "../../plug-api/types/datastore.ts"; export async function allTests(db: KvPrimitives) { await db.batchSet([ diff --git a/lib/data/kv_primitives.ts b/lib/data/kv_primitives.ts index 53d310f1..ca72ee89 100644 --- a/lib/data/kv_primitives.ts +++ b/lib/data/kv_primitives.ts @@ -1,4 +1,4 @@ -import type { KV, KvKey } from "../../type/datastore.ts"; +import type { KV, KvKey } from "../../plug-api/types/datastore.ts"; export type KvQueryOptions = { prefix?: KvKey; diff --git a/lib/data/memory_kv_primitives.test.ts b/lib/data/memory_kv_primitives.test.ts index 4be89d9c..313ca6ae 100644 --- a/lib/data/memory_kv_primitives.test.ts +++ b/lib/data/memory_kv_primitives.test.ts @@ -2,7 +2,7 @@ import { assertEquals } from "@std/assert"; import { MemoryKvPrimitives } from "./memory_kv_primitives.ts"; import { allTests } from "./kv_primitives.test.ts"; -import type { KV } from "../../type/datastore.ts"; +import type { KV } from "../../plug-api/types/datastore.ts"; Deno.test("MemoryKvPrimitives loads from non-existent file without error", async () => { const tempPath = await Deno.makeTempFile() + "_nonexistent"; diff --git a/lib/data/memory_kv_primitives.ts b/lib/data/memory_kv_primitives.ts index f38594fb..75f55ed6 100644 --- a/lib/data/memory_kv_primitives.ts +++ b/lib/data/memory_kv_primitives.ts @@ -1,7 +1,7 @@ import type { KvPrimitives, KvQueryOptions } from "./kv_primitives.ts"; import { throttle } from "../async.ts"; -import type { KV, KvKey } from "../../type/datastore.ts"; +import type { KV, KvKey } from "../../plug-api/types/datastore.ts"; const memoryKeySeparator = "\0"; diff --git a/lib/data/mq.datastore.test.ts b/lib/data/mq.datastore.test.ts index 350ffa8b..f05a1b10 100644 --- a/lib/data/mq.datastore.test.ts +++ b/lib/data/mq.datastore.test.ts @@ -5,7 +5,7 @@ import { DataStore } from "./datastore.ts"; import { PrefixedKvPrimitives } from "./prefixed_kv_primitives.ts"; import { FakeTime } from "@std/testing/time"; -import type { MQMessage } from "../../type/datastore.ts"; +import type { MQMessage } from "../../plug-api/types/datastore.ts"; Deno.test("DataStore MQ", async () => { const time = new FakeTime(); diff --git a/lib/data/mq.datastore.ts b/lib/data/mq.datastore.ts index 57fb4f86..8e425da4 100644 --- a/lib/data/mq.datastore.ts +++ b/lib/data/mq.datastore.ts @@ -7,7 +7,7 @@ import type { MQMessage, MQStats, MQSubscribeOptions, -} from "../../type/datastore.ts"; +} from "../../plug-api/types/datastore.ts"; import { race, sleep } from "../async.ts"; export type ProcessingMessage = MQMessage & { diff --git a/lib/data/prefixed_kv_primitives.ts b/lib/data/prefixed_kv_primitives.ts index 3f112aa0..98f1bcbf 100644 --- a/lib/data/prefixed_kv_primitives.ts +++ b/lib/data/prefixed_kv_primitives.ts @@ -1,6 +1,6 @@ import type { KvPrimitives, KvQueryOptions } from "./kv_primitives.ts"; -import type { KV, KvKey } from "../../type/datastore.ts"; +import type { KV, KvKey } from "../../plug-api/types/datastore.ts"; /** * Turns any KvPrimitives into a KvPrimitives that automatically prefixes all keys (and removes them again when reading) diff --git a/lib/plugos/hooks/mq.ts b/lib/plugos/hooks/mq.ts index 5f89b602..ba6535fc 100644 --- a/lib/plugos/hooks/mq.ts +++ b/lib/plugos/hooks/mq.ts @@ -3,7 +3,7 @@ import type { System } from "../system.ts"; import { throttle } from "../../async.ts"; import type { MQHookT } from "../../manifest.ts"; import type { DataStoreMQ, QueueWorker } from "../../data/mq.datastore.ts"; -import type { MQMessage } from "../../../type/datastore.ts"; +import type { MQMessage } from "../../../plug-api/types/datastore.ts"; export class MQHook implements Hook<MQHookT> { subscriptions: QueueWorker[] = []; diff --git a/cmd/compile.ts b/lib/plugos/plug_compile.ts similarity index 86% rename from cmd/compile.ts rename to lib/plugos/plug_compile.ts index f25da09f..c1cecafb 100644 --- a/cmd/compile.ts +++ b/lib/plugos/plug_compile.ts @@ -2,9 +2,9 @@ import * as path from "@std/path"; import * as YAML from "@std/yaml"; import { denoPlugins } from "@luca/esbuild-deno-loader"; import * as esbuild from "esbuild"; -import { bundleAssets } from "../lib/asset_bundle/builder.ts"; -import type { Manifest } from "../lib/plugos/types.ts"; -import { version } from "../version.ts"; +import { bundleAssets } from "../asset_bundle/builder.ts"; +import type { Manifest } from "./types.ts"; +import { version } from "../../version.ts"; // const workerRuntimeUrl = new URL( // "../lib/plugos/worker_runtime.ts", @@ -20,8 +20,6 @@ export type CompileOptions = { configPath?: string; // path to import map importMap?: string; - // Reload plug import cache - reload?: boolean; // Print info on bundle size info?: boolean; }; @@ -138,7 +136,6 @@ setupMessageListener(functionMapping, manifest, self.postMessage); export async function compileManifests( manifestFiles: string[], dist: string, - watch: boolean, options: CompileOptions = {}, ) { let building = false; @@ -170,25 +167,37 @@ export async function compileManifests( } await buildAll(); - - if (watch) { - console.log("Watching for changes..."); - const watcher = Deno.watchFs(manifestFiles.map((p) => path.dirname(p))); - for await (const event of watcher) { - if (event.paths.length > 0) { - if ( - event.paths[0].endsWith(".json") || event.paths[0].startsWith(dist) - ) { - continue; - } - } - console.log("Change detected, rebuilding..."); - await buildAll(); - } - } } export function patchDenoLibJS(code: string): string { // The Deno std lib has one occurence of a regex that Webkit JS doesn't (yet parse), we'll strip it because it's likely never invoked anyway, YOLO return code.replaceAll("/(?<=\\n)/", "/()/"); } + +export async function plugCompileCommand( + { dist, debug, info, importmap, config, runtimeUrl }: { + dist: string; + debug: boolean; + info: boolean; + importmap?: string; + config?: string; + runtimeUrl?: string; + }, + ...manifestPaths: string[] +) { + await compileManifests( + manifestPaths, + dist, + { + debug: debug, + info: info, + runtimeUrl, + importMap: importmap + ? new URL(importmap, `file://${Deno.cwd()}/`).toString() + : undefined, + configPath: config, + }, + ); + esbuild.stop(); + Deno.exit(0); +} diff --git a/lib/plugos/syscalls/asset.ts b/lib/plugos/syscalls/asset.ts index f8a5a3ab..6208c856 100644 --- a/lib/plugos/syscalls/asset.ts +++ b/lib/plugos/syscalls/asset.ts @@ -1,5 +1,5 @@ import type { SysCallMapping, System } from "../system.ts"; -import type { FileMeta } from "../../../type/index.ts"; +import type { FileMeta } from "../../../plug-api/types/index.ts"; export default function assetSyscalls(system: System<any>): SysCallMapping { return { diff --git a/lib/space_lua/parse.ts b/lib/space_lua/parse.ts index ad46881c..11ce15b0 100644 --- a/lib/space_lua/parse.ts +++ b/lib/space_lua/parse.ts @@ -1,4 +1,4 @@ -import { lezerToParseTree } from "../../web/markdown_parser/parse_tree.ts"; +import { lezerToParseTree } from "../../client/markdown_parser/parse_tree.ts"; import { cleanTree, type ParseTree, diff --git a/lib/space_lua/query_collection.ts b/lib/space_lua/query_collection.ts index 5cdf4afb..07e3284d 100644 --- a/lib/space_lua/query_collection.ts +++ b/lib/space_lua/query_collection.ts @@ -5,9 +5,9 @@ import { asyncQuickSort } from "./util.ts"; import type { DataStore } from "../data/datastore.ts"; import type { KvPrimitives } from "../data/kv_primitives.ts"; -import type { QueryCollationConfig } from "../../type/config.ts"; +import type { QueryCollationConfig } from "../../plug-api/types/config.ts"; -import type { KvKey } from "../../type/datastore.ts"; +import type { KvKey } from "../../plug-api/types/datastore.ts"; export function buildItemEnv( objectVariable: string | undefined, diff --git a/lib/spaces/asset_bundle_space_primitives.ts b/lib/spaces/asset_bundle_space_primitives.ts index 96de9fa6..613346ec 100644 --- a/lib/spaces/asset_bundle_space_primitives.ts +++ b/lib/spaces/asset_bundle_space_primitives.ts @@ -1,7 +1,7 @@ import type { SpacePrimitives } from "./space_primitives.ts"; import type { AssetBundle } from "../../lib/asset_bundle/bundle.ts"; -import type { FileMeta } from "../../type/index.ts"; +import type { FileMeta } from "../../plug-api/types/index.ts"; export class AssetBundlePlugSpacePrimitives implements SpacePrimitives { constructor( diff --git a/lib/spaces/datastore_space_primitives.ts b/lib/spaces/datastore_space_primitives.ts index e4dcdd8f..0eb28f0d 100644 --- a/lib/spaces/datastore_space_primitives.ts +++ b/lib/spaces/datastore_space_primitives.ts @@ -1,6 +1,6 @@ import type { SpacePrimitives } from "./space_primitives.ts"; import { mime } from "mimetypes"; -import type { FileMeta } from "../../type/index.ts"; +import type { FileMeta } from "../../plug-api/types/index.ts"; import { notFoundError } from "../constants.ts"; import type { KvPrimitives } from "../data/kv_primitives.ts"; diff --git a/lib/spaces/disk_space_primitives.ts b/lib/spaces/disk_space_primitives.ts index ed31b0a4..2e514318 100644 --- a/lib/spaces/disk_space_primitives.ts +++ b/lib/spaces/disk_space_primitives.ts @@ -3,7 +3,7 @@ import { readAll } from "@std/io/read-all"; import type { SpacePrimitives } from "./space_primitives.ts"; import { mime } from "mimetypes"; -import type { FileMeta } from "../../type/index.ts"; +import type { FileMeta } from "../../plug-api/types/index.ts"; import { notFoundError } from "../constants.ts"; function lookupContentType(path: string): string { diff --git a/lib/spaces/evented_space_primitives.ts b/lib/spaces/evented_space_primitives.ts index 7ea7f822..3bf76144 100644 --- a/lib/spaces/evented_space_primitives.ts +++ b/lib/spaces/evented_space_primitives.ts @@ -1,7 +1,7 @@ -import type { EventHook } from "../../web/hooks/event.ts"; +import type { EventHook } from "../../client/hooks/event.ts"; import type { SpacePrimitives } from "./space_primitives.ts"; -import type { FileMeta } from "../../type/index.ts"; +import type { FileMeta } from "../../plug-api/types/index.ts"; import type { DataStore } from "../data/datastore.ts"; import { notFoundError } from "../constants.ts"; diff --git a/lib/spaces/filtered_space_primitives.ts b/lib/spaces/filtered_space_primitives.ts index 493b64c3..de934706 100644 --- a/lib/spaces/filtered_space_primitives.ts +++ b/lib/spaces/filtered_space_primitives.ts @@ -1,5 +1,5 @@ import type { SpacePrimitives } from "./space_primitives.ts"; -import type { FileMeta } from "../../type/index.ts"; +import type { FileMeta } from "../../plug-api/types/index.ts"; export class FilteredSpacePrimitives implements SpacePrimitives { constructor( diff --git a/lib/spaces/http_space_primitives.ts b/lib/spaces/http_space_primitives.ts index 720e8739..e61cc377 100644 --- a/lib/spaces/http_space_primitives.ts +++ b/lib/spaces/http_space_primitives.ts @@ -2,10 +2,10 @@ import type { SpacePrimitives } from "./space_primitives.ts"; import { encodePageURI } from "@silverbulletmd/silverbullet/lib/ref"; import { flushCachesAndUnregisterServiceWorker, -} from "../../web/service_worker/util.ts"; -import type { FileMeta } from "../../type/index.ts"; +} from "../../client/service_worker/util.ts"; +import type { FileMeta } from "../../plug-api/types/index.ts"; import { notFoundError, offlineError } from "../constants.ts"; -import { headersToFileMeta } from "../../server/util.ts"; +import { headersToFileMeta } from "../../lib/util.ts"; const defaultFetchTimeout = 30000; // 30 seconds diff --git a/lib/spaces/plug_space_primitives.ts b/lib/spaces/plug_space_primitives.ts index 50afeb93..0a83d06f 100644 --- a/lib/spaces/plug_space_primitives.ts +++ b/lib/spaces/plug_space_primitives.ts @@ -1,7 +1,7 @@ import type { SpacePrimitives } from "./space_primitives.ts"; import type { NamespaceOperation } from "../plugos/namespace.ts"; -import type { PlugNamespaceHook } from "../../web/hooks/plug_namespace.ts"; -import type { FileMeta } from "../../type/index.ts"; +import type { PlugNamespaceHook } from "../../client/hooks/plug_namespace.ts"; +import type { FileMeta } from "../../plug-api/types/index.ts"; export class PlugSpacePrimitives implements SpacePrimitives { constructor( diff --git a/lib/spaces/ro_space_primitives.ts b/lib/spaces/ro_space_primitives.ts index 91211b70..7c05da9e 100644 --- a/lib/spaces/ro_space_primitives.ts +++ b/lib/spaces/ro_space_primitives.ts @@ -1,5 +1,5 @@ import type { SpacePrimitives } from "./space_primitives.ts"; -import type { FileMeta } from "../../type/index.ts"; +import type { FileMeta } from "../../plug-api/types/index.ts"; export class ReadOnlySpacePrimitives implements SpacePrimitives { wrapped: SpacePrimitives; diff --git a/lib/spaces/space_primitives.test.ts b/lib/spaces/space_primitives.test.ts index 67253ae9..855b19eb 100644 --- a/lib/spaces/space_primitives.test.ts +++ b/lib/spaces/space_primitives.test.ts @@ -1,6 +1,6 @@ import { assert, assertEquals } from "@std/assert"; import type { SpacePrimitives } from "./space_primitives.ts"; -import type { FileMeta } from "../../type/index.ts"; +import type { FileMeta } from "../../plug-api/types/index.ts"; import { notFoundError } from "../constants.ts"; export async function testSpacePrimitives(spacePrimitives: SpacePrimitives) { diff --git a/lib/spaces/space_primitives.ts b/lib/spaces/space_primitives.ts index 6685b19d..4ed22c57 100644 --- a/lib/spaces/space_primitives.ts +++ b/lib/spaces/space_primitives.ts @@ -1,4 +1,4 @@ -import type { FileMeta } from "../../type/index.ts"; +import type { FileMeta } from "../../plug-api/types/index.ts"; /** * A generic interface used by `Space` to interact with the underlying storage, designed to be easy to implement for different storage backends diff --git a/lib/spaces/sync.ts b/lib/spaces/sync.ts index 5480c1a0..04d47277 100644 --- a/lib/spaces/sync.ts +++ b/lib/spaces/sync.ts @@ -1,11 +1,11 @@ import type { SpacePrimitives } from "./space_primitives.ts"; import { EventEmitter } from "../plugos/event.ts"; import { plugPrefix } from "./constants.ts"; -import type { FileMeta } from "../../type/index.ts"; +import type { FileMeta } from "../../plug-api/types/index.ts"; import { notFoundError } from "../constants.ts"; import { processWithConcurrency } from "../async.ts"; -const syncConcurrency = 5; +const syncConcurrency = 3; type SyncHash = number; diff --git a/server/util.ts b/lib/util.ts similarity index 100% rename from server/util.ts rename to lib/util.ts diff --git a/Library/Std.md b/libraries/Library/Std.md similarity index 100% rename from Library/Std.md rename to libraries/Library/Std.md diff --git a/Library/Std/Action Button.md b/libraries/Library/Std/Action Button.md similarity index 100% rename from Library/Std/Action Button.md rename to libraries/Library/Std/Action Button.md diff --git a/Library/Std/Command.md b/libraries/Library/Std/Command.md similarity index 100% rename from Library/Std/Command.md rename to libraries/Library/Std/Command.md diff --git a/Library/Std/Config.md b/libraries/Library/Std/Config.md similarity index 100% rename from Library/Std/Config.md rename to libraries/Library/Std/Config.md diff --git a/Library/Std/DOM.md b/libraries/Library/Std/DOM.md similarity index 100% rename from Library/Std/DOM.md rename to libraries/Library/Std/DOM.md diff --git a/Library/Std/Date.md b/libraries/Library/Std/Date.md similarity index 100% rename from Library/Std/Date.md rename to libraries/Library/Std/Date.md diff --git a/Library/Std/Embed.md b/libraries/Library/Std/Embed.md similarity index 100% rename from Library/Std/Embed.md rename to libraries/Library/Std/Embed.md diff --git a/Library/Std/Export.md b/libraries/Library/Std/Export.md similarity index 100% rename from Library/Std/Export.md rename to libraries/Library/Std/Export.md diff --git a/Library/Std/Gist.md b/libraries/Library/Std/Gist.md similarity index 100% rename from Library/Std/Gist.md rename to libraries/Library/Std/Gist.md diff --git a/Library/Std/Github.md b/libraries/Library/Std/Github.md similarity index 100% rename from Library/Std/Github.md rename to libraries/Library/Std/Github.md diff --git a/Library/Std/Import.md b/libraries/Library/Std/Import.md similarity index 100% rename from Library/Std/Import.md rename to libraries/Library/Std/Import.md diff --git a/Library/Std/Lua.md b/libraries/Library/Std/Lua.md similarity index 100% rename from Library/Std/Lua.md rename to libraries/Library/Std/Lua.md diff --git a/Library/Std/Page Templates.md b/libraries/Library/Std/Page Templates.md similarity index 100% rename from Library/Std/Page Templates.md rename to libraries/Library/Std/Page Templates.md diff --git a/Library/Std/Page Templates/Page Template.md b/libraries/Library/Std/Page Templates/Page Template.md similarity index 100% rename from Library/Std/Page Templates/Page Template.md rename to libraries/Library/Std/Page Templates/Page Template.md diff --git a/Library/Std/Page Templates/Quick Note.md b/libraries/Library/Std/Page Templates/Quick Note.md similarity index 100% rename from Library/Std/Page Templates/Quick Note.md rename to libraries/Library/Std/Page Templates/Quick Note.md diff --git a/Library/Std/Page Templates/Slash Template.md b/libraries/Library/Std/Page Templates/Slash Template.md similarity index 100% rename from Library/Std/Page Templates/Slash Template.md rename to libraries/Library/Std/Page Templates/Slash Template.md diff --git a/Library/Std/Pages/Maintenance.md b/libraries/Library/Std/Pages/Maintenance.md similarity index 100% rename from Library/Std/Pages/Maintenance.md rename to libraries/Library/Std/Pages/Maintenance.md diff --git a/Library/Std/Query Templates.md b/libraries/Library/Std/Query Templates.md similarity index 100% rename from Library/Std/Query Templates.md rename to libraries/Library/Std/Query Templates.md diff --git a/Library/Std/Schema.md b/libraries/Library/Std/Schema.md similarity index 100% rename from Library/Std/Schema.md rename to libraries/Library/Std/Schema.md diff --git a/Library/Std/Slash Templates.md b/libraries/Library/Std/Slash Templates.md similarity index 100% rename from Library/Std/Slash Templates.md rename to libraries/Library/Std/Slash Templates.md diff --git a/Library/Std/Slash Templates/code.md b/libraries/Library/Std/Slash Templates/code.md similarity index 100% rename from Library/Std/Slash Templates/code.md rename to libraries/Library/Std/Slash Templates/code.md diff --git a/Library/Std/Slash Templates/danger-admonition.md b/libraries/Library/Std/Slash Templates/danger-admonition.md similarity index 100% rename from Library/Std/Slash Templates/danger-admonition.md rename to libraries/Library/Std/Slash Templates/danger-admonition.md diff --git a/Library/Std/Slash Templates/func.md b/libraries/Library/Std/Slash Templates/func.md similarity index 100% rename from Library/Std/Slash Templates/func.md rename to libraries/Library/Std/Slash Templates/func.md diff --git a/Library/Std/Slash Templates/hr.md b/libraries/Library/Std/Slash Templates/hr.md similarity index 100% rename from Library/Std/Slash Templates/hr.md rename to libraries/Library/Std/Slash Templates/hr.md diff --git a/Library/Std/Slash Templates/lua-command.md b/libraries/Library/Std/Slash Templates/lua-command.md similarity index 100% rename from Library/Std/Slash Templates/lua-command.md rename to libraries/Library/Std/Slash Templates/lua-command.md diff --git a/Library/Std/Slash Templates/lua-query.md b/libraries/Library/Std/Slash Templates/lua-query.md similarity index 100% rename from Library/Std/Slash Templates/lua-query.md rename to libraries/Library/Std/Slash Templates/lua-query.md diff --git a/Library/Std/Slash Templates/lua-slash-command.md b/libraries/Library/Std/Slash Templates/lua-slash-command.md similarity index 100% rename from Library/Std/Slash Templates/lua-slash-command.md rename to libraries/Library/Std/Slash Templates/lua-slash-command.md diff --git a/Library/Std/Slash Templates/note-admonition.md b/libraries/Library/Std/Slash Templates/note-admonition.md similarity index 100% rename from Library/Std/Slash Templates/note-admonition.md rename to libraries/Library/Std/Slash Templates/note-admonition.md diff --git a/Library/Std/Slash Templates/query.md b/libraries/Library/Std/Slash Templates/query.md similarity index 100% rename from Library/Std/Slash Templates/query.md rename to libraries/Library/Std/Slash Templates/query.md diff --git a/Library/Std/Slash Templates/success-admonition.md b/libraries/Library/Std/Slash Templates/success-admonition.md similarity index 100% rename from Library/Std/Slash Templates/success-admonition.md rename to libraries/Library/Std/Slash Templates/success-admonition.md diff --git a/Library/Std/Slash Templates/table.md b/libraries/Library/Std/Slash Templates/table.md similarity index 100% rename from Library/Std/Slash Templates/table.md rename to libraries/Library/Std/Slash Templates/table.md diff --git a/Library/Std/Slash Templates/tpl.md b/libraries/Library/Std/Slash Templates/tpl.md similarity index 100% rename from Library/Std/Slash Templates/tpl.md rename to libraries/Library/Std/Slash Templates/tpl.md diff --git a/Library/Std/Slash Templates/warning-admonition.md b/libraries/Library/Std/Slash Templates/warning-admonition.md similarity index 100% rename from Library/Std/Slash Templates/warning-admonition.md rename to libraries/Library/Std/Slash Templates/warning-admonition.md diff --git a/Library/Std/Slash.md b/libraries/Library/Std/Slash.md similarity index 100% rename from Library/Std/Slash.md rename to libraries/Library/Std/Slash.md diff --git a/Library/Std/Space Overview.md b/libraries/Library/Std/Space Overview.md similarity index 100% rename from Library/Std/Space Overview.md rename to libraries/Library/Std/Space Overview.md diff --git a/Library/Std/Table.md b/libraries/Library/Std/Table.md similarity index 100% rename from Library/Std/Table.md rename to libraries/Library/Std/Table.md diff --git a/Library/Std/Tag.md b/libraries/Library/Std/Tag.md similarity index 100% rename from Library/Std/Tag.md rename to libraries/Library/Std/Tag.md diff --git a/Library/Std/Template.md b/libraries/Library/Std/Template.md similarity index 100% rename from Library/Std/Template.md rename to libraries/Library/Std/Template.md diff --git a/Library/Std/Widget.md b/libraries/Library/Std/Widget.md similarity index 100% rename from Library/Std/Widget.md rename to libraries/Library/Std/Widget.md diff --git a/Library/Std/Widgets.md b/libraries/Library/Std/Widgets.md similarity index 100% rename from Library/Std/Widgets.md rename to libraries/Library/Std/Widgets.md diff --git a/main.go b/main.go deleted file mode 100644 index 6e465b33..00000000 --- a/main.go +++ /dev/null @@ -1,9 +0,0 @@ -package main - -import ( - "fmt" -) - -func main() { - fmt.Println("Nothing here yet") -} diff --git a/netlify.toml b/netlify.toml deleted file mode 100644 index 81133584..00000000 --- a/netlify.toml +++ /dev/null @@ -1,3 +0,0 @@ -[build] -publish = "website_build" -command = "./scripts/build_website.sh" diff --git a/plug-api/lib/attribute.test.ts b/plug-api/lib/attribute.test.ts index 58ec116c..800c150d 100644 --- a/plug-api/lib/attribute.test.ts +++ b/plug-api/lib/attribute.test.ts @@ -1,12 +1,9 @@ import "./syscall_mock.ts"; -import { parse } from "../../web/markdown_parser/parse_tree.ts"; -import { - cleanAttributes, - extractAttributes, -} from "@silverbulletmd/silverbullet/lib/attribute"; +import { parse } from "../../client/markdown_parser/parse_tree.ts"; +import { extractAttributes } from "@silverbulletmd/silverbullet/lib/attribute"; import { assertEquals } from "@std/assert"; import { renderToText } from "./tree.ts"; -import { extendedMarkdownLanguage } from "../../web/markdown_parser/parser.ts"; +import { extendedMarkdownLanguage } from "../../client/markdown_parser/parser.ts"; const inlineAttributeSample = ` # My document @@ -18,16 +15,6 @@ Top level attributes: [name:: sup] [age:: 42] [children: [pete, "john", mary]] 1. Itemized list [tag:: baz] `; -const cleanedInlineAttributeSample = ` -# My document -Top level attributes: - -* [ ] Attribute in a task -* Regular item - -1. Itemized list -`; - Deno.test("Test attribute extraction", async () => { const tree = parse(extendedMarkdownLanguage, inlineAttributeSample); const toplevelAttributes = await extractAttributes(tree); @@ -37,7 +24,4 @@ Deno.test("Test attribute extraction", async () => { assertEquals(toplevelAttributes.children, ["pete", "john", "mary"]); // Check if the attributes are still there assertEquals(renderToText(tree), inlineAttributeSample); - // And now clean - cleanAttributes(tree); - assertEquals(renderToText(tree), cleanedInlineAttributeSample); }); diff --git a/plug-api/lib/tags.ts b/plug-api/lib/tags.ts index 67657686..04507298 100644 --- a/plug-api/lib/tags.ts +++ b/plug-api/lib/tags.ts @@ -3,8 +3,8 @@ import { type ParseTree, traverseTree, } from "@silverbulletmd/silverbullet/lib/tree"; -import { tagRegex } from "../../web/markdown_parser/constants.ts"; -import type { ObjectValue } from "../../type/index.ts"; +import { tagRegex } from "../../client/markdown_parser/constants.ts"; +import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; export function updateITags<T>(obj: ObjectValue<T>, frontmatter: FrontMatter) { const itags = [obj.tag, ...frontmatter.tags || []]; diff --git a/plug-api/lib/tree.test.ts b/plug-api/lib/tree.test.ts index e6f93633..be4e19eb 100644 --- a/plug-api/lib/tree.test.ts +++ b/plug-api/lib/tree.test.ts @@ -9,8 +9,8 @@ import { replaceNodesMatching, } from "./tree.ts"; import { assertEquals, assertNotEquals } from "@std/assert"; -import { parse } from "../../web/markdown_parser/parse_tree.ts"; -import { extendedMarkdownLanguage } from "../../web/markdown_parser/parser.ts"; +import { parse } from "../../client/markdown_parser/parse_tree.ts"; +import { extendedMarkdownLanguage } from "../../client/markdown_parser/parser.ts"; const mdTest1 = ` # Heading diff --git a/plug-api/syscalls/asset.ts b/plug-api/syscalls/asset.ts index 1e854ad7..8b3aeff9 100644 --- a/plug-api/syscalls/asset.ts +++ b/plug-api/syscalls/asset.ts @@ -1,7 +1,7 @@ import { base64DecodeDataUrl } from "../../lib/crypto.ts"; import { syscall } from "../syscall.ts"; -import type { FileMeta } from "../../type/index.ts"; +import type { FileMeta } from "../../plug-api/types/index.ts"; /** * Reads an asset embedded in a plug (via the `assets` field in the plug manifest). diff --git a/plug-api/syscalls/datastore.ts b/plug-api/syscalls/datastore.ts index a482c892..6827c356 100644 --- a/plug-api/syscalls/datastore.ts +++ b/plug-api/syscalls/datastore.ts @@ -1,7 +1,7 @@ import type { LuaCollectionQuery } from "../../lib/space_lua/query_collection.ts"; import { syscall } from "../syscall.ts"; -import type { KV, KvKey, KvQuery } from "../../type/datastore.ts"; +import type { KV, KvKey, KvQuery } from "../../plug-api/types/datastore.ts"; /** * Exposes a key value story with query capabilities. diff --git a/plug-api/syscalls/editor.ts b/plug-api/syscalls/editor.ts index 41eeef73..fea6353b 100644 --- a/plug-api/syscalls/editor.ts +++ b/plug-api/syscalls/editor.ts @@ -1,10 +1,7 @@ import { syscall } from "../syscall.ts"; -import type { - FilterOption, - UploadFile, -} from "@silverbulletmd/silverbullet/type/client"; +import type { FilterOption, UploadFile } from "../../plug-api/types/client.ts"; import type { Path, Ref } from "../lib/ref.ts"; -import type { PageMeta } from "../../type/index.ts"; +import type { PageMeta } from "../../plug-api/types/index.ts"; /** * Exposes various editor utilities. diff --git a/plug-api/syscalls/index.ts b/plug-api/syscalls/index.ts index 4ccbf066..27f5abd0 100644 --- a/plug-api/syscalls/index.ts +++ b/plug-api/syscalls/index.ts @@ -1,6 +1,6 @@ import type { LuaCollectionQuery } from "../../lib/space_lua/query_collection.ts"; import { syscall } from "@silverbulletmd/silverbullet/syscall"; -import type { ObjectValue } from "../../type/index.ts"; +import type { ObjectValue } from "../../plug-api/types/index.ts"; /** * Exposes the SilverBullet object indexing system diff --git a/plug-api/syscalls/mq.ts b/plug-api/syscalls/mq.ts index dd604fee..6cb2b73e 100644 --- a/plug-api/syscalls/mq.ts +++ b/plug-api/syscalls/mq.ts @@ -1,6 +1,6 @@ import { syscall } from "../syscall.ts"; -import type { MQStats } from "../../type/datastore.ts"; +import type { MQStats } from "../../plug-api/types/datastore.ts"; /** * Implements a simple Message Queue system. diff --git a/plug-api/syscalls/space.ts b/plug-api/syscalls/space.ts index ae6d6a4e..5ab4d284 100644 --- a/plug-api/syscalls/space.ts +++ b/plug-api/syscalls/space.ts @@ -1,6 +1,10 @@ import { syscall } from "../syscall.ts"; -import type { DocumentMeta, FileMeta, PageMeta } from "../../type/index.ts"; +import type { + DocumentMeta, + FileMeta, + PageMeta, +} from "../../plug-api/types/index.ts"; /** * Exposes the space with its pages, documents and plugs. diff --git a/plug-api/syscalls/system.ts b/plug-api/syscalls/system.ts index abefd968..c61568e7 100644 --- a/plug-api/syscalls/system.ts +++ b/plug-api/syscalls/system.ts @@ -1,6 +1,6 @@ import { syscall } from "../syscall.ts"; import type { CommandDef } from "../../lib/manifest.ts"; -import type { SyscallMeta } from "../../type/index.ts"; +import type { SyscallMeta } from "../../plug-api/types/index.ts"; /** * System level syscalls diff --git a/type/client.ts b/plug-api/types/client.ts similarity index 100% rename from type/client.ts rename to plug-api/types/client.ts diff --git a/type/config.ts b/plug-api/types/config.ts similarity index 100% rename from type/config.ts rename to plug-api/types/config.ts diff --git a/type/datastore.ts b/plug-api/types/datastore.ts similarity index 100% rename from type/datastore.ts rename to plug-api/types/datastore.ts diff --git a/type/event.ts b/plug-api/types/event.ts similarity index 87% rename from type/event.ts rename to plug-api/types/event.ts index de3c9803..7ffa1b38 100644 --- a/type/event.ts +++ b/plug-api/types/event.ts @@ -1,5 +1,5 @@ import type { ParseTree } from "@silverbulletmd/silverbullet/lib/tree"; -import type { EventSubscriptionDef } from "../lib/manifest.ts"; +import type { EventSubscriptionDef } from "../../lib/manifest.ts"; export type IndexTreeEvent = { name: string; diff --git a/type/index.ts b/plug-api/types/index.ts similarity index 100% rename from type/index.ts rename to plug-api/types/index.ts diff --git a/plugs/editor/complete.ts b/plugs/editor/complete.ts index 7b673c08..bcb26ff3 100644 --- a/plugs/editor/complete.ts +++ b/plugs/editor/complete.ts @@ -1,7 +1,7 @@ import { folderName } from "@silverbulletmd/silverbullet/lib/resolve"; import { queryLuaObjects } from "../index/api.ts"; import { language, lua } from "@silverbulletmd/silverbullet/syscalls"; -import type { DocumentMeta, PageMeta } from "../../type/index.ts"; +import type { DocumentMeta, PageMeta } from "../../plug-api/types/index.ts"; import type { CompleteEvent } from "@silverbulletmd/silverbullet/type/client"; // Page completion diff --git a/plugs/emoji/emoji.ts b/plugs/emoji/emoji.ts index 6f773802..6c68b957 100644 --- a/plugs/emoji/emoji.ts +++ b/plugs/emoji/emoji.ts @@ -1,7 +1,7 @@ import emojiBlob from "./emoji.json" with { type: "json" }; import { editor, system } from "@silverbulletmd/silverbullet/syscalls"; -import type { EmojiConfig } from "../../type/config.ts"; +import type { EmojiConfig } from "../../plug-api/types/config.ts"; import type { CompleteEvent } from "@silverbulletmd/silverbullet/type/client"; let emojiConfig: EmojiConfig = { aliases: [] }; diff --git a/plugs/index/api.ts b/plugs/index/api.ts index fc4b7a27..80b3f520 100644 --- a/plugs/index/api.ts +++ b/plugs/index/api.ts @@ -11,9 +11,9 @@ import { renderToText, } from "@silverbulletmd/silverbullet/lib/tree"; import { applyPatches, type SetKeyPatch } from "../../lib/yaml.ts"; -import type { ObjectValue } from "../../type/index.ts"; +import type { ObjectValue } from "../../plug-api/types/index.ts"; -import type { KV, KvKey, KvQuery } from "../../type/datastore.ts"; +import type { KV, KvKey, KvQuery } from "../../plug-api/types/datastore.ts"; const indexKey = "idx"; const pageKey = "ridx"; diff --git a/plugs/index/attributes.ts b/plugs/index/attributes.ts index 9dd72ca2..47a42891 100644 --- a/plugs/index/attributes.ts +++ b/plugs/index/attributes.ts @@ -1,4 +1,4 @@ -import type { ObjectValue } from "../../type/index.ts"; +import type { ObjectValue } from "../../plug-api/types/index.ts"; export type SimpleJSONType = { type?: "string" | "number" | "boolean" | "any" | "array" | "object" | "null"; diff --git a/plugs/index/data.ts b/plugs/index/data.ts index ef228f58..891dd176 100644 --- a/plugs/index/data.ts +++ b/plugs/index/data.ts @@ -3,12 +3,12 @@ import { collectNodesOfType, findNodeOfType, } from "@silverbulletmd/silverbullet/lib/tree"; -import type { IndexTreeEvent } from "../../type/event.ts"; +import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; import { indexObjects } from "./api.ts"; import type { TagObject } from "./tags.ts"; import { extractFrontMatter } from "@silverbulletmd/silverbullet/lib/frontmatter"; import { updateITags } from "@silverbulletmd/silverbullet/lib/tags"; -import type { ObjectValue } from "../../type/index.ts"; +import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; type DataObject = ObjectValue< { diff --git a/plugs/index/document.ts b/plugs/index/document.ts index b1f81093..5173d3b1 100644 --- a/plugs/index/document.ts +++ b/plugs/index/document.ts @@ -1,6 +1,6 @@ import { space, system } from "@silverbulletmd/silverbullet/syscalls"; import { indexObjects } from "./api.ts"; -import type { DocumentMeta } from "../../type/index.ts"; +import type { DocumentMeta } from "../../plug-api/types/index.ts"; // Note: clearFileIndex is not called but since this is the only attachmet:index listener, this should be fine (famous last words) export async function indexDocument(name: string) { diff --git a/plugs/index/header.ts b/plugs/index/header.ts index b27c8fe3..b72a8cf3 100644 --- a/plugs/index/header.ts +++ b/plugs/index/header.ts @@ -3,13 +3,16 @@ import { collectNodesOfType, renderToText, } from "@silverbulletmd/silverbullet/lib/tree"; -import type { IndexTreeEvent } from "../../type/event.ts"; +import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; import { indexObjects, queryLuaObjects } from "./api.ts"; -import { getNameFromPath, parseToRef } from "../../plug-api/lib/ref.ts"; +import { + getNameFromPath, + parseToRef, +} from "@silverbulletmd/silverbullet/lib/ref"; import { extractAttributes } from "@silverbulletmd/silverbullet/lib/attribute"; -import { extractHashtag } from "../../plug-api/lib/tags.ts"; +import { extractHashtag } from "@silverbulletmd/silverbullet/lib/tags"; import { lua } from "@silverbulletmd/silverbullet/syscalls"; -import type { ObjectValue } from "../../type/index.ts"; +import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; import type { CompleteEvent } from "@silverbulletmd/silverbullet/type/client"; type HeaderObject = ObjectValue< diff --git a/plugs/index/item.test.ts b/plugs/index/item.test.ts index 2dfe4459..37970d54 100644 --- a/plugs/index/item.test.ts +++ b/plugs/index/item.test.ts @@ -1,5 +1,5 @@ import "../../plug-api/lib/syscall_mock.ts"; -import { parseMarkdown } from "../../web/markdown_parser/parser.ts"; +import { parseMarkdown } from "../../client/markdown_parser/parser.ts"; import { extractItems } from "./item.ts"; import { assertEquals } from "@std/assert"; diff --git a/plugs/index/item.ts b/plugs/index/item.ts index 1879ec2d..678de03a 100644 --- a/plugs/index/item.ts +++ b/plugs/index/item.ts @@ -1,4 +1,4 @@ -import type { IndexTreeEvent } from "../../type/event.ts"; +import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; import { findParentMatching, @@ -21,7 +21,7 @@ import { type FrontMatter, } from "@silverbulletmd/silverbullet/lib/frontmatter"; import { deepClone } from "@silverbulletmd/silverbullet/lib/json"; -import type { ObjectValue } from "../../type/index.ts"; +import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; import { system } from "@silverbulletmd/silverbullet/syscalls"; export type ItemObject = ObjectValue< diff --git a/plugs/index/page.ts b/plugs/index/page.ts index 477dc3ef..edb5b857 100644 --- a/plugs/index/page.ts +++ b/plugs/index/page.ts @@ -1,4 +1,4 @@ -import type { IndexTreeEvent } from "../../type/event.ts"; +import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; import { editor, lua, @@ -22,7 +22,7 @@ import { } from "@silverbulletmd/silverbullet/lib/tree"; import { updateITags } from "@silverbulletmd/silverbullet/lib/tags"; import type { AspiringPageObject } from "./page_links.ts"; -import type { PageMeta } from "../../type/index.ts"; +import type { PageMeta } from "@silverbulletmd/silverbullet/type/index"; import type { LintDiagnostic } from "@silverbulletmd/silverbullet/type/client"; export async function indexPage({ name, tree }: IndexTreeEvent) { diff --git a/plugs/index/page_links.ts b/plugs/index/page_links.ts index 6e066195..ce46bb6a 100644 --- a/plugs/index/page_links.ts +++ b/plugs/index/page_links.ts @@ -4,7 +4,7 @@ import { renderToText, traverseTree, } from "@silverbulletmd/silverbullet/lib/tree"; -import type { IndexTreeEvent } from "../../type/event.ts"; +import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; import { isLocalURL, resolveMarkdownLink, @@ -21,9 +21,9 @@ import { extractSnippetAroundIndex } from "./snippet_extractor.ts"; import { mdLinkRegex, wikiLinkRegex, -} from "../../web/markdown_parser/constants.ts"; +} from "../../client/markdown_parser/constants.ts"; import { lua, space } from "@silverbulletmd/silverbullet/syscalls"; -import type { ObjectValue } from "../../type/index.ts"; +import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; export type LinkObject = ObjectValue< { diff --git a/plugs/index/paragraph.ts b/plugs/index/paragraph.ts index 1c073dfb..61435f94 100644 --- a/plugs/index/paragraph.ts +++ b/plugs/index/paragraph.ts @@ -1,16 +1,16 @@ -import type { IndexTreeEvent } from "../../type/event.ts"; +import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; import { indexObjects } from "./api.ts"; import { collectNodesOfType, findParentMatching, renderToText, traverseTreeAsync, -} from "../../plug-api/lib/tree.ts"; +} from "@silverbulletmd/silverbullet/lib/tree"; import { extractAttributes } from "@silverbulletmd/silverbullet/lib/attribute"; import { updateITags } from "@silverbulletmd/silverbullet/lib/tags"; import { extractFrontMatter } from "@silverbulletmd/silverbullet/lib/frontmatter"; -import { extractHashtag } from "../../plug-api/lib/tags.ts"; -import type { ObjectValue } from "../../type/index.ts"; +import { extractHashtag } from "@silverbulletmd/silverbullet/lib/tags"; +import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; import { system } from "@silverbulletmd/silverbullet/syscalls"; /** ParagraphObject An index object for the top level text nodes */ diff --git a/plugs/index/plug_api.ts b/plugs/index/plug_api.ts index 7f280d22..aa6dda0d 100644 --- a/plugs/index/plug_api.ts +++ b/plugs/index/plug_api.ts @@ -1,7 +1,7 @@ import { system } from "@silverbulletmd/silverbullet/syscalls"; -import type { ObjectValue } from "../../type/index.ts"; +import type { ObjectValue } from "../../plug-api/types/index.ts"; -import type { KV, KvQuery } from "../../type/datastore.ts"; +import type { KV, KvQuery } from "../../plug-api/types/datastore.ts"; export function indexObjects<T>( page: string, diff --git a/plugs/index/queue.ts b/plugs/index/queue.ts index e695d493..62f2963c 100644 --- a/plugs/index/queue.ts +++ b/plugs/index/queue.ts @@ -7,7 +7,7 @@ import { system, } from "@silverbulletmd/silverbullet/syscalls"; import { sleep } from "../../lib/async.ts"; -import type { MQMessage } from "../../type/datastore.ts"; +import type { MQMessage } from "../../plug-api/types/datastore.ts"; import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; import { clearFileIndex } from "./api.ts"; diff --git a/plugs/index/script.ts b/plugs/index/script.ts index d56c8e5a..312b91cd 100644 --- a/plugs/index/script.ts +++ b/plugs/index/script.ts @@ -1,7 +1,10 @@ -import type { IndexTreeEvent } from "../../type/event.ts"; -import { collectNodesOfType, findNodeOfType } from "../../plug-api/lib/tree.ts"; +import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; +import { + collectNodesOfType, + findNodeOfType, +} from "@silverbulletmd/silverbullet/lib/tree"; import { indexObjects } from "./api.ts"; -import type { ObjectValue } from "../../type/index.ts"; +import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; export type ScriptObject = ObjectValue<{ script: string; diff --git a/plugs/index/style.ts b/plugs/index/style.ts index 593f870c..69811981 100644 --- a/plugs/index/style.ts +++ b/plugs/index/style.ts @@ -1,7 +1,10 @@ -import type { IndexTreeEvent } from "../../type/event.ts"; -import { collectNodesOfType, findNodeOfType } from "../../plug-api/lib/tree.ts"; +import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; +import { + collectNodesOfType, + findNodeOfType, +} from "@silverbulletmd/silverbullet/lib/tree"; import { indexObjects } from "./api.ts"; -import type { ObjectValue } from "../../type/index.ts"; +import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; export type StyleObject = ObjectValue<{ style: string; diff --git a/plugs/index/table.ts b/plugs/index/table.ts index d5f4dbcc..9fc7a2d1 100644 --- a/plugs/index/table.ts +++ b/plugs/index/table.ts @@ -1,13 +1,16 @@ -import type { IndexTreeEvent } from "../../type/event.ts"; -import { renderToText, replaceNodesMatching } from "../../plug-api/lib/tree.ts"; -import { extractHashtag } from "../../plug-api/lib/tags.ts"; +import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; +import { + renderToText, + replaceNodesMatching, +} from "@silverbulletmd/silverbullet/lib/tree"; +import { extractHashtag } from "@silverbulletmd/silverbullet/lib/tags"; import { collectNodesMatching, collectNodesOfType, type ParseTree, } from "@silverbulletmd/silverbullet/lib/tree"; import { indexObjects } from "./api.ts"; -import type { ObjectValue } from "../../type/index.ts"; +import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; type TableRowObject = & ObjectValue<{ diff --git a/plugs/index/tags.ts b/plugs/index/tags.ts index ec87b926..f60450b8 100644 --- a/plugs/index/tags.ts +++ b/plugs/index/tags.ts @@ -1,4 +1,4 @@ -import type { IndexTreeEvent } from "../../type/event.ts"; +import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; import { extractFrontMatter } from "@silverbulletmd/silverbullet/lib/frontmatter"; import { indexObjects, queryLuaObjects } from "./api.ts"; import { @@ -6,8 +6,11 @@ import { collectNodesOfType, findParentMatching, } from "@silverbulletmd/silverbullet/lib/tree"; -import { extractHashtag, renderHashtag } from "../../plug-api/lib/tags.ts"; -import type { ObjectValue } from "../../type/index.ts"; +import { + extractHashtag, + renderHashtag, +} from "@silverbulletmd/silverbullet/lib/tags"; +import type { ObjectValue } from "@silverbulletmd/silverbullet/type/index"; import type { CompleteEvent } from "@silverbulletmd/silverbullet/type/client"; export type TagObject = ObjectValue<{ diff --git a/plugs/tasks/task.test.ts b/plugs/tasks/task.test.ts index 3bcfddcf..b7710fe8 100644 --- a/plugs/tasks/task.test.ts +++ b/plugs/tasks/task.test.ts @@ -1,5 +1,5 @@ import "../../plug-api/lib/syscall_mock.ts"; -import { parseMarkdown } from "../../web/markdown_parser/parser.ts"; +import { parseMarkdown } from "../../client/markdown_parser/parser.ts"; import { extractTasks } from "./task.ts"; import { assertEquals } from "@std/assert"; diff --git a/plugs/tasks/task.ts b/plugs/tasks/task.ts index 9a3f1239..51ab9ef7 100644 --- a/plugs/tasks/task.ts +++ b/plugs/tasks/task.ts @@ -1,4 +1,4 @@ -import type { IndexTreeEvent } from "../../type/event.ts"; +import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/type/event"; import { editor, diff --git a/scripts/build_website.sh b/scripts/build_website.sh deleted file mode 100755 index 70dd9253..00000000 --- a/scripts/build_website.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -e - -if [ "$1" != "local" ]; then - echo "Install Deno" - curl -fsSL https://deno.land/install.sh | sh - export PATH=~/.deno/bin:$PATH - export DENO_DIR=$PWD/deno_cache - echo "DENO_DIR: $DENO_DIR" - mkdir -p $DENO_DIR - -fi - -deno task clean -deno task build -deno task bundle -mkdir -p website_build -cp dist/silverbullet.js website_build/ -cp install.sh website_build/ \ No newline at end of file diff --git a/scripts/generate.sh b/scripts/generate.sh deleted file mode 100755 index 23c23d34..00000000 --- a/scripts/generate.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -e - -LUA_GRAMMAR=lib/space_lua/lua.grammar -LEZER_GENERATOR_VERSION=1.5.1 - -deno run -A npm:@lezer/generator@$LEZER_GENERATOR_VERSION $LUA_GRAMMAR -o lib/space_lua/parse-lua.js \ No newline at end of file diff --git a/server/auth.go b/server/auth.go new file mode 100644 index 00000000..5b19d80f --- /dev/null +++ b/server/auth.go @@ -0,0 +1,256 @@ +package server + +import ( + "fmt" + "html/template" + "log" + "net/http" + "path" + "strings" + "time" + + "github.com/go-chi/chi/v5" +) + +// path to auth page in the client bundle +const authPagePath = ".client/auth.html" + +// These endpoints are not protected by auth +var excludedPaths = []string{ + "/service_worker.js", // because browser fetch this without sending any cookie (auth) headers + "/.client/", // to allow a basic UI to load, not 100% sure this is necessary to exclude + "/.auth", // otherwise the authentication UI would be behind authentication + "/.ping", // because docker needs to be able to access this for its health check +} + +func addAuthEndpoints(r chi.Router, config *ServerConfig) { + // Logout + r.Get("/.logout", func(w http.ResponseWriter, r *http.Request) { + host := extractHost(r) + spaceConfig := spaceConfigFromContext(r.Context()) + cookieOptions := CookieOptions{ + Path: fmt.Sprintf("%s/", spaceConfig.HostURLPrefix), + } + + deleteCookie(w, authCookieName(host), cookieOptions) + deleteCookie(w, "refreshLogin", cookieOptions) + + http.Redirect(w, r, applyURLPrefix("/.auth", spaceConfig.HostURLPrefix), http.StatusFound) + }) + + // Auth page + r.Get("/.auth", func(w http.ResponseWriter, r *http.Request) { + spaceConfig := spaceConfigFromContext(r.Context()) + if err := spaceConfig.InitAuth(); err != nil { + http.Error(w, "Failed to initialize authentication", http.StatusInternalServerError) + return + } + + data, _, err := config.ClientBundle.ReadFile(authPagePath) + if err != nil { + http.Error(w, "Auth page not found", http.StatusNotFound) + return + } + + tpl := template.Must(template.New("auth").Parse(string(data))) + + templateData := struct { + HostPrefix string + SpaceName string + }{ + HostPrefix: spaceConfig.HostURLPrefix, + SpaceName: spaceConfig.SpaceName, + } + + w.Header().Set("Content-type", "text/html") + w.WriteHeader(http.StatusOK) + if err := tpl.Execute(w, templateData); err != nil { + log.Printf("Could not render auth page: %v", err) + w.Write([]byte("Server error")) + } + }) + + // Auth POST endpoint + r.Post("/.auth", func(w http.ResponseWriter, r *http.Request) { + spaceConfig := spaceConfigFromContext(r.Context()) + if err := spaceConfig.InitAuth(); err != nil { + http.Error(w, "Failed to initialize authentication", http.StatusInternalServerError) + return + } + if err := r.ParseForm(); err != nil { + http.Error(w, "Failed to parse form", http.StatusBadRequest) + return + } + + username := r.FormValue("username") + password := r.FormValue("password") + rememberMe := r.FormValue("rememberMe") + from := r.FormValue("from") + + if username == "" || password == "" { + http.Redirect(w, r, applyURLPrefix("/.auth?error=0", spaceConfig.HostURLPrefix), http.StatusFound) + return + } + + if spaceConfig.LockoutTimer.IsLocked() { + fmt.Println("Authentication locked out, redirecting to auth page.") + http.Redirect(w, r, applyURLPrefix("/.auth?error=2", spaceConfig.HostURLPrefix), http.StatusFound) + return + } + + if username == spaceConfig.Auth.User && password == spaceConfig.Auth.Pass { + // Generate JWT with username (not currently used) + payload := map[string]any{ + "username": username, + } + + var jwt string + var err error + if rememberMe != "" { + jwt, err = spaceConfig.JwtIssuer.CreateJWT(payload) // No expiry + } else { + jwt, err = spaceConfig.JwtIssuer.CreateJWT(payload, authenticationExpirySeconds) + } + + if err != nil { + fmt.Printf("Failed to create JWT: %v\n", err) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + + host := extractHost(r) + inAWeek := time.Now().Add(time.Duration(authenticationExpirySeconds) * time.Second) + + cookieOptions := CookieOptions{ + Path: fmt.Sprintf("%s/", spaceConfig.HostURLPrefix), + Expires: inAWeek, + } + + setCookie(w, authCookieName(host), jwt, cookieOptions) + + if rememberMe != "" { + setCookie(w, "refreshLogin", "true", cookieOptions) + } + + redirectPath := "/" + if from != "" { + redirectPath = from + } + + http.Redirect(w, r, applyURLPrefix(redirectPath, spaceConfig.HostURLPrefix), http.StatusFound) + } else { + fmt.Println("Authentication failed, redirecting to auth page.") + spaceConfig.LockoutTimer.AddCount() + + http.Redirect(w, r, applyURLPrefix("/.auth?error=1", spaceConfig.HostURLPrefix), http.StatusFound) + } + }) +} + +func (spaceConfig *SpaceConfig) InitAuth() error { + if spaceConfig.JwtIssuer == nil { + var err error + // Need to do some initialization + spaceConfig.JwtIssuer, err = CreateAuthenticator(path.Join(spaceConfig.SpaceFolderPath, ".silverbullet.auth.json"), spaceConfig.Auth) + if err != nil { + return err + } + + // Initialize lockout timer + if spaceConfig.Auth.LockoutLimit > 0 { + spaceConfig.LockoutTimer = NewLockoutTimer(spaceConfig.Auth.LockoutTime*1000, spaceConfig.Auth.LockoutLimit) + } else { + spaceConfig.LockoutTimer = NewLockoutTimer(0, 0) // disabled + } + } + return nil +} + +// authMiddleware provides authentication middleware +func authMiddleware(config *ServerConfig) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var err error + spaceConfig := spaceConfigFromContext(r.Context()) + if spaceConfig.Auth == nil { + // No authentication to do, moving on + next.ServeHTTP(w, r) + return + } + + path := removeURLPrefix(r.URL.Path, spaceConfig.HostURLPrefix) + host := extractHost(r) + + if isExcludedPath(path) { + next.ServeHTTP(w, r) + return + } + + if err := spaceConfig.InitAuth(); err != nil { + http.Error(w, "Failed to initialize authentication", http.StatusInternalServerError) + return + } + + authCookie := getCookie(r, authCookieName(host)) + + // Try Bearer token authentication if no cookie + if authCookie == "" && spaceConfig.Auth.AuthToken != "" { + authHeader := r.Header.Get("Authorization") + if after, ok := strings.CutPrefix(authHeader, "Bearer "); ok { + authToken := after + if authToken == spaceConfig.Auth.AuthToken { + next.ServeHTTP(w, r) + return + } else { + fmt.Println("Unauthorized token access") + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + } + } + + if authCookie == "" { + log.Printf("Unauthorized access to %s, redirecting to auth page", path) + redirectToAuth(w, "/.auth", path, spaceConfig.HostURLPrefix) + return + } + + // Verify JWT + claims, err := spaceConfig.JwtIssuer.VerifyAndDecodeJWT(authCookie) + if err != nil { + log.Printf("Error verifying JWT on %s, redirecting to auth page: %v\n", path, err) + redirectToAuth(w, "/.auth", path, spaceConfig.HostURLPrefix) + return + } + + username, ok := claims["username"].(string) + if !ok || username != spaceConfig.Auth.User { + log.Printf("Username mismatch in JWT on %s", path) + redirectToAuth(w, "/.auth", path, spaceConfig.HostURLPrefix) + return + } + + refreshLogin(w, r, config, host) + next.ServeHTTP(w, r) + }) + } +} + +// refreshLogin refreshes the login cookie if needed +func refreshLogin(w http.ResponseWriter, r *http.Request, config *ServerConfig, host string) { + if getCookie(r, "refreshLogin") != "" { + spaceConfig := spaceConfigFromContext(r.Context()) + inAWeek := time.Now().Add(time.Duration(authenticationExpirySeconds) * time.Second) + jwt := getCookie(r, authCookieName(host)) + + if jwt != "" { + cookieOptions := CookieOptions{ + Path: fmt.Sprintf("%s/", spaceConfig.HostURLPrefix), + Expires: inAWeek, + } + + setCookie(w, authCookieName(host), jwt, cookieOptions) + setCookie(w, "refreshLogin", "true", cookieOptions) + } + } +} diff --git a/server/auth_utils.go b/server/auth_utils.go new file mode 100644 index 00000000..afbe337f --- /dev/null +++ b/server/auth_utils.go @@ -0,0 +1,129 @@ +package server + +import ( + "fmt" + "net/http" + "regexp" + "strings" + "time" +) + +const authenticationExpirySeconds = 60 * 60 * 24 * 7 // 1 week + +// authCookieName generates a cookie name based on the host +func authCookieName(host string) string { + // Replace non-word characters with underscores + re := regexp.MustCompile(`\W`) + cleanHost := re.ReplaceAllString(host, "_") + return fmt.Sprintf("auth_%s", cleanHost) +} + +// utcDateString converts a timestamp to UTC date string +func utcDateString(mtime int64) string { + return time.Unix(mtime/1000, 0).UTC().Format(time.RFC1123) +} + +// setCookie sets an HTTP cookie +func setCookie(w http.ResponseWriter, name, value string, options CookieOptions) { + cookie := &http.Cookie{ + Name: name, + Value: value, + Path: options.Path, + } + + if !options.Expires.IsZero() { + cookie.Expires = options.Expires + } + + http.SetCookie(w, cookie) +} + +// getCookie gets an HTTP cookie value +func getCookie(r *http.Request, name string) string { + cookie, err := r.Cookie(name) + if err != nil { + return "" + } + return cookie.Value +} + +// deleteCookie deletes an HTTP cookie +func deleteCookie(w http.ResponseWriter, name string, options CookieOptions) { + cookie := &http.Cookie{ + Name: name, + Value: "", + Path: options.Path, + Expires: time.Unix(0, 0), + MaxAge: -1, + } + http.SetCookie(w, cookie) +} + +// CookieOptions represents cookie configuration options +type CookieOptions struct { + Path string + Expires time.Time + HttpOnly bool + Secure bool + SameSite string +} + +func extractHost(r *http.Request) string { + host := r.Host + if host == "" { + host = r.Header.Get("Host") + } + return host +} + +// redirectToAuth creates a redirect response to the auth page +func redirectToAuth(w http.ResponseWriter, authPath, fromPath, hostURLPrefix string) { + var redirectURL string + + // Try filtering api paths + if strings.HasPrefix(fromPath, "/.") || strings.HasSuffix(fromPath, ".md") { + w.Header().Set("Location", applyURLPrefix(authPath, hostURLPrefix)) + w.WriteHeader(http.StatusUnauthorized) + return + } else { + redirectURL = fmt.Sprintf("%s?from=%s", authPath, fromPath) + } + + w.Header().Set("Location", applyURLPrefix(redirectURL, hostURLPrefix)) + w.WriteHeader(http.StatusFound) +} + +// applyURLPrefix applies a URL prefix to a path +func applyURLPrefix(path, prefix string) string { + if prefix == "" { + return path + } + return prefix + path +} + +// removeURLPrefix removes a URL prefix from a path +func removeURLPrefix(path, prefix string) string { + if prefix == "" { + return path + } + if strings.HasPrefix(path, prefix) { + return path[len(prefix):] + } + return path +} + +// isExcludedPath checks if a path should be excluded from authentication +func isExcludedPath(path string) bool { + // Check if path starts with any item in excludedPaths + for _, excludedPath := range excludedPaths { + if strings.HasPrefix(path, excludedPath) { + return true + } + } + return false +} + +// parseFormValue safely extracts a form value +func parseFormValue(r *http.Request, key string) string { + return r.FormValue(key) +} diff --git a/server/cmd/server.go b/server/cmd/server.go new file mode 100644 index 00000000..1afee71c --- /dev/null +++ b/server/cmd/server.go @@ -0,0 +1,191 @@ +package cmd + +import ( + "io/fs" + "log" + "net/http" + "os" + "strconv" + "strings" + "time" + + "github.com/silverbulletmd/silverbullet/server" + "github.com/spf13/cobra" +) + +func buildConfig(bundledFiles fs.FS, args []string) *server.ServerConfig { + serverConfig := &server.ServerConfig{ + BindHost: "127.0.0.1", + Port: 3000, + } + + rootSpaceConfig := &server.SpaceConfig{ + IndexPage: "index", + SpaceName: "SilverBullet", + SpaceDescription: "Powerful and programmable note taking app", + } + + // For now just point every request to the rootSpaceConfig + serverConfig.SpaceConfigResolver = func(r *http.Request) *server.SpaceConfig { + return rootSpaceConfig + } + + if os.Getenv("SB_HOSTNAME") != "" { + serverConfig.BindHost = os.Getenv("SB_HOSTNAME") + } + + if os.Getenv("SB_PORT") != "" { + // Parse integer from string + port, err := strconv.ParseInt(os.Getenv("SB_PORT"), 10, 32) + if err != nil { + log.Fatalf("Could not parse numeric port from SB_PORT: %v", err) + } + serverConfig.Port = int(port) + } + + if os.Getenv("SB_INDEX_PAGE") != "" { + rootSpaceConfig.IndexPage = os.Getenv("SB_INDEX_PAGE") + } + if len(args) > 0 { + rootSpaceConfig.SpaceFolderPath = args[0] + } + if os.Getenv("SB_FOLDER") != "" { + rootSpaceConfig.SpaceFolderPath = os.Getenv("SB_FOLDER") + } + + if rootSpaceConfig.SpaceFolderPath == "" { + log.Fatal("No folder specified. Please pass a folder as an argument or set SB_FOLDER environment variable.") + } + + var spacePrimitives server.SpacePrimitives + spacePrimitives, err := server.NewDiskSpacePrimitives(rootSpaceConfig.SpaceFolderPath, os.Getenv("SB_GIT_IGNORE")) + + if err != nil { + log.Fatal(err) + } + + serverConfig.EnableHTTPLogging = os.Getenv("SB_HTTP_LOGGING") != "" + + if os.Getenv("SB_USER") != "" { + pieces := strings.Split(os.Getenv("SB_USER"), ":") + + rootSpaceConfig.Auth = &server.AuthOptions{ + User: pieces[0], + Pass: pieces[1], + AuthToken: os.Getenv("SB_AUTH_TOKEN"), + LockoutLimit: 10, + LockoutTime: 60, + } + + if os.Getenv("SB_LOCKOUT_LIMIT") != "" { + rootSpaceConfig.Auth.LockoutLimit, err = strconv.Atoi(os.Getenv("SB_LOCKOUT_LIMIT")) + if err != nil { + log.Fatalf("Could not parse SB_LOCKOUT_LIMIT as number: %v", err) + } + + } + + if os.Getenv("SB_LOCKOUT_TIME") != "" { + rootSpaceConfig.Auth.LockoutTime, err = strconv.Atoi(os.Getenv("SB_LOCKOUT_TIME")) + if err != nil { + log.Fatalf("Could not parse SB_LOCKOUT_TIME as number: %v", err) + } + } + + log.Printf("User authentication enabled for user \"%s\" with lockout limit %d and lockout time %ds", + pieces[0], rootSpaceConfig.Auth.LockoutLimit, rootSpaceConfig.Auth.LockoutTime) + } + + rootSpaceConfig.ReadOnlyMode = os.Getenv("SB_READ_ONLY") != "" + + if os.Getenv("SB_NAME") != "" { + rootSpaceConfig.SpaceName = os.Getenv("SB_NAME") + } + + if os.Getenv("SB_DESCRIPTION") != "" { + rootSpaceConfig.SpaceDescription = os.Getenv("SB_DESCRIPTION") + } + + if rootSpaceConfig.ReadOnlyMode { + log.Println("Starting in read-only mode.") + } + + rootSpaceConfig.GitIgnore = os.Getenv("SB_GIT_IGNORE") + + if os.Getenv("SB_SPACE_IGNORE") != "" { + log.Printf("Ignoring files matching: %s", os.Getenv("SB_SPACE_IGNORE")) + } + + if os.Getenv("SB_URL_PREFIX") != "" { + hostUrlPrefix := os.Getenv("SB_URL_PREFIX") + // Normalize the host URL prefix (if set) + if !strings.HasPrefix(hostUrlPrefix, "/") { + hostUrlPrefix = "/" + hostUrlPrefix + } + hostUrlPrefix = strings.TrimRight(hostUrlPrefix, "/") + + if hostUrlPrefix != "" { + log.Printf("Host URL Prefix: %s", hostUrlPrefix) + rootSpaceConfig.HostURLPrefix = hostUrlPrefix + } + } + + // Extract the last modified time from the main binary, best effort + bundlePathDate := time.Now() + if executablePath, err := os.Executable(); err == nil { + if stat, err := os.Stat(executablePath); err == nil { + bundlePathDate = stat.ModTime() + } + } + + serverConfig.ClientBundle = server.NewReadOnlyFallthroughSpacePrimitives(bundledFiles, "dist_client_bundle", bundlePathDate, nil) + rootSpaceConfig.SpacePrimitives = server.NewReadOnlyFallthroughSpacePrimitives(bundledFiles, "dist_base_fs_bundle", bundlePathDate, spacePrimitives) + + log.Printf("Starting SilverBullet binding to %s:%d", serverConfig.BindHost, serverConfig.Port) + if serverConfig.BindHost == "127.0.0.1" { + log.Println("SilverBullet will only be available locally, to allow outside connections, pass -L0.0.0.0 as a flag, and put a TLS terminator on top.") + } + + // Initialize shell backend + backendConfig := os.Getenv("SB_SHELL_BACKEND") + if backendConfig == "" && !rootSpaceConfig.ReadOnlyMode { + localShell := server.NewLocalShell(rootSpaceConfig.SpaceFolderPath, os.Getenv("SB_SHELL_WHITELIST")) + rootSpaceConfig.ShellBackend = localShell + if localShell.AllowAllCmds { + log.Println("Local shell command execution enabled for ALL commands.") + } else { + log.Printf("Local shell command execution enabled for: %v", localShell.CmdWhiteList) + } + } else { + log.Println("Shell running disabled.") + rootSpaceConfig.ShellBackend = server.NewNotSupportedShell() + } + + return serverConfig +} + +func ServerCommand(bundledFiles fs.FS) *cobra.Command { + var hostname string + var port int + var c = &cobra.Command{ + Use: "silverbullet", + Short: "Run the Silverbullet server", + Args: cobra.MinimumNArgs(0), + Run: func(cmd *cobra.Command, args []string) { + serverConfig := buildConfig(bundledFiles, args) + if port != 0 { + serverConfig.Port = port + } + if hostname != "" { + serverConfig.BindHost = hostname + } + if err := server.RunServer(serverConfig); err != nil { + log.Fatal(err) + } + }, + } + c.Flags().StringVarP(&hostname, "hostname", "L", "127.0.0.1", "Host or address to listen to") + c.Flags().IntVarP(&port, "port", "p", 3000, "Port to listen to") + + return c +} diff --git a/server/cmd/upgrade.go b/server/cmd/upgrade.go new file mode 100644 index 00000000..f75dd5d4 --- /dev/null +++ b/server/cmd/upgrade.go @@ -0,0 +1,157 @@ +package cmd + +import ( + "archive/zip" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "runtime" + + "github.com/spf13/cobra" +) + +// We can add more now that this will be Go compiled (we can do ARM32 too, for instance) +var archMappings map[string]string = map[string]string{ + "amd64": "x86_64", + "arm64": "aarch64", +} + +func UpgradeCommand() *cobra.Command { + return &cobra.Command{ + Use: "upgrade", + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Now going to attempt an upgrade...") + if err := upgrade("https://github.com/silverbulletmd/silverbullet/releases/latest/download"); err != nil { + log.Fatal(err) + } + }, + } +} + +func UpgradeEdgeCommand() *cobra.Command { + return &cobra.Command{ + Use: "upgrade-edge", + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("Now going to attempt an upgrade...") + if err := upgrade("https://github.com/silverbulletmd/silverbullet/releases/download/edge"); err != nil { + log.Fatal(err) + } + }, + } +} + +func upgrade(urlPrefix string) error { + // Get the directory of the current executable + execPath, err := os.Executable() + if err != nil { + return fmt.Errorf("failed to get executable path: %w", err) + } + installDir := filepath.Dir(execPath) + + fmt.Println("Install dir:", installDir) + + // Create temporary directory + tmpDir, err := os.MkdirTemp("", "silverbullet-upgrade") + if err != nil { + return fmt.Errorf("failed to create temp dir: %w", err) + } + defer os.RemoveAll(tmpDir) + + // Construct download URL + zipURL := fmt.Sprintf("%s/silverbullet-server-%s-%s.zip", urlPrefix, runtime.GOOS, archMappings[runtime.GOARCH]) + zipPath := filepath.Join(tmpDir, "silverbullet.zip") + + fmt.Println("Downloading from", zipURL) + + // Download the zip file + resp, err := http.Get(zipURL) + if err != nil { + return fmt.Errorf("failed to download: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download failed with status: %d", resp.StatusCode) + } + + // Create the zip file + zipFile, err := os.Create(zipPath) + if err != nil { + return fmt.Errorf("failed to create zip file: %w", err) + } + defer zipFile.Close() + + // Copy response body to file + _, err = io.Copy(zipFile, resp.Body) + if err != nil { + return fmt.Errorf("failed to save zip file: %w", err) + } + zipFile.Close() + + fmt.Printf("Now going to replace the existing silverbullet binary in %s\n", installDir) + + // Extract the zip file + err = extractZip(zipPath, installDir) + if err != nil { + return fmt.Errorf("failed to extract zip: %w", err) + } + + // Set executable permissions on the binary + binaryPath := filepath.Join(installDir, "silverbullet") + if runtime.GOOS == "windows" { + binaryPath += ".exe" + } + err = os.Chmod(binaryPath, 0755) + if err != nil { + return fmt.Errorf("failed to set executable permissions: %w", err) + } + + fmt.Println("And done! Restart your server to get the latest and greatest!") + return nil +} + +func extractZip(src, dest string) error { + reader, err := zip.OpenReader(src) + if err != nil { + return err + } + defer reader.Close() + + for _, file := range reader.File { + rc, err := file.Open() + if err != nil { + return err + } + defer rc.Close() + + path := filepath.Join(dest, file.Name) + + // Check for directory + if file.FileInfo().IsDir() { + os.MkdirAll(path, file.FileInfo().Mode()) + continue + } + + // Create parent directories + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return err + } + + // Create the file + outFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.FileInfo().Mode()) + if err != nil { + return err + } + defer outFile.Close() + + _, err = io.Copy(outFile, rc) + if err != nil { + return err + } + } + + return nil +} diff --git a/server/cmd/version.go b/server/cmd/version.go new file mode 100644 index 00000000..92a1df4b --- /dev/null +++ b/server/cmd/version.go @@ -0,0 +1,22 @@ +package cmd + +import ( + "fmt" + "log" + + "github.com/silverbulletmd/silverbullet/server" + "github.com/spf13/cobra" +) + +func VersionCommand(versionFileText string) *cobra.Command { + return &cobra.Command{ + Use: "version", + Run: func(cmd *cobra.Command, args []string) { + version, err := server.ParseVersionFromTypeScript(versionFileText) + if err != nil { + log.Fatal(err) + } + fmt.Println(version) + }, + } +} diff --git a/server/crypto.go b/server/crypto.go new file mode 100644 index 00000000..e837bb80 --- /dev/null +++ b/server/crypto.go @@ -0,0 +1,146 @@ +package server + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "maps" + "os" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +// Authenticator handles JWT creation and verification +type Authenticator struct { + path string + SecretKey string `json:"secret_key"` + AuthHash string `json:"auth_hash"` +} + +func CreateAuthenticator(path string, authOptions *AuthOptions) (*Authenticator, error) { + // Load from JSON + jsonBytes, err := os.ReadFile(path) + // If the file is not found, just start fresh + if os.IsNotExist(err) { + auth := &Authenticator{path: path} + + if err := auth.init(authOptions); err != nil { + return nil, err + } + return auth, nil + } + if err != nil { + return nil, err + } + + var auth Authenticator + auth.path = path + if err := json.Unmarshal(jsonBytes, &auth); err != nil { + return nil, err + } + if err := auth.init(authOptions); err != nil { + return nil, err + } + + return &auth, nil +} + +func (j *Authenticator) save() error { + jsonBytes, err := json.Marshal(j) + if err != nil { + return err + } + + return os.WriteFile(j.path, jsonBytes, 0600) +} + +// Init initializes the JWT issuer with an auth string for validation +func (j *Authenticator) init(authConfig *AuthOptions) error { + if j.SecretKey == "" { + fmt.Println("Generating new JWT secret key") + if err := j.generateNewKey(); err != nil { + return err + } + } + + newAuthHash := j.hashOptions(authConfig) + if j.AuthHash != newAuthHash { + fmt.Println("Authentication has changed since last run, so invalidating all existing tokens") + // Generate new key to invalidate all existing tokens + if err := j.generateNewKey(); err != nil { + return err + } + } + + j.AuthHash = newAuthHash + + return j.save() + +} + +// generateNewKey creates and stores a new secret key +func (j *Authenticator) generateNewKey() error { + // Generate 32 bytes (256 bits) of random data for the secret + secretKey := make([]byte, 32) + if _, err := rand.Read(secretKey); err != nil { + return fmt.Errorf("failed to generate secret key: %v", err) + } + + j.SecretKey = base64.StdEncoding.EncodeToString(secretKey) + + return nil +} + +func (j *Authenticator) hashOptions(authOptions *AuthOptions) string { + data, err := json.Marshal(authOptions) + if err != nil { + panic(err) + } + hash := sha256.Sum256(data) + return hex.EncodeToString(hash[:]) +} + +// CreateJWT creates a new JWT token with the given payload +func (j *Authenticator) CreateJWT(payload map[string]any, expirySeconds ...int) (string, error) { + claims := jwt.MapClaims{} + + // Copy payload to claims + maps.Copy(claims, payload) + + // Add expiry if specified + if len(expirySeconds) > 0 && expirySeconds[0] > 0 { + claims["exp"] = time.Now().Add(time.Duration(expirySeconds[0]) * time.Second).Unix() + } + + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + key, err := base64.StdEncoding.DecodeString(j.SecretKey) + if err != nil { + return "", err + } + return token.SignedString(key) +} + +// VerifyAndDecodeJWT verifies and decodes a JWT token +func (j *Authenticator) VerifyAndDecodeJWT(tokenString string) (map[string]any, error) { + token, err := jwt.Parse(tokenString, func(token *jwt.Token) (any, error) { + // Validate the signing method + if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok { + return nil, fmt.Errorf("unexpected signing method: %v", token.Header["alg"]) + } + return base64.StdEncoding.DecodeString(j.SecretKey) + }) + + if err != nil { + return nil, err + } + + if claims, ok := token.Claims.(jwt.MapClaims); ok && token.Valid { + return claims, nil + } + + return nil, fmt.Errorf("invalid token") +} diff --git a/server/crypto.test.ts b/server/crypto.test.ts deleted file mode 100644 index 5908f2e8..00000000 --- a/server/crypto.test.ts +++ /dev/null @@ -1,27 +0,0 @@ -import { sleep } from "../lib/async.ts"; -import { MemoryKvPrimitives } from "../lib/data/memory_kv_primitives.ts"; -import { assertEquals } from "@std/assert"; -import { JWTIssuer } from "./crypto.ts"; - -Deno.test("Test JWT crypto", async () => { - const db = new MemoryKvPrimitives(); - const jwt = new JWTIssuer(db); - await jwt.init("test"); - // Timeout value is 0 seconds, which means it should expire immediately with a 1 second grace period - const token = await jwt.createJWT({ user: "pete", role: "admin" }, 0); - const verified = await jwt.verifyAndDecodeJWT(token); - assertEquals(verified.user, "pete"); - try { - await jwt.verifyAndDecodeJWT(token + "bla"); - assertEquals(true, false, "Should have thrown invalid signature"); - } catch { - // expected - } - await sleep(1500); - try { - await jwt.verifyAndDecodeJWT(token); - assertEquals(true, false, "Should have thrown a timeout"); - } catch { - // expected - } -}); diff --git a/server/crypto.ts b/server/crypto.ts deleted file mode 100644 index 8f9f989b..00000000 --- a/server/crypto.ts +++ /dev/null @@ -1,78 +0,0 @@ -import { create, getNumericDate, verify } from "djwt"; -import type { KvPrimitives } from "../lib/data/kv_primitives.ts"; - -import type { KvKey } from "../type/datastore.ts"; -import { hashSHA256 } from "../lib/crypto.ts"; - -const jwtSecretKey: KvKey = ["jwtSecretKey"]; - -export class JWTIssuer { - private key!: CryptoKey; - - constructor(readonly kv: KvPrimitives) { - } - - // authString is only used to compare hashes to see if the auth has changed - async init(authString: string) { - const [secret] = await this.kv.batchGet([jwtSecretKey]); - if (!secret) { - console.log("Generating new JWT secret key"); - return this.generateNewKey(); - } else { - this.key = await crypto.subtle.importKey( - "jwk", - secret, - { name: "HMAC", hash: "SHA-512" }, - true, - ["sign", "verify"], - ); - } - - // Check if the authentication has changed since last run - const [currentAuthHash] = await this.kv.batchGet([[ - "authHash", - ]]); - const newAuthHash = await hashSHA256(authString); - if (currentAuthHash && currentAuthHash !== newAuthHash) { - console.log( - "Authentication has changed since last run, so invalidating all existing tokens", - ); - // It has, so we need to generate a new key to invalidate all existing tokens - await this.generateNewKey(); - } - if (currentAuthHash !== newAuthHash) { - // Persist new auth hash - await this.kv.batchSet([{ - key: ["authHash"], - value: newAuthHash, - }]); - } - } - - async generateNewKey() { - this.key = await crypto.subtle.generateKey( - { name: "HMAC", hash: "SHA-512" }, - true, - ["sign", "verify"], - ); - await this.kv.batchSet([{ - key: jwtSecretKey, - value: await crypto.subtle.exportKey("jwk", this.key), - }]); - } - - createJWT( - payload: Record<string, unknown>, - expirySeconds?: number, - ): Promise<string> { - const jwtPayload = { ...payload }; - if (expirySeconds) { - jwtPayload.exp = getNumericDate(expirySeconds); - } - return create({ alg: "HS512", typ: "JWT" }, jwtPayload, this.key); - } - - verifyAndDecodeJWT(jwt: string): Promise<Record<string, unknown>> { - return verify(jwt, this.key); - } -} diff --git a/server/crypto_test.go b/server/crypto_test.go new file mode 100644 index 00000000..2a98040b --- /dev/null +++ b/server/crypto_test.go @@ -0,0 +1,259 @@ +package server + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCreateAuthenticator_NewFile(t *testing.T) { + tmpDir := t.TempDir() + authFile := filepath.Join(tmpDir, "auth.json") + + authOptions := &AuthOptions{ + User: "testuser", + Pass: "testpass", + LockoutTime: 300, + LockoutLimit: 5, + } + + auth, err := CreateAuthenticator(authFile, authOptions) + assert.NoError(t, err, "Should create new authenticator without error") + assert.NotNil(t, auth, "Authenticator should not be nil") + assert.NotEmpty(t, auth.SecretKey, "Secret key should be generated") + assert.NotEmpty(t, auth.AuthHash, "Auth hash should be generated") + + // Verify file was created + _, err = os.Stat(authFile) + assert.False(t, os.IsNotExist(err), "Auth file should be created") +} + +func TestCreateAuthenticator_ExistingFile(t *testing.T) { + tmpDir := t.TempDir() + authFile := filepath.Join(tmpDir, "auth.json") + + // Create initial authenticator + authOptions := &AuthOptions{ + User: "testuser", + Pass: "testpass", + LockoutTime: 300, + LockoutLimit: 5, + } + + auth1, err := CreateAuthenticator(authFile, authOptions) + require.NoError(t, err) + originalKey := auth1.SecretKey + + // Load from existing file with same options + auth2, err := CreateAuthenticator(authFile, authOptions) + assert.NoError(t, err, "Should load existing authenticator without error") + assert.Equal(t, originalKey, auth2.SecretKey, "Secret key should remain the same") +} + +func TestCreateAuthenticator_AuthOptionsChanged(t *testing.T) { + tmpDir := t.TempDir() + authFile := filepath.Join(tmpDir, "auth.json") + + // Create initial authenticator + authOptions1 := &AuthOptions{ + User: "testuser", + Pass: "testpass", + LockoutTime: 300, + LockoutLimit: 5, + } + + auth1, err := CreateAuthenticator(authFile, authOptions1) + require.NoError(t, err) + originalKey := auth1.SecretKey + + // Load with different auth options + authOptions2 := &AuthOptions{ + User: "testuser", + Pass: "newpass", // Changed password + LockoutTime: 300, + LockoutLimit: 5, + } + + auth2, err := CreateAuthenticator(authFile, authOptions2) + assert.NoError(t, err, "Should load authenticator with changed options") + assert.NotEqual(t, originalKey, auth2.SecretKey, "Secret key should change when auth options change") +} + +func TestAuthenticator_CreateJWT(t *testing.T) { + tmpDir := t.TempDir() + authFile := filepath.Join(tmpDir, "auth.json") + + authOptions := &AuthOptions{ + User: "testuser", + Pass: "testpass", + LockoutTime: 300, + LockoutLimit: 5, + } + + auth, err := CreateAuthenticator(authFile, authOptions) + require.NoError(t, err) + + payload := map[string]any{ + "user": "testuser", + "role": "admin", + } + + token, err := auth.CreateJWT(payload) + assert.NoError(t, err, "Should create JWT without error") + assert.NotEmpty(t, token, "Token should not be empty") + + // Verify token structure (should have 3 parts separated by dots) + parts := len([]rune(token)) + assert.Greater(t, parts, 0, "Token should not be empty") +} + +func TestAuthenticator_CreateJWT_WithExpiry(t *testing.T) { + tmpDir := t.TempDir() + authFile := filepath.Join(tmpDir, "auth.json") + + authOptions := &AuthOptions{ + User: "testuser", + Pass: "testpass", + LockoutTime: 300, + LockoutLimit: 5, + } + + auth, err := CreateAuthenticator(authFile, authOptions) + require.NoError(t, err) + + payload := map[string]any{ + "user": "testuser", + } + + token, err := auth.CreateJWT(payload, 3600) // 1 hour expiry + assert.NoError(t, err, "Should create JWT with expiry without error") + assert.NotEmpty(t, token, "Token should not be empty") + + // Verify the token and check expiry + claims, err := auth.VerifyAndDecodeJWT(token) + assert.NoError(t, err, "Should verify token") + + exp, ok := claims["exp"] + assert.True(t, ok, "Token should have expiry claim") + assert.NotNil(t, exp, "Expiry should not be nil") +} + +func TestAuthenticator_VerifyAndDecodeJWT(t *testing.T) { + tmpDir := t.TempDir() + authFile := filepath.Join(tmpDir, "auth.json") + + authOptions := &AuthOptions{ + User: "testuser", + Pass: "testpass", + LockoutTime: 300, + LockoutLimit: 5, + } + + auth, err := CreateAuthenticator(authFile, authOptions) + require.NoError(t, err) + + payload := map[string]any{ + "user": "testuser", + "role": "admin", + "id": 123, + } + + token, err := auth.CreateJWT(payload) + require.NoError(t, err) + + // Verify and decode the token + claims, err := auth.VerifyAndDecodeJWT(token) + assert.NoError(t, err, "Should verify and decode JWT without error") + assert.Equal(t, "testuser", claims["user"], "Should preserve user claim") + assert.Equal(t, "admin", claims["role"], "Should preserve role claim") + assert.Equal(t, float64(123), claims["id"], "Should preserve id claim") +} + +func TestAuthenticator_VerifyAndDecodeJWT_InvalidToken(t *testing.T) { + tmpDir := t.TempDir() + authFile := filepath.Join(tmpDir, "auth.json") + + authOptions := &AuthOptions{ + User: "testuser", + Pass: "testpass", + LockoutTime: 300, + LockoutLimit: 5, + } + + auth, err := CreateAuthenticator(authFile, authOptions) + require.NoError(t, err) + + // Test invalid token + _, err = auth.VerifyAndDecodeJWT("invalid.token.here") + assert.Error(t, err, "Should return error for invalid token") + + // Test empty token + _, err = auth.VerifyAndDecodeJWT("") + assert.Error(t, err, "Should return error for empty token") +} + +func TestAuthenticator_VerifyAndDecodeJWT_ExpiredToken(t *testing.T) { + tmpDir := t.TempDir() + authFile := filepath.Join(tmpDir, "auth.json") + + authOptions := &AuthOptions{ + User: "testuser", + Pass: "testpass", + LockoutTime: 300, + LockoutLimit: 5, + } + + auth, err := CreateAuthenticator(authFile, authOptions) + require.NoError(t, err) + + payload := map[string]any{ + "user": "testuser", + } + + // Create token with 1 second expiry + token, err := auth.CreateJWT(payload, 1) + require.NoError(t, err) + + // Wait for token to expire + time.Sleep(2 * time.Second) + + // Try to verify expired token + _, err = auth.VerifyAndDecodeJWT(token) + assert.Error(t, err, "Should return error for expired token") +} + +func TestAuthenticator_VerifyAndDecodeJWT_WrongKey(t *testing.T) { + tmpDir := t.TempDir() + authFile1 := filepath.Join(tmpDir, "auth1.json") + authFile2 := filepath.Join(tmpDir, "auth2.json") + + authOptions := &AuthOptions{ + User: "testuser", + Pass: "testpass", + LockoutTime: 300, + LockoutLimit: 5, + } + + // Create two different authenticators (different keys) + auth1, err := CreateAuthenticator(authFile1, authOptions) + require.NoError(t, err) + + auth2, err := CreateAuthenticator(authFile2, authOptions) + require.NoError(t, err) + + payload := map[string]any{ + "user": "testuser", + } + + // Create token with first authenticator + token, err := auth1.CreateJWT(payload) + require.NoError(t, err) + + // Try to verify with second authenticator (different key) + _, err = auth2.VerifyAndDecodeJWT(token) + assert.Error(t, err, "Should return error when verifying token with wrong key") +} diff --git a/server_go/disk_space_primitives.go b/server/disk_space_primitives.go similarity index 84% rename from server_go/disk_space_primitives.go rename to server/disk_space_primitives.go index e621dd20..d51696a3 100644 --- a/server_go/disk_space_primitives.go +++ b/server/disk_space_primitives.go @@ -1,24 +1,27 @@ -package server_go +package server import ( "fmt" "io/fs" - "mime" "os" "path/filepath" "strings" "time" + + ignore "github.com/Diogenesoftoronto/go-gitignore" + "github.com/djherbis/times" ) // DiskSpacePrimitives implements SpacePrimitives for local disk storage type DiskSpacePrimitives struct { - rootPath string + rootPath string + gitIgnore *ignore.GitIgnore } var _ SpacePrimitives = &DiskSpacePrimitives{} // NewDiskSpacePrimitives creates a new DiskSpacePrimitives instance -func NewDiskSpacePrimitives(rootPath string) (*DiskSpacePrimitives, error) { +func NewDiskSpacePrimitives(rootPath string, gitIgnore string) (*DiskSpacePrimitives, error) { // Resolve to absolute path absPath, err := filepath.Abs(rootPath) if err != nil { @@ -26,12 +29,14 @@ func NewDiskSpacePrimitives(rootPath string) (*DiskSpacePrimitives, error) { } // Ensure the directory exists - if err := os.MkdirAll(absPath, 0755); err != nil { - return nil, fmt.Errorf("failed to create root directory: %w", err) + if stat, err := os.Stat(absPath); err != nil || !stat.IsDir() { + return nil, fmt.Errorf("no such directory: %w", err) } + ignore := ignore.CompileIgnoreLines(strings.Split(gitIgnore, "\n")...) return &DiskSpacePrimitives{ - rootPath: absPath, + rootPath: absPath, + gitIgnore: ignore, }, nil } @@ -86,35 +91,14 @@ func (d *DiskSpacePrimitives) pathToFilename(fullPath string) string { return strings.ReplaceAll(relativePath, string(filepath.Separator), "/") } -// lookupContentType determines MIME type based on file extension -func lookupContentType(path string) string { - contentType := mime.TypeByExtension(filepath.Ext(path)) - if contentType == "" { - return "application/octet-stream" - } - return contentType -} - // fileInfoToFileMeta converts os.FileInfo to FileMeta func (d *DiskSpacePrimitives) fileInfoToFileMeta(path string, info os.FileInfo) FileMeta { - var created int64 - var lastModified int64 - - lastModified = info.ModTime().UnixMilli() - - // Try to get creation time (platform-specific) - if stat := getCreationTime(info); stat != nil { - created = stat.UnixMilli() - } else { - created = lastModified - } - return FileMeta{ Name: path, Size: info.Size(), - ContentType: lookupContentType(path), - Created: created, - LastModified: lastModified, + ContentType: LookupContentTypeFromPath(path), + Created: getCreationTime(info).UnixMilli(), + LastModified: info.ModTime().UnixMilli(), Perm: "rw", } } @@ -134,6 +118,18 @@ func (d *DiskSpacePrimitives) FetchFileList() ([]FileMeta, error) { return nil } + relativePath := d.pathToFilename(path) + + // Exclude ignored files + if d.gitIgnore.MatchesPath(relativePath) { + return nil + } + + // Skip hidden folders in root + if strings.HasPrefix(relativePath, ".") { + return nil + } + // Get file info info, err := entry.Info() if err != nil { @@ -141,8 +137,6 @@ func (d *DiskSpacePrimitives) FetchFileList() ([]FileMeta, error) { return nil } - // Convert to relative path with forward slashes - relativePath := d.pathToFilename(path) fileMeta := d.fileInfoToFileMeta(relativePath, info) allFiles = append(allFiles, fileMeta) @@ -164,6 +158,7 @@ func (d *DiskSpacePrimitives) GetFileMeta(path string, observing bool) (FileMeta } info, err := os.Stat(localPath) + if err != nil { if os.IsNotExist(err) { return FileMeta{}, ErrNotFound @@ -263,3 +258,13 @@ func (d *DiskSpacePrimitives) cleanOrphaned(deletedFilePath string) { current = filepath.Dir(current) } } + +// This is tricky and OS specific, luckily we found the "times" package that abstracts from the OS-specific details +func getCreationTime(info os.FileInfo) time.Time { + t := times.Get(info) + if t.HasBirthTime() { + return t.BirthTime() + } else { + return t.ChangeTime() + } +} diff --git a/server/disk_space_primitives_test.go b/server/disk_space_primitives_test.go new file mode 100644 index 00000000..3775291a --- /dev/null +++ b/server/disk_space_primitives_test.go @@ -0,0 +1,89 @@ +package server + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestDiskSpacePrimitives(t *testing.T) { + // Create DiskSpacePrimitives instance + space, err := NewDiskSpacePrimitives(t.TempDir(), "") + assert.NoError(t, err, "Failed to create DiskSpacePrimitives") + + // Run all test suites using local testing utilities + TestSpacePrimitives(t, space) +} + +// Test path safety - specific to disk implementation +func TestSafePath(t *testing.T) { + space, err := NewDiskSpacePrimitives(t.TempDir(), "") + assert.NoError(t, err, "Failed to create DiskSpacePrimitives") + + // Test that paths outside root are rejected + _, err = space.safePath("../outside") + assert.Equal(t, ErrPathOutsideRoot, err, "Expected ErrPathOutsideRoot for '../outside'") + + _, err = space.safePath("/etc/passwd") + assert.Equal(t, ErrPathOutsideRoot, err, "Expected ErrPathOutsideRoot for '/etc/passwd'") + + // Test that valid paths work + _, err = space.safePath("valid/path.txt") + assert.NoError(t, err, "Valid path should not error") + + _, err = space.safePath("./valid.txt") + assert.NoError(t, err, "Valid relative path should not error") +} + +func TestGitIngore(t *testing.T) { + space, err := NewDiskSpacePrimitives(t.TempDir(), "*.txt\n*.jpg") + assert.NoError(t, err, "Failed to create DiskSpacePrimitives") + + // Initially no files + fileList, err := space.FetchFileList() + assert.NoError(t, err) + assert.Len(t, fileList, 0) + + // Write a file matching the gitignore pattern + _, err = space.WriteFile("test.txt", []byte("Test"), nil) + assert.NoError(t, err) + // And one that doesn't + _, err = space.WriteFile("test.md", []byte("Test"), nil) + assert.NoError(t, err) + + fileList, err = space.FetchFileList() + assert.NoError(t, err) + // Only 1 file + assert.Len(t, fileList, 1) + assert.Equal(t, fileList[0].Name, "test.md") + +} + +// Test directory cleanup - specific to disk implementation +func TestCleanOrphaned(t *testing.T) { + tmpDir := t.TempDir() + space, err := NewDiskSpacePrimitives(tmpDir, "") + assert.NoError(t, err, "Failed to create DiskSpacePrimitives") + + // Create nested directory structure with a file + _, err = space.WriteFile("deep/nested/dir/file.txt", []byte("test"), nil) + assert.NoError(t, err, "Failed to write nested file") + + // Verify the directories were created + nestedPath := filepath.Join(tmpDir, "deep", "nested", "dir") + _, err = os.Stat(nestedPath) + assert.False(t, os.IsNotExist(err), "Nested directory should exist") + + // Delete the file + err = space.DeleteFile("deep/nested/dir/file.txt") + assert.NoError(t, err, "Failed to delete nested file") + + _, err = os.Stat(nestedPath) + assert.True(t, os.IsNotExist(err), "Empty nested directory should be cleaned up") + + deepPath := filepath.Join(tmpDir, "deep") + _, err = os.Stat(deepPath) + assert.True(t, os.IsNotExist(err), "Empty parent directory should be cleaned up") +} diff --git a/server/embed_space_primitives.go b/server/embed_space_primitives.go new file mode 100644 index 00000000..7a34866c --- /dev/null +++ b/server/embed_space_primitives.go @@ -0,0 +1,177 @@ +package server + +import ( + "errors" + "fmt" + "io/fs" + pathLib "path" + "strings" + "time" +) + +// Implements a simple read-only fallthrough implementation of SpacePrimitives, used to serve static files embedde in the Go binary +type ReadOnlyFallthroughSpacePrimitives struct { + fsys fs.FS + fallthroughSpacePrimitives SpacePrimitives + rootPath string // Root path within the fs.FS + timeStamp time.Time // Fake timestamp to use for all files +} + +var _ SpacePrimitives = &ReadOnlyFallthroughSpacePrimitives{} + +func NewReadOnlyFallthroughSpacePrimitives(fsys fs.FS, rootPath string, timeStamp time.Time, wrapped SpacePrimitives) *ReadOnlyFallthroughSpacePrimitives { + return &ReadOnlyFallthroughSpacePrimitives{ + fsys: fsys, + fallthroughSpacePrimitives: wrapped, + rootPath: rootPath, + timeStamp: timeStamp, + } +} + +// pathToEmbedPath converts a relative path to an fs.FS path +func (e *ReadOnlyFallthroughSpacePrimitives) pathToEmbedPath(path string) string { + return pathLib.Join(e.rootPath, path) +} + +// Inverse of pathToEmbedPath +func (e *ReadOnlyFallthroughSpacePrimitives) embedPathToPath(path string) string { + return strings.TrimPrefix(path, e.rootPath+"/") +} + +// fileInfoToFileMeta converts fs.FileInfo to FileMeta for fs.FS files +func (e *ReadOnlyFallthroughSpacePrimitives) fileInfoToFileMeta(path string, info fs.FileInfo) FileMeta { + return FileMeta{ + Name: path, + Size: info.Size(), + ContentType: LookupContentTypeFromPath(path), + Created: e.timeStamp.UnixMilli(), + LastModified: e.timeStamp.UnixMilli(), + Perm: "ro", + } +} + +// FetchFileList implements SpacePrimitives.FetchFileList +// Lists all files from the filesystem first, then combines files from the fallback. +func (e *ReadOnlyFallthroughSpacePrimitives) FetchFileList() ([]FileMeta, error) { + allFiles := make([]FileMeta, 0, 1000) + + // First, collect files from the filesystem + err := fs.WalkDir(e.fsys, e.rootPath, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + // Skip files that can't be accessed + return nil + } + + // Skip directories + if entry.IsDir() { + return nil + } + + // Get file info + info, err := entry.Info() + if err != nil { + // Skip files we can't stat + return nil + } + + // Convert to our path format (forward slashes) + relativePath := e.embedPathToPath(path) + fileMeta := e.fileInfoToFileMeta(relativePath, info) + allFiles = append(allFiles, fileMeta) + + return nil + }) + + if err != nil { + fmt.Printf("Something went wrong listing files in the FS: %v", err) + } + + wrappedFiles, err := e.fallthroughSpacePrimitives.FetchFileList() + if err != nil { + return allFiles, err + } + + allFiles = append(allFiles, wrappedFiles...) + + return allFiles, nil +} + +// GetFileMeta implements SpacePrimitives.GetFileMeta +func (e *ReadOnlyFallthroughSpacePrimitives) GetFileMeta(path string, observing bool) (FileMeta, error) { + embedPath := e.pathToEmbedPath(path) + + // Try to get file info from filesystem + info, err := fs.Stat(e.fsys, embedPath) + if err == nil && !info.IsDir() { + return e.fileInfoToFileMeta(path, info), nil + } + + if e.fallthroughSpacePrimitives == nil { + return FileMeta{}, errors.New("Not found") + } + + // If not found in fs.FS, fall back + return e.fallthroughSpacePrimitives.GetFileMeta(path, observing) +} + +// ReadFile implements SpacePrimitives.ReadFile +func (e *ReadOnlyFallthroughSpacePrimitives) ReadFile(path string) ([]byte, FileMeta, error) { + embedPath := e.pathToEmbedPath(path) + + // Try to read from filesystem + data, err := fs.ReadFile(e.fsys, embedPath) + if err == nil { + // Get file info for metadata + info, statErr := fs.Stat(e.fsys, embedPath) + if statErr != nil { + return nil, FileMeta{}, statErr + } + + return data, e.fileInfoToFileMeta(path, info), nil + } + + if e.fallthroughSpacePrimitives == nil { + return nil, FileMeta{}, errors.New("Not found") + } + + // If not found in fs.FS, fall back + return e.fallthroughSpacePrimitives.ReadFile(path) +} + +// WriteFile implements SpacePrimitives.WriteFile +// Fails if file exists in filesystem, otherwise delegates to fallback +func (e *ReadOnlyFallthroughSpacePrimitives) WriteFile(path string, data []byte, meta *FileMeta) (FileMeta, error) { + embedPath := e.pathToEmbedPath(path) + + // Check if file exists in filesystem + _, err := fs.Stat(e.fsys, embedPath) + if err == nil { + // File exists in filesystem, cannot write + return FileMeta{}, fmt.Errorf("Cannot write file %s: read-only", path) + } + + if e.fallthroughSpacePrimitives == nil { + return FileMeta{}, errors.New("Not found") + } + + return e.fallthroughSpacePrimitives.WriteFile(path, data, meta) +} + +// DeleteFile implements SpacePrimitives.DeleteFile +// Fails if file exists in filesystem, otherwise delegates to fallback +func (e *ReadOnlyFallthroughSpacePrimitives) DeleteFile(path string) error { + embedPath := e.pathToEmbedPath(path) + + // Check if file exists in filesystem + _, err := fs.Stat(e.fsys, embedPath) + if err == nil { + // File exists in filesystem, cannot delete + return fmt.Errorf("cannot delete file %s: read-only file", path) + } + + if e.fallthroughSpacePrimitives == nil { + return errors.New("Not found") + } + + return e.fallthroughSpacePrimitives.DeleteFile(path) +} diff --git a/server/embed_space_primitives_test.go b/server/embed_space_primitives_test.go new file mode 100644 index 00000000..dca201c4 --- /dev/null +++ b/server/embed_space_primitives_test.go @@ -0,0 +1,50 @@ +package server + +import ( + "testing" + "testing/fstest" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestFSSpacePrimitivesWithMapFS(t *testing.T) { + // Test with a MapFS that has embedded files + mapFS := fstest.MapFS{ + "readme.txt": &fstest.MapFile{Data: []byte("Hello World")}, + "config.json": &fstest.MapFile{Data: []byte(`{"test": true}`)}, + } + + // Note: t.TempDir() automatically deletes the folder after the tests runs + fallback, err := NewDiskSpacePrimitives(t.TempDir(), "") + assert.NoError(t, err, "Failed to create fallback") + + primitives := NewReadOnlyFallthroughSpacePrimitives(mapFS, "", time.Now(), fallback) + + // Test reading embedded files + data, meta, err := primitives.ReadFile("readme.txt") + assert.NoError(t, err, "Should read embedded file") + assert.Equal(t, []byte("Hello World"), data, "Content should match") + assert.Equal(t, "readme.txt", meta.Name, "Name should match") + assert.Equal(t, "ro", meta.Perm, "Embedded files should be read-only") + + // Test that writing to embedded file names fails + _, err = primitives.WriteFile("readme.txt", []byte("new content"), nil) + assert.Error(t, err, "Should not overwrite embedded files") + assert.Contains(t, err.Error(), "read-only") + + // Test that deleting embedded files fails + err = primitives.DeleteFile("config.json") + assert.Error(t, err, "Should not delete embedded files") + assert.Contains(t, err.Error(), "read-only") + + // Test writing to non-embedded file names (should work via fallback) + _, err = primitives.WriteFile("new_file.txt", []byte("fallback content"), nil) + assert.NoError(t, err, "Should write to fallback for non-embedded files") + + // Test that fallback file can be read back + data, meta, err = primitives.ReadFile("new_file.txt") + assert.NoError(t, err, "Should read fallback file") + assert.Equal(t, []byte("fallback content"), data, "Fallback content should match") + assert.Equal(t, "new_file.txt", meta.Name, "Fallback file name should match") +} diff --git a/server/fs.go b/server/fs.go new file mode 100644 index 00000000..b674b963 --- /dev/null +++ b/server/fs.go @@ -0,0 +1,175 @@ +package server + +import ( + "fmt" + "io" + "log" + "net/http" + "net/url" + "strconv" + + "github.com/go-chi/chi/v5" + "github.com/go-chi/render" +) + +// buildFsRoutes creates the filesystem API routes +func buildFsRoutes() http.Handler { + fsRouter := chi.NewRouter() + + // File list endpoint + fsRouter.Get("/", handleFsList) + + // File operations + fsRouter.Get("/*", handleFsGet) + fsRouter.Put("/*", handleFsPut) + fsRouter.Delete("/*", handleFsDelete) + fsRouter.Options("/*", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Allow", "GET, PUT, DELETE, OPTIONS") + w.WriteHeader(http.StatusOK) + }) + + return fsRouter +} + +func handleFsList(w http.ResponseWriter, r *http.Request) { + spaceConfig := spaceConfigFromContext(r.Context()) + if r.Header.Get("X-Sync-Mode") != "" { + // Handle direct requests for JSON representation of file list + files, err := spaceConfig.SpacePrimitives.FetchFileList() + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("X-Space-Path", spaceConfig.SpaceFolderPath) + render.JSON(w, r, files) + } else { + // Otherwise, redirect to the UI + http.Redirect(w, r, "/", http.StatusTemporaryRedirect) + } +} + +// handleFsGet handles GET requests for individual files +func handleFsGet(w http.ResponseWriter, r *http.Request) { + path, err := getPath(w, r) + + spaceConfig := spaceConfigFromContext(r.Context()) + + if err != nil { + // Handled by getPath + return + } + + // log.Printf("Got this path: %s", path) + + if r.Header.Get("X-Get-Meta") != "" { + // Getting meta via GET request + meta, err := spaceConfig.SpacePrimitives.GetFileMeta(path, false) + if err != nil { + if err == ErrNotFound { + http.NotFound(w, r) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + setFileMetaHeaders(w, meta) + w.WriteHeader(http.StatusOK) + return + } + + // Read file content + data, meta, err := spaceConfig.SpacePrimitives.ReadFile(path) + if err != nil { + if err == ErrNotFound { + http.NotFound(w, r) + } else { + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + setFileMetaHeaders(w, meta) + w.WriteHeader(http.StatusOK) + w.Write(data) +} + +// handleFsPut handles PUT requests for writing files +func handleFsPut(w http.ResponseWriter, r *http.Request) { + path, err := getPath(w, r) + + spaceConfig := spaceConfigFromContext(r.Context()) + + if err != nil { + // Handled by getPath + return + } + + // Read request body + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "Failed to read request body", http.StatusBadRequest) + return + } + + // Write file + meta, err := spaceConfig.SpacePrimitives.WriteFile(path, body, nil) + if err != nil { + fmt.Printf("Write failed: %v\n", err) + http.Error(w, "Write failed", http.StatusInternalServerError) + return + } + + setFileMetaHeaders(w, meta) + w.WriteHeader(http.StatusOK) + w.Write([]byte("OK")) +} + +// handleFsDelete handles DELETE requests for removing files +func handleFsDelete(w http.ResponseWriter, r *http.Request) { + path, err := getPath(w, r) + + spaceConfig := spaceConfigFromContext(r.Context()) + + if err != nil { + // Handled by getPath + return + } + + fmt.Printf("Deleting file: %s\n", path) + + err = spaceConfig.SpacePrimitives.DeleteFile(path) + if err != nil { + if err == ErrNotFound { + http.NotFound(w, r) + } else { + fmt.Printf("Error deleting file: %v\n", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + } + return + } + + w.WriteHeader(http.StatusOK) + w.Write([]byte("OK")) +} + +// setFileMetaHeaders sets HTTP headers based on FileMeta +func setFileMetaHeaders(w http.ResponseWriter, meta FileMeta) { + w.Header().Set("Content-Type", meta.ContentType) + w.Header().Set("X-Created", strconv.FormatInt(meta.Created, 10)) + w.Header().Set("X-Last-Modified", strconv.FormatInt(meta.LastModified, 10)) + w.Header().Set("X-Content-Length", strconv.FormatInt(meta.Size, 10)) + w.Header().Set("X-Permission", meta.Perm) + w.Header().Set("Cache-Contro", "no-cache") +} + +func getPath(w http.ResponseWriter, r *http.Request) (string, error) { + path, err := url.QueryUnescape(chi.URLParam(r, "*")) + + if err != nil { + log.Printf("Could not decode URL: %s: %v", path, err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return "", err + } + return path, nil +} diff --git a/server/http_server.ts b/server/http_server.ts deleted file mode 100644 index 0e165f27..00000000 --- a/server/http_server.ts +++ /dev/null @@ -1,601 +0,0 @@ -import { type Context, Hono } from "hono"; -import { deleteCookie, getCookie, setCookie } from "hono/cookie"; -import { validator } from "hono/validator"; -import type { AssetBundle } from "../lib/asset_bundle/bundle.ts"; -import { handleShellEndpoint } from "./shell_endpoint.ts"; -import type { KvPrimitives } from "../lib/data/kv_primitives.ts"; -import { compile as gitIgnoreCompiler } from "gitignore-parser"; -import { decodePageURI } from "@silverbulletmd/silverbullet/lib/ref"; -import { LockoutTimer } from "./lockout.ts"; -import type { AuthOptions } from "../cmd/server.ts"; -import type { BootConfig } from "../web/ui_types.ts"; -import { applyUrlPrefix, removeUrlPrefix } from "../lib/url_prefix.ts"; -import { authCookieName, fileMetaToHeaders, utcDateString } from "./util.ts"; -import { renderHtmlPage } from "./serverside_render.ts"; -import { FilteredSpacePrimitives } from "../lib/spaces/filtered_space_primitives.ts"; -import { AssetBundlePlugSpacePrimitives } from "../lib/spaces/asset_bundle_space_primitives.ts"; -import { determineStorageBackend } from "./storage_backend.ts"; -import { ReadOnlySpacePrimitives } from "../lib/spaces/ro_space_primitives.ts"; -import type { SpacePrimitives } from "../lib/spaces/space_primitives.ts"; -import { JWTIssuer } from "./crypto.ts"; -import { - determineShellBackend, - NotSupportedShell, - type ShellBackend, -} from "./shell_backend.ts"; -import { CONFIG_TEMPLATE, INDEX_TEMPLATE } from "../web/PAGE_TEMPLATES.ts"; -import { - CheckPathSpacePrimitives, -} from "../lib/spaces/checked_space_primitives.ts"; -import { notFoundError } from "../lib/constants.ts"; - -const authenticationExpirySeconds = 60 * 60 * 24 * 7; // 1 week - -export type ServerOptions = { - hostname: string; - port: number; - hostUrlPrefix?: string; - auth?: AuthOptions; - spaceIgnore?: string; - pagesPath: string; - shellBackend: string; - shellCommandWhiteList?: string[]; - readOnly: boolean; - indexPage: string; -}; - -export class HttpServer { - private abortController?: AbortController; - private readonly hostname: string; - private readonly port: number; - private app: Hono; - private readonly spacePrimitives: SpacePrimitives; - private jwtIssuer: JWTIssuer; - private readonly shellBackend: ShellBackend; - - constructor( - readonly options: ServerOptions, - private clientAssetBundle: AssetBundle, - private plugAssetBundle: AssetBundle, - public baseKvPrimitives: KvPrimitives, - ) { - this.app = new Hono().basePath(options.hostUrlPrefix ?? ""); - this.hostname = options.hostname; - this.port = options.port; - - let fileFilterFn: (s: string) => boolean = () => true; - if (options.spaceIgnore) { - fileFilterFn = gitIgnoreCompiler(options.spaceIgnore).accepts; - } - - this.spacePrimitives = new CheckPathSpacePrimitives( - new FilteredSpacePrimitives( - new AssetBundlePlugSpacePrimitives( - determineStorageBackend(options.pagesPath), - this.plugAssetBundle, - ), - (meta) => fileFilterFn(meta.name), - ), - ); - - if (options.readOnly) { - this.spacePrimitives = new ReadOnlySpacePrimitives(this.spacePrimitives); - } - this.shellBackend = options.readOnly - ? new NotSupportedShell() // No shell for read only mode - : determineShellBackend(options); - this.jwtIssuer = new JWTIssuer(baseKvPrimitives); - } - - async start() { - if (this.options.auth) { - // Initialize JWT issuer - await this.jwtIssuer.init( - JSON.stringify({ auth: this.options.auth }), - ); - } - await this.ensureBasicPages(); - // Serve static files (javascript, css, html) - this.serveStatic(); - this.addAuth(); - this.app.route("/.fs", this.buildFsRoutes()); - - // Fallback, serve the UI index.html - this.app.use("*", (c) => { - const url = new URL(this.unprefixedUrl(c.req.url)); - const pageName = decodePageURI(url.pathname.slice(1)); - return renderHtmlPage( - this.spacePrimitives, - pageName, - c, - this.options, - this.clientAssetBundle, - ); - }); - - this.abortController = new AbortController(); - const listenOptions: any = { - hostname: this.hostname, - port: this.port, - signal: this.abortController.signal, - }; - - // Start the actual server - Deno.serve(listenOptions, this.app.fetch); - - const visibleHostname = this.hostname === "0.0.0.0" - ? "localhost" - : this.hostname; - console.log( - `SilverBullet is now running: http://${visibleHostname}:${this.port}`, - ); - } - - serveStatic() { - this.app.use("*", (c, next): Promise<void | Response> => { - const req = c.req; - const url = new URL(this.unprefixedUrl(req.url)); - if ( - url.pathname === "/" - ) { - // Serve the UI (index.html) - const indexPage = this.options.indexPage ?? "index"; - return renderHtmlPage( - this.spacePrimitives, - indexPage, - c, - this.options, - this.clientAssetBundle, - ); - } - try { - const assetName = url.pathname.slice(1); - if (!this.clientAssetBundle.has(assetName)) { - return next(); - } - if ( - this.clientAssetBundle.has(assetName) && - req.header("If-Modified-Since") === - utcDateString(this.clientAssetBundle.getMtime(assetName)) - ) { - return Promise.resolve(c.body(null, 304)); - } - c.status(200); - c.header("Content-type", this.clientAssetBundle.getMimeType(assetName)); - const data = this.clientAssetBundle.readFileSync( - assetName, - ); - c.header("Content-length", "" + data.length); - c.header( - "Last-Modified", - utcDateString(this.clientAssetBundle.getMtime(assetName)), - ); - - if (req.method === "GET") { - return Promise.resolve(c.body(new Uint8Array(data).buffer)); - } // else e.g. HEAD, OPTIONS, don't send body - } catch { - return next(); - } - return Promise.resolve(); - }); - } - - stop() { - if (this.abortController) { - this.abortController.abort(); - console.log("stopped server"); - } - } - - private addAuth() { - const excludedPaths = [ - "/manifest.json", - "/favicon.png", - "/logo.png", - "/.auth", - ]; - - // Since we're a single user app, we can use a single lockout timer to prevent brute force attacks - const lockoutTimer = this.options.auth?.lockoutLimit - ? new LockoutTimer( - // Turn into ms - this.options.auth.lockoutTime * 1000, - this.options.auth.lockoutLimit!, - ) - : new LockoutTimer(0, 0); // disabled - - this.app.get("/.logout", (c) => { - const url = new URL(this.unprefixedUrl(c.req.url)); - deleteCookie(c, authCookieName(url.host), { - path: `${this.options.hostUrlPrefix ?? ""}/`, - }); - deleteCookie(c, "refreshLogin", { - path: `${this.options.hostUrlPrefix ?? ""}/`, - }); - - return c.redirect(this.prefixedUrl("/.auth")); - }); - - // Authentication endpoints - this.app.get("/.auth", (c) => { - const html = this.clientAssetBundle.readTextFileSync(".client/auth.html") - .replaceAll("{{HOST_URL_PREFIX}}", this.options.hostUrlPrefix ?? ""); - - return c.html(html); - }).post( - validator("form", (value: any, c: Context) => { - const username = value["username"]; - const password = value["password"]; - const rememberMe = value["rememberMe"]; - - if ( - !username || typeof username !== "string" || - !password || typeof password !== "string" || - (rememberMe && typeof rememberMe !== "string") - ) { - return c.redirect(this.prefixedUrl("/.auth?error=0")); - } - - return { username, password, rememberMe }; - }), - async (c) => { - const req = c.req; - const url = new URL(this.unprefixedUrl(req.url)); - const { username, password, rememberMe } = req.valid("form"); - - const { - user: expectedUser, - pass: expectedPassword, - } = this.options.auth!; - - if (lockoutTimer.isLocked()) { - console.error("Authentication locked out, redirecting to auth page."); - return c.redirect(this.prefixedUrl("/.auth?error=2")); - } - - if (username === expectedUser && password === expectedPassword) { - // Generate a JWT and set it as a cookie - const jwt = rememberMe - ? await this.jwtIssuer.createJWT({ username }) - : await this.jwtIssuer.createJWT( - { username }, - authenticationExpirySeconds, - ); - console.log("Successful auth"); - const inAWeek = new Date( - Date.now() + authenticationExpirySeconds * 1000, - ); - setCookie(c, authCookieName(url.host), jwt, { - path: `${this.options.hostUrlPrefix ?? ""}/`, - expires: inAWeek, - }); - if (rememberMe) { - setCookie(c, "refreshLogin", "true", { - path: `${this.options.hostUrlPrefix ?? ""}/`, - expires: inAWeek, - }); - } - const values = await req.parseBody(); - const from = values["from"]; - return c.redirect( - this.prefixedUrl(typeof from === "string" ? from : "/"), - ); - } else { - console.error("Authentication failed, redirecting to auth page."); - lockoutTimer.addCount(); - return c.redirect(this.prefixedUrl("/.auth?error=1")); - } - }, - ).all((c) => { - return c.redirect(this.prefixedUrl("/.auth")); - }); - - // Simple ping health endpoint - this.app.get("/.ping", (c) => { - return c.text("OK", 200, { - "Cache-Control": "no-cache", - }); - }); - - // Check auth on every other request - this.app.use("*", async (c, next) => { - if (!this.options.auth) { - // Auth disabled in this config, skip - return next(); - } - const req = c.req; - const url = new URL(this.unprefixedUrl(req.url)); - const path = this.unprefixedUrl(req.path); - const host = url.host; - const redirectToAuth = () => { - // Try filtering api paths - if (path.startsWith("/.") || path.endsWith(".md")) { - return c.redirect(this.prefixedUrl("/.auth"), 401 as any); - } else { - return c.redirect( - this.prefixedUrl(`/.auth?from=${path}`), - 302 as any, - ); - } - }; - if (!excludedPaths.includes(url.pathname)) { - const authCookie = getCookie(c, authCookieName(host)); - - if (!authCookie && this.options.auth?.authToken) { - // Attempt Bearer Authorization based authentication - const authHeader = req.header("Authorization"); - if (authHeader && authHeader.startsWith("Bearer ")) { - const authToken = authHeader.slice("Bearer ".length); - if (authToken === this.options.auth.authToken) { - // All good, let's proceed - this.refreshLogin(c, host); - return next(); - } else { - console.log( - "Unauthorized token access, redirecting to auth page", - ); - return c.text("Unauthorized", 401); - } - } - } - if (!authCookie) { - console.log("Unauthorized access, redirecting to auth page"); - return redirectToAuth(); - } - const { user: expectedUser } = this.options.auth!; - - try { - const verifiedJwt = await this.jwtIssuer - .verifyAndDecodeJWT( - authCookie, - ); - if (verifiedJwt.username !== expectedUser) { - throw new Error("Username mismatch"); - } - } catch (e: any) { - console.error( - "Error verifying JWT, redirecting to auth page", - e.message, - ); - return redirectToAuth(); - } - } - this.refreshLogin(c, host); - return next(); - }); - - // Fetch config - this.app.get("/.config", (c) => { - const clientConfig: BootConfig = { - readOnly: this.options.readOnly, - spaceFolderPath: this.options.pagesPath, - indexPage: this.options.indexPage, - }; - return c.json(clientConfig, 200, { - "Cache-Control": "no-cache", - }); - }); - - // Shell command endpoint - this.app.post("/.shell", (c) => { - return handleShellEndpoint( - c, - this.shellBackend, - this.options.readOnly, - ); - }); - - // HTTP Proxy endpoint - const proxyPathRegex = "/.proxy/:uri{.+}"; - this.app.all( - proxyPathRegex, - async (c) => { - const req = c.req; - if (this.options.readOnly) { - return c.text("Read only mode, no proxy allowed", 405); - } - - // Get the full URL including query parameters - const originalUrl = new URL(req.url); - let url = req.param("uri")! + originalUrl.search; - - // Assume https unless this is localhost or an IP address - if ( - url.startsWith("localhost") || url.match(/^\d+\./) - ) { - url = `http://${url}`; - } else { - url = `https://${url}`; - } - console.log("Proxying to", url); - try { - const safeRequestHeaders = new Headers(); - // List all headers - for ( - const headerName of ["Authorization", "Accept", "Content-Type"] - ) { - if (req.header(headerName)) { - safeRequestHeaders.set( - headerName, - req.header(headerName)!, - ); - } - } - // List all headers starting with X-Proxy-Header-, remove the prefix and add to the safe headers - for (const [key, value] of Object.entries(req.header())) { - if (key.startsWith("x-proxy-header-")) { - safeRequestHeaders.set( - key.slice("x-proxy-header-".length), // corrected casing of header prefix - value, - ); - } - } - const body = await req.arrayBuffer(); - return fetch(url, { - method: req.method, - headers: safeRequestHeaders, - body: body.byteLength > 0 ? body : undefined, - }); - } catch (e: any) { - console.error("Error fetching federated link", e); - return c.text(e.message, 500); - } - }, - ); - } - - private refreshLogin(c: Context, host: string) { - if (getCookie(c, "refreshLogin")) { - const inAWeek = new Date( - Date.now() + authenticationExpirySeconds * 1000, - ); - const jwt = getCookie(c, authCookieName(host)); - if (jwt) { - setCookie(c, authCookieName(host), jwt, { - path: `${this.options.hostUrlPrefix ?? ""}/`, - expires: inAWeek, - // sameSite: "Strict", - // httpOnly: true, - }); - setCookie(c, "refreshLogin", "true", { - path: `${this.options.hostUrlPrefix ?? ""}/`, - expires: inAWeek, - }); - } - } - } - - private buildFsRoutes(): Hono { - const fsRoutes = new Hono(); - // File list - fsRoutes.get("/", async (c) => { - const req = c.req; - if (req.header("X-Sync-Mode")) { - // Only handle direct requests for a JSON representation of the file list - const files = await this.spacePrimitives.fetchFileList(); - return c.json(files, 200, { - "X-Space-Path": this.options.pagesPath, - }); - } else { - // Otherwise, redirect to the UI - // The reason to do this is to handle authentication systems like Authelia nicely - return c.redirect(this.prefixedUrl("/")); - } - }); - - fsRoutes.get("/:path{.*}", this.handleGet.bind(this)).put( - this.handlePut.bind(this), - ).delete(this.handleDelete.bind(this)).options(); - - return fsRoutes; - } - - private async handleDelete(c: Context<any>) { - const req = c.req; - const name = req.param("path")!; - if (this.options.readOnly) { - return c.text("Read only mode, no writes allowed", 405); - } - console.log("Deleting file", name); - try { - await this.spacePrimitives.deleteFile(name); - return c.text("OK"); - } catch (e: any) { - console.error("Error deleting document", e); - return c.text(e.message, 500); - } - } - - private async handlePut(c: Context<any>) { - const req = c.req; - const path = req.param("path")!; - if (this.options.readOnly) { - return c.text("Read only mode, no writes allowed", 405); - } - console.log("Writing file", path); - - const body = await req.arrayBuffer(); - - try { - const meta = await this.spacePrimitives.writeFile( - path, - new Uint8Array(body), - ); - return c.text("OK", 200, fileMetaToHeaders(meta)); - } catch (err) { - console.error("Write failed", err); - return c.text("Write failed", 500); - } - } - - async handleGet(c: Context<any>) { - const req = c.req; - const name = req.param("path")!; - - try { - if (req.header("X-Get-Meta")) { - // Getting meta via GET request - const fileData = await this.spacePrimitives.getFileMeta( - name, - ); - return c.text("", 200, fileMetaToHeaders(fileData)); - } - const fileData = await this.spacePrimitives.readFile(name); - const lastModifiedHeader = new Date(fileData.meta.lastModified) - .toUTCString(); - if ( - req.header("If-Modified-Since") === lastModifiedHeader - ) { - return c.body(null, 304); - } - return c.body(new Uint8Array(fileData.data).buffer, 200, { - ...fileMetaToHeaders(fileData.meta), - "Last-Modified": lastModifiedHeader, - }); - } catch (e: any) { - console.error("Error GETting file", name, e.message); - return c.notFound(); - } - } - - private prefixedUrl(url: string): string { - return applyUrlPrefix(url, this.options.hostUrlPrefix); - } - - private unprefixedUrl(url: string): string { - return removeUrlPrefix(url, this.options.hostUrlPrefix); - } - - async ensureBasicPages() { - await this.ensurePageWithContent( - `${this.options.indexPage}.md`, - INDEX_TEMPLATE, - ); - - const files = await this.spacePrimitives.fetchFileList(); - const hasConfig = files.some( - (f) => f.name === "CONFIG.md" || f.name.endsWith("/CONFIG.md"), - ); - if (!hasConfig) { - await this.ensurePageWithContent("CONFIG.md", CONFIG_TEMPLATE); - } - } - - private async ensurePageWithContent(path: string, content: string) { - try { - // This will blow up if the page doesn't exist - await this.spacePrimitives.getFileMeta(path); - } catch (e: any) { - if (e.message === notFoundError.message) { - console.info(path, "page not found, creating..."); - await this.spacePrimitives.writeFile( - path, - new TextEncoder().encode(content), - ); - } else { - throw e; - } - } - } -} diff --git a/server/lockout.go b/server/lockout.go new file mode 100644 index 00000000..6b26b42b --- /dev/null +++ b/server/lockout.go @@ -0,0 +1,69 @@ +package server + +import ( + "math" + "sync" + "time" +) + +// LockoutTimer implements a simple rate limiter to prevent brute force attacks +type LockoutTimer struct { + mutex sync.Mutex + bucketTime int64 + bucketCount int + bucketSize int64 // duration in milliseconds + limit int + disabled bool +} + +// NewLockoutTimer creates a new lockout timer +// countPeriodMs: time window in milliseconds +// limit: maximum attempts allowed in the time window +func NewLockoutTimer(countPeriodMs int, limit int) *LockoutTimer { + disabled := math.IsNaN(float64(countPeriodMs)) || math.IsNaN(float64(limit)) || + countPeriodMs < 1 || limit < 1 + + return &LockoutTimer{ + bucketSize: int64(countPeriodMs), + limit: limit, + disabled: disabled, + } +} + +// updateBucketTime updates the current bucket time and resets count if needed +func (lt *LockoutTimer) updateBucketTime() { + currentBucketTime := time.Now().UnixMilli() / lt.bucketSize + if lt.bucketTime == currentBucketTime { + return + } + // the bucket is too old - empty it + lt.bucketTime = currentBucketTime + lt.bucketCount = 0 +} + +// IsLocked checks if the timer is currently locked due to too many attempts +func (lt *LockoutTimer) IsLocked() bool { + if lt.disabled { + return false + } + + lt.mutex.Lock() + defer lt.mutex.Unlock() + + lt.updateBucketTime() + return lt.bucketCount >= lt.limit +} + +// AddCount increments the attempt counter +// IsLocked() should be called first to keep bucketTime current +func (lt *LockoutTimer) AddCount() { + if lt.disabled { + return + } + + lt.mutex.Lock() + defer lt.mutex.Unlock() + + // updateBucketTime should have been called by IsLocked first + lt.bucketCount++ +} diff --git a/server/lockout.test.ts b/server/lockout.test.ts deleted file mode 100644 index 4e0e2a3b..00000000 --- a/server/lockout.test.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { assert, assertEquals } from "@std/assert"; -import { LockoutTimer } from "./lockout.ts"; -import { FakeTime } from "@std/testing/time"; - -const lockoutTime = 60000; -const lockoutLimit = 10; - -Deno.test("Lockout - failed login rate limiter", () => { - using time = new FakeTime(); - - testDisabled(new LockoutTimer(NaN, lockoutLimit), "by period"); - testDisabled(new LockoutTimer(lockoutTime, NaN), "by limit"); - - testLockout(new LockoutTimer(lockoutTime, 10), "explicit params"); - testLockoutPerMS(new LockoutTimer(lockoutTime, 10), "explicit params"); - - function testDisabled(timer: LockoutTimer, txt: string) { - for (let i = 0; i < 100; i++) { - assertEquals( - timer.isLocked(), - false, - `Should be unlocked - disabled ${txt} loop ${i}`, - ); - timer.addCount(); - } - } - - function testLockoutPerMS(timer: LockoutTimer, txt: string) { - assert( - lockoutTime > lockoutLimit, - `testLockoutPerMS assumes lockoutTime(${lockoutTime}) > lockoutLimit(${lockoutLimit}), so it can fill a bucket in 1ms jumps`, - ); - - const bucketPasses = 2; - - let countLocked = 0; - let countUnLocked = 0; - - const totalTests = lockoutTime * bucketPasses; - - for (let i = 0; i < totalTests; i++) { - if (timer.isLocked()) { - countLocked++; - } else { - countUnLocked++; - } - timer.addCount(); - time.tick(1); - } - - // usually this will be bucketPasses+1 buckets - // but if time aligns could be just bucketPasses buckets - // and could be in between if it doesn't have time to completely fill the extra bucket - - const expectedUnlocked = lockoutLimit * (bucketPasses + 1); - const expectedLocked = totalTests - expectedUnlocked; - - const expectedMinUnlocked = lockoutLimit * bucketPasses; - const expectedMaxLocked = totalTests - expectedMinUnlocked; - - assert( - countUnLocked >= expectedMinUnlocked && countUnLocked <= expectedUnlocked, - `Expected between ${expectedMinUnlocked} and ${expectedUnlocked} unlocks in ${bucketPasses} passes of ${lockoutTime}ms, but got ${countUnLocked} (${txt})`, - ); - - assert( - countLocked >= expectedLocked && countLocked <= expectedMaxLocked, - `Expected between ${expectedLocked} and ${expectedMaxLocked} locks in ${bucketPasses} passes of ${lockoutTime}ms, but got ${countLocked} (${txt})`, - ); - } - - function testLockout(timer: LockoutTimer, txt: string) { - for (let pass = 0; pass < 3; pass++) { - for (let i = 0; i < lockoutLimit; i++) { - assertEquals( - timer.isLocked(), - false, - `Should be unlocked pass ${pass} loop ${i} ${txt}`, - ); - timer.addCount(); - } - // count = lockoutLimit - - for (let i = 0; i < 10; i++) { - assertEquals( - timer.isLocked(), - true, - `Should be locked before bucket rollover. pass ${pass} loop ${i} ${txt}`, - ); - timer.addCount(); - } - // count = lockoutLimit + 10 - - time.tick(lockoutTime); - // bucket should rollover, count=0 - } - } -}); diff --git a/server/lockout.ts b/server/lockout.ts deleted file mode 100644 index e9e54de7..00000000 --- a/server/lockout.ts +++ /dev/null @@ -1,45 +0,0 @@ -export class LockoutTimer { - // A very simple, quick, inexact lockout timer - // For each request isLocked() or updateBucketTime() must be called before addCount() - // counts in buckets of countPeriodMs - // each period starts with an empty bucket - // suggest SB_LOCKOUT_TIME_MS=60000 SB_LOCKOUT_LIMIT=10 - bucketTime: number = 0; - bucketCount: number = 0; - bucketSize: number; - limit: number; - disabled: boolean; - - constructor( - countPeriodMs: number, - limit: number, - ) { - this.disabled = isNaN(countPeriodMs) || isNaN(limit) || countPeriodMs < 1 || - limit < 1; - this.bucketSize = countPeriodMs; - this.limit = limit; - } - - updateBucketTime(): void { - const currentBucketTime = Math.floor(Date.now() / this.bucketSize); - if (this.bucketTime === currentBucketTime) { - return; - } - // the bucket is too old - empty it - this.bucketTime = currentBucketTime; - this.bucketCount = 0; - } - - isLocked(): boolean { - if (this.disabled) { - return false; - } - this.updateBucketTime(); - return this.bucketCount >= this.limit; - } - - addCount(): void { - // isLocked or updateBucketTime must be called first to keep bucketTime current - this.bucketCount++; - } -} diff --git a/server/manifest.go b/server/manifest.go new file mode 100644 index 00000000..8d7bb574 --- /dev/null +++ b/server/manifest.go @@ -0,0 +1,61 @@ +package server + +import ( + "net/http" + + "github.com/go-chi/render" +) + +// ManifestIcon represents an icon in the PWA manifest +type ManifestIcon struct { + Src string `json:"src"` + Type string `json:"type"` + Sizes string `json:"sizes"` +} + +// Manifest represents the structure of a PWA manifest.json file +type Manifest struct { + ShortName string `json:"short_name"` + Name string `json:"name"` + Icons []ManifestIcon `json:"icons"` + CaptureLinks string `json:"capture_links"` + StartURL string `json:"start_url"` + Display string `json:"display"` + DisplayOverride []string `json:"display_override"` + Scope string `json:"scope"` + ThemeColor string `json:"theme_color"` + Description string `json:"description"` +} + +// Generate PWA manifest.json with dynamic values +func manifestHandler(config *ServerConfig) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + spaceConfig := spaceConfigFromContext(r.Context()) + manifest := Manifest{ + ShortName: spaceConfig.SpaceName, + Name: spaceConfig.SpaceName, + Icons: []ManifestIcon{ + { + Src: "/.client/logo-dock.png", + Type: "image/png", + Sizes: "512x512", + }, + }, + CaptureLinks: "new-client", + StartURL: "/#boot", + Display: "standalone", + DisplayOverride: []string{"window-controls-overlay"}, + Scope: "/", + ThemeColor: "#e1e1e1", + Description: spaceConfig.SpaceDescription, + } + + if spaceConfig.HostURLPrefix != "" { + manifest.Icons[0].Src = spaceConfig.HostURLPrefix + manifest.Icons[0].Src + manifest.StartURL = spaceConfig.HostURLPrefix + manifest.StartURL + manifest.Scope = spaceConfig.HostURLPrefix + manifest.Scope + } + + render.JSON(w, r, manifest) + } +} diff --git a/server/proxy.go b/server/proxy.go new file mode 100644 index 00000000..045893a0 --- /dev/null +++ b/server/proxy.go @@ -0,0 +1,96 @@ +package server + +import ( + "fmt" + "io" + "net/http" + "regexp" + "strings" + + "github.com/go-chi/chi/v5" +) + +var localhostRegex *regexp.Regexp + +func init() { + localhostRegex = regexp.MustCompile(`^(localhost|127\.0\.0\.1|\d+\.\d+\.\d+\.\d+)`) +} + +func proxyHandler(config *ServerConfig) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + spaceConfig := spaceConfigFromContext(r.Context()) + if spaceConfig.ReadOnlyMode { + http.Error(w, "Read only mode, no proxy allowed", http.StatusMethodNotAllowed) + return + } + + // Extract the URI from the path + path := chi.URLParam(r, "*") + if path == "" { + http.Error(w, "No URI provided", http.StatusBadRequest) + return + } + + // Add query parameters if they exist + if r.URL.RawQuery != "" { + path += "?" + r.URL.RawQuery + } + + // Determine protocol based on hostname + var targetURL string + if localhostRegex.MatchString(path) { + targetURL = "http://" + path + } else { + targetURL = "https://" + path + } + + fmt.Printf("Proxying to %s\n", targetURL) + + // Create the proxy request + req, err := http.NewRequest(r.Method, targetURL, r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Copy safe headers + safeHeaders := []string{"Authorization", "Accept", "Content-Type"} + for _, header := range safeHeaders { + if value := r.Header.Get(header); value != "" { + req.Header.Set(header, value) + } + } + + // Copy headers starting with X-Proxy-Header-, removing the prefix + for key, values := range r.Header { + if strings.HasPrefix(strings.ToLower(key), "x-proxy-header-") { + headerName := key[len("x-proxy-header-"):] + for _, value := range values { + req.Header.Set(headerName, value) + } + } + } + + // Make the request + client := http.DefaultClient + + resp, err := client.Do(req) + if err != nil { + fmt.Printf("Error fetching proxied URL: %v\n", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + defer resp.Body.Close() + + // Copy response headers + for key, values := range resp.Header { + for _, value := range values { + w.Header().Set(key, value) + } + } + + // Set status code and copy body + w.WriteHeader(resp.StatusCode) + io.Copy(w, resp.Body) + } +} diff --git a/server/server.go b/server/server.go new file mode 100644 index 00000000..a4bfc7c4 --- /dev/null +++ b/server/server.go @@ -0,0 +1,102 @@ +package server + +import ( + "fmt" + "log" + "net/http" + + "github.com/go-chi/chi/v5" + "github.com/go-chi/chi/v5/middleware" + "github.com/go-chi/render" +) + +const indexHtmlPath = ".client/index.html" + +// BootConfig represents the client configuration +type BootConfig struct { + SpaceFolderPath string `json:"spaceFolderPath"` + IndexPage string `json:"indexPage"` + ReadOnly bool `json:"readOnly"` +} + +func RunServer(config *ServerConfig) error { + r := chi.NewRouter() + + if config.EnableHTTPLogging { + r.Use(middleware.Logger) + } + + // Expose space primitives and path to the request + r.Use(spaceMiddleware(config)) + + // Authentication middleware (applies to all routes after this point) + r.Use(authMiddleware(config)) + + // Authentication endpoints (must come before auth middleware) + addAuthEndpoints(r, config) + + r.Get("/.ping", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Cache-Control", "no-cache") + w.WriteHeader(http.StatusOK) + w.Write([]byte("OK")) + }) + + // Mount filesystem routes under /.fs + r.Mount("/.fs", buildFsRoutes()) + + // Config endpoint + r.Get("/.config", func(w http.ResponseWriter, r *http.Request) { + spaceConfig := spaceConfigFromContext(r.Context()) + clientConfig := &BootConfig{ + SpaceFolderPath: spaceConfig.SpaceFolderPath, + IndexPage: spaceConfig.IndexPage, + ReadOnly: spaceConfig.ReadOnlyMode, + } + + w.Header().Set("Cache-Control", "no-cache") + render.JSON(w, r, clientConfig) + }) + + // Shell endpoint + r.Post("/.shell", handleShellEndpoint(config)) + + // Proxy endpoint + r.HandleFunc("/.proxy/*", proxyHandler(config)) + + // Manifest endpoint + r.HandleFunc("/.client/manifest.json", manifestHandler(config)) + + r.HandleFunc("/*", func(w http.ResponseWriter, r *http.Request) { + path := chi.URLParam(r, "*") + + spaceConfig := spaceConfigFromContext(r.Context()) + + // See if it's in the client bundle + data, meta, err := config.ClientBundle.ReadFile(path) + if err == nil { + // File is in the bundle, let's serve it + if r.Header.Get("If-Modified-Since") == utcDateString(meta.LastModified) { + w.WriteHeader(304) + return + } + w.Header().Set("Content-Type", meta.ContentType) + w.Header().Set("Last-Modified", utcDateString(meta.LastModified)) + w.WriteHeader(200) + w.Write(data) + return + } + + // TODO: handle request types + ServerSideRender(config, spaceConfig, path, w, r) + }) + + // Display the final server running message + visibleHostname := config.BindHost + if config.BindHost == "127.0.0.1" { + visibleHostname = "localhost" + } + log.Printf("SilverBullet is now running: http://%s:%d", visibleHostname, config.Port) + + http.ListenAndServe(fmt.Sprintf("%s:%d", config.BindHost, config.Port), r) + return nil +} diff --git a/server/serverside_render.ts b/server/serverside_render.ts deleted file mode 100644 index ff4f4552..00000000 --- a/server/serverside_render.ts +++ /dev/null @@ -1,93 +0,0 @@ -import type { Context } from "hono"; -import { utcDateString } from "./util.ts"; -import { - getNameFromPath, - isMarkdownPath, - parseToRef, -} from "@silverbulletmd/silverbullet/lib/ref"; -import { parse } from "../web/markdown_parser/parse_tree.ts"; -import { extendedMarkdownLanguage } from "../web/markdown_parser/parser.ts"; -import { renderMarkdownToHtml } from "../web/markdown/markdown_render.ts"; -import type { ServerOptions } from "./http_server.ts"; -import type { AssetBundle } from "../lib/asset_bundle/bundle.ts"; -import { htmlEscape } from "../web/markdown/html_render.ts"; -import type { SpacePrimitives } from "../lib/spaces/space_primitives.ts"; -import { notFoundError } from "../lib/constants.ts"; - -// Server-side renders a markdown file to HTML -export async function renderHtmlPage( - spacePrimitives: SpacePrimitives, - path: string, - c: Context, - options: ServerOptions, - clientAssetBundle: AssetBundle, -): Promise<Response> { - const ref = parseToRef(path) ?? parseToRef(options.indexPage)!; - - let html = ""; - let title = "SilverBullet"; // Default to simply SilverBullet initially - let lastModified = utcDateString(Date.now()); - if (!options.auth && options.readOnly) { - // Only attempt server-side rendering when this site is not protected by auth and running in read-only mode - if (isMarkdownPath(ref.path)) { - title = getNameFromPath(ref.path); - try { - const { data, meta } = await spacePrimitives.readFile( - ref.path, - ); - lastModified = utcDateString(meta.lastModified); - - if (c.req.header("If-Modified-Since") === lastModified) { - // Not modified, empty body status 304 - return c.body(null, 304); - } - const text = new TextDecoder().decode(data); - const tree = parse(extendedMarkdownLanguage, text); - html = renderMarkdownToHtml(tree); - } catch (e: any) { - if (e.message !== notFoundError.message) { - console.error("Error server-side rendering page", e); - } - } - } else { - // If it it's a file with an extension and it doesn't exist we can't really create a new one/recover - try { - await spacePrimitives.getFileMeta(ref.path); - } catch (e: any) { - if (e.message !== notFoundError.message) { - return c.notFound(); - } - } - } - } - - const templateData = { - TITLE: title, - DESCRIPTION: stripHtml(html).substring(0, 255), - CONTENT: html, - HOST_URL_PREFIX: options.hostUrlPrefix ?? "", - }; - - html = clientAssetBundle.readTextFileSync(".client/index.html"); - - // Replace each template variable with its corresponding value - for (const [key, value] of Object.entries(templateData)) { - const placeholder = `{{${key}}}`; - const stringValue = typeof value === "boolean" - ? (value ? "true" : "false") - : (key === "DESCRIPTION" ? htmlEscape(value) : String(value)); - html = html.replace(placeholder, stringValue); - } - return c.html( - html, - 200, - { - "Last-Modified": lastModified, - }, - ); -} - -function stripHtml(html: string): string { - const regex = /<[^>]*>/g; - return html.replace(regex, ""); -} diff --git a/server/shell.test.ts b/server/shell.test.ts deleted file mode 100644 index 4787fef7..00000000 --- a/server/shell.test.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { LocalShell } from "./shell_backend.ts"; -import { assert, assertEquals } from "@std/assert"; - -Deno.test("Test shell backend", async () => { - const shell = new LocalShell("."); - - // ls test - let r = await shell.handle({ - cmd: "ls", - args: [], - }); - assertEquals(r.code, 0, "successful exit"); - assert(r.stdout.length > 0, "some ls output"); - - // cat test - r = await shell.handle({ - cmd: "cat", - args: [], - stdin: "hello", - }); - assertEquals(r.code, 0, "successful exit"); - assert(r.stdout, "hello"); - console.log("Output", r); -}); diff --git a/server/shell_backend.go b/server/shell_backend.go new file mode 100644 index 00000000..d988a5ca --- /dev/null +++ b/server/shell_backend.go @@ -0,0 +1,99 @@ +package server + +import ( + "bytes" + "fmt" + "os/exec" + "slices" + "strings" +) + +// LocalShell implements shell execution on the local system +type LocalShell struct { + Cwd string + CmdWhiteList []string + AllowAllCmds bool +} + +// NewLocalShell creates a new LocalShell instance +func NewLocalShell(cwd string, cmdWhiteList string) *LocalShell { + whiteListedCommands := strings.Split(cmdWhiteList, " ") + if cmdWhiteList == "" { + whiteListedCommands = []string{} + } + return &LocalShell{ + Cwd: cwd, + CmdWhiteList: whiteListedCommands, + AllowAllCmds: len(whiteListedCommands) == 0, + } +} + +// Handle executes a shell command and returns the result +func (ls *LocalShell) Handle(request ShellRequest) (ShellResponse, error) { + // Check if command is whitelisted + if !ls.AllowAllCmds && !slices.Contains(ls.CmdWhiteList, request.Cmd) { + fmt.Printf("Not running shell command because not in whitelist: %s\n", request.Cmd) + return ShellResponse{ + Code: -1, + Stdout: "", + Stderr: "Not allowed, command not in whitelist", + }, nil + } + + fmt.Printf("Running shell command: %s %s\n", request.Cmd, strings.Join(request.Args, " ")) + + // Create the command + cmd := exec.Command(request.Cmd, request.Args...) + cmd.Dir = ls.Cwd + + // Set up stdin if provided + if request.Stdin != nil { + cmd.Stdin = strings.NewReader(*request.Stdin) + } + + // Set up separate buffers for stdout and stderr + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + // Execute the command + err := cmd.Run() + + var exitCode int + if err != nil { + // Try to get the exit code + if exitError, ok := err.(*exec.ExitError); ok { + exitCode = exitError.ExitCode() + } else { + // Command failed to start + return ShellResponse{ + Code: -1, + Stdout: "", + Stderr: err.Error(), + }, nil + } + } + + return ShellResponse{ + Code: exitCode, + Stdout: stdout.String(), + Stderr: stderr.String(), + }, nil +} + +// NotSupportedShell implements a shell backend that doesn't support execution +type NotSupportedShell struct{} + +// NewNotSupportedShell creates a new NotSupportedShell instance +func NewNotSupportedShell() *NotSupportedShell { + return &NotSupportedShell{} +} + +// Handle always returns a "not supported" error +func (nss *NotSupportedShell) Handle(request ShellRequest) (ShellResponse, error) { + return ShellResponse{ + Code: 1, + Stdout: "", + Stderr: "Not supported", + }, nil +} diff --git a/server/shell_backend.ts b/server/shell_backend.ts deleted file mode 100644 index 3616078f..00000000 --- a/server/shell_backend.ts +++ /dev/null @@ -1,111 +0,0 @@ -import type { ShellRequest, ShellResponse } from "../type/rpc.ts"; -import type { ServerOptions } from "./http_server.ts"; - -/** - * Configuration via environment variables: - * - SB_SHELL_BACKEND: "local" or "off" - */ -export function determineShellBackend( - serverOptions: ServerOptions, -): ShellBackend { - const backendConfig = Deno.env.get("SB_SHELL_BACKEND") || "local"; - switch (backendConfig) { - case "local": - if (serverOptions.shellCommandWhiteList) { - console.info( - "Running with the following shell commands enabled:", - serverOptions.shellCommandWhiteList, - ); - } else { - console.info("Running with ALL shell commands enabled."); - } - return new LocalShell( - serverOptions.pagesPath, - serverOptions.shellCommandWhiteList, - ); - default: - console.info( - "Running in shell-less mode, meaning shell commands are disabled", - ); - return new NotSupportedShell(); - } -} - -export interface ShellBackend { - handle(shellRequest: ShellRequest): Promise<ShellResponse>; -} - -export class NotSupportedShell implements ShellBackend { - handle(): Promise<ShellResponse> { - return Promise.resolve({ - stdout: "", - stderr: "Not supported", - code: 1, - }); - } -} - -export class LocalShell implements ShellBackend { - constructor(private cwd: string, private cmdWhiteList?: string[]) { - } - - async handle(shellRequest: ShellRequest): Promise<ShellResponse> { - if (this.cmdWhiteList && !this.cmdWhiteList.includes(shellRequest.cmd)) { - console.error( - "Not running shell command because not in whitelist", - shellRequest, - ); - return { - code: -1, - stdout: "", - stderr: "Not allowed, command not in whitelist", - }; - } - console.log( - "Running shell command:", - shellRequest.cmd, - shellRequest.args, - ); - const p = new Deno.Command(shellRequest.cmd, { - cwd: this.cwd, - args: shellRequest.args, - stdin: "piped", - stdout: "piped", - stderr: "piped", - }).spawn(); - if (shellRequest.stdin) { - // Write a string to stdin - const encoder = new TextEncoder(); - const writer = p.stdin.getWriter(); - await writer.write(encoder.encode(shellRequest.stdin)); - writer.releaseLock(); - } - await p.stdin.close(); - - // Capture stdout and stderr - const [stdout, stderr] = await Promise.all([ - readAll(p.stdout.getReader()), - readAll(p.stderr.getReader()), - ]); - - // Get the exit status - const status = await p.status; - - return { - stderr, - stdout, - code: status.code, - }; - } -} - -async function readAll(r: ReadableStreamDefaultReader) { - const chunks: string[] = []; - const decoder = new TextDecoder(); - while (true) { - const { value, done } = await r.read(); - if (done) break; - chunks.push(decoder.decode(value)); - } - return chunks.join(); -} diff --git a/server/shell_backend_test.go b/server/shell_backend_test.go new file mode 100644 index 00000000..856f3e4a --- /dev/null +++ b/server/shell_backend_test.go @@ -0,0 +1,118 @@ +package server + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLocalShell_HandleValidCommand(t *testing.T) { + tmpDir := t.TempDir() + shell := NewLocalShell(tmpDir, "") // No whitelist = allow all commands + + request := ShellRequest{ + Cmd: "echo", + Args: []string{"Hello World"}, + } + + response, err := shell.Handle(request) + assert.NoError(t, err, "Should handle valid command without error") + assert.Equal(t, 0, response.Code, "Echo command should exit with code 0") + assert.Equal(t, "Hello World\n", response.Stdout, "Should return stdout") + assert.Empty(t, response.Stderr, "Should have empty stderr") +} + +func TestLocalShell_HandleCommandWithStdin(t *testing.T) { + tmpDir := t.TempDir() + shell := NewLocalShell(tmpDir, "") + + input := "test input" + request := ShellRequest{ + Cmd: "cat", + Args: []string{}, + Stdin: &input, + } + + response, err := shell.Handle(request) + assert.NoError(t, err, "Should handle command with stdin without error") + assert.Equal(t, 0, response.Code, "cat command should exit with code 0") + assert.Equal(t, "test input", response.Stdout, "Should echo stdin to stdout") + assert.Empty(t, response.Stderr, "Should have empty stderr") +} + +func TestLocalShell_HandleWorkingDirectory(t *testing.T) { + tmpDir := t.TempDir() + shell := NewLocalShell(tmpDir, "") + + // Create a test file in the temp directory + testFile := filepath.Join(tmpDir, "test.txt") + err := os.WriteFile(testFile, []byte("test content"), 0644) + assert.NoError(t, err, "Should create test file") + + request := ShellRequest{ + Cmd: "ls", + Args: []string{}, + } + + response, err := shell.Handle(request) + assert.NoError(t, err, "Should handle ls command without error") + assert.Equal(t, 0, response.Code, "ls command should exit with code 0") + assert.Contains(t, response.Stdout, "test.txt", "Should list the test file") + assert.Empty(t, response.Stderr, "Should have empty stderr") +} + +func TestLocalShell_HandleWithWhitelist(t *testing.T) { + tmpDir := t.TempDir() + shell := NewLocalShell(tmpDir, "echo pwd") + + // Test allowed command + request := ShellRequest{ + Cmd: "echo", + Args: []string{"allowed"}, + } + + response, err := shell.Handle(request) + assert.NoError(t, err, "Should handle whitelisted command without error") + assert.Equal(t, 0, response.Code, "Whitelisted command should succeed") + assert.Equal(t, "allowed\n", response.Stdout, "Should return expected output") + + // Test disallowed command + request = ShellRequest{ + Cmd: "ls", + Args: []string{}, + } + + response, err = shell.Handle(request) + assert.NoError(t, err, "Should handle request without error") + assert.Equal(t, -1, response.Code, "Non-whitelisted command should return -1") + assert.Empty(t, response.Stdout, "Should have empty stdout") + assert.Equal(t, "Not allowed, command not in whitelist", response.Stderr, "Should return whitelist error") +} + +func TestLocalShell_HandleCommandWithBothStreams(t *testing.T) { + tmpDir := t.TempDir() + shell := NewLocalShell(tmpDir, "") + + // Create a shell script that outputs to both stdout and stderr + script := `#!/bin/sh +echo "stdout message" +echo "stderr message" >&2 +exit 0` + + scriptPath := filepath.Join(tmpDir, "test_script.sh") + err := os.WriteFile(scriptPath, []byte(script), 0755) + assert.NoError(t, err, "Should create test script") + + request := ShellRequest{ + Cmd: "sh", + Args: []string{scriptPath}, + } + + response, err := shell.Handle(request) + assert.NoError(t, err, "Should handle script without error") + assert.Equal(t, 0, response.Code, "Script should exit with code 0") + assert.Equal(t, "stdout message\n", response.Stdout, "Should capture stdout") + assert.Equal(t, "stderr message\n", response.Stderr, "Should capture stderr") +} diff --git a/server/shell_endpoint.go b/server/shell_endpoint.go new file mode 100644 index 00000000..75bd35ab --- /dev/null +++ b/server/shell_endpoint.go @@ -0,0 +1,36 @@ +package server + +import ( + "encoding/json" + "fmt" + "net/http" +) + +// handleShellEndpoint handles POST requests to /.shell for executing shell commands +func handleShellEndpoint(config *ServerConfig) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + spaceConfig := spaceConfigFromContext(r.Context()) + // Parse the request body + var shellRequest ShellRequest + if err := json.NewDecoder(r.Body).Decode(&shellRequest); err != nil { + http.Error(w, "Invalid JSON request", http.StatusBadRequest) + return + } + + // Execute the shell command + shellResponse, err := spaceConfig.ShellBackend.Handle(shellRequest) + if err != nil { + fmt.Printf("Shell error: %v\n", err) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Return the response as JSON + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(shellResponse); err != nil { + fmt.Printf("Error encoding shell response: %v\n", err) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + } +} diff --git a/server/shell_endpoint.ts b/server/shell_endpoint.ts deleted file mode 100644 index 542641da..00000000 --- a/server/shell_endpoint.ts +++ /dev/null @@ -1,27 +0,0 @@ -import type { Context } from "hono"; -import type { ShellRequest } from "@silverbulletmd/silverbullet/type/rpc"; -import type { ShellBackend } from "./shell_backend.ts"; - -/** - * Handles the /.shell endpoint for running shell commands - */ -export async function handleShellEndpoint( - c: Context, - shellBackend: ShellBackend, - readOnly: boolean, -): Promise<Response> { - const req = c.req; - const body = await req.json(); - try { - if (readOnly) { - return c.text("Read only mode, no shell commands allowed", 405); - } - const shellCommand: ShellRequest = body; - // Note: in read-only this is set to NoShellSupport, so don't worry - const shellResponse = await shellBackend.handle(shellCommand); - return c.json(shellResponse); - } catch (e: any) { - console.error("Shell error", e); - return c.text(e.message, 500); - } -} diff --git a/server_go/space_primitive_testing.go b/server/space_primitive_testing.go similarity index 51% rename from server_go/space_primitive_testing.go rename to server/space_primitive_testing.go index df01ed4c..7b607033 100644 --- a/server_go/space_primitive_testing.go +++ b/server/space_primitive_testing.go @@ -1,8 +1,10 @@ -package server_go +package server import ( "bytes" "testing" + + "github.com/stretchr/testify/assert" ) // TestSpacePrimitives runs a comprehensive test suite against any SpacePrimitives implementation @@ -34,23 +36,15 @@ func TestSpacePrimitives(t *testing.T, spacePrimitives SpacePrimitives) { // Ensure clean state at the end finalFiles, err := spacePrimitives.FetchFileList() - if err != nil { - t.Fatalf("Failed to fetch final file list: %v", err) - } - if len(finalFiles) != 0 { - t.Errorf("Expected empty file list at end, got %d files", len(finalFiles)) - } + assert.NoError(t, err, "Failed to fetch final file list") + assert.Empty(t, finalFiles, "Expected empty file list at end") } func testBasicOperations(t *testing.T, spacePrimitives SpacePrimitives) { // Test initial empty state files, err := spacePrimitives.FetchFileList() - if err != nil { - t.Fatalf("Failed to fetch file list: %v", err) - } - if len(files) != 0 { - t.Errorf("Expected empty file list, got %d files", len(files)) - } + assert.NoError(t, err, "Failed to fetch file list") + assert.Empty(t, files, "Expected empty file list") // Write text file testContent := []byte("Hello World") @@ -64,36 +58,22 @@ func testBasicOperations(t *testing.T, spacePrimitives SpacePrimitives) { } _, err = spacePrimitives.WriteFile("test.txt", testContent, fileMeta) - if err != nil { - t.Fatalf("Failed to write file: %v", err) - } + assert.NoError(t, err, "Failed to write file") // Read the file back retrievedData, retrievedMeta, err := spacePrimitives.ReadFile("test.txt") - if err != nil { - t.Fatalf("Failed to read file: %v", err) - } + assert.NoError(t, err, "Failed to read file") - if !bytes.Equal(retrievedData, testContent) { - t.Errorf("Retrieved data doesn't match: expected %s, got %s", testContent, retrievedData) - } + assert.Equal(t, testContent, retrievedData, "Retrieved data doesn't match") // Check that the meta data is persisted (lastModified should be preserved) - if retrievedMeta.LastModified != 20 { - t.Errorf("Expected lastModified to be 20, got %d", retrievedMeta.LastModified) - } + assert.Equal(t, int64(20), retrievedMeta.LastModified, "Expected lastModified to be 20") // Check file list fileList, err := spacePrimitives.FetchFileList() - if err != nil { - t.Fatalf("Failed to fetch file list: %v", err) - } - if len(fileList) != 1 { - t.Errorf("Expected 1 file in list, got %d", len(fileList)) - } - if fileList[0].Name != "test.txt" { - t.Errorf("Expected file name 'test.txt', got '%s'", fileList[0].Name) - } + assert.NoError(t, err, "Failed to fetch file list") + assert.Len(t, fileList, 1, "Expected 1 file in list") + assert.Equal(t, "test.txt", fileList[0].Name, "Expected file name 'test.txt'") // Write binary file buf := make([]byte, 1024*1024) @@ -104,116 +84,72 @@ func testBasicOperations(t *testing.T, spacePrimitives SpacePrimitives) { buf[4] = 5 _, err = spacePrimitives.WriteFile("test.bin", buf, nil) - if err != nil { - t.Fatalf("Failed to write binary file: %v", err) - } + assert.NoError(t, err, "Failed to write binary file") fileData, _, err := spacePrimitives.ReadFile("test.bin") - if err != nil { - t.Fatalf("Failed to read binary file: %v", err) - } + assert.NoError(t, err, "Failed to read binary file") - if len(fileData) != 1024*1024 { - t.Errorf("Expected binary file size %d, got %d", 1024*1024, len(fileData)) - } + assert.Len(t, fileData, 1024*1024, "Expected binary file size to match") fileList, err = spacePrimitives.FetchFileList() - if err != nil { - t.Fatalf("Failed to fetch file list: %v", err) - } - if len(fileList) != 2 { - t.Errorf("Expected 2 files in list, got %d", len(fileList)) - } + assert.NoError(t, err, "Failed to fetch file list") + assert.Len(t, fileList, 2, "Expected 2 files in list") // Delete binary file err = spacePrimitives.DeleteFile("test.bin") - if err != nil { - t.Fatalf("Failed to delete binary file: %v", err) - } + assert.NoError(t, err, "Failed to delete binary file") fileList, err = spacePrimitives.FetchFileList() - if err != nil { - t.Fatalf("Failed to fetch file list: %v", err) - } - if len(fileList) != 1 { - t.Errorf("Expected 1 file in list after deletion, got %d", len(fileList)) - } + assert.NoError(t, err, "Failed to fetch file list") + assert.Len(t, fileList, 1, "Expected 1 file in list after deletion") // Clean up err = spacePrimitives.DeleteFile("test.txt") - if err != nil { - t.Fatalf("Failed to delete test file: %v", err) - } + assert.NoError(t, err, "Failed to delete test file") fileList, err = spacePrimitives.FetchFileList() - if err != nil { - t.Fatalf("Failed to fetch file list: %v", err) - } - if len(fileList) != 0 { - t.Errorf("Expected empty file list after cleanup, got %d files", len(fileList)) - } + assert.NoError(t, err, "Failed to fetch file list") + assert.Empty(t, fileList, "Expected empty file list after cleanup") // Test weird file names weirdContent := []byte("Hello world!") _, err = spacePrimitives.WriteFile("test+'s.txt", weirdContent, nil) - if err != nil { - t.Fatalf("Failed to write file with weird name: %v", err) - } + assert.NoError(t, err, "Failed to write file with weird name") readWeirdData, _, err := spacePrimitives.ReadFile("test+'s.txt") - if err != nil { - t.Fatalf("Failed to read file with weird name: %v", err) - } + assert.NoError(t, err, "Failed to read file with weird name") - if !bytes.Equal(readWeirdData, weirdContent) { - t.Errorf("Weird file content doesn't match: expected %s, got %s", weirdContent, readWeirdData) - } + assert.Equal(t, weirdContent, readWeirdData, "Weird file content doesn't match") err = spacePrimitives.DeleteFile("test+'s.txt") - if err != nil { - t.Fatalf("Failed to delete weird file: %v", err) - } + assert.NoError(t, err, "Failed to delete weird file") // Check deletion of weird file name _, _, err = spacePrimitives.ReadFile("test+'s.txt") - if err != ErrNotFound { - t.Errorf("Expected ErrNotFound when reading deleted file, got: %v", err) - } + assert.Equal(t, ErrNotFound, err, "Expected ErrNotFound when reading deleted file") } func testFileOverwriting(t *testing.T, spacePrimitives SpacePrimitives) { // Test overwriting existing files originalContent := []byte("Original") _, err := spacePrimitives.WriteFile("overwrite.txt", originalContent, nil) - if err != nil { - t.Fatalf("Failed to write original file: %v", err) - } + assert.NoError(t, err, "Failed to write original file") _, err = spacePrimitives.GetFileMeta("overwrite.txt", false) - if err != nil { - t.Fatalf("Failed to get original meta: %v", err) - } + assert.NoError(t, err, "Failed to get original meta") updatedContent := []byte("Updated") _, err = spacePrimitives.WriteFile("overwrite.txt", updatedContent, nil) - if err != nil { - t.Fatalf("Failed to overwrite file: %v", err) - } + assert.NoError(t, err, "Failed to overwrite file") updatedData, _, err := spacePrimitives.ReadFile("overwrite.txt") - if err != nil { - t.Fatalf("Failed to read overwritten file: %v", err) - } + assert.NoError(t, err, "Failed to read overwritten file") - if !bytes.Equal(updatedData, updatedContent) { - t.Errorf("Overwritten content doesn't match: expected %s, got %s", updatedContent, updatedData) - } + assert.Equal(t, updatedContent, updatedData, "Overwritten content doesn't match") // File list should still have only one entry for this file filesAfterOverwrite, err := spacePrimitives.FetchFileList() - if err != nil { - t.Fatalf("Failed to fetch files after overwrite: %v", err) - } + assert.NoError(t, err, "Failed to fetch files after overwrite") overwriteFiles := 0 for _, f := range filesAfterOverwrite { @@ -222,64 +158,54 @@ func testFileOverwriting(t *testing.T, spacePrimitives SpacePrimitives) { } } - if overwriteFiles != 1 { - t.Errorf("Expected 1 instance of overwrite.txt, got %d", overwriteFiles) - } + assert.Equal(t, 1, overwriteFiles, "Expected 1 instance of overwrite.txt") err = spacePrimitives.DeleteFile("overwrite.txt") - if err != nil { - t.Fatalf("Failed to delete overwritten file: %v", err) - } + assert.NoError(t, err, "Failed to delete overwritten file") + + // Test writing files to new folders + writeMeta, err := spacePrimitives.WriteFile("some/folder/test.dat", []byte("Testing"), nil) + assert.NoError(t, err, "Writing to a nested folder should work") + + // Read it back + data, meta, err := spacePrimitives.ReadFile("some/folder/test.dat") + assert.NoError(t, err, "Reading should work") + assert.Equal(t, writeMeta.LastModified, meta.LastModified, "Meta should be the same") + assert.Equal(t, []byte("Testing"), data, "Did not get back what I put in") + + err = spacePrimitives.DeleteFile("some/folder/test.dat") + assert.NoError(t, err, "Deletion should work") } func testEmptyFiles(t *testing.T, spacePrimitives SpacePrimitives) { // Test empty file emptyData := []byte{} _, err := spacePrimitives.WriteFile("empty.txt", emptyData, nil) - if err != nil { - t.Fatalf("Failed to write empty file: %v", err) - } + assert.NoError(t, err, "Failed to write empty file") emptyFileData, emptyFileMeta, err := spacePrimitives.ReadFile("empty.txt") - if err != nil { - t.Fatalf("Failed to read empty file: %v", err) - } + assert.NoError(t, err, "Failed to read empty file") - if len(emptyFileData) != 0 { - t.Errorf("Expected empty file data length 0, got %d", len(emptyFileData)) - } - - if emptyFileMeta.Size != 0 { - t.Errorf("Expected empty file meta size 0, got %d", emptyFileMeta.Size) - } + assert.Empty(t, emptyFileData, "Expected empty file data") + assert.Equal(t, int64(0), emptyFileMeta.Size, "Expected empty file meta size 0") err = spacePrimitives.DeleteFile("empty.txt") - if err != nil { - t.Fatalf("Failed to delete empty file: %v", err) - } + assert.NoError(t, err, "Failed to delete empty file") } func testUnicodeContent(t *testing.T, spacePrimitives SpacePrimitives) { // Test files with Unicode characters unicodeContent := []byte("Hello 世界! 🌍 Здравствуй мир!") _, err := spacePrimitives.WriteFile("unicode.txt", unicodeContent, nil) - if err != nil { - t.Fatalf("Failed to write unicode file: %v", err) - } + assert.NoError(t, err, "Failed to write unicode file") unicodeData, _, err := spacePrimitives.ReadFile("unicode.txt") - if err != nil { - t.Fatalf("Failed to read unicode file: %v", err) - } + assert.NoError(t, err, "Failed to read unicode file") - if !bytes.Equal(unicodeData, unicodeContent) { - t.Errorf("Unicode content doesn't match: expected %s, got %s", unicodeContent, unicodeData) - } + assert.Equal(t, unicodeContent, unicodeData, "Unicode content doesn't match") err = spacePrimitives.DeleteFile("unicode.txt") - if err != nil { - t.Fatalf("Failed to delete unicode file: %v", err) - } + assert.NoError(t, err, "Failed to delete unicode file") } func testSpecialFileNames(t *testing.T, spacePrimitives SpacePrimitives) { @@ -297,26 +223,18 @@ func testSpecialFileNames(t *testing.T, spacePrimitives SpacePrimitives) { for _, fileName := range specialNames { content := []byte("Content of " + fileName) _, err := spacePrimitives.WriteFile(fileName, content, nil) - if err != nil { - t.Fatalf("Failed to write file %s: %v", fileName, err) - } + assert.NoError(t, err, "Failed to write file %s", fileName) fileData, _, err := spacePrimitives.ReadFile(fileName) - if err != nil { - t.Fatalf("Failed to read file %s: %v", fileName, err) - } + assert.NoError(t, err, "Failed to read file %s", fileName) expectedContent := "Content of " + fileName - if string(fileData) != expectedContent { - t.Errorf("Content mismatch for %s: expected %s, got %s", fileName, expectedContent, string(fileData)) - } + assert.Equal(t, expectedContent, string(fileData), "Content mismatch for %s", fileName) } // Verify all special files are in the list allFiles, err := spacePrimitives.FetchFileList() - if err != nil { - t.Fatalf("Failed to fetch file list: %v", err) - } + assert.NoError(t, err, "Failed to fetch file list") for _, fileName := range specialNames { found := false @@ -326,36 +244,26 @@ func testSpecialFileNames(t *testing.T, spacePrimitives SpacePrimitives) { break } } - if !found { - t.Errorf("File %s should be in the file list", fileName) - } + assert.True(t, found, "File %s should be in the file list", fileName) } // Clean up special files for _, fileName := range specialNames { err := spacePrimitives.DeleteFile(fileName) - if err != nil { - t.Fatalf("Failed to delete file %s: %v", fileName, err) - } + assert.NoError(t, err, "Failed to delete file %s", fileName) } } func testErrorHandling(t *testing.T, spacePrimitives SpacePrimitives) { // Test error cases _, _, err := spacePrimitives.ReadFile("nonexistent.txt") - if err != ErrNotFound { - t.Errorf("Expected ErrNotFound for non-existent file, got: %v", err) - } + assert.Equal(t, ErrNotFound, err, "Expected ErrNotFound for non-existent file") err = spacePrimitives.DeleteFile("nonexistent.txt") - if err != ErrNotFound { - t.Errorf("Expected ErrNotFound when deleting non-existent file, got: %v", err) - } + assert.Equal(t, ErrNotFound, err, "Expected ErrNotFound when deleting non-existent file") _, err = spacePrimitives.GetFileMeta("nonexistent.txt", false) - if err != ErrNotFound { - t.Errorf("Expected ErrNotFound for non-existent file meta, got: %v", err) - } + assert.Equal(t, ErrNotFound, err, "Expected ErrNotFound for non-existent file meta") } func testLargeFiles(t *testing.T, spacePrimitives SpacePrimitives) { @@ -366,22 +274,13 @@ func testLargeFiles(t *testing.T, spacePrimitives SpacePrimitives) { } _, err := spacePrimitives.WriteFile("large.bin", largeContent, nil) - if err != nil { - t.Fatalf("Failed to write large file: %v", err) - } + assert.NoError(t, err, "Failed to write large file") largeData, largeMeta, err := spacePrimitives.ReadFile("large.bin") - if err != nil { - t.Fatalf("Failed to read large file: %v", err) - } + assert.NoError(t, err, "Failed to read large file") - if len(largeData) != len(largeContent) { - t.Errorf("Large file size mismatch: expected %d, got %d", len(largeContent), len(largeData)) - } - - if largeMeta.Size != int64(len(largeContent)) { - t.Errorf("Large file meta size mismatch: expected %d, got %d", len(largeContent), largeMeta.Size) - } + assert.Len(t, largeData, len(largeContent), "Large file size mismatch") + assert.Equal(t, int64(len(largeContent)), largeMeta.Size, "Large file meta size mismatch") // Verify content integrity (check first 1000 bytes) checkSize := 1000 @@ -389,17 +288,10 @@ func testLargeFiles(t *testing.T, spacePrimitives SpacePrimitives) { checkSize = len(largeContent) } - for i := 0; i < checkSize; i++ { - if largeData[i] != largeContent[i] { - t.Errorf("Large file content mismatch at index %d: expected %d, got %d", i, largeContent[i], largeData[i]) - break - } - } + assert.True(t, bytes.Equal(largeData[:checkSize], largeContent[:checkSize]), "Large file content mismatch in first %d bytes", checkSize) err = spacePrimitives.DeleteFile("large.bin") - if err != nil { - t.Fatalf("Failed to delete large file: %v", err) - } + assert.NoError(t, err, "Failed to delete large file") } func testMetadataPreservation(t *testing.T, spacePrimitives SpacePrimitives) { @@ -415,30 +307,16 @@ func testMetadataPreservation(t *testing.T, spacePrimitives SpacePrimitives) { } _, err := spacePrimitives.WriteFile("meta-test.txt", testContent, customMeta) - if err != nil { - t.Fatalf("Failed to write file with custom meta: %v", err) - } + assert.NoError(t, err, "Failed to write file with custom meta") _, metaFile, err := spacePrimitives.ReadFile("meta-test.txt") - if err != nil { - t.Fatalf("Failed to read file with meta: %v", err) - } + assert.NoError(t, err, "Failed to read file with meta") // Check that some metadata is preserved (implementations may handle timestamps differently) - if metaFile.LastModified <= 0 { - t.Error("LastModified timestamp should be set") - } - - if metaFile.Name != "meta-test.txt" { - t.Errorf("Expected name 'meta-test.txt', got '%s'", metaFile.Name) - } - - if metaFile.Size != int64(len(testContent)) { - t.Errorf("Expected size %d, got %d", len(testContent), metaFile.Size) - } + assert.Greater(t, metaFile.LastModified, int64(0), "LastModified timestamp should be set") + assert.Equal(t, "meta-test.txt", metaFile.Name, "Expected name 'meta-test.txt'") + assert.Equal(t, int64(len(testContent)), metaFile.Size, "Expected size to match content length") err = spacePrimitives.DeleteFile("meta-test.txt") - if err != nil { - t.Fatalf("Failed to delete meta test file: %v", err) - } + assert.NoError(t, err, "Failed to delete meta test file") } diff --git a/server/spaces.go b/server/spaces.go new file mode 100644 index 00000000..1aede02b --- /dev/null +++ b/server/spaces.go @@ -0,0 +1,22 @@ +package server + +import ( + "context" + "net/http" +) + +const spaceConfigKey = "spaceConfig" + +func spaceMiddleware(config *ServerConfig) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + spaceConfig := config.SpaceConfigResolver(r) + augmentedRequest := r.WithContext(context.WithValue(r.Context(), spaceConfigKey, spaceConfig)) + next.ServeHTTP(w, augmentedRequest) + }) + } +} + +func spaceConfigFromContext(ctx context.Context) *SpaceConfig { + return ctx.Value(spaceConfigKey).(*SpaceConfig) +} diff --git a/server/ssr.go b/server/ssr.go new file mode 100644 index 00000000..760800a8 --- /dev/null +++ b/server/ssr.go @@ -0,0 +1,94 @@ +package server + +import ( + "fmt" + "html/template" + "log" + "net/http" + pathLib "path" + "regexp" + "strings" + + "github.com/gomarkdown/markdown" + "github.com/gomarkdown/markdown/html" + "github.com/gomarkdown/markdown/parser" +) + +// This implements very crude server-side rendered markdown using a Go markdown library +// Note: this not even nearly supports all SilverBullet markdown features. It's best effort. +// The ONLY use case of this is search engine indexing +// It is only activated when the space is both read-only and has no authentication, so basically the silverbullet.md scenario. + +// Very crude markdown link replacement +func convertWikiLinksToMarkdown(input string) string { + // Regular expression to match [[text]] patterns + re := regexp.MustCompile(`\[\[([^\]]+)\]\]`) + + // Replace all matches with [text](text) format + result := re.ReplaceAllStringFunc(input, func(match string) string { + // Extract the text between [[ and ]] + text := strings.TrimPrefix(match, "[[") + text = strings.TrimSuffix(text, "]]") + + return fmt.Sprintf("[%s](%s)", text, text) + }) + + return result +} + +func ServerSideRender(config *ServerConfig, spaceConfig *SpaceConfig, pageName string, w http.ResponseWriter, r *http.Request) { + // Let's fetch the index.html text + indexHtmlBuf, _, _ := config.ClientBundle.ReadFile(indexHtmlPath) + + if pageName == "" { + pageName = spaceConfig.IndexPage + } + + tpl := template.New("index_html") + tpl.Funcs(template.FuncMap{ + "markdown": func(text string) template.HTML { + if text == "" { + // Let's not even bother doing anything + return "" + } + + // Setup markdown renderer + var markdownParser = parser.NewWithExtensions(parser.CommonExtensions) + var markdownRenderer = html.NewRenderer(html.RendererOptions{}) + text = convertWikiLinksToMarkdown(text) + doc := markdownParser.Parse([]byte(text)) + return template.HTML(markdown.Render(doc, markdownRenderer)) + }, + }) + tpl = template.Must(tpl.Parse(string(indexHtmlBuf))) + + templateData := struct { + HostPrefix string + Title string + Description string + Content string + }{ + Title: "SilverBullet", + HostPrefix: spaceConfig.HostURLPrefix, + } + + if spaceConfig.ReadOnlyMode && spaceConfig.Auth == nil { + // Read only and without out, let's do some server side rendering + templateData.Title = pathLib.Base(pageName) + + // Read page content + pageData, _, err := spaceConfig.SpacePrimitives.ReadFile(fmt.Sprintf("%s.md", pageName)) + if err != nil { + log.Printf("Could not read content for page %s", pageName) + } else { + templateData.Content = string(pageData) + } + } + + w.Header().Set("Content-type", "text/html") + w.WriteHeader(http.StatusOK) + if err := tpl.Execute(w, templateData); err != nil { + log.Printf("SSR error for %s: %v", pageName, err) + w.Write([]byte("Server error")) + } +} diff --git a/server/storage_backend.ts b/server/storage_backend.ts deleted file mode 100644 index 11c66505..00000000 --- a/server/storage_backend.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { DiskSpacePrimitives } from "../lib/spaces/disk_space_primitives.ts"; -import type { SpacePrimitives } from "../lib/spaces/space_primitives.ts"; -import { resolve } from "@std/path"; -import { HttpSpacePrimitives } from "../lib/spaces/http_space_primitives.ts"; - -export function determineStorageBackend( - folder: string, -): SpacePrimitives { - if (folder.startsWith("http://") || folder.startsWith("https://")) { - return new HttpSpacePrimitives( - folder, - "", - (err) => { - console.error("[HTTP space primitives error]", err); - }, - Deno.env.get("SB_AUTH_TOKEN"), - ); - } else { - folder = resolve(Deno.cwd(), folder); - console.info(`Using local disk as a storage backend: ${folder}`); - return new DiskSpacePrimitives(folder); - } -} diff --git a/server/types.go b/server/types.go new file mode 100644 index 00000000..d2dda568 --- /dev/null +++ b/server/types.go @@ -0,0 +1,107 @@ +package server + +import ( + "errors" + "net/http" +) + +type ServerConfig struct { + SpaceConfigResolver ConfigResolver + + BindHost string + Port int + EnableHTTPLogging bool + + ClientBundle SpacePrimitives +} + +type SpaceConfig struct { + Hostname string + HostURLPrefix string + Auth *AuthOptions + + SpacePrimitives SpacePrimitives + + SpaceFolderPath string + IndexPage string + GitIgnore string + ReadOnlyMode bool + + // Used for the PWA manifest and login page + SpaceName string + SpaceDescription string + + // Shell configuration + ShellBackend ShellBackend + + // Auth temporary objects + JwtIssuer *Authenticator + LockoutTimer *LockoutTimer +} + +type ConfigResolver func(r *http.Request) *SpaceConfig + +// FileMeta represents metadata for a file in the space +type FileMeta struct { + Name string `json:"name"` + Created int64 `json:"created"` + LastModified int64 `json:"lastModified"` + ContentType string `json:"contentType"` + Size int64 `json:"size"` + Perm string `json:"perm"` // "ro" or "rw" +} + +// SpacePrimitives defines the interface for storage backends +type SpacePrimitives interface { + // FetchFileList retrieves a list of all files in the space + FetchFileList() ([]FileMeta, error) + + // GetFileMeta retrieves metadata for a specific file + // observing parameter hints that this file is under regular observation + GetFileMeta(path string, observing bool) (FileMeta, error) + + // ReadFile reads a file and returns its data and metadata + ReadFile(path string) ([]byte, FileMeta, error) + + // WriteFile writes data to a file with optional metadata + // Returns the actual metadata of the written file + WriteFile(path string, data []byte, meta *FileMeta) (FileMeta, error) + + // DeleteFile removes a file from the space + DeleteFile(path string) error +} + +// Shell request/response types +type ShellRequest struct { + Cmd string `json:"cmd"` + Args []string `json:"args"` + Stdin *string `json:"stdin,omitempty"` +} + +type ShellResponse struct { + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` + Code int `json:"code"` +} + +// Shell backend interface +type ShellBackend interface { + Handle(request ShellRequest) (ShellResponse, error) +} + +// Auth options for user authentication +type AuthOptions struct { + AuthToken string `json:"authToken,omitempty"` + User string `json:"user"` + Pass string `json:"pass"` + LockoutTime int `json:"lockoutTime"` // in seconds + LockoutLimit int `json:"lockoutLimit"` +} + +// Common errors +var ( + ErrNotFound = errors.New("Not found") + ErrPathOutsideRoot = errors.New("Path not in space") + ErrCouldNotWrite = errors.New("Could not write file") + ErrCouldNotGetMeta = errors.New("Could not get file metadata") +) diff --git a/server/util.go b/server/util.go new file mode 100644 index 00000000..59c83fdb --- /dev/null +++ b/server/util.go @@ -0,0 +1,19 @@ +package server + +import ( + "mime" + "path/filepath" +) + +// lookupContentTypeFromPath determines MIME type based on file extension +func LookupContentTypeFromPath(path string) string { + ext := filepath.Ext(path) + if ext == ".md" { + return "text/markdown" + } + contentType := mime.TypeByExtension(ext) + if contentType == "" { + return "application/octet-stream" + } + return contentType +} diff --git a/server/version.go b/server/version.go new file mode 100644 index 00000000..1d33898a --- /dev/null +++ b/server/version.go @@ -0,0 +1,24 @@ +package server + +import ( + "fmt" + "regexp" + "strings" +) + +func ParseVersionFromTypeScript(code string) (string, error) { + // Parse the version using regex + re := regexp.MustCompile(`"([^"]+)"`) + matches := re.FindStringSubmatch(code) + + if len(matches) < 2 { + return "", fmt.Errorf("version not found in file") + } + + version := strings.TrimSpace(matches[1]) + if version == "" { + return "", fmt.Errorf("empty version found") + } + + return version, nil +} diff --git a/server/version_test.go b/server/version_test.go new file mode 100644 index 00000000..5112472a --- /dev/null +++ b/server/version_test.go @@ -0,0 +1,13 @@ +package server + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestVersionParsing(t *testing.T) { + version, err := ParseVersionFromTypeScript(`export const publicVersion = "2.0.0"`) + assert.NoError(t, err) + assert.Equal(t, "2.0.0", version) +} diff --git a/server_go/creation_time.go b/server_go/creation_time.go deleted file mode 100644 index d3fe91af..00000000 --- a/server_go/creation_time.go +++ /dev/null @@ -1,32 +0,0 @@ -package server_go - -import ( - "os" - "runtime" - "syscall" - "time" -) - -// getCreationTime extracts creation time from file info -// This is platform-specific and may not be available on all systems -func getCreationTime(info os.FileInfo) *time.Time { - if stat, ok := info.Sys().(*syscall.Stat_t); ok { - var creationTime time.Time - - switch runtime.GOOS { - case "darwin": - case "linux": - // On macOS, use Ctimespec which is available - creationTime = time.Unix(stat.Ctimespec.Sec, stat.Ctimespec.Nsec) - default: - // For other Unix systems, fall back to modification time - return nil - } - - return &creationTime - } - - // If we can't get creation time, return nil - // The caller will fall back to modification time - return nil -} diff --git a/server_go/disk_space_primitives_test.go b/server_go/disk_space_primitives_test.go deleted file mode 100644 index 6a08784a..00000000 --- a/server_go/disk_space_primitives_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package server_go - -import ( - "os" - "path/filepath" - "testing" -) - -func TestDiskSpacePrimitives(t *testing.T) { - // Create a temporary directory for testing - tempDir, err := os.MkdirTemp("", "disk_space_test") - if err != nil { - t.Fatalf("Failed to create temp dir: %v", err) - } - defer os.RemoveAll(tempDir) - - // Create DiskSpacePrimitives instance - space, err := NewDiskSpacePrimitives(tempDir) - if err != nil { - t.Fatalf("Failed to create DiskSpacePrimitives: %v", err) - } - - // Run all test suites using local testing utilities - TestSpacePrimitives(t, space) -} - -// Test path safety - specific to disk implementation -func TestSafePath(t *testing.T) { - tempDir, err := os.MkdirTemp("", "safe_path_test") - if err != nil { - t.Fatalf("Failed to create temp dir: %v", err) - } - defer os.RemoveAll(tempDir) - - space, err := NewDiskSpacePrimitives(tempDir) - if err != nil { - t.Fatalf("Failed to create DiskSpacePrimitives: %v", err) - } - - // Test that paths outside root are rejected - _, err = space.safePath("../outside") - if err != ErrPathOutsideRoot { - t.Errorf("Expected ErrPathOutsideRoot for '../outside', got: %v", err) - } - - _, err = space.safePath("/etc/passwd") - if err != ErrPathOutsideRoot { - t.Errorf("Expected ErrPathOutsideRoot for '/etc/passwd', got: %v", err) - } - - // Test that valid paths work - _, err = space.safePath("valid/path.txt") - if err != nil { - t.Errorf("Valid path should not error: %v", err) - } - - _, err = space.safePath("./valid.txt") - if err != nil { - t.Errorf("Valid relative path should not error: %v", err) - } -} - -// Test directory cleanup - specific to disk implementation -func TestCleanOrphaned(t *testing.T) { - tempDir, err := os.MkdirTemp("", "clean_test") - if err != nil { - t.Fatalf("Failed to create temp dir: %v", err) - } - defer os.RemoveAll(tempDir) - - space, err := NewDiskSpacePrimitives(tempDir) - if err != nil { - t.Fatalf("Failed to create DiskSpacePrimitives: %v", err) - } - - // Create nested directory structure with a file - _, err = space.WriteFile("deep/nested/dir/file.txt", []byte("test"), nil) - if err != nil { - t.Fatalf("Failed to write nested file: %v", err) - } - - // Verify the directories were created - nestedPath := filepath.Join(tempDir, "deep", "nested", "dir") - if _, err := os.Stat(nestedPath); os.IsNotExist(err) { - t.Error("Nested directory should exist") - } - - // Delete the file - err = space.DeleteFile("deep/nested/dir/file.txt") - if err != nil { - t.Fatalf("Failed to delete nested file: %v", err) - } - - if _, err := os.Stat(nestedPath); !os.IsNotExist(err) { - t.Error("Empty nested directory should be cleaned up") - } - - deepPath := filepath.Join(tempDir, "deep") - if _, err := os.Stat(deepPath); !os.IsNotExist(err) { - t.Error("Empty parent directory should be cleaned up") - } -} diff --git a/server_go/types.go b/server_go/types.go deleted file mode 100644 index c8cbd717..00000000 --- a/server_go/types.go +++ /dev/null @@ -1,43 +0,0 @@ -package server_go - -import ( - "errors" -) - -// FileMeta represents metadata for a file in the space -type FileMeta struct { - Name string `json:"name"` - Created int64 `json:"created"` - LastModified int64 `json:"lastModified"` - ContentType string `json:"contentType"` - Size int64 `json:"size"` - Perm string `json:"perm"` // "ro" or "rw" -} - -// SpacePrimitives defines the interface for storage backends -type SpacePrimitives interface { - // FetchFileList retrieves a list of all files in the space - FetchFileList() ([]FileMeta, error) - - // GetFileMeta retrieves metadata for a specific file - // observing parameter hints that this file is under regular observation - GetFileMeta(path string, observing bool) (FileMeta, error) - - // ReadFile reads a file and returns its data and metadata - ReadFile(path string) ([]byte, FileMeta, error) - - // WriteFile writes data to a file with optional metadata - // Returns the actual metadata of the written file - WriteFile(path string, data []byte, meta *FileMeta) (FileMeta, error) - - // DeleteFile removes a file from the space - DeleteFile(path string) error -} - -// Common errors -var ( - ErrNotFound = errors.New("not found") - ErrPathOutsideRoot = errors.New("path is not in the space") - ErrCouldNotWrite = errors.New("could not write file") - ErrCouldNotGetMeta = errors.New("could not get file metadata") -) diff --git a/silverbullet.go b/silverbullet.go new file mode 100644 index 00000000..392cf559 --- /dev/null +++ b/silverbullet.go @@ -0,0 +1,22 @@ +package main + +import ( + "embed" + + "github.com/silverbulletmd/silverbullet/server/cmd" +) + +// Embed client files, plugs and version files into the binary +// +//go:embed dist_client_bundle/* +//go:embed dist_base_fs_bundle/* +var bundledFiles embed.FS + +//go:embed public_version.ts +var versionFileText string + +func main() { + c := cmd.ServerCommand(bundledFiles) + c.AddCommand(cmd.VersionCommand(versionFileText), cmd.UpgradeCommand(), cmd.UpgradeEdgeCommand()) + c.Execute() +} diff --git a/silverbullet.ts b/silverbullet.ts deleted file mode 100755 index a924b62c..00000000 --- a/silverbullet.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { Command } from "@cliffy/command"; - -import { publicVersion } from "./public_version.ts"; - -import { upgradeCommand, upgradeEdgeCommand } from "./cmd/upgrade.ts"; -import { versionCommand } from "./cmd/version.ts"; -import { serveCommand } from "./cmd/server.ts"; -import { syncCommand } from "./cmd/sync.ts"; - -// Unhandled rejection, let's not crash -globalThis.addEventListener("unhandledrejection", (event) => { - console.error("Unhandled rejection:", event); - event.preventDefault(); -}); - -await new Command() - .name("silverbullet") - .description("Workshop for the mind") - .version(publicVersion) - .helpOption(false) - .usage("<options> <folder> | <command> (see below)") - // Main command - .arguments("[folder:string]") - .option( - "-L, --hostname <hostname:string>", - "Hostname or address to listen on", - ) - .option("-p, --port <port:number>", "Port to listen on") - .option( - "--user <user:string>", - "'username:password' combo for authentication", - ) - .action(serveCommand) - // upgrade - .command("upgrade") - .description("Upgrade SilverBullet") - .action(upgradeCommand) - // upgrade-edge - .command("upgrade-edge") - .description("Upgrade SilverBullet to the latest edge build") - .action(upgradeEdgeCommand) - // sync - .command("sync") - .description("Synchronize two spaces") - .option( - "--snapshot <snapshot:string>", - "Path to state file to use", - ) - .option( - "--wipe-secondary", - "Wipe secondary and perform a full sync", - ) - .arguments("<primary:string> <secondary:string>") - .action(syncCommand) - // version - .command("version") - .description("Get current version") - .action(versionCommand) - .parse(Deno.args); diff --git a/type/rpc.ts b/type/rpc.ts deleted file mode 100644 index a3260b92..00000000 --- a/type/rpc.ts +++ /dev/null @@ -1,11 +0,0 @@ -export type ShellRequest = { - cmd: string; - args: string[]; - stdin?: string; -}; - -export type ShellResponse = { - stdout: string; - stderr: string; - code: number; -}; diff --git a/web/auth.html b/web/auth.html deleted file mode 100644 index e371b5a9..00000000 --- a/web/auth.html +++ /dev/null @@ -1,224 +0,0 @@ -<!DOCTYPE html> -<html lang="en"> - -<head> - <meta charset="utf-8"/> - <meta content="width=device-width, initial-scale=1, maximum-scale=1" name="viewport"/> - <base href="{{HOST_URL_PREFIX}}/"/> - <link href="favicon.png" rel="icon" type="image/x-icon"/> - <title>Login to SilverBullet - - - - -
-
-

- Login to - SilverBullet -

-
-
-
- - -
-
- - -
-
- - -
-
- -
-
- -
-
- - - - - \ No newline at end of file diff --git a/web/index.html b/web/index.html deleted file mode 100644 index 45ad60ca..00000000 --- a/web/index.html +++ /dev/null @@ -1,74 +0,0 @@ - - - - - - - - - - - - {{TITLE}} - - - - - - - - - - - -
-
-
-
-
- {{CONTENT}} -
-
-
-
-
-
- - diff --git a/website/CHANGELOG.md b/website/CHANGELOG.md index 8f23e288..e621f044 100644 --- a/website/CHANGELOG.md +++ b/website/CHANGELOG.md @@ -3,11 +3,12 @@ An attempt at documenting the changes/new features introduced in each release. ## Edge These are changes live on the edge builds: +* New server written in Go, should be functionally equivalent to the old Deno-based backend, but is much smaller, faster and uses less memory. * Sync engine re-architecture: see [[Architecture]] and [[Sync]] * More configuring what to index (see [[^Library/Std/Config]] under the `index` section) for the purpose of reducing local storage size and needless CPU waste. Some useful ones: * `config.set("index.search.enable", false)` to disable [[Full Text Search]] entirely (saves on processing and storage if you don’t use it) * `config.set("index.paragraph.all", false)` to disable indexing all (untagged) paragraphs. This is also somewhat wasteful if you don’t query these. -* Parallel sync (5 files simultaneously) +* Parallel sync * Disable ability to rename pages in read-only mode (by [Jelenkee](https://github.com/silverbulletmd/silverbullet/pull/1509)) * Improved docker build + health check (by [Zef](https://github.com/silverbulletmd/silverbullet/issues/1515)) * Added `templates.tagItem` template (by [Andy Costanza](https://github.com/silverbulletmd/silverbullet/commit/6d4f964a6e2a4f7dae04aa7558defcaa9f1f1a86)) diff --git a/website/Development.md b/website/Development.md index 7d6f54b9..337fff1e 100644 --- a/website/Development.md +++ b/website/Development.md @@ -1,42 +1,86 @@ # Stack -SilverBullet is written in [TypeScript](https://www.typescriptlang.org/) and built on top of the excellent [CodeMirror 6](https://codemirror.net/) editor component. Additional UI is built using [Preact](https://preactjs.com/). [ES Build](https://esbuild.github.io) is used to build both the front-end and back-end bundles. The server backend runs as an HTTP server on [Deno](https://deno.land/) using [Hono](https://hono.dev). +SilverBullet’s client is written in [TypeScript](https://www.typescriptlang.org/) and built on top of the excellent [CodeMirror 6](https://codemirror.net/) editor component. Additional UI is built using [Preact](https://preactjs.com/). [ES Build](https://esbuild.github.io) is used to build the frontend. + +[[Plugs]] are also written in TypeScript. + +The SilverBullet server is written in [Go](https://go.dev/). + +# Code structure +* `client/`: The SilverBullet client, implemented with TypeScript +* `server/`: The SilverBullet server, written in Go +* `plugs`: Set of built-in plugs that are distributed with SilverBullet +* `libraries`: A set of libraries (space scripts, page templates, slash templates) distributed with SilverBullet +* `plug-api/`: Useful APIs for use in plugs + * `lib/`: Useful libraries to be used in plugs + * `syscalls/`: TypeScript wrappers around syscalls + * `types/`: Various (client) types that can be references from plugs +* `bin` + * `plug_compile.ts` the plug compiler +* `scripts/`: Useful scripts +* `website/`: silverbullet.md website content # Development -Requirements: [Deno](https://deno.land/) 2.2 or newer. +Requirements: +* [Deno](https://deno.land/) 2.4 or newer. +* [Go](https://go.dev/) 1.25 or newer +* Make -Clone the repository from GitHub: +It's convenient to also install [air](https://github.com/air-verse/air) for development, this tool will watch your code base for changes and automatically rebuild: ```shell -git clone git@github.com:silverbulletmd/silverbullet.git -cd silverbullet +go install github.com/air-verse/air@latest ``` -And build it: +Make sure your `$GOPATH/bin` is in your $PATH. + +To build everything and run the server using air: ```shell -deno task build +air ``` -For convenience, replace your `silverbullet` install with the one from this repo via: +Note, that if you want to pass arguments to your SilverBullet binary like `-p` or `-L` you need to this as follows: ```shell -deno task install +air -- -L 0.0.0.0 ``` -You can now run the server in “watch mode” (automatically restarting when you change source files) with: + +Alternatively, to build the project without air: ```shell -deno task watch-server +make build ``` -It's convenient to run three commands in parallel (in separate terminals): +To run the resulting server: ```shell -deno task watch-web -deno task watch-server -deno task watch-plugs +./silverbullet ``` -All of these watch for file changes and a rebuild should trigger automatically. +### Useful development tasks + +```shell +# Clean all generated files +make clean +# Typecheck and lint all code +make check +# Format all code +make fmt +# Run all tests +make test +``` + +### Build a docker container +Note, you do not need Deno nor Go locally installed for this to work: + +```shell +docker build -t silverbullet . +``` + +To run: + +```shell +docker run -p 3000:3000 -v :/space silverbullet +``` -Note that there are dependencies between these builds. Any change to any of the built-in _plugs_ requires a rebuild of the web app. Any rebuild of the web app will only be picked up by the server after it restarts (which should happen automatically). diff --git a/website/Install/Configuration.md b/website/Install/Configuration.md index 7228ef3c..bfede70d 100644 --- a/website/Install/Configuration.md +++ b/website/Install/Configuration.md @@ -4,6 +4,7 @@ SilverBullet is primarily configured via environment variables. This page gives * `SB_INDEX_PAGE`: Sets the default page to load, defaults to `index`. * `SB_SPACE_IGNORE`: Ignore certain path patterns based on a .gitignore style format, e.g. `SB_SPACE_IGNORE="IgnoreMe/*"`. +* `SB_HTTP_LOGGING`: Set to any value to enable HTTP logging # Network * `SB_HOSTNAME`: Set to the hostname to bind to (defaults to `127.0.0.0`, set to `0.0.0.0` to accept outside connections for the local deno setup, defaults to `0.0.0.0` for docker) @@ -43,7 +44,7 @@ Configuration only relevant to docker deployments: * `PGID`: Runs the server process with the specified GID (default: whatever group owns the `/space` mapped folder)\ # Web app manifest -Configure aspects of web app appearance: +Configure aspects of web app appearance as well as the authentication page: * `SB_NAME`: Sets `name` and `short_name` members of web app manifest to whatever specified in `SB_NAME` * `SB_DESCRIPTION`: Sets `description` member of web app manifest to whatever specified in `SB_DESCRIPTION` diff --git a/website/Space Style.md b/website/Space Style.md index 33722bae..7401fa9e 100644 --- a/website/Space Style.md +++ b/website/Space Style.md @@ -4,7 +4,7 @@ This can be used to achieve various things, such as overriding the default edito To apply the updated styles, either reload the client or run the ${widgets.commandButton("System: Reload")} command. -Many styles can be set with [variables](https://github.com/silverbulletmd/silverbullet/blob/main/web/styles/theme.scss) but not everything is covered. You’ll have to reverse-engineer those parts, unfortunately. +Many styles can be set with [variables](https://github.com/silverbulletmd/silverbullet/blob/main/client/styles/theme.scss) but not everything is covered. You’ll have to reverse-engineer those parts, unfortunately. # Load order You can tweak the CSS load order style by including a `/* priority: number */` comment: