From 5d1e616585f72d1c3dd08a8b6f14b0156f52cf4a Mon Sep 17 00:00:00 2001 From: Haoqing Wang <78337154+hqhq1025@users.noreply.github.com> Date: Fri, 17 Apr 2026 11:02:30 +0800 Subject: [PATCH] fix(web): strip CJK punctuation from auto-linked URLs (#479) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(web): strip CJK punctuation from auto-linked URLs (#478) remark-gfm auto-links bare URLs but only handles ASCII trailing punctuation. When a URL is followed by CJK punctuation like ，or 。 without whitespace, the punctuation gets included in the link. Add a remark plugin that walks the MDAST after GFM and moves any trailing CJK/fullwidth punctuation out of the link node into a sibling text node. via [HAPI](https://hapi.run) Co-Authored-By: HAPI * fix: add non-null assertion for link.children in test via [HAPI](https://hapi.run) Co-Authored-By: HAPI * fix: only strip CJK punctuation from auto-linked URLs, not explicit links Only process links where the text content matches the URL (auto-links). Explicit markdown links like [text](url) are left untouched, preventing unintended mutation of deliberately authored URLs. via [HAPI](https://hapi.run) Co-Authored-By: HAPI * fix: add non-null assertion for textChild.value via [HAPI](https://hapi.run) Co-Authored-By: HAPI * fix: remove duplicate unicode escapes from CJK punctuation regex 7 characters were listed twice (once as literals, once as \uXXXX escapes). Keep only the literals and the 2 unique escapes (\u3000 ideographic space, \uFF0E fullwidth full stop). via [HAPI](https://hapi.run) Co-Authored-By: HAPI --------- Co-authored-by: HAPI --- .../components/assistant-ui/markdown-text.tsx | 3 +- web/src/lib/remark-strip-cjk-autolink.test.ts | 110 ++++++++++++++++++ web/src/lib/remark-strip-cjk-autolink.ts | 69 +++++++++++ 3 files changed, 181 insertions(+), 1 deletion(-) create mode 100644 web/src/lib/remark-strip-cjk-autolink.test.ts create mode 100644 web/src/lib/remark-strip-cjk-autolink.ts diff --git a/web/src/components/assistant-ui/markdown-text.tsx b/web/src/components/assistant-ui/markdown-text.tsx index 3bfaad3d..5d49ce3c 100644 --- a/web/src/components/assistant-ui/markdown-text.tsx +++ b/web/src/components/assistant-ui/markdown-text.tsx @@ -9,6 +9,7 @@ import remarkGfm from 'remark-gfm' import remarkMath from 'remark-math' import rehypeKatex from 'rehype-katex' import remarkDisableIndentedCode from '@/lib/remark-disable-indented-code' +import remarkStripCjkAutolink from '@/lib/remark-strip-cjk-autolink' import { cn } from '@/lib/utils' import { SyntaxHighlighter } from '@/components/assistant-ui/shiki-highlighter' import { useCopyToClipboard } from '@/hooks/useCopyToClipboard' @@ -16,7 +17,7 @@ import { CopyIcon, CheckIcon } from '@/components/icons' import type { MarkdownTextPrimitiveProps } from '@assistant-ui/react-markdown' -export const MARKDOWN_PLUGINS = [remarkGfm, remarkMath, remarkDisableIndentedCode] satisfies NonNullable +export const MARKDOWN_PLUGINS = [remarkGfm, remarkStripCjkAutolink, remarkMath, remarkDisableIndentedCode] satisfies NonNullable export const MARKDOWN_REHYPE_PLUGINS = [rehypeKatex] satisfies NonNullable function CodeHeader(props: CodeHeaderProps) { diff --git a/web/src/lib/remark-strip-cjk-autolink.test.ts b/web/src/lib/remark-strip-cjk-autolink.test.ts new file mode 100644 index 00000000..f6bf9ba1 --- /dev/null +++ b/web/src/lib/remark-strip-cjk-autolink.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, it } from 'vitest' +import remarkStripCjkAutolink from '@/lib/remark-strip-cjk-autolink' + +/** + * Unit tests for the remark-strip-cjk-autolink plugin. + * + * We test the tree-transform function directly by feeding it MDAST + * structures that remark-gfm would produce for auto-linked URLs. + */ +describe('remarkStripCjkAutolink', () => { + const transform = remarkStripCjkAutolink() + + function makeAutolink(url: string) { + return { + type: 'root', + children: [ + { + type: 'paragraph', + children: [ + { type: 'text', value: 'See ' }, + { + type: 'link', + url, + children: [{ type: 'text', value: url }] + } + ] + } + ] + } + } + + it('strips trailing fullwidth comma from URL', () => { + const tree = makeAutolink('https://example.com/path，') + transform(tree) + + const paragraph = tree.children[0] + const link = paragraph.children[1] + expect(link.url).toBe('https://example.com/path') + expect(link.children![0].value).toBe('https://example.com/path') + + // Punctuation moved to a new text node after the link + const punct = paragraph.children[2] + expect(punct.type).toBe('text') + expect(punct.value).toBe('，') + }) + + it('strips trailing ideographic full stop', () => { + const tree = makeAutolink('https://example.com。') + transform(tree) + + const paragraph = tree.children[0] + const link = paragraph.children[1] + expect(link.url).toBe('https://example.com') + expect(paragraph.children[2].value).toBe('。') + }) + + it('strips multiple trailing CJK punctuation characters', () => { + const tree = makeAutolink('https://example.com，。') + transform(tree) + + const paragraph = tree.children[0] + const link = paragraph.children[1] + expect(link.url).toBe('https://example.com') + expect(paragraph.children[2].value).toBe('，。') + }) + + it('does not modify URLs without CJK trailing punctuation', () => { + const tree = makeAutolink('https://example.com/path') + transform(tree) + + const link = tree.children[0].children[1] + expect(link.url).toBe('https://example.com/path') + // No extra node inserted + expect(tree.children[0].children.length).toBe(2) + }) + + it('does not strip CJK characters that are part of the URL path', () => { + const tree = makeAutolink('https://example.com/路径/page') + transform(tree) + + const link = tree.children[0].children[1] + expect(link.url).toBe('https://example.com/路径/page') + expect(tree.children[0].children.length).toBe(2) + }) + + it('does not modify explicit markdown links', () => { + // Explicit markdown link: [click here](https://example.com/path）) + // The link text differs from the URL, so it's not an autolink + const tree = { + type: 'root', + children: [ + { + type: 'paragraph', + children: [ + { + type: 'link', + url: 'https://example.com/path）', + children: [{ type: 'text', value: 'click here' }] + } + ] + } + ] + } + transform(tree) + + const link = tree.children[0].children[0] + expect(link.url).toBe('https://example.com/path）') + expect(tree.children[0].children.length).toBe(1) + }) +}) diff --git a/web/src/lib/remark-strip-cjk-autolink.ts b/web/src/lib/remark-strip-cjk-autolink.ts new file mode 100644 index 00000000..ab4fff26 --- /dev/null +++ b/web/src/lib/remark-strip-cjk-autolink.ts @@ -0,0 +1,69 @@ +/** + * Remark plugin that strips CJK/fullwidth punctuation from the end of + * auto-linked URLs. + * + * `remark-gfm` auto-links bare URLs but its boundary detection only + * handles ASCII punctuation. When a URL is followed by CJK punctuation + * (e.g. `，`、`。`) without whitespace, the punctuation is swallowed + * into the link. This plugin walks the MDAST after GFM runs and moves + * any trailing CJK punctuation out of the link node into a sibling text + * node. + */ + +// Common CJK / fullwidth punctuation that should never be part of a URL. +// Includes: fullwidth comma/period/semicolon/colon/exclamation/question/parens, +// ideographic comma/period, CJK brackets, ideographic space, fullwidth full stop. +const TRAILING_CJK_PUNCT = /[，。、；：！？（）【】「」『』《》〈〉\u3000\uFF0E]+$/ + +interface MdastNode { + type: string + url?: string + value?: string + children?: MdastNode[] +} + +function visitLinks(node: MdastNode): void { + if (!node.children) return + + for (let i = 0; i < node.children.length; i++) { + const child = node.children[i] + + if (child.type === 'link' && typeof child.url === 'string') { + // Only process auto-linked URLs (where the link text matches the URL). + // Explicit markdown links like [text](url) should not be modified. + const textChild = child.children?.[0] + const isAutolink = child.children?.length === 1 + && textChild?.type === 'text' + && typeof textChild.value === 'string' + && textChild.value === child.url + + if (isAutolink) { + const match = child.url.match(TRAILING_CJK_PUNCT) + if (match) { + const punct = match[0] + + // Strip punctuation from the URL + child.url = child.url.slice(0, -punct.length) + + // Strip from the link's text child + textChild!.value = textChild!.value!.slice(0, -punct.length) + + // Insert the punctuation as a plain text node after the link + const punctNode: MdastNode = { type: 'text', value: punct } + node.children.splice(i + 1, 0, punctNode) + // Skip the newly inserted node + i++ + } + } + } + + // Recurse into children + visitLinks(child) + } +} + +export default function remarkStripCjkAutolink() { + return (tree: MdastNode) => { + visitLinks(tree) + } +}