fix(web): strip CJK punctuation from auto-linked URLs (#479)

* fix(web): strip CJK punctuation from auto-linked URLs (#478) remark-gfm auto-links bare URLs but only handles ASCII trailing punctuation. When a URL is followed by CJK punctuation like ，or 。 without whitespace, the punctuation gets included in the link. Add a remark plugin that walks the MDAST after GFM and moves any trailing CJK/fullwidth punctuation out of the link node into a sibling text node. via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> * fix: add non-null assertion for link.children in test via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> * fix: only strip CJK punctuation from auto-linked URLs, not explicit links Only process links where the text content matches the URL (auto-links). Explicit markdown links like [text](url) are left untouched, preventing unintended mutation of deliberately authored URLs. via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> * fix: add non-null assertion for textChild.value via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> * fix: remove duplicate unicode escapes from CJK punctuation regex 7 characters were listed twice (once as literals, once as \uXXXX escapes). Keep only the literals and the 2 unique escapes (\u3000 ideographic space, \uFF0E fullwidth full stop). via [HAPI](https://hapi.run) Co-Authored-By: HAPI <noreply@hapi.run> --------- Co-authored-by: HAPI <noreply@hapi.run>
2026-08-05 06:24:37 +00:00 · 2026-04-17 11:02:30 +08:00
parent a67558e9e9
commit 5d1e616585
3 changed files with 181 additions and 1 deletions
@@ -9,6 +9,7 @@ import remarkGfm from 'remark-gfm'
 import remarkMath from 'remark-math'
 import rehypeKatex from 'rehype-katex'
 import remarkDisableIndentedCode from '@/lib/remark-disable-indented-code'
+import remarkStripCjkAutolink from '@/lib/remark-strip-cjk-autolink'
 import { cn } from '@/lib/utils'
 import { SyntaxHighlighter } from '@/components/assistant-ui/shiki-highlighter'
 import { useCopyToClipboard } from '@/hooks/useCopyToClipboard'
@@ -16,7 +17,7 @@ import { CopyIcon, CheckIcon } from '@/components/icons'

 import type { MarkdownTextPrimitiveProps } from '@assistant-ui/react-markdown'

-export const MARKDOWN_PLUGINS = [remarkGfm, remarkMath, remarkDisableIndentedCode] satisfies NonNullable<MarkdownTextPrimitiveProps['remarkPlugins']>
+export const MARKDOWN_PLUGINS = [remarkGfm, remarkStripCjkAutolink, remarkMath, remarkDisableIndentedCode] satisfies NonNullable<MarkdownTextPrimitiveProps['remarkPlugins']>
 export const MARKDOWN_REHYPE_PLUGINS = [rehypeKatex] satisfies NonNullable<MarkdownTextPrimitiveProps['rehypePlugins']>

 function CodeHeader(props: CodeHeaderProps) {
@@ -0,0 +1,110 @@
+import { describe, expect, it } from 'vitest'
+import remarkStripCjkAutolink from '@/lib/remark-strip-cjk-autolink'
+
+/**
+ * Unit tests for the remark-strip-cjk-autolink plugin.
+ *
+ * We test the tree-transform function directly by feeding it MDAST
+ * structures that remark-gfm would produce for auto-linked URLs.
+ */
+describe('remarkStripCjkAutolink', () => {
+    const transform = remarkStripCjkAutolink()
+
+    function makeAutolink(url: string) {
+        return {
+            type: 'root',
+            children: [
+                {
+                    type: 'paragraph',
+                    children: [
+                        { type: 'text', value: 'See ' },
+                        {
+                            type: 'link',
+                            url,
+                            children: [{ type: 'text', value: url }]
+                        }
+                    ]
+                }
+            ]
+        }
+    }
+
+    it('strips trailing fullwidth comma from URL', () => {
+        const tree = makeAutolink('https://example.com/path，')
+        transform(tree)
+
+        const paragraph = tree.children[0]
+        const link = paragraph.children[1]
+        expect(link.url).toBe('https://example.com/path')
+        expect(link.children![0].value).toBe('https://example.com/path')
+
+        // Punctuation moved to a new text node after the link
+        const punct = paragraph.children[2]
+        expect(punct.type).toBe('text')
+        expect(punct.value).toBe('，')
+    })
+
+    it('strips trailing ideographic full stop', () => {
+        const tree = makeAutolink('https://example.com。')
+        transform(tree)
+
+        const paragraph = tree.children[0]
+        const link = paragraph.children[1]
+        expect(link.url).toBe('https://example.com')
+        expect(paragraph.children[2].value).toBe('。')
+    })
+
+    it('strips multiple trailing CJK punctuation characters', () => {
+        const tree = makeAutolink('https://example.com，。')
+        transform(tree)
+
+        const paragraph = tree.children[0]
+        const link = paragraph.children[1]
+        expect(link.url).toBe('https://example.com')
+        expect(paragraph.children[2].value).toBe('，。')
+    })
+
+    it('does not modify URLs without CJK trailing punctuation', () => {
+        const tree = makeAutolink('https://example.com/path')
+        transform(tree)
+
+        const link = tree.children[0].children[1]
+        expect(link.url).toBe('https://example.com/path')
+        // No extra node inserted
+        expect(tree.children[0].children.length).toBe(2)
+    })
+
+    it('does not strip CJK characters that are part of the URL path', () => {
+        const tree = makeAutolink('https://example.com/路径/page')
+        transform(tree)
+
+        const link = tree.children[0].children[1]
+        expect(link.url).toBe('https://example.com/路径/page')
+        expect(tree.children[0].children.length).toBe(2)
+    })
+
+    it('does not modify explicit markdown links', () => {
+        // Explicit markdown link: [click here](https://example.com/path）)
+        // The link text differs from the URL, so it's not an autolink
+        const tree = {
+            type: 'root',
+            children: [
+                {
+                    type: 'paragraph',
+                    children: [
+                        {
+                            type: 'link',
+                            url: 'https://example.com/path）',
+                            children: [{ type: 'text', value: 'click here' }]
+                        }
+                    ]
+                }
+            ]
+        }
+        transform(tree)
+
+        const link = tree.children[0].children[0]
+        expect(link.url).toBe('https://example.com/path）')
+        expect(tree.children[0].children.length).toBe(1)
+    })
+})
@@ -0,0 +1,69 @@
+/**
+ * Remark plugin that strips CJK/fullwidth punctuation from the end of
+ * auto-linked URLs.
+ *
+ * `remark-gfm` auto-links bare URLs but its boundary detection only
+ * handles ASCII punctuation.  When a URL is followed by CJK punctuation
+ * (e.g. `，`、`。`) without whitespace, the punctuation is swallowed
+ * into the link.  This plugin walks the MDAST after GFM runs and moves
+ * any trailing CJK punctuation out of the link node into a sibling text
+ * node.
+ */
+
+// Common CJK / fullwidth punctuation that should never be part of a URL.
+// Includes: fullwidth comma/period/semicolon/colon/exclamation/question/parens,
+// ideographic comma/period, CJK brackets, ideographic space, fullwidth full stop.
+const TRAILING_CJK_PUNCT = /[，。、；：！？（）【】「」『』《》〈〉\u3000\uFF0E]+$/
+
+interface MdastNode {
+    type: string
+    url?: string
+    value?: string
+    children?: MdastNode[]
+}
+
+function visitLinks(node: MdastNode): void {
+    if (!node.children) return
+
+    for (let i = 0; i < node.children.length; i++) {
+        const child = node.children[i]
+
+        if (child.type === 'link' && typeof child.url === 'string') {
+            // Only process auto-linked URLs (where the link text matches the URL).
+            // Explicit markdown links like [text](url) should not be modified.
+            const textChild = child.children?.[0]
+            const isAutolink = child.children?.length === 1
+                && textChild?.type === 'text'
+                && typeof textChild.value === 'string'
+                && textChild.value === child.url
+
+            if (isAutolink) {
+                const match = child.url.match(TRAILING_CJK_PUNCT)
+                if (match) {
+                    const punct = match[0]
+
+                    // Strip punctuation from the URL
+                    child.url = child.url.slice(0, -punct.length)
+
+                    // Strip from the link's text child
+                    textChild!.value = textChild!.value!.slice(0, -punct.length)
+
+                    // Insert the punctuation as a plain text node after the link
+                    const punctNode: MdastNode = { type: 'text', value: punct }
+                    node.children.splice(i + 1, 0, punctNode)
+                    // Skip the newly inserted node
+                    i++
+                }
+            }
+        }
+
+        // Recurse into children
+        visitLinks(child)
+    }
+}
+
+export default function remarkStripCjkAutolink() {
+    return (tree: MdastNode) => {
+        visitLinks(tree)
+    }
+}