diff --git a/web/src/components/assistant-ui/markdown-text.tsx b/web/src/components/assistant-ui/markdown-text.tsx index 4b7fe53d..b8cd4024 100644 --- a/web/src/components/assistant-ui/markdown-text.tsx +++ b/web/src/components/assistant-ui/markdown-text.tsx @@ -13,6 +13,7 @@ import remarkBreaks from 'remark-breaks' import remarkMath from 'remark-math' import rehypeKatex from 'rehype-katex' import remarkDisableIndentedCode from '@/lib/remark-disable-indented-code' +import remarkRepairTables from '@/lib/remark-repair-tables' import { useNavigate } from '@tanstack/react-router' import remarkStripCjkAutolink from '@/lib/remark-strip-cjk-autolink' import remarkNonHttpsAutolink from '@/lib/remark-non-https-autolink' @@ -28,7 +29,9 @@ import { UriConfirmDialog } from '@/components/UriConfirmDialog' import type { MarkdownTextPrimitiveProps } from '@assistant-ui/react-markdown' // ── Plugin array ──────────────────────────────────────────────────────────── -// Order: remarkGfm → remarkNonHttpsAutolink → remarkStripCjkAutolink → remarkMath → remarkDisableIndentedCode → remarkFilePathLinks +// Order: remarkGfm → remarkRepairTables → remarkNonHttpsAutolink → remarkStripCjkAutolink → remarkMath → remarkDisableIndentedCode → remarkFilePathLinks +// remarkRepairTables must run immediately after remarkGfm — it reads file.value +// (raw source) to pad short separator rows before remark-gfm parses the table. // remarkNonHttpsAutolink must run BEFORE remarkStripCjkAutolink so that the // CJK strip plugin sees the new link nodes and can trim trailing CJK punctuation // from them. Both must come before remarkMath (to avoid treating TeX as URI). @@ -51,6 +54,7 @@ const MARKDOWN_PLUGIN_TAIL = [ export const MARKDOWN_PLUGINS = [ remarkGfm, + remarkRepairTables, ...MARKDOWN_PLUGIN_TAIL, ] satisfies NonNullable @@ -58,6 +62,7 @@ export const MARKDOWN_PLUGINS = [ // changing assistant/tool markdown behavior globally. export const MARKDOWN_PLUGINS_WITH_BREAKS = [ remarkGfm, + remarkRepairTables, remarkBreaks, ...MARKDOWN_PLUGIN_TAIL, ] satisfies NonNullable diff --git a/web/src/lib/remark-repair-tables.test.ts b/web/src/lib/remark-repair-tables.test.ts new file mode 100644 index 00000000..abe95c00 --- /dev/null +++ b/web/src/lib/remark-repair-tables.test.ts @@ -0,0 +1,245 @@ +import { describe, expect, it } from 'vitest' +import remarkParse from 'remark-parse' +import remarkGfm from 'remark-gfm' +import remarkStringify from 'remark-stringify' +import { unified } from 'unified' +import remarkRepairTables, { repairMarkdownTables } from './remark-repair-tables' + +function process(md: string): string { + return unified() + .use(remarkParse) + .use(remarkGfm) + .use(remarkRepairTables) + .use(remarkStringify) + .processSync(md) + .toString() +} + +/** + * Returns table rows from the stringified output. + * A proper table row starts with | (not \| which is escaped paragraph content). + */ +function tableRows(md: string): string[] { + return md.split('\n').filter(l => { + const t = l.trim() + return t.startsWith('|') && !t.startsWith('\\|') + }) +} + +// ── String-level function ──────────────────────────────────────────────────── + +describe('repairMarkdownTables (string)', () => { + it('pads a 2-cell separator for a 3-column header', () => { + const input = '| A | B | C |\n|---|---|\n| x | y | z |\n' + const out = repairMarkdownTables(input) + expect(out).not.toBe(input) + // Separator line should now have 3 cells + const sepLine = out.split('\n')[1] + expect(sepLine.split('|').filter(c => c.trim()).length).toBe(3) + }) + + it('pads a 1-cell separator for a 4-column header', () => { + const input = '| W | X | Y | Z |\n|---|\n| a | b | c | d |\n' + const out = repairMarkdownTables(input) + const sepLine = out.split('\n')[1] + expect(sepLine.split('|').filter(c => c.trim()).length).toBe(4) + }) + + it('returns the source unchanged when separator already matches', () => { + const input = '| A | B | C |\n|---|---|---|\n| x | y | z |\n' + expect(repairMarkdownTables(input)).toBe(input) + }) + + it('does not modify separator lines not following a |-starting row', () => { + const input = 'Some prose\n|---|---|\n| x | y | z |\n' + expect(repairMarkdownTables(input)).toBe(input) + }) + + it('does not modify table-like lines inside a fenced code block', () => { + const input = [ + 'Here is an example:', + '```', + '| A | B | C |', + '|---|---|', + '| x | y | z |', + '```', + '', + ].join('\n') + expect(repairMarkdownTables(input)).toBe(input) + }) + + it('does not modify table-like lines inside a ~~~ fenced code block', () => { + const input = '~~~\n| A | B | C |\n|---|---|\n| x | y | z |\n~~~\n' + expect(repairMarkdownTables(input)).toBe(input) + }) + + it('does not pad a valid table whose header contains a code span with a pipe', () => { + // | `a | b` | c | is a 2-column header; separator has 2 cells — valid + const input = '| `a | b` | c |\n|---|---|\n| x | y |\n' + expect(repairMarkdownTables(input)).toBe(input) + }) + + it('does not close a ```` fence on a ``` line (closer must be >= opener length)', () => { + const input = [ + '````', + '```', + '| A | B | C |', + '|---|---|', + '| x | y | z |', + '```', + '````', + '', + ].join('\n') + expect(repairMarkdownTables(input)).toBe(input) + }) + + it('does not close a ~~~~ fence on a ~~~ line (closer must be >= opener length)', () => { + const input = [ + '~~~~', + '~~~', + '| A | B | C |', + '|---|---|', + '| x | y | z |', + '~~~', + '~~~~', + '', + ].join('\n') + expect(repairMarkdownTables(input)).toBe(input) + }) + + it('does not flip fence state when ``` appears inside a ~~~ block', () => { + const input = [ + '~~~', + '```', + '| A | B | C |', + '|---|---|', + '| x | y | z |', + '```', + '~~~', + '', + ].join('\n') + expect(repairMarkdownTables(input)).toBe(input) + }) + + it('does not close a fence on a same-marker line that has info text', () => { + // ```ts inside a ``` fence is not a valid closing marker — only whitespace may follow + const input = [ + '```', + '```ts', + '| A | B | C |', + '|---|---|', + '| x | y | z |', + '```ts', + '```', + '', + ].join('\n') + expect(repairMarkdownTables(input)).toBe(input) + }) + + it('repairs a broken table after a fenced code block closes', () => { + const input = [ + '```', + '| A | B | C |', + '|---|---|', + '```', + '| A | B | C |', + '|---|---|', + '| x | y | z |', + ].join('\n') + const out = repairMarkdownTables(input) + // Lines inside the fence should be unchanged + expect(out.split('\n')[2]).toBe('|---|---|') + // The real table after the fence should be repaired + const sepLine = out.split('\n')[5] + expect(sepLine.split('|').filter(c => c.trim()).length).toBe(3) + }) +}) + +// ── Plugin (parse + transform + stringify) ──────────────────────────────────── + +describe('remarkRepairTables (plugin)', () => { + it('leaves a valid 3-column table unchanged', () => { + const md = '| A | B | C |\n|---|---|---|\n| x | y | z |\n' + const out = process(md) + // Must render as table rows (no escaped \|) + const rows = tableRows(out) + expect(rows.length).toBeGreaterThanOrEqual(3) + expect(out).toContain('| A |') + expect(out).toContain('| C |') + }) + + it('repairs separator with 2 cells for a 3-column header', () => { + const md = '| A | B | C |\n|---|---|\n| x | y | z |\n' + const out = process(md) + // Must render as a table — no escaped \| prefix + const rows = tableRows(out) + expect(rows.length).toBeGreaterThanOrEqual(2) + // All 3 header columns survive as proper table cells + expect(out).toContain('| A |') + expect(out).toContain('| B |') + expect(out).toContain('| C |') + }) + + it('repairs separator with 1 cell for a 4-column header', () => { + const md = '| W | X | Y | Z |\n|---|\n| a | b | c | d |\n' + const out = process(md) + expect(out).toContain('| W |') + expect(out).toContain('| Z |') + expect(tableRows(out).length).toBeGreaterThanOrEqual(2) + }) + + it('preserves alignment hints in existing separator cells', () => { + const md = '| A | B | C |\n|:---|---:|\n| x | y | z |\n' + const out = process(md) + expect(out).toContain('| C |') + const rows = tableRows(out) + expect(rows.length).toBeGreaterThanOrEqual(2) + }) + + it('does not corrupt a valid table with an escaped pipe in the header', () => { + // | A \| B | C | is a 2-column header (the \| is a literal pipe, not a delimiter) + // separator has 2 cells — valid, must not be padded to 3 + const md = '| A \\| B | C |\n|---|---|\n| x | y |\n' + const out = process(md) + const sepRow = out.split('\n').find(l => /^\|[\s|:|-]+\|$/.test(l.trim())) + expect(sepRow).toBeDefined() + expect(sepRow!.split('|').filter(c => c.trim()).length).toBe(2) + }) + + it('does not modify a table where separator already matches', () => { + const md = '| A | B |\n|---|---|\n| x | y |\n' + const out = process(md) + expect(out).toContain('| A |') + expect(out).toContain('| B |') + }) + + it('handles multiple tables — repairs broken, leaves valid untouched', () => { + const md = [ + '| A | B | C |', + '|---|---|', + '| x | y | z |', + '', + '| P | Q |', + '|---|---|', + '| 1 | 2 |', + ].join('\n') + '\n' + const out = process(md) + // First table repaired — C must be in a proper table row + expect(out).toContain('| C |') + // Second table unchanged and intact + expect(out).toContain('| P |') + expect(out).toContain('| Q |') + }) + + it('does not touch pipe characters in code spans or prose', () => { + const md = 'Use `foo | bar` for piping.\n' + const out = process(md) + expect(out).toContain('foo | bar') + }) + + it('ignores a paragraph that merely contains pipe characters', () => { + const md = 'Run: `jq \'.[] | select(.active)\'`\n' + const out = process(md) + expect(out).toContain('jq') + }) +}) diff --git a/web/src/lib/remark-repair-tables.ts b/web/src/lib/remark-repair-tables.ts new file mode 100644 index 00000000..2d47e645 --- /dev/null +++ b/web/src/lib/remark-repair-tables.ts @@ -0,0 +1,158 @@ +/** + * Remark plugin that repairs GFM tables where the separator row has fewer + * columns than the header row. + * + * Background: remark-gfm 4.x follows the GFM spec strictly — if the delimiter + * row has fewer cells than the header row, the entire block is degraded to a + * paragraph (no table node is produced at all). The previous approach of + * visiting `table` AST nodes could never trigger because remark-gfm never + * produced one. This version operates at the source level: it scans file.value + * for broken separator rows and pads them BEFORE remark-gfm parses, so the + * table is preserved with all columns intact. + */ + +import type { Processor } from 'unified' +import type { Root } from 'mdast' +import type { VFile } from 'vfile' + +// ── Helpers ────────────────────────────────────────────────────────────────── + +/** Count pipe-delimited cells in one table row line of raw source. + * Strips backtick code spans first so pipes inside them are not counted. + * Skips escaped pipes (\|) which are literal characters, not cell boundaries. */ +function countSourceCells(line: string): number { + // Replace code spans with a placeholder so any | inside them is invisible + const trimmed = line.trim().replace(/`+[^`]*?`+/g, '\x00') + const inner = trimmed.startsWith('|') ? trimmed.slice(1) : trimmed + const stripped = inner.endsWith('|') ? inner.slice(0, -1) : inner + let cells = 1 + let escaped = false + for (const ch of stripped) { + if (escaped) { escaped = false; continue } + if (ch === '\\') { escaped = true; continue } + if (ch === '|') cells++ + } + return cells +} + +/** Returns true if every pipe-delimited cell in the line matches the GFM separator pattern. */ +function isSeparatorLine(line: string): boolean { + const trimmed = line.trim() + if (!trimmed.includes('-')) return false + const inner = trimmed.startsWith('|') ? trimmed.slice(1) : trimmed + const stripped = inner.endsWith('|') ? inner.slice(0, -1) : inner + const cells = stripped.split('|') + return cells.length > 0 && cells.every(c => /^\s*:?-+:?\s*$/.test(c)) +} + +/** Count cells in a separator line (returns null if line is not a separator). */ +function countSeparatorCells(line: string): number | null { + if (!isSeparatorLine(line)) return null + const trimmed = line.trim() + const inner = trimmed.startsWith('|') ? trimmed.slice(1) : trimmed + const stripped = inner.endsWith('|') ? inner.slice(0, -1) : inner + return stripped.split('|').length +} + +/** + * Pad `sepLine` to have `targetCols` cells, preserving any existing alignment + * hints in the cells that are already there. Returns the repaired line, or + * null if the line already has enough cells or is not a valid separator. + */ +function padSeparatorLine(sepLine: string, targetCols: number): string | null { + const trimmed = sepLine.trim() + if (!trimmed) return null + + const hasLeading = trimmed.startsWith('|') + const hasTrailing = trimmed.endsWith('|') + + const inner = hasLeading ? trimmed.slice(1) : trimmed + const stripped = inner.endsWith('|') ? inner.slice(0, -1) : inner + const cells = stripped.split('|') + + if (cells.length >= targetCols) return null + if (!cells.every(c => /^\s*:?-+:?\s*$/.test(c))) return null + + const extra = Array(targetCols - cells.length).fill(' --- ') + const paddedInner = [...cells, ...extra].join('|') + return (hasLeading ? '|' : '') + paddedInner + (hasTrailing ? '|' : '') +} + +// ── String-level preprocessor ───────────────────────────────────────────────── + +/** + * Scan raw markdown for broken table separators and pad them in-place. + * This must run before any markdown parser sees the source, because + * remark-gfm 4.x degrades a mismatched-separator table block to a paragraph. + * + * Tracks fenced code blocks so table-like lines inside ``` or ~~~ fences are + * never modified. Also preserves leading whitespace when replacing the + * separator line so indented tables are not affected. + */ +export function repairMarkdownTables(source: string): string { + const lines = source.split('\n') + let changed = false + // Track fence character AND opening length: a ```` fence must not be closed + // by ``` (GFM §4.5: closer must match the opening marker family AND be at + // least as long). Also ignore the opposite marker family (backtick vs tilde). + let fenceChar: '`' | '~' | null = null + let fenceLength = 0 + + for (let i = 0; i < lines.length; i++) { + // Capture marker + everything after so we can check the closing-fence rule: + // openers may have an info string (```ts), but closers must be whitespace-only. + const fenceMatch = lines[i].match(/^ {0,3}(`{3,}|~{3,})(.*)$/) + if (fenceMatch) { + const ch = fenceMatch[1][0] as '`' | '~' + const len = fenceMatch[1].length + const rest = fenceMatch[2] + if (fenceChar === null) { + fenceChar = ch + fenceLength = len + } else if (ch === fenceChar && len >= fenceLength && /^\s*$/.test(rest)) { + fenceChar = null + fenceLength = 0 + } + continue + } + if (fenceChar !== null) continue + if (i === 0) continue + + const sep = lines[i] + if (!isSeparatorLine(sep)) continue + + const hdr = lines[i - 1] + // Only repair when the header row starts with | (the common LLM output form) + if (!hdr.trim().startsWith('|')) continue + + const headerCols = countSourceCells(hdr) + const sepCols = countSeparatorCells(sep) + if (sepCols === null || sepCols >= headerCols) continue + + const repaired = padSeparatorLine(sep, headerCols) + if (repaired !== null) { + // Preserve original leading whitespace so indented tables are unchanged + const prefix = sep.match(/^\s*/)?.[0] ?? '' + lines[i] = prefix + repaired + changed = true + } + } + + return changed ? lines.join('\n') : source +} + +// ── Plugin ─────────────────────────────────────────────────────────────────── + +export default function remarkRepairTables(this: Processor) { + const processor = this + return (tree: Root, file: VFile) => { + const original = String(file.value) + const repaired = repairMarkdownTables(original) + if (repaired === original) return + + // Re-parse with the repaired source so remark-gfm produces table nodes + // processor.parse() runs only the parse phase, not transformers + const newTree = processor.parse(repaired) as Root + Object.assign(tree, newTree) + } +}