300 lines
9.7 KiB
TypeScript
300 lines
9.7 KiB
TypeScript
export type SyntaxTokenKind =
|
|
| 'comment'
|
|
| 'string'
|
|
| 'keyword'
|
|
| 'number'
|
|
| 'type'
|
|
| 'function'
|
|
| 'property'
|
|
| 'operator'
|
|
| 'plain'
|
|
|
|
export interface SyntaxToken {
|
|
kind: SyntaxTokenKind
|
|
text: string
|
|
}
|
|
|
|
export interface SyntaxPalette {
|
|
background: string
|
|
headerBackground: string
|
|
border: string
|
|
headerText: string
|
|
plain: string
|
|
comment: string
|
|
string: string
|
|
keyword: string
|
|
number: string
|
|
type: string
|
|
function: string
|
|
property: string
|
|
operator: string
|
|
}
|
|
|
|
export const LIGHT_CODE_PALETTE: SyntaxPalette = {
|
|
background: '#F6F8FB',
|
|
headerBackground: '#EDF1F7',
|
|
border: '#E3E8F0',
|
|
headerText: '#59606D',
|
|
plain: '#1F2328',
|
|
comment: '#6E7781',
|
|
string: '#0A7B4F',
|
|
keyword: '#9333EA',
|
|
number: '#B45309',
|
|
type: '#0284C7',
|
|
function: '#3B5BDB',
|
|
property: '#0F766E',
|
|
operator: '#57606A',
|
|
}
|
|
|
|
export const DARK_CODE_PALETTE: SyntaxPalette = {
|
|
background: '#131720',
|
|
headerBackground: '#1B212E',
|
|
border: '#273041',
|
|
headerText: '#9AA3B5',
|
|
plain: '#E6EDF3',
|
|
comment: '#7B8496',
|
|
string: '#7EE787',
|
|
keyword: '#C792EA',
|
|
number: '#F78C6C',
|
|
type: '#79C0FF',
|
|
function: '#82AAFF',
|
|
property: '#7FDBCA',
|
|
operator: '#9AA3B5',
|
|
}
|
|
|
|
const MAX_HIGHLIGHT_CHARS = 16_000
|
|
const MAX_HIGHLIGHT_LINES = 400
|
|
|
|
const COMMON_KEYWORDS = new Set([
|
|
'abstract', 'and', 'as', 'async', 'await', 'break', 'case', 'catch', 'class',
|
|
'const', 'continue', 'debugger', 'def', 'default', 'defer', 'delete', 'do',
|
|
'elif', 'else', 'enum', 'except', 'export', 'extends', 'extern', 'finally',
|
|
'fn', 'for', 'from', 'fun', 'func', 'function', 'go', 'goto', 'if', 'impl',
|
|
'import', 'in', 'instanceof', 'interface', 'is', 'lambda', 'let', 'loop',
|
|
'match', 'mod', 'mut', 'namespace', 'new', 'not', 'of', 'or', 'override',
|
|
'package', 'pass', 'private', 'protected', 'pub', 'public', 'raise', 'readonly',
|
|
'return', 'select', 'self', 'static', 'struct', 'super', 'switch', 'this',
|
|
'throw', 'throws', 'trait', 'try', 'type', 'typedef', 'typeof', 'use', 'using',
|
|
'val', 'var', 'void', 'where', 'while', 'with', 'yield',
|
|
// SQL & shell keywords
|
|
'SELECT', 'FROM', 'WHERE', 'JOIN', 'LEFT', 'RIGHT', 'INNER', 'ON', 'GROUP',
|
|
'BY', 'ORDER', 'LIMIT', 'INSERT', 'INTO', 'VALUES', 'UPDATE', 'SET', 'DELETE',
|
|
'CREATE', 'TABLE', 'ALTER', 'DROP', 'AS', 'AND', 'OR', 'NOT', 'NULL', 'DISTINCT',
|
|
'echo', 'export', 'local', 'fi', 'then', 'done', 'esac', 'cd', 'grep', 'git',
|
|
])
|
|
|
|
const LITERAL_CONSTANTS = new Set([
|
|
'true', 'false', 'null', 'undefined', 'nil', 'None', 'True', 'False', 'NaN', 'Infinity',
|
|
])
|
|
|
|
const BUILTIN_TYPES = new Set([
|
|
'string', 'number', 'boolean', 'unknown', 'never', 'any', 'object', 'symbol', 'bigint',
|
|
'int', 'float', 'double', 'char', 'bool', 'str', 'bytes', 'list', 'dict', 'set', 'tuple',
|
|
'u8', 'u16', 'u32', 'u64', 'usize', 'i8', 'i16', 'i32', 'i64', 'isize', 'f32', 'f64',
|
|
'Promise', 'Array', 'Record', 'Map', 'Set', 'Error', 'Date', 'RegExp', 'View', 'Text', 'ReactNode',
|
|
])
|
|
|
|
function isIdentStart(ch: number): boolean {
|
|
return (ch >= 65 && ch <= 90) || (ch >= 97 && ch <= 122) || ch === 95 || ch === 36
|
|
}
|
|
|
|
function isIdentPart(ch: number): boolean {
|
|
return isIdentStart(ch) || (ch >= 48 && ch <= 57)
|
|
}
|
|
|
|
function isDigit(ch: number): boolean {
|
|
return ch >= 48 && ch <= 57
|
|
}
|
|
|
|
function isHexDigit(ch: number): boolean {
|
|
return isDigit(ch) || (ch >= 65 && ch <= 70) || (ch >= 97 && ch <= 102) || ch === 95
|
|
}
|
|
|
|
function isWhitespace(ch: number): boolean {
|
|
return ch === 32 || ch === 9 || ch === 10 || ch === 13
|
|
}
|
|
|
|
function isOperatorChar(ch: number): boolean {
|
|
return ch === 61 || ch === 43 || ch === 45 || ch === 42 || ch === 47 || ch === 37
|
|
|| ch === 60 || ch === 62 || ch === 33 || ch === 38 || ch === 124 || ch === 94
|
|
|| ch === 126 || ch === 63 || ch === 58
|
|
}
|
|
|
|
function pushToken(out: SyntaxToken[], kind: SyntaxTokenKind, text: string) {
|
|
if (!text) return
|
|
const prev = out[out.length - 1]
|
|
if (prev && prev.kind === kind) {
|
|
prev.text += text
|
|
} else {
|
|
out.push({ kind, text })
|
|
}
|
|
}
|
|
|
|
function normalizeLang(language?: string): string {
|
|
return (language || '').trim().toLowerCase()
|
|
}
|
|
|
|
function supportsHashComments(lang: string): boolean {
|
|
return lang === 'py' || lang === 'python' || lang === 'sh' || lang === 'bash'
|
|
|| lang === 'zsh' || lang === 'shell' || lang === 'yaml' || lang === 'yml'
|
|
|| lang === 'toml' || lang === 'ruby' || lang === 'rb' || lang === 'dockerfile'
|
|
|| lang === 'makefile' || lang === 'ini' || lang === 'env'
|
|
}
|
|
|
|
/**
|
|
* Single-pass linear tokenizer for code blocks.
|
|
* Guaranteed O(n) with no regex backtracking; falls back to plain text on huge blocks.
|
|
*/
|
|
export function highlightCode(code: string, language?: string): SyntaxToken[] {
|
|
if (!code) return []
|
|
if (code.length > MAX_HIGHLIGHT_CHARS) return [{ kind: 'plain', text: code }]
|
|
let lines = 1
|
|
for (let i = 0; i < code.length; i += 1) {
|
|
if (code.charCodeAt(i) === 10 && ++lines > MAX_HIGHLIGHT_LINES) {
|
|
return [{ kind: 'plain', text: code }]
|
|
}
|
|
}
|
|
|
|
const lang = normalizeLang(language)
|
|
const isJson = lang === 'json' || lang === 'jsonc' || lang === 'json5'
|
|
const allowHashComment = supportsHashComments(lang) || lang === ''
|
|
const allowSlashComment = !isJson || lang === 'jsonc' || lang === 'json5'
|
|
const tokens: SyntaxToken[] = []
|
|
const len = code.length
|
|
let i = 0
|
|
|
|
while (i < len) {
|
|
const ch = code.charCodeAt(i)
|
|
const next = i + 1 < len ? code.charCodeAt(i + 1) : 0
|
|
|
|
// Line comments: // ... or -- ... (SQL) or # ...
|
|
if ((allowSlashComment && ch === 47 && next === 47)
|
|
|| ((lang === 'sql' || lang === 'lua' || lang === 'haskell') && ch === 45 && next === 45)
|
|
|| (allowHashComment && ch === 35)) {
|
|
let end = i + 1
|
|
while (end < len && code.charCodeAt(end) !== 10) end += 1
|
|
pushToken(tokens, 'comment', code.slice(i, end))
|
|
i = end
|
|
continue
|
|
}
|
|
|
|
// Block comments: /* ... */
|
|
if (allowSlashComment && ch === 47 && next === 42) {
|
|
let end = i + 2
|
|
while (end + 1 < len && !(code.charCodeAt(end) === 42 && code.charCodeAt(end + 1) === 47)) {
|
|
end += 1
|
|
}
|
|
end = end + 1 < len ? end + 2 : len
|
|
pushToken(tokens, 'comment', code.slice(i, end))
|
|
i = end
|
|
continue
|
|
}
|
|
|
|
// Strings: ', ", `
|
|
if (ch === 34 || ch === 39 || ch === 96) {
|
|
const quote = ch
|
|
let end = i + 1
|
|
while (end < len) {
|
|
const c = code.charCodeAt(end)
|
|
if (c === 92) {
|
|
end += 2
|
|
continue
|
|
}
|
|
if (c === quote) {
|
|
end += 1
|
|
break
|
|
}
|
|
if (c === 10 && quote !== 96) {
|
|
break
|
|
}
|
|
end += 1
|
|
}
|
|
const raw = code.slice(i, end)
|
|
// Check if JSON / object property key followed by ':'
|
|
let k = end
|
|
while (k < len && (code.charCodeAt(k) === 32 || code.charCodeAt(k) === 9)) k += 1
|
|
const isKey = isJson && code.charCodeAt(k) === 58
|
|
pushToken(tokens, isKey ? 'property' : 'string', raw)
|
|
i = end
|
|
continue
|
|
}
|
|
|
|
// Numbers (only at token boundaries)
|
|
if (isDigit(ch) || (ch === 46 && isDigit(next))) {
|
|
let end = i
|
|
if (ch === 48 && (next === 120 || next === 88 || next === 98 || next === 66 || next === 111 || next === 79)) {
|
|
end += 2
|
|
while (end < len && isHexDigit(code.charCodeAt(end))) end += 1
|
|
} else {
|
|
while (end < len && (isDigit(code.charCodeAt(end)) || code.charCodeAt(end) === 95)) end += 1
|
|
if (end < len && code.charCodeAt(end) === 46 && end + 1 < len && isDigit(code.charCodeAt(end + 1))) {
|
|
end += 1
|
|
while (end < len && (isDigit(code.charCodeAt(end)) || code.charCodeAt(end) === 95)) end += 1
|
|
}
|
|
if (end < len && (code.charCodeAt(end) === 101 || code.charCodeAt(end) === 69)) {
|
|
let exp = end + 1
|
|
if (exp < len && (code.charCodeAt(exp) === 43 || code.charCodeAt(exp) === 45)) exp += 1
|
|
if (exp < len && isDigit(code.charCodeAt(exp))) {
|
|
end = exp
|
|
while (end < len && isDigit(code.charCodeAt(end))) end += 1
|
|
}
|
|
}
|
|
}
|
|
pushToken(tokens, 'number', code.slice(i, end))
|
|
i = end
|
|
continue
|
|
}
|
|
|
|
// Identifiers, keywords, types, functions, properties
|
|
if (isIdentStart(ch)) {
|
|
let end = i + 1
|
|
while (end < len && isIdentPart(code.charCodeAt(end))) end += 1
|
|
const word = code.slice(i, end)
|
|
|
|
let lookahead = end
|
|
while (lookahead < len && (code.charCodeAt(lookahead) === 32 || code.charCodeAt(lookahead) === 9)) {
|
|
lookahead += 1
|
|
}
|
|
const nextNonWs = lookahead < len ? code.charCodeAt(lookahead) : 0
|
|
const prevChar = i > 0 ? code.charCodeAt(i - 1) : 0
|
|
|
|
if (LITERAL_CONSTANTS.has(word)) {
|
|
pushToken(tokens, 'number', word)
|
|
} else if (COMMON_KEYWORDS.has(word)) {
|
|
pushToken(tokens, 'keyword', word)
|
|
} else if (BUILTIN_TYPES.has(word) || (ch >= 65 && ch <= 90 && word.length > 1 && word !== word.toUpperCase())) {
|
|
pushToken(tokens, 'type', word)
|
|
} else if (nextNonWs === 40) {
|
|
// Followed by '(' -> function call or declaration
|
|
pushToken(tokens, 'function', word)
|
|
} else if (prevChar === 46 || (nextNonWs === 58 && !isJson)) {
|
|
// Preceded by '.' or followed by ':' -> property / key
|
|
pushToken(tokens, 'property', word)
|
|
} else {
|
|
pushToken(tokens, 'plain', word)
|
|
}
|
|
i = end
|
|
continue
|
|
}
|
|
|
|
// Operators
|
|
if (isOperatorChar(ch)) {
|
|
let end = i + 1
|
|
while (end < len && isOperatorChar(code.charCodeAt(end)) && end - i < 3) end += 1
|
|
pushToken(tokens, 'operator', code.slice(i, end))
|
|
i = end
|
|
continue
|
|
}
|
|
|
|
// Whitespace or punctuation
|
|
let end = i + 1
|
|
if (isWhitespace(ch)) {
|
|
while (end < len && isWhitespace(code.charCodeAt(end))) end += 1
|
|
}
|
|
pushToken(tokens, 'plain', code.slice(i, end))
|
|
i = end
|
|
}
|
|
|
|
return tokens
|
|
}
|