776 lines
41 KiB
JavaScript
776 lines
41 KiB
JavaScript
// Search provider `auto` for dsh's web seam (`ctx.web`). Each query first tries native search
|
||
// behind the asking Session's own model route, with that route's own key, on that route's own
|
||
// origin. Native search is found by protocol, not by vendor: the route's wire protocol defines a
|
||
// standard search tool (Anthropic Messages, OpenAI Responses, Gemini), and a host may also serve
|
||
// a separate search API, declared as data (SEARCH_APIS, or `searchApis` in config). Whether an
|
||
// endpoint supports either is only knowable by trying, so every outcome is remembered. It never
|
||
// borrows another route's key: every key bills its own account. When native search is
|
||
// unavailable, a keyless fallback parses a public results page fetched through ctx.web.fetch.
|
||
// Every result says which path answered it; when both fail, the error tells the model not to
|
||
// guess. Independent of the Bot team code in index.js.
|
||
import { WebError } from '@deepseek-ai/dsh-web'
|
||
|
||
export const name = 'web-search-auto'
|
||
export const inject = ['web']
|
||
|
||
export const AUTO_PROVIDER_ID = 'auto'
|
||
export const DEFAULTS = Object.freeze({
|
||
native: true,
|
||
fallback: true,
|
||
engines: ['bing', 'duckduckgo'],
|
||
searchApis: [],
|
||
nativeTimeoutMs: 40_000,
|
||
fallbackTimeoutMs: 15_000,
|
||
})
|
||
|
||
const USER_AGENT = 'deepseek-harness/0.0.1'
|
||
const SNIPPET_CHARS = 500
|
||
const PROMPT = query => `Perform a web search for the query: ${query}`
|
||
|
||
const MINUTE = 60_000
|
||
// How long one outcome keeps a candidate from being tried again.
|
||
const SKIP_FOR = {
|
||
absent: 24 * 60 * MINUTE,
|
||
// The query itself may be what was refused (length, content moderation).
|
||
rejected: 30 * MINUTE,
|
||
// The user may enable search for the key right away.
|
||
denied: 30 * MINUTE,
|
||
// Each retry of an endpoint that answers without searching costs a model call.
|
||
ignored: 6 * 60 * MINUTE,
|
||
}
|
||
const STATUS_OUTCOMES = { 404: 'absent', 405: 'absent', 501: 'absent', 400: 'rejected', 415: 'rejected', 422: 'rejected', 401: 'denied', 402: 'denied', 403: 'denied' }
|
||
|
||
// Routes whose profile leaves protocol, endpoint, or key name to the adapter: dsh's DeepSeek
|
||
// adapters, and the pi-ai 0.87.1 catalog routes that sign in with an API key
|
||
// (models.generated.js, env-api-keys.js). The protocol is undefined where it differs per model.
|
||
const CATALOG = {
|
||
'deepseek-official': ['anthropic-messages', 'https://api.deepseek.com/anthropic', 'DEEPSEEK_API_KEY'],
|
||
'deepseek-account': ['anthropic-messages', 'https://api.deepseek.com/anthropic', undefined],
|
||
'ant-ling': ['openai-completions', 'https://api.ant-ling.com/v1', 'ANT_LING_API_KEY'],
|
||
anthropic: ['anthropic-messages', 'https://api.anthropic.com', 'ANTHROPIC_API_KEY'],
|
||
baseten: ['openai-completions', 'https://inference.baseten.co/v1', 'BASETEN_API_KEY'],
|
||
cerebras: ['openai-completions', 'https://api.cerebras.ai/v1', 'CEREBRAS_API_KEY'],
|
||
deepseek: ['openai-completions', 'https://api.deepseek.com', 'DEEPSEEK_API_KEY'],
|
||
fireworks: [undefined, 'https://api.fireworks.ai/inference/v1', 'FIREWORKS_API_KEY'],
|
||
google: ['google-generative-ai', 'https://generativelanguage.googleapis.com/v1beta', 'GEMINI_API_KEY'],
|
||
groq: ['openai-completions', 'https://api.groq.com/openai/v1', 'GROQ_API_KEY'],
|
||
huggingface: ['openai-completions', 'https://router.huggingface.co/v1', 'HF_TOKEN'],
|
||
'kimi-coding': ['anthropic-messages', 'https://api.kimi.com/coding', 'KIMI_API_KEY'],
|
||
meta: ['openai-responses', 'https://api.meta.ai/v1', 'META_API_KEY'],
|
||
minimax: ['anthropic-messages', 'https://api.minimax.io/anthropic', 'MINIMAX_API_KEY'],
|
||
'minimax-cn': ['anthropic-messages', 'https://api.minimaxi.com/anthropic', 'MINIMAX_CN_API_KEY'],
|
||
moonshotai: ['openai-completions', 'https://api.moonshot.ai/v1', 'MOONSHOT_API_KEY'],
|
||
'moonshotai-cn': ['openai-completions', 'https://api.moonshot.cn/v1', 'MOONSHOT_API_KEY'],
|
||
nvidia: ['openai-completions', 'https://integrate.api.nvidia.com/v1', 'NVIDIA_API_KEY'],
|
||
openai: ['openai-responses', 'https://api.openai.com/v1', 'OPENAI_API_KEY'],
|
||
opencode: [undefined, 'https://opencode.ai/zen/v1', 'OPENCODE_API_KEY'],
|
||
'opencode-go': [undefined, 'https://opencode.ai/zen/go/v1', 'OPENCODE_API_KEY'],
|
||
openrouter: [undefined, 'https://openrouter.ai/api/v1', 'OPENROUTER_API_KEY'],
|
||
'qwen-token-plan': ['openai-completions', 'https://token-plan.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1', 'QWEN_TOKEN_PLAN_API_KEY'],
|
||
'qwen-token-plan-cn': ['openai-completions', 'https://token-plan.cn-beijing.maas.aliyuncs.com/compatible-mode/v1', 'QWEN_TOKEN_PLAN_CN_API_KEY'],
|
||
'qwen-token-plan-individual': ['openai-completions', 'https://token-plan.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1', 'QWEN_TOKEN_PLAN_API_KEY'],
|
||
together: ['openai-completions', 'https://api.together.ai/v1', 'TOGETHER_API_KEY'],
|
||
'vercel-ai-gateway': ['anthropic-messages', 'https://ai-gateway.vercel.sh', 'AI_GATEWAY_API_KEY'],
|
||
xai: ['openai-responses', 'https://api.x.ai/v1', 'XAI_API_KEY'],
|
||
xiaomi: ['openai-completions', 'https://api.xiaomimimo.com/v1', 'XIAOMI_API_KEY'],
|
||
'xiaomi-token-plan-ams': ['openai-completions', 'https://token-plan-ams.xiaomimimo.com/v1', 'XIAOMI_TOKEN_PLAN_AMS_API_KEY'],
|
||
'xiaomi-token-plan-cn': ['openai-completions', 'https://token-plan-cn.xiaomimimo.com/v1', 'XIAOMI_TOKEN_PLAN_CN_API_KEY'],
|
||
'xiaomi-token-plan-sgp': ['openai-completions', 'https://token-plan-sgp.xiaomimimo.com/v1', 'XIAOMI_TOKEN_PLAN_SGP_API_KEY'],
|
||
zai: ['openai-completions', 'https://api.z.ai/api/coding/paas/v4', 'ZAI_API_KEY'],
|
||
'zai-coding-cn': ['openai-completions', 'https://open.bigmodel.cn/api/coding/paas/v4', 'ZAI_CODING_CN_API_KEY'],
|
||
}
|
||
const DSH_DEEPSEEK_ROUTES = new Set(['deepseek-official', 'deepseek-account'])
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Small helpers
|
||
|
||
const str = value => (typeof value === 'string' && value.trim() !== '' ? value.trim() : undefined)
|
||
const clamp = (value, low, high) => Math.min(high, Math.max(low, Math.floor(value)))
|
||
const isRecord = value => value !== null && typeof value === 'object' && !Array.isArray(value)
|
||
const httpUrl = (value) => {
|
||
if (typeof value !== 'string') return undefined
|
||
try {
|
||
const url = new URL(value)
|
||
return url.protocol === 'http:' || url.protocol === 'https:' ? url.href : undefined
|
||
} catch { return undefined }
|
||
}
|
||
const clip = (text, max) => (text.length > max ? `${text.slice(0, max - 1)}…` : text)
|
||
const reasonOf = error => (error?.cause?.message && error.message === 'fetch failed' ? error.cause.message : error?.message ?? String(error))
|
||
const seconds = ms => `${Math.round(ms / 1000)}s`
|
||
const trimSlash = url => url.replace(/\/+$/, '')
|
||
|
||
const NAMED_ENTITIES = {
|
||
amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', ensp: ' ', emsp: ' ', thinsp: ' ',
|
||
middot: '·', hellip: '…', mdash: '—', ndash: '–', laquo: '«', raquo: '»', lsquo: '‘', rsquo: '’',
|
||
ldquo: '“', rdquo: '”', bull: '•', copy: '©', reg: '®', trade: '™', zwj: '', zwnj: '',
|
||
}
|
||
export const decodeEntities = text => text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (whole, ref) => {
|
||
if (ref[0] === '#') {
|
||
const code = ref[1] === 'x' || ref[1] === 'X' ? Number.parseInt(ref.slice(2), 16) : Number(ref.slice(1))
|
||
return Number.isInteger(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : whole
|
||
}
|
||
return NAMED_ENTITIES[ref.toLowerCase()] ?? whole
|
||
})
|
||
// Inline tags vanish so "<strong>deep</strong>seek" stays one word; block ends become spaces.
|
||
export const textOf = html => decodeEntities(html
|
||
.replace(/<(script|style)\b[\s\S]*?<\/\1>/gi, ' ')
|
||
.replace(/<(br|\/p|\/div|\/li|\/h\d|\/td|\/tr)\b[^>]*>/gi, ' ')
|
||
.replace(/<[^>]*>/g, ''))
|
||
.replace(/\s+/g, ' ')
|
||
.trim()
|
||
const attrOf = (attrs, attr) => {
|
||
const match = new RegExp(`\\b${attr}\\s*=\\s*("([^"]*)"|'([^']*)')`, 'i').exec(attrs)
|
||
return match === null ? undefined : decodeEntities(match[2] ?? match[3] ?? '')
|
||
}
|
||
|
||
const plain = value => (typeof value === 'string' ? value.replace(/\s+/g, ' ').trim() : '')
|
||
|
||
/** One portable source from plain-text fields; empty optional fields are omitted rather than invented. */
|
||
const sourceOf = (url, title, snippet, publishedAt) => {
|
||
const href = httpUrl(url)
|
||
if (href === undefined) return undefined
|
||
const cleanTitle = plain(title)
|
||
const cleanSnippet = plain(snippet)
|
||
const date = str(publishedAt)
|
||
return {
|
||
url: href,
|
||
...cleanTitle ? { title: clip(cleanTitle, 300) } : {},
|
||
...cleanSnippet ? { snippet: clip(cleanSnippet, SNIPPET_CHARS) } : {},
|
||
...date ? { publishedAt: date } : {},
|
||
}
|
||
}
|
||
const dedupe = (sources) => {
|
||
const seen = new Map()
|
||
for (const source of sources) {
|
||
if (source === undefined) continue
|
||
const known = seen.get(source.url)
|
||
if (known === undefined) seen.set(source.url, source)
|
||
else seen.set(source.url, { ...source, ...known })
|
||
}
|
||
return [...seen.values()]
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Native search: response readers. Each returns `{ ran, sources, error? }`: whether the endpoint
|
||
// actually ran a search, and only the structured sources it reported (URL, title, excerpt,
|
||
// date), never the answer text a model wrote around them.
|
||
|
||
/** Anthropic Messages with the `web_search_20250305` server tool. */
|
||
export function readMessages(body) {
|
||
const blocks = Array.isArray(body?.content) ? body.content : []
|
||
const results = blocks.filter(block => block?.type === 'web_search_tool_result')
|
||
const excerpts = new Map()
|
||
for (const block of blocks) {
|
||
if (block?.type !== 'text' || !Array.isArray(block.citations)) continue
|
||
for (const cite of block.citations) {
|
||
if (typeof cite?.url === 'string' && str(cite.cited_text) && !excerpts.has(cite.url)) excerpts.set(cite.url, cite.cited_text)
|
||
}
|
||
}
|
||
const sources = dedupe(results.flatMap(block => (Array.isArray(block.content) ? block.content : []))
|
||
.filter(item => item?.type === 'web_search_result')
|
||
.map(item => sourceOf(item.url, item.title, excerpts.get(item.url) ?? (typeof item.content === 'string' ? item.content : undefined), item.page_age)))
|
||
const failed = results.map(block => block.content).find(content => content?.type === 'web_search_tool_result_error')
|
||
return {
|
||
ran: results.length > 0 || blocks.some(block => block?.type === 'server_tool_use'),
|
||
cut: body?.stop_reason === 'max_tokens',
|
||
sources,
|
||
...sources.length === 0 && failed !== undefined ? { error: `web search failed: ${failed.error_code ?? 'unknown error'}` } : {},
|
||
}
|
||
}
|
||
|
||
/** OpenAI Responses with the `web_search` tool. */
|
||
export function readResponses(body) {
|
||
const output = Array.isArray(body?.output) ? body.output : []
|
||
const calls = output.filter(item => item?.type === 'web_search_call')
|
||
const found = []
|
||
for (const call of calls) {
|
||
for (const source of Array.isArray(call.action?.sources) ? call.action.sources : []) found.push(sourceOf(source?.url, source?.title))
|
||
}
|
||
for (const item of output) {
|
||
if (item?.type !== 'message' || !Array.isArray(item.content)) continue
|
||
for (const part of item.content) {
|
||
for (const note of Array.isArray(part?.annotations) ? part.annotations : []) {
|
||
if (note?.type === 'url_citation') found.push(sourceOf(note.url, note.title))
|
||
}
|
||
}
|
||
}
|
||
for (const cite of Array.isArray(body?.citations) ? body.citations : []) {
|
||
found.push(typeof cite === 'string' ? sourceOf(cite) : sourceOf(cite?.url, cite?.title))
|
||
}
|
||
const sources = dedupe(found)
|
||
return { ran: calls.length > 0 || sources.length > 0, cut: body?.status === 'incomplete', sources }
|
||
}
|
||
|
||
/** Gemini `generateContent` with the `google_search` tool. */
|
||
export function readGemini(body) {
|
||
const candidates = Array.isArray(body?.candidates) ? body.candidates : []
|
||
return {
|
||
ran: candidates.some(candidate => candidate?.groundingMetadata),
|
||
cut: candidates.some(candidate => candidate?.finishReason === 'MAX_TOKENS'),
|
||
sources: dedupe(candidates
|
||
.flatMap(candidate => candidate?.groundingMetadata?.groundingChunks ?? [])
|
||
.map(chunk => sourceOf(chunk?.web?.uri, chunk?.web?.title))),
|
||
}
|
||
}
|
||
|
||
const URL_KEYS = ['url', 'uri', 'link']
|
||
const TITLE_KEYS = ['title', 'name', 'site_name']
|
||
const SNIPPET_KEYS = ['snippet', 'summary', 'content', 'description', 'cited_text', 'text']
|
||
const DATE_KEYS = ['published_date', 'publish_date', 'publish_time', 'page_age', 'date', 'last_updated']
|
||
// Media links are not results, and a response that echoes the request's tools is not a search.
|
||
const NOT_SOURCES = new Set(['image_url', 'images', 'image', 'icon', 'favicon', 'logo', 'logo_url', 'thumbnail', 'tools'])
|
||
const SEARCH_MARK = /search|grounding|citation/i
|
||
const firstText = (record, keys) => keys.map(key => record[key]).find(value => typeof value === 'string' && value.trim() !== '')
|
||
|
||
/** Every object in a JSON body that carries an http(s) URL, as sources: the shape of any search API. */
|
||
export function findSources(body) {
|
||
const found = []
|
||
const visit = (value, depth) => {
|
||
if (depth > 16 || value === null || typeof value !== 'object') return
|
||
if (Array.isArray(value)) { for (const item of value) visit(item, depth + 1); return }
|
||
const url = URL_KEYS.map(key => httpUrl(value[key])).find(Boolean)
|
||
if (url !== undefined) found.push(sourceOf(url, firstText(value, TITLE_KEYS), firstText(value, SNIPPET_KEYS), firstText(value, DATE_KEYS)))
|
||
for (const [key, child] of Object.entries(value)) if (!NOT_SOURCES.has(key)) visit(child, depth + 1)
|
||
}
|
||
visit(body, 0)
|
||
return dedupe(found)
|
||
}
|
||
|
||
/** Whether a JSON body shows a search ran, read from its keys and `type` tags, never from prose. */
|
||
export function showsSearch(body) {
|
||
const visit = (value, depth) => {
|
||
if (depth > 16 || value === null || typeof value !== 'object') return false
|
||
if (Array.isArray(value)) return value.some(item => visit(item, depth + 1))
|
||
return Object.entries(value).some(([key, child]) => !NOT_SOURCES.has(key) && (
|
||
SEARCH_MARK.test(key) || key === 'results'
|
||
|| (key === 'type' && typeof child === 'string' && SEARCH_MARK.test(child))
|
||
|| visit(child, depth + 1)))
|
||
}
|
||
return visit(body, 0)
|
||
}
|
||
|
||
/** A declared search API's response: any structured sources, wherever they sit. */
|
||
export function readDeclared(body) {
|
||
const sources = findSources(body)
|
||
return { ran: sources.length > 0 || showsSearch(body), sources }
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Native search: what to try for one route. Nothing here names a vendor except SEARCH_APIS,
|
||
// which is data in the same shape as `searchApis` config.
|
||
|
||
const bearer = key => ({ authorization: `Bearer ${key}` })
|
||
export const AUTH_STYLES = { bearer, 'x-api-key': key => ({ 'x-api-key': key }), 'api-key': key => ({ 'api-key': key }) }
|
||
// One trailing /v1 belongs to the root, as in dsh's DeepSeek adapter and pi-ai's model listing.
|
||
const messagesRoot = base => (new URL(base).pathname.replace(/\/+$/, '').endsWith('/v1') ? trimSlash(base) : `${trimSlash(base)}/v1`)
|
||
|
||
// Bounds what an endpoint that ignores the search tool can cost, while leaving a thinking model
|
||
// room to reach the search: dsh's own DeepSeek search uses the same cap.
|
||
const OUTPUT_CAP = 4096
|
||
|
||
/** The standard search tool of each wire protocol. */
|
||
export const PROTOCOL_SEARCHES = {
|
||
messages: {
|
||
name: 'Anthropic Messages web_search tool',
|
||
url: base => `${messagesRoot(base)}/messages`,
|
||
// Anthropic reads x-api-key; compatible gateways often read Bearer.
|
||
auth: key => ({ 'x-api-key': key, ...bearer(key) }),
|
||
request: ({ model, query }) => ({
|
||
headers: { 'anthropic-version': '2023-06-01' },
|
||
body: {
|
||
model,
|
||
max_tokens: OUTPUT_CAP,
|
||
messages: [{ role: 'user', content: [{ type: 'text', text: PROMPT(query) }] }],
|
||
tools: [{ type: 'web_search_20250305', name: 'web_search', max_uses: 1 }],
|
||
},
|
||
}),
|
||
read: readMessages,
|
||
},
|
||
responses: {
|
||
name: 'OpenAI Responses web_search tool',
|
||
url: base => `${trimSlash(base)}/responses`,
|
||
auth: bearer,
|
||
request: ({ model, query }) => ({ body: { model, input: PROMPT(query), tools: [{ type: 'web_search' }], max_output_tokens: OUTPUT_CAP } }),
|
||
read: readResponses,
|
||
},
|
||
gemini: {
|
||
name: 'Gemini google_search tool',
|
||
url: (base, model) => `${trimSlash(base)}/models/${encodeURIComponent(model)}:generateContent`,
|
||
auth: key => ({ 'x-goog-api-key': key }),
|
||
request: ({ query }) => ({
|
||
body: { contents: [{ role: 'user', parts: [{ text: PROMPT(query) }] }], tools: [{ google_search: {} }], generationConfig: { maxOutputTokens: OUTPUT_CAP } },
|
||
}),
|
||
read: readGemini,
|
||
},
|
||
}
|
||
const BY_PROTOCOL = {
|
||
'anthropic-messages': ['messages'],
|
||
'openai-responses': ['responses'],
|
||
// Many OpenAI-compatible servers also serve Responses and Messages under the same base.
|
||
'openai-completions': ['responses', 'messages'],
|
||
'google-generative-ai': ['gemini'],
|
||
}
|
||
const UNKNOWN_PROTOCOL = ['responses', 'messages']
|
||
|
||
/**
|
||
* Search APIs a host serves beside its model protocol. No standard advertises these, so they
|
||
* are declared; `searchApis` in config adds more in the same shape. `$query`, `$model`, and
|
||
* `$maxResults` in a body are replaced per call. Each path is joined to the route's own origin.
|
||
*/
|
||
export const SEARCH_APIS = Object.freeze([
|
||
// docs.ollama.com/capabilities/web-search
|
||
{ id: 'ollama', name: 'Ollama web search API', hosts: ['ollama.com'], path: '/api/web_search', maxResults: 10, body: { query: '$query', max_results: '$maxResults' } },
|
||
// platform.kimi.ai; an empty result is not billed.
|
||
{ id: 'kimi', name: 'Kimi search API', hosts: ['api.moonshot.ai', 'api.moonshot.cn'], path: '/v1/tools/search', maxResults: 20, body: { text_query: '$query', limit: '$maxResults', timeout_seconds: 30 } },
|
||
// docs.bigmodel.cn; the API rejects queries over 70 characters.
|
||
{ id: 'zhipu', name: 'Zhipu web search API', hosts: ['open.bigmodel.cn', 'api.z.ai'], path: '/api/paas/v4/web_search', maxResults: 50, maxQueryChars: 70, body: { search_query: '$query', search_engine: 'search_std', count: '$maxResults' } },
|
||
// MiMo searches only through this Chat Completions plugin tool, once the Web Search plugin is
|
||
// enabled for the key. Each keyword is one billed search, so one keyword per query; the answer
|
||
// text is discarded, so it stays short and without thinking.
|
||
{
|
||
id: 'mimo', name: 'MiMo web_search plugin', hosts: ['xiaomimimo.com', '*.xiaomimimo.com'], path: '/v1/chat/completions', auth: ['bearer', 'api-key'], maxResults: 5,
|
||
body: {
|
||
model: '$model',
|
||
messages: [{ role: 'user', content: '$query' }],
|
||
tools: [{ type: 'web_search', force_search: true, max_keyword: 1, limit: '$maxResults' }],
|
||
max_completion_tokens: 256,
|
||
thinking: { type: 'disabled' },
|
||
stream: false,
|
||
},
|
||
},
|
||
])
|
||
|
||
const hostMatches = (patterns, host) => patterns.some(pattern => (pattern.startsWith('*.') ? host.endsWith(pattern.slice(1)) : host === pattern.toLowerCase()))
|
||
const fill = (template, values) => {
|
||
if (Array.isArray(template)) return template.map(item => fill(item, values))
|
||
if (isRecord(template)) return Object.fromEntries(Object.entries(template).map(([key, value]) => [key, fill(value, values)]))
|
||
return typeof template === 'string' && Object.hasOwn(values, template) ? values[template] : template
|
||
}
|
||
|
||
const usesModel = api => JSON.stringify(api.body).includes('"$model"')
|
||
|
||
function declaredCandidate(api, origin, model) {
|
||
const url = `${origin}${api.path}`
|
||
// A path such as `//other.example` must not move the key to another origin.
|
||
if (new URL(url).origin !== origin) return undefined
|
||
const styles = (api.auth ?? ['bearer']).map(style => AUTH_STYLES[style])
|
||
return {
|
||
id: api.id,
|
||
name: api.name ?? `${api.id} search API`,
|
||
url,
|
||
model: usesModel(api) ? model : undefined,
|
||
auth: key => Object.assign({}, ...styles.map(style => style(key))),
|
||
request: ({ query, maxResults }) => ({
|
||
body: fill(api.body, {
|
||
$query: api.maxQueryChars === undefined ? query : [...query].slice(0, api.maxQueryChars).join(''),
|
||
$model: model,
|
||
$maxResults: clamp(maxResults ?? 8, 1, api.maxResults ?? 10),
|
||
}),
|
||
}),
|
||
read: readDeclared,
|
||
}
|
||
}
|
||
|
||
function protocolCandidate(id, base, model) {
|
||
const shape = PROTOCOL_SEARCHES[id]
|
||
return { id, name: shape.name, url: shape.url(base, model), model, auth: shape.auth, request: call => shape.request({ ...call, model }), read: shape.read }
|
||
}
|
||
|
||
/**
|
||
* Everything worth trying for one route, cheapest first: search APIs declared for its host
|
||
* (no model call), then the standard search tool of its protocol.
|
||
* @returns `{ candidates }`, with `reason` when there is nothing to try.
|
||
*/
|
||
export function planNative(route, searchApis = SEARCH_APIS) {
|
||
const base = httpUrl(route.baseURL)
|
||
if (base === undefined) return { candidates: [], reason: `route "${route.provider}" has no known endpoint` }
|
||
const url = new URL(base)
|
||
const model = str(route.model)
|
||
const candidates = []
|
||
for (const api of searchApis) {
|
||
if (!hostMatches(api.hosts, url.hostname)) continue
|
||
if (model === undefined && usesModel(api)) continue
|
||
const candidate = declaredCandidate(api, url.origin, model)
|
||
if (candidate !== undefined) candidates.push(candidate)
|
||
}
|
||
const shapes = route.api === undefined ? UNKNOWN_PROTOCOL : BY_PROTOCOL[route.api] ?? []
|
||
if (model !== undefined) for (const id of shapes) candidates.push(protocolCandidate(id, base, model))
|
||
if (candidates.length > 0) return { candidates }
|
||
return {
|
||
candidates,
|
||
reason: model === undefined && shapes.length > 0
|
||
? `route "${route.provider}" has no current model to search with`
|
||
: `route "${route.provider}" speaks ${route.api}, which has no standard search tool, and ${url.host} declares no search API`,
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Keyless fallback: public results pages, parsed into sources.
|
||
|
||
/** Bing wraps result links in `bing.com/ck/a?…&u=a1<base64url>`; recover the target. */
|
||
export function decodeBingUrl(href) {
|
||
const url = httpUrl(href)
|
||
if (url === undefined) return undefined
|
||
const parsed = new URL(url)
|
||
if (!/(^|\.)bing\.com$/i.test(parsed.hostname)) return url
|
||
const target = parsed.searchParams.get('u')
|
||
if (parsed.pathname !== '/ck/a' || target === null || !target.startsWith('a1')) return undefined
|
||
try { return httpUrl(Buffer.from(target.slice(2), 'base64url').toString('utf8')) } catch { return undefined }
|
||
}
|
||
|
||
/** Bing results page → `{ sources }`, or `{ error }` when the page is not a results page. */
|
||
export function parseBing(html) {
|
||
const starts = [...html.matchAll(/<li\b[^>]*\bclass\s*=\s*"[^"]*\bb_algo\b[^"]*"[^>]*>/gi)].map(match => match.index)
|
||
const sources = []
|
||
starts.forEach((start, index) => {
|
||
const block = html.slice(start, starts[index + 1] ?? start + 12_000)
|
||
const heading = /<h2\b[^>]*>\s*<a\b([^>]*)>([\s\S]*?)<\/a>/i.exec(block)
|
||
if (heading === null) return
|
||
const href = attrOf(heading[1], 'href')
|
||
const caption = block.slice(Math.max(0, block.search(/<div\b[^>]*\bclass\s*=\s*"[^"]*\bb_caption\b/i)))
|
||
const paragraph = /<p\b[^>]*>([\s\S]*?)<\/p>/i.exec(caption)
|
||
sources.push(sourceOf(decodeBingUrl(href), textOf(heading[2]), paragraph ? textOf(paragraph[1]) : undefined))
|
||
})
|
||
const parsed = dedupe(sources)
|
||
if (parsed.length > 0) return { sources: parsed }
|
||
if (/\bclass\s*=\s*"[^"]*\bb_no\b/i.test(html)) return { sources: [] }
|
||
return { error: /id\s*=\s*"b_results"/i.test(html) ? 'results page had no parsable results' : 'not a results page (bot check or changed layout)' }
|
||
}
|
||
|
||
/** DuckDuckGo HTML results page → `{ sources }`, or `{ error }`. */
|
||
export function parseDuckDuckGo(html) {
|
||
if (/anomaly-modal|bots use DuckDuckGo too|challenge-form/i.test(html)) return { error: 'bot check page' }
|
||
const links = [...html.matchAll(/<a\b([^>]*\bclass\s*=\s*"[^"]*\bresult__a\b[^"]*"[^>]*)>([\s\S]*?)<\/a>/gi)]
|
||
const sources = links.map((link, index) => {
|
||
const href = attrOf(link[1], 'href') ?? ''
|
||
let target
|
||
try {
|
||
const parsed = new URL(href, 'https://duckduckgo.com')
|
||
// `/y.js` links are ads.
|
||
if (/(^|\.)duckduckgo\.com$/i.test(parsed.hostname)) target = parsed.pathname === '/l/' ? parsed.searchParams.get('uddg') : undefined
|
||
else target = parsed.href
|
||
} catch { target = undefined }
|
||
const rest = html.slice(link.index + link[0].length, links[index + 1]?.index ?? link.index + 6000)
|
||
const snippet = /<(?:a|div|td)\b[^>]*\bclass\s*=\s*"[^"]*\bresult__snippet\b[^"]*"[^>]*>([\s\S]*?)<\/(?:a|div|td)>/i.exec(rest)
|
||
return sourceOf(target, textOf(link[2]), snippet ? textOf(snippet[1]) : undefined)
|
||
})
|
||
const parsed = dedupe(sources)
|
||
if (parsed.length > 0) return { sources: parsed }
|
||
if (/\bresult--no-results\b|\bno-results\b/i.test(html)) return { sources: [] }
|
||
return { error: 'not a results page (bot check or changed layout)' }
|
||
}
|
||
|
||
export const ENGINES = {
|
||
// No market or language parameter: Bing picks them from the network, as it does for a browser.
|
||
// `regional` is the hosts Bing itself may redirect to (mainland networks go to cn.bing.com).
|
||
bing: { name: 'Bing', url: query => `https://www.bing.com/search?q=${encodeURIComponent(query)}`, parse: parseBing, regional: /^https:\/\/([a-z0-9-]+\.)*bing\.com$/i },
|
||
duckduckgo: { name: 'DuckDuckGo', url: query => `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`, parse: parseDuckDuckGo },
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// The provider
|
||
|
||
const aborted = signal => new WebError('web search aborted', 'WEB_ABORTED', { cause: signal?.reason })
|
||
|
||
// AbortSignal.timeout's wait is unref'd, so under Node 22's test runner a pending subtest
|
||
// is cancelled as soon as nothing ref'd holds the loop — and a mocked fetch holds nothing.
|
||
// A plain setTimeout keeps the loop honest; the abort reason stays a TimeoutError.
|
||
const timeoutAfter = (ms) => {
|
||
const timeout = new AbortController()
|
||
const timer = setTimeout(() => timeout.abort(new DOMException('The operation timed out.', 'TimeoutError')), ms)
|
||
return { timeout, timer }
|
||
}
|
||
|
||
function checkSearchApi(api, index) {
|
||
const where = `web-search-auto: searchApis[${index}]`
|
||
if (!isRecord(api)) throw new Error(`${where} must be an object`)
|
||
if (str(api.id) === undefined) throw new Error(`${where} needs an id`)
|
||
if (!Array.isArray(api.hosts) || api.hosts.length === 0 || !api.hosts.every(host => typeof host === 'string' && /^(\*\.)?[a-z0-9-]+(\.[a-z0-9-]+)*$/i.test(host))) {
|
||
throw new Error(`${where}.hosts must be a list of host names; a leading *. matches subdomains`)
|
||
}
|
||
if (typeof api.path !== 'string' || !/^\/[^/]/.test(api.path)) throw new Error(`${where}.path must be a path on the route's host, starting with one /`)
|
||
if (!isRecord(api.body)) throw new Error(`${where}.body must be an object`)
|
||
if (api.auth !== undefined && (!Array.isArray(api.auth) || api.auth.length === 0 || api.auth.some(style => !Object.hasOwn(AUTH_STYLES, style)))) {
|
||
throw new Error(`${where}.auth must be a list of ${Object.keys(AUTH_STYLES).join(', ')}`)
|
||
}
|
||
for (const key of ['maxResults', 'maxQueryChars']) {
|
||
if (api[key] !== undefined && !(Number.isInteger(api[key]) && api[key] >= 1)) throw new Error(`${where}.${key} must be a positive integer`)
|
||
}
|
||
if (api.name !== undefined && str(api.name) === undefined) throw new Error(`${where}.name must be text`)
|
||
}
|
||
|
||
/**
|
||
* Normalize plugin config. Invalid entries fail loud at load rather than silently searching less.
|
||
* Declared `searchApis` come before the built-in SEARCH_APIS, so a user entry wins on its host.
|
||
*/
|
||
export function resolveConfig(config = {}) {
|
||
const resolved = { ...DEFAULTS, ...Object.fromEntries(Object.entries(config ?? {}).filter(([, value]) => value !== undefined)) }
|
||
if (!Array.isArray(resolved.engines) || resolved.engines.some(engine => !Object.hasOwn(ENGINES, engine))) {
|
||
throw new Error(`web-search-auto: engines must be a list of ${Object.keys(ENGINES).join(', ')}`)
|
||
}
|
||
for (const key of ['nativeTimeoutMs', 'fallbackTimeoutMs']) {
|
||
if (!Number.isFinite(resolved[key]) || resolved[key] < 1000) throw new Error(`web-search-auto: ${key} must be at least 1000`)
|
||
}
|
||
if (!Array.isArray(resolved.searchApis)) throw new Error('web-search-auto: searchApis must be a list')
|
||
resolved.searchApis.forEach(checkSearchApi)
|
||
return {
|
||
...resolved,
|
||
native: resolved.native !== false,
|
||
fallback: resolved.fallback !== false,
|
||
searchApis: [...resolved.searchApis, ...SEARCH_APIS],
|
||
}
|
||
}
|
||
|
||
const errorDetail = (text) => {
|
||
try {
|
||
const parsed = JSON.parse(text)
|
||
return str(typeof parsed?.error === 'string' ? parsed.error : parsed?.error?.message ?? parsed?.message ?? parsed?.msg) ?? ''
|
||
} catch { return '' }
|
||
}
|
||
|
||
/**
|
||
* Build the `auto` search provider over injected effects, so tests run it without a host.
|
||
* @param deps.route - the asking Session's route: `{ provider, model, api, baseURL, apiKeyEnv, account, headers }`.
|
||
* @param deps.resolveKey - credential reference → key, or undefined.
|
||
* @param deps.resolveAccountToken - endpoint → DeepSeek account token, or undefined.
|
||
* @param deps.fetch - the HTTP client for native searches.
|
||
* @param deps.webFetch - `ctx.web.fetch`.
|
||
*/
|
||
export function createAutoSearchProvider(deps, config = {}) {
|
||
const options = resolveConfig(config)
|
||
const now = deps.now ?? Date.now
|
||
// route + endpoint + model → `{ works: true }` or `{ until, reason }`, for this process.
|
||
const memory = new Map()
|
||
|
||
async function credentialFor(route, endpoint) {
|
||
if (route.account) {
|
||
let token
|
||
try { token = str(await deps.resolveAccountToken?.(endpoint)) } catch (error) { return { reason: `the DeepSeek account token could not be read: ${reasonOf(error)}` } }
|
||
return token === undefined
|
||
? { reason: 'it is not signed in to a DeepSeek account' }
|
||
: { headers: () => ({ 'x-dsh-auth-token': token }), via: 'the DeepSeek account sign-in' }
|
||
}
|
||
if (route.apiKeyEnv === undefined) return { reason: 'it names no API key' }
|
||
let key
|
||
try { key = str(await deps.resolveKey(route.apiKeyEnv)) } catch (error) { return { reason: `the key ${route.apiKeyEnv} could not be read: ${reasonOf(error)}` } }
|
||
return key === undefined
|
||
? { reason: `no API key is stored for ${route.apiKeyEnv}` }
|
||
: { headers: candidate => candidate.auth(key), via: `the API key ${route.apiKeyEnv}` }
|
||
}
|
||
|
||
async function attempt(candidate, route, credential, request, signal, budget, known) {
|
||
const call = candidate.request({ query: request.query, maxResults: request.maxResults })
|
||
const { timeout, timer } = timeoutAfter(budget)
|
||
let response
|
||
let text
|
||
try {
|
||
response = await deps.fetch(candidate.url, {
|
||
method: 'POST',
|
||
// Credential-bearing requests never follow redirects (deepseek-harness/packages/web/AGENTS.md).
|
||
redirect: 'error',
|
||
headers: { 'content-type': 'application/json', accept: 'application/json', 'user-agent': USER_AGENT, ...route.headers, ...call.headers, ...credential.headers(candidate) },
|
||
body: JSON.stringify(call.body),
|
||
signal: signal === undefined ? timeout.signal : AbortSignal.any([signal, timeout.signal]),
|
||
})
|
||
text = await response.text()
|
||
} catch (error) {
|
||
if (signal?.aborted) throw aborted(signal)
|
||
return timeout.signal.aborted ? { reason: `timed out after ${seconds(budget)}`, timedOut: true } : { reason: `request failed: ${reasonOf(error)}` }
|
||
} finally {
|
||
clearTimeout(timer)
|
||
}
|
||
if (!response.ok) {
|
||
const detail = errorDetail(text)
|
||
return { reason: `HTTP ${response.status}${detail ? `: ${clip(detail, 200)}` : ''}`, skip: STATUS_OUTCOMES[response.status] }
|
||
}
|
||
let body
|
||
try { body = JSON.parse(text) } catch { return { reason: 'answered with a body that is not JSON', skip: 'ignored' } }
|
||
const read = candidate.read(body)
|
||
if (read.sources.length > 0) return { sources: read.sources }
|
||
if (read.error !== undefined) return { reason: read.error, final: true }
|
||
if (read.ran || known) return { reason: 'returned no results', final: true }
|
||
// The cap says nothing about support, so a cut-off answer is not remembered.
|
||
if (read.cut) return { reason: 'stopped at the output cap before searching' }
|
||
return { reason: 'answered without running a search, so it does not seem to support this one', skip: 'ignored' }
|
||
}
|
||
|
||
async function runNative(request, signal) {
|
||
if (!options.native) return { reason: 'native search is turned off (web-search-auto native: false)' }
|
||
let route
|
||
try { route = deps.route() } catch (error) { return { reason: `the conversation's model route could not be read: ${reasonOf(error)}` } }
|
||
if (route === undefined) return { reason: 'no conversation model route is known for this search' }
|
||
const plan = planNative(route, options.searchApis)
|
||
if (plan.candidates.length === 0) return { reason: plan.reason }
|
||
const keyOf = candidate => [route.provider, candidate.url, candidate.model ?? ''].join('\n')
|
||
const works = candidate => memory.get(keyOf(candidate))?.works === true
|
||
const ordered = [...plan.candidates.filter(works), ...plan.candidates.filter(candidate => !works(candidate))]
|
||
let deadline = now() + options.nativeTimeoutMs
|
||
const notes = []
|
||
let credential
|
||
for (const candidate of ordered) {
|
||
const where = `${candidate.name} (${new URL(candidate.url).host}${new URL(candidate.url).pathname})`
|
||
const memo = memory.get(keyOf(candidate))
|
||
if (memo?.until !== undefined && memo.until > now()) {
|
||
notes.push(`${where}: ${memo.reason}; not retried for ${Math.ceil((memo.until - now()) / MINUTE)} min`)
|
||
continue
|
||
}
|
||
credential ??= await credentialFor(route, candidate.url)
|
||
if (signal?.aborted) throw aborted(signal)
|
||
if (credential.reason !== undefined) return { reason: `route "${route.provider}" could not search natively: ${credential.reason}` }
|
||
const remaining = deadline - now()
|
||
if (remaining <= 0) { notes.push(`${where}: skipped, out of time`); continue }
|
||
const outcome = await attempt(candidate, route, credential, request, signal, remaining, memo?.works === true)
|
||
if (outcome.sources !== undefined) {
|
||
memory.set(keyOf(candidate), { works: true })
|
||
return {
|
||
result: {
|
||
content: `Search backend: native search of this conversation's model route "${route.provider}": ${where}, billed to ${credential.via}.`,
|
||
sources: outcome.sources,
|
||
truncated: false,
|
||
},
|
||
}
|
||
}
|
||
notes.push(`${where}: ${outcome.reason}`)
|
||
if (outcome.skip !== undefined) memory.set(keyOf(candidate), { until: now() + SKIP_FOR[outcome.skip], reason: outcome.reason })
|
||
if (outcome.final) break
|
||
// A timed-out attempt had all the time that was left, but the timer and now() use
|
||
// different clocks, so now() can still read a millisecond before the deadline.
|
||
if (outcome.timedOut) deadline = now()
|
||
}
|
||
return { reason: `route "${route.provider}": ${notes.join('; ')}` }
|
||
}
|
||
|
||
async function fetchPage(engine, url, signal) {
|
||
try {
|
||
return { page: await deps.webFetch({ url }, signal), url }
|
||
} catch (error) {
|
||
// The fetch provider refuses cross-origin redirects and names the target; follow one hop
|
||
// when it is the engine's own regional host.
|
||
const origin = error?.code === 'WEB_REDIRECT_BLOCKED' ? /https?:\/\/[a-z0-9.-]+(:\d+)?/i.exec(error.message ?? '')?.[0] : undefined
|
||
if (origin === undefined || engine.regional?.test(origin) !== true || origin === new URL(url).origin) throw error
|
||
const { pathname, search } = new URL(url)
|
||
const regional = `${origin}${pathname}${search}`
|
||
return { page: await deps.webFetch({ url: regional }, signal), url: regional }
|
||
}
|
||
}
|
||
|
||
async function runFallback(request, signal) {
|
||
const attempts = []
|
||
let empty
|
||
const deadline = now() + options.fallbackTimeoutMs
|
||
for (const [index, id] of options.engines.entries()) {
|
||
const engine = ENGINES[id]
|
||
const url = engine.url(request.query)
|
||
const host = new URL(url).host
|
||
const remaining = deadline - now()
|
||
if (remaining <= 0) { attempts.push(`${host}: skipped, out of time`); continue }
|
||
// Leave later engines a share of the budget when an earlier one hangs.
|
||
const { timeout, timer } = timeoutAfter(index === options.engines.length - 1 ? remaining : Math.min(remaining, Math.ceil(options.fallbackTimeoutMs / 2)))
|
||
let fetched
|
||
try {
|
||
fetched = await fetchPage(engine, url, signal === undefined ? timeout.signal : AbortSignal.any([signal, timeout.signal]))
|
||
} catch (error) {
|
||
if (signal?.aborted) throw aborted(signal)
|
||
attempts.push(`${host}: ${timeout.signal.aborted ? 'timed out' : `${error?.code ? `${error.code} ` : ''}${reasonOf(error)}`}`)
|
||
continue
|
||
} finally {
|
||
clearTimeout(timer)
|
||
}
|
||
const at = new URL(fetched.url).host
|
||
if (fetched.page.statusCode !== 200) { attempts.push(`${at}: HTTP ${fetched.page.statusCode}`); continue }
|
||
const parsed = engine.parse(fetched.page.body?.content ?? '')
|
||
if (parsed.error !== undefined) { attempts.push(`${at}: ${parsed.error}`); continue }
|
||
if (parsed.sources.length === 0) { empty ??= { engine, at }; attempts.push(`${at}: no results`); continue }
|
||
return { result: { engine, at, sources: parsed.sources } }
|
||
}
|
||
return empty === undefined ? { attempts } : { result: { ...empty, sources: [] } }
|
||
}
|
||
|
||
return {
|
||
id: AUTO_PROVIDER_ID,
|
||
available: () => options.native || options.fallback,
|
||
async search(request, signal) {
|
||
if (signal?.aborted) throw aborted(signal)
|
||
const native = await runNative(request, signal)
|
||
if (native.result !== undefined) return native.result
|
||
let fallbackReason = 'turned off (web-search-auto fallback: false)'
|
||
if (options.fallback) {
|
||
const fallback = await runFallback(request, signal)
|
||
if (fallback.result !== undefined) {
|
||
return {
|
||
content: `Search backend: keyless fallback, parsed from the public ${fallback.result.engine.name} results page at ${fallback.result.at} (no API key, no charge; less reliable than an API search). Native search was not used: ${native.reason}.`,
|
||
sources: fallback.result.sources,
|
||
truncated: false,
|
||
}
|
||
}
|
||
fallbackReason = fallback.attempts.join('; ')
|
||
}
|
||
throw new WebError([
|
||
'Web search is unavailable for this query.',
|
||
`Native search: ${native.reason}.`,
|
||
`Keyless fallback: ${fallbackReason}.`,
|
||
'Do not guess URLs or present pages you remember as search results. If you have a browser tool, open a search engine in it; otherwise tell the user that web search is unavailable and why.',
|
||
].join('\n'), 'WEB_SEARCH_UNAVAILABLE')
|
||
},
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Host wiring: read the asking Session's route through public dsh services only.
|
||
|
||
const stringRecord = value => (isRecord(value) ? Object.fromEntries(Object.entries(value).filter(([, item]) => typeof item === 'string')) : {})
|
||
|
||
// OpenCode's gateways turn away a request that does not name its conversation. DSH's
|
||
// adapter names it by the Session id, so a search for that Session does the same,
|
||
// unless the provider settings name one.
|
||
const OPENCODE_SESSION = 'x-opencode-session'
|
||
const OPENCODE_ROUTES = new Set(['opencode', 'opencode-go'])
|
||
const withConversation = (provider, headers, sessionId) => (!OPENCODE_ROUTES.has(provider) || sessionId === undefined
|
||
|| Object.keys(headers).some(name => name.toLowerCase() === OPENCODE_SESSION)
|
||
? headers
|
||
: { ...headers, [OPENCODE_SESSION]: sessionId })
|
||
|
||
/** The route of the Session that started the current tool call: protocol, endpoint, key name, and headers. */
|
||
export function routeOf(ctx) {
|
||
const session = ctx.get('agents')?.currentInitiator()?.session
|
||
const request = session?.requestContext()
|
||
const provider = str(request?.provider)
|
||
if (provider === undefined) return undefined
|
||
let section
|
||
try {
|
||
const entry = ctx.get('llm')?.listConfigurableProviders().find(item => item.provider === provider)
|
||
if (entry !== undefined) {
|
||
const descriptor = ctx.get('settings')?.describe().find(item => item.ns === entry.settingsNs)
|
||
section = entry.settingsPath.reduce((value, key) => value?.[key], descriptor?.value)
|
||
}
|
||
} catch { section = undefined }
|
||
const [catalogApi, catalogBase, catalogKey] = CATALOG[provider] ?? []
|
||
const account = provider === 'deepseek-account'
|
||
return {
|
||
provider,
|
||
model: str(request.model),
|
||
api: str(section?.api) ?? catalogApi,
|
||
baseURL: str(section?.baseURL) ?? (DSH_DEEPSEEK_ROUTES.has(provider) ? str(envOf(ctx)('DEEPSEEK_BASE_URL')) : undefined) ?? catalogBase,
|
||
apiKeyEnv: account ? undefined : str(section?.apiKeyEnv) ?? catalogKey,
|
||
account,
|
||
headers: withConversation(provider, stringRecord(section?.headers), str(session.id)),
|
||
}
|
||
}
|
||
|
||
const envOf = ctx => name => ctx.get('launchEnvironment')?.get(name)?.value ?? process.env[name]
|
||
|
||
export function apply(ctx, config = {}) {
|
||
const env = envOf(ctx)
|
||
ctx.web.registerSearchProvider(createAutoSearchProvider({
|
||
route: () => routeOf(ctx),
|
||
resolveKey: async (ref) => {
|
||
const credentials = ctx.get('credentials')
|
||
// Without the seam the environment is the whole credential plane, as in dsh's own adapters.
|
||
return credentials === undefined ? env(ref) : (await credentials.resolve(ref))?.value
|
||
},
|
||
resolveAccountToken: endpoint => ctx.get('deepseekAccount')?.resolveToken(endpoint),
|
||
fetch: (url, init) => fetch(url, init),
|
||
webFetch: (request, signal) => ctx.web.fetch(request, signal),
|
||
}, config))
|
||
}
|