// Search provider `auto` for dsh's web seam (`ctx.web`). Each query first tries native search // behind the asking Session's own model route, with that route's own key, on that route's own // origin. Native search is found by protocol, not by vendor: the route's wire protocol defines a // standard search tool (Anthropic Messages, OpenAI Responses, Gemini), and a host may also serve // a separate search API, declared as data (SEARCH_APIS, or `searchApis` in config). Whether an // endpoint supports either is only knowable by trying, so every outcome is remembered. It never // borrows another route's key: every key bills its own account. When native search is // unavailable, a keyless fallback parses a public results page fetched through ctx.web.fetch. // Every result says which path answered it; when both fail, the error tells the model not to // guess. Independent of the Bot team code in index.js. import { WebError } from '@deepseek-ai/dsh-web' export const name = 'web-search-auto' export const inject = ['web'] export const AUTO_PROVIDER_ID = 'auto' export const DEFAULTS = Object.freeze({ native: true, fallback: true, engines: ['bing', 'duckduckgo'], searchApis: [], nativeTimeoutMs: 40_000, fallbackTimeoutMs: 15_000, }) const USER_AGENT = 'deepseek-harness/0.0.1' const SNIPPET_CHARS = 500 const PROMPT = query => `Perform a web search for the query: ${query}` const MINUTE = 60_000 // How long one outcome keeps a candidate from being tried again. const SKIP_FOR = { absent: 24 * 60 * MINUTE, // The query itself may be what was refused (length, content moderation). rejected: 30 * MINUTE, // The user may enable search for the key right away. denied: 30 * MINUTE, // Each retry of an endpoint that answers without searching costs a model call. ignored: 6 * 60 * MINUTE, } const STATUS_OUTCOMES = { 404: 'absent', 405: 'absent', 501: 'absent', 400: 'rejected', 415: 'rejected', 422: 'rejected', 401: 'denied', 402: 'denied', 403: 'denied' } // Routes whose profile leaves protocol, endpoint, or key name to the adapter: dsh's DeepSeek // adapters, and the pi-ai 0.87.1 catalog routes that sign in with an API key // (models.generated.js, env-api-keys.js). The protocol is undefined where it differs per model. const CATALOG = { 'deepseek-official': ['anthropic-messages', 'https://api.deepseek.com/anthropic', 'DEEPSEEK_API_KEY'], 'deepseek-account': ['anthropic-messages', 'https://api.deepseek.com/anthropic', undefined], 'ant-ling': ['openai-completions', 'https://api.ant-ling.com/v1', 'ANT_LING_API_KEY'], anthropic: ['anthropic-messages', 'https://api.anthropic.com', 'ANTHROPIC_API_KEY'], baseten: ['openai-completions', 'https://inference.baseten.co/v1', 'BASETEN_API_KEY'], cerebras: ['openai-completions', 'https://api.cerebras.ai/v1', 'CEREBRAS_API_KEY'], deepseek: ['openai-completions', 'https://api.deepseek.com', 'DEEPSEEK_API_KEY'], fireworks: [undefined, 'https://api.fireworks.ai/inference/v1', 'FIREWORKS_API_KEY'], google: ['google-generative-ai', 'https://generativelanguage.googleapis.com/v1beta', 'GEMINI_API_KEY'], groq: ['openai-completions', 'https://api.groq.com/openai/v1', 'GROQ_API_KEY'], huggingface: ['openai-completions', 'https://router.huggingface.co/v1', 'HF_TOKEN'], 'kimi-coding': ['anthropic-messages', 'https://api.kimi.com/coding', 'KIMI_API_KEY'], meta: ['openai-responses', 'https://api.meta.ai/v1', 'META_API_KEY'], minimax: ['anthropic-messages', 'https://api.minimax.io/anthropic', 'MINIMAX_API_KEY'], 'minimax-cn': ['anthropic-messages', 'https://api.minimaxi.com/anthropic', 'MINIMAX_CN_API_KEY'], moonshotai: ['openai-completions', 'https://api.moonshot.ai/v1', 'MOONSHOT_API_KEY'], 'moonshotai-cn': ['openai-completions', 'https://api.moonshot.cn/v1', 'MOONSHOT_API_KEY'], nvidia: ['openai-completions', 'https://integrate.api.nvidia.com/v1', 'NVIDIA_API_KEY'], openai: ['openai-responses', 'https://api.openai.com/v1', 'OPENAI_API_KEY'], opencode: [undefined, 'https://opencode.ai/zen/v1', 'OPENCODE_API_KEY'], 'opencode-go': [undefined, 'https://opencode.ai/zen/go/v1', 'OPENCODE_API_KEY'], openrouter: [undefined, 'https://openrouter.ai/api/v1', 'OPENROUTER_API_KEY'], 'qwen-token-plan': ['openai-completions', 'https://token-plan.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1', 'QWEN_TOKEN_PLAN_API_KEY'], 'qwen-token-plan-cn': ['openai-completions', 'https://token-plan.cn-beijing.maas.aliyuncs.com/compatible-mode/v1', 'QWEN_TOKEN_PLAN_CN_API_KEY'], 'qwen-token-plan-individual': ['openai-completions', 'https://token-plan.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1', 'QWEN_TOKEN_PLAN_API_KEY'], together: ['openai-completions', 'https://api.together.ai/v1', 'TOGETHER_API_KEY'], 'vercel-ai-gateway': ['anthropic-messages', 'https://ai-gateway.vercel.sh', 'AI_GATEWAY_API_KEY'], xai: ['openai-responses', 'https://api.x.ai/v1', 'XAI_API_KEY'], xiaomi: ['openai-completions', 'https://api.xiaomimimo.com/v1', 'XIAOMI_API_KEY'], 'xiaomi-token-plan-ams': ['openai-completions', 'https://token-plan-ams.xiaomimimo.com/v1', 'XIAOMI_TOKEN_PLAN_AMS_API_KEY'], 'xiaomi-token-plan-cn': ['openai-completions', 'https://token-plan-cn.xiaomimimo.com/v1', 'XIAOMI_TOKEN_PLAN_CN_API_KEY'], 'xiaomi-token-plan-sgp': ['openai-completions', 'https://token-plan-sgp.xiaomimimo.com/v1', 'XIAOMI_TOKEN_PLAN_SGP_API_KEY'], zai: ['openai-completions', 'https://api.z.ai/api/coding/paas/v4', 'ZAI_API_KEY'], 'zai-coding-cn': ['openai-completions', 'https://open.bigmodel.cn/api/coding/paas/v4', 'ZAI_CODING_CN_API_KEY'], } const DSH_DEEPSEEK_ROUTES = new Set(['deepseek-official', 'deepseek-account']) // --------------------------------------------------------------------------- // Small helpers const str = value => (typeof value === 'string' && value.trim() !== '' ? value.trim() : undefined) const clamp = (value, low, high) => Math.min(high, Math.max(low, Math.floor(value))) const isRecord = value => value !== null && typeof value === 'object' && !Array.isArray(value) const httpUrl = (value) => { if (typeof value !== 'string') return undefined try { const url = new URL(value) return url.protocol === 'http:' || url.protocol === 'https:' ? url.href : undefined } catch { return undefined } } const clip = (text, max) => (text.length > max ? `${text.slice(0, max - 1)}…` : text) const reasonOf = error => (error?.cause?.message && error.message === 'fetch failed' ? error.cause.message : error?.message ?? String(error)) const seconds = ms => `${Math.round(ms / 1000)}s` const trimSlash = url => url.replace(/\/+$/, '') const NAMED_ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', ensp: ' ', emsp: ' ', thinsp: ' ', middot: '·', hellip: '…', mdash: '—', ndash: '–', laquo: '«', raquo: '»', lsquo: '‘', rsquo: '’', ldquo: '“', rdquo: '”', bull: '•', copy: '©', reg: '®', trade: '™', zwj: '', zwnj: '', } export const decodeEntities = text => text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (whole, ref) => { if (ref[0] === '#') { const code = ref[1] === 'x' || ref[1] === 'X' ? Number.parseInt(ref.slice(2), 16) : Number(ref.slice(1)) return Number.isInteger(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : whole } return NAMED_ENTITIES[ref.toLowerCase()] ?? whole }) // Inline tags vanish so "deepseek" stays one word; block ends become spaces. export const textOf = html => decodeEntities(html .replace(/<(script|style)\b[\s\S]*?<\/\1>/gi, ' ') .replace(/<(br|\/p|\/div|\/li|\/h\d|\/td|\/tr)\b[^>]*>/gi, ' ') .replace(/<[^>]*>/g, '')) .replace(/\s+/g, ' ') .trim() const attrOf = (attrs, attr) => { const match = new RegExp(`\\b${attr}\\s*=\\s*("([^"]*)"|'([^']*)')`, 'i').exec(attrs) return match === null ? undefined : decodeEntities(match[2] ?? match[3] ?? '') } const plain = value => (typeof value === 'string' ? value.replace(/\s+/g, ' ').trim() : '') /** One portable source from plain-text fields; empty optional fields are omitted rather than invented. */ const sourceOf = (url, title, snippet, publishedAt) => { const href = httpUrl(url) if (href === undefined) return undefined const cleanTitle = plain(title) const cleanSnippet = plain(snippet) const date = str(publishedAt) return { url: href, ...cleanTitle ? { title: clip(cleanTitle, 300) } : {}, ...cleanSnippet ? { snippet: clip(cleanSnippet, SNIPPET_CHARS) } : {}, ...date ? { publishedAt: date } : {}, } } const dedupe = (sources) => { const seen = new Map() for (const source of sources) { if (source === undefined) continue const known = seen.get(source.url) if (known === undefined) seen.set(source.url, source) else seen.set(source.url, { ...source, ...known }) } return [...seen.values()] } // --------------------------------------------------------------------------- // Native search: response readers. Each returns `{ ran, sources, error? }`: whether the endpoint // actually ran a search, and only the structured sources it reported (URL, title, excerpt, // date), never the answer text a model wrote around them. /** Anthropic Messages with the `web_search_20250305` server tool. */ export function readMessages(body) { const blocks = Array.isArray(body?.content) ? body.content : [] const results = blocks.filter(block => block?.type === 'web_search_tool_result') const excerpts = new Map() for (const block of blocks) { if (block?.type !== 'text' || !Array.isArray(block.citations)) continue for (const cite of block.citations) { if (typeof cite?.url === 'string' && str(cite.cited_text) && !excerpts.has(cite.url)) excerpts.set(cite.url, cite.cited_text) } } const sources = dedupe(results.flatMap(block => (Array.isArray(block.content) ? block.content : [])) .filter(item => item?.type === 'web_search_result') .map(item => sourceOf(item.url, item.title, excerpts.get(item.url) ?? (typeof item.content === 'string' ? item.content : undefined), item.page_age))) const failed = results.map(block => block.content).find(content => content?.type === 'web_search_tool_result_error') return { ran: results.length > 0 || blocks.some(block => block?.type === 'server_tool_use'), cut: body?.stop_reason === 'max_tokens', sources, ...sources.length === 0 && failed !== undefined ? { error: `web search failed: ${failed.error_code ?? 'unknown error'}` } : {}, } } /** OpenAI Responses with the `web_search` tool. */ export function readResponses(body) { const output = Array.isArray(body?.output) ? body.output : [] const calls = output.filter(item => item?.type === 'web_search_call') const found = [] for (const call of calls) { for (const source of Array.isArray(call.action?.sources) ? call.action.sources : []) found.push(sourceOf(source?.url, source?.title)) } for (const item of output) { if (item?.type !== 'message' || !Array.isArray(item.content)) continue for (const part of item.content) { for (const note of Array.isArray(part?.annotations) ? part.annotations : []) { if (note?.type === 'url_citation') found.push(sourceOf(note.url, note.title)) } } } for (const cite of Array.isArray(body?.citations) ? body.citations : []) { found.push(typeof cite === 'string' ? sourceOf(cite) : sourceOf(cite?.url, cite?.title)) } const sources = dedupe(found) return { ran: calls.length > 0 || sources.length > 0, cut: body?.status === 'incomplete', sources } } /** Gemini `generateContent` with the `google_search` tool. */ export function readGemini(body) { const candidates = Array.isArray(body?.candidates) ? body.candidates : [] return { ran: candidates.some(candidate => candidate?.groundingMetadata), cut: candidates.some(candidate => candidate?.finishReason === 'MAX_TOKENS'), sources: dedupe(candidates .flatMap(candidate => candidate?.groundingMetadata?.groundingChunks ?? []) .map(chunk => sourceOf(chunk?.web?.uri, chunk?.web?.title))), } } const URL_KEYS = ['url', 'uri', 'link'] const TITLE_KEYS = ['title', 'name', 'site_name'] const SNIPPET_KEYS = ['snippet', 'summary', 'content', 'description', 'cited_text', 'text'] const DATE_KEYS = ['published_date', 'publish_date', 'publish_time', 'page_age', 'date', 'last_updated'] // Media links are not results, and a response that echoes the request's tools is not a search. const NOT_SOURCES = new Set(['image_url', 'images', 'image', 'icon', 'favicon', 'logo', 'logo_url', 'thumbnail', 'tools']) const SEARCH_MARK = /search|grounding|citation/i const firstText = (record, keys) => keys.map(key => record[key]).find(value => typeof value === 'string' && value.trim() !== '') /** Every object in a JSON body that carries an http(s) URL, as sources: the shape of any search API. */ export function findSources(body) { const found = [] const visit = (value, depth) => { if (depth > 16 || value === null || typeof value !== 'object') return if (Array.isArray(value)) { for (const item of value) visit(item, depth + 1); return } const url = URL_KEYS.map(key => httpUrl(value[key])).find(Boolean) if (url !== undefined) found.push(sourceOf(url, firstText(value, TITLE_KEYS), firstText(value, SNIPPET_KEYS), firstText(value, DATE_KEYS))) for (const [key, child] of Object.entries(value)) if (!NOT_SOURCES.has(key)) visit(child, depth + 1) } visit(body, 0) return dedupe(found) } /** Whether a JSON body shows a search ran, read from its keys and `type` tags, never from prose. */ export function showsSearch(body) { const visit = (value, depth) => { if (depth > 16 || value === null || typeof value !== 'object') return false if (Array.isArray(value)) return value.some(item => visit(item, depth + 1)) return Object.entries(value).some(([key, child]) => !NOT_SOURCES.has(key) && ( SEARCH_MARK.test(key) || key === 'results' || (key === 'type' && typeof child === 'string' && SEARCH_MARK.test(child)) || visit(child, depth + 1))) } return visit(body, 0) } /** A declared search API's response: any structured sources, wherever they sit. */ export function readDeclared(body) { const sources = findSources(body) return { ran: sources.length > 0 || showsSearch(body), sources } } // --------------------------------------------------------------------------- // Native search: what to try for one route. Nothing here names a vendor except SEARCH_APIS, // which is data in the same shape as `searchApis` config. const bearer = key => ({ authorization: `Bearer ${key}` }) export const AUTH_STYLES = { bearer, 'x-api-key': key => ({ 'x-api-key': key }), 'api-key': key => ({ 'api-key': key }) } // One trailing /v1 belongs to the root, as in dsh's DeepSeek adapter and pi-ai's model listing. const messagesRoot = base => (new URL(base).pathname.replace(/\/+$/, '').endsWith('/v1') ? trimSlash(base) : `${trimSlash(base)}/v1`) // Bounds what an endpoint that ignores the search tool can cost, while leaving a thinking model // room to reach the search: dsh's own DeepSeek search uses the same cap. const OUTPUT_CAP = 4096 /** The standard search tool of each wire protocol. */ export const PROTOCOL_SEARCHES = { messages: { name: 'Anthropic Messages web_search tool', url: base => `${messagesRoot(base)}/messages`, // Anthropic reads x-api-key; compatible gateways often read Bearer. auth: key => ({ 'x-api-key': key, ...bearer(key) }), request: ({ model, query }) => ({ headers: { 'anthropic-version': '2023-06-01' }, body: { model, max_tokens: OUTPUT_CAP, messages: [{ role: 'user', content: [{ type: 'text', text: PROMPT(query) }] }], tools: [{ type: 'web_search_20250305', name: 'web_search', max_uses: 1 }], }, }), read: readMessages, }, responses: { name: 'OpenAI Responses web_search tool', url: base => `${trimSlash(base)}/responses`, auth: bearer, request: ({ model, query }) => ({ body: { model, input: PROMPT(query), tools: [{ type: 'web_search' }], max_output_tokens: OUTPUT_CAP } }), read: readResponses, }, gemini: { name: 'Gemini google_search tool', url: (base, model) => `${trimSlash(base)}/models/${encodeURIComponent(model)}:generateContent`, auth: key => ({ 'x-goog-api-key': key }), request: ({ query }) => ({ body: { contents: [{ role: 'user', parts: [{ text: PROMPT(query) }] }], tools: [{ google_search: {} }], generationConfig: { maxOutputTokens: OUTPUT_CAP } }, }), read: readGemini, }, } const BY_PROTOCOL = { 'anthropic-messages': ['messages'], 'openai-responses': ['responses'], // Many OpenAI-compatible servers also serve Responses and Messages under the same base. 'openai-completions': ['responses', 'messages'], 'google-generative-ai': ['gemini'], } const UNKNOWN_PROTOCOL = ['responses', 'messages'] /** * Search APIs a host serves beside its model protocol. No standard advertises these, so they * are declared; `searchApis` in config adds more in the same shape. `$query`, `$model`, and * `$maxResults` in a body are replaced per call. Each path is joined to the route's own origin. */ export const SEARCH_APIS = Object.freeze([ // docs.ollama.com/capabilities/web-search { id: 'ollama', name: 'Ollama web search API', hosts: ['ollama.com'], path: '/api/web_search', maxResults: 10, body: { query: '$query', max_results: '$maxResults' } }, // platform.kimi.ai; an empty result is not billed. { id: 'kimi', name: 'Kimi search API', hosts: ['api.moonshot.ai', 'api.moonshot.cn'], path: '/v1/tools/search', maxResults: 20, body: { text_query: '$query', limit: '$maxResults', timeout_seconds: 30 } }, // docs.bigmodel.cn; the API rejects queries over 70 characters. { id: 'zhipu', name: 'Zhipu web search API', hosts: ['open.bigmodel.cn', 'api.z.ai'], path: '/api/paas/v4/web_search', maxResults: 50, maxQueryChars: 70, body: { search_query: '$query', search_engine: 'search_std', count: '$maxResults' } }, // MiMo searches only through this Chat Completions plugin tool, once the Web Search plugin is // enabled for the key. Each keyword is one billed search, so one keyword per query; the answer // text is discarded, so it stays short and without thinking. { id: 'mimo', name: 'MiMo web_search plugin', hosts: ['xiaomimimo.com', '*.xiaomimimo.com'], path: '/v1/chat/completions', auth: ['bearer', 'api-key'], maxResults: 5, body: { model: '$model', messages: [{ role: 'user', content: '$query' }], tools: [{ type: 'web_search', force_search: true, max_keyword: 1, limit: '$maxResults' }], max_completion_tokens: 256, thinking: { type: 'disabled' }, stream: false, }, }, ]) const hostMatches = (patterns, host) => patterns.some(pattern => (pattern.startsWith('*.') ? host.endsWith(pattern.slice(1)) : host === pattern.toLowerCase())) const fill = (template, values) => { if (Array.isArray(template)) return template.map(item => fill(item, values)) if (isRecord(template)) return Object.fromEntries(Object.entries(template).map(([key, value]) => [key, fill(value, values)])) return typeof template === 'string' && Object.hasOwn(values, template) ? values[template] : template } const usesModel = api => JSON.stringify(api.body).includes('"$model"') function declaredCandidate(api, origin, model) { const url = `${origin}${api.path}` // A path such as `//other.example` must not move the key to another origin. if (new URL(url).origin !== origin) return undefined const styles = (api.auth ?? ['bearer']).map(style => AUTH_STYLES[style]) return { id: api.id, name: api.name ?? `${api.id} search API`, url, model: usesModel(api) ? model : undefined, auth: key => Object.assign({}, ...styles.map(style => style(key))), request: ({ query, maxResults }) => ({ body: fill(api.body, { $query: api.maxQueryChars === undefined ? query : [...query].slice(0, api.maxQueryChars).join(''), $model: model, $maxResults: clamp(maxResults ?? 8, 1, api.maxResults ?? 10), }), }), read: readDeclared, } } function protocolCandidate(id, base, model) { const shape = PROTOCOL_SEARCHES[id] return { id, name: shape.name, url: shape.url(base, model), model, auth: shape.auth, request: call => shape.request({ ...call, model }), read: shape.read } } /** * Everything worth trying for one route, cheapest first: search APIs declared for its host * (no model call), then the standard search tool of its protocol. * @returns `{ candidates }`, with `reason` when there is nothing to try. */ export function planNative(route, searchApis = SEARCH_APIS) { const base = httpUrl(route.baseURL) if (base === undefined) return { candidates: [], reason: `route "${route.provider}" has no known endpoint` } const url = new URL(base) const model = str(route.model) const candidates = [] for (const api of searchApis) { if (!hostMatches(api.hosts, url.hostname)) continue if (model === undefined && usesModel(api)) continue const candidate = declaredCandidate(api, url.origin, model) if (candidate !== undefined) candidates.push(candidate) } const shapes = route.api === undefined ? UNKNOWN_PROTOCOL : BY_PROTOCOL[route.api] ?? [] if (model !== undefined) for (const id of shapes) candidates.push(protocolCandidate(id, base, model)) if (candidates.length > 0) return { candidates } return { candidates, reason: model === undefined && shapes.length > 0 ? `route "${route.provider}" has no current model to search with` : `route "${route.provider}" speaks ${route.api}, which has no standard search tool, and ${url.host} declares no search API`, } } // --------------------------------------------------------------------------- // Keyless fallback: public results pages, parsed into sources. /** Bing wraps result links in `bing.com/ck/a?…&u=a1`; recover the target. */ export function decodeBingUrl(href) { const url = httpUrl(href) if (url === undefined) return undefined const parsed = new URL(url) if (!/(^|\.)bing\.com$/i.test(parsed.hostname)) return url const target = parsed.searchParams.get('u') if (parsed.pathname !== '/ck/a' || target === null || !target.startsWith('a1')) return undefined try { return httpUrl(Buffer.from(target.slice(2), 'base64url').toString('utf8')) } catch { return undefined } } /** Bing results page → `{ sources }`, or `{ error }` when the page is not a results page. */ export function parseBing(html) { const starts = [...html.matchAll(/]*\bclass\s*=\s*"[^"]*\bb_algo\b[^"]*"[^>]*>/gi)].map(match => match.index) const sources = [] starts.forEach((start, index) => { const block = html.slice(start, starts[index + 1] ?? start + 12_000) const heading = /]*>\s*]*)>([\s\S]*?)<\/a>/i.exec(block) if (heading === null) return const href = attrOf(heading[1], 'href') const caption = block.slice(Math.max(0, block.search(/]*\bclass\s*=\s*"[^"]*\bb_caption\b/i))) const paragraph = /]*>([\s\S]*?)<\/p>/i.exec(caption) sources.push(sourceOf(decodeBingUrl(href), textOf(heading[2]), paragraph ? textOf(paragraph[1]) : undefined)) }) const parsed = dedupe(sources) if (parsed.length > 0) return { sources: parsed } if (/\bclass\s*=\s*"[^"]*\bb_no\b/i.test(html)) return { sources: [] } return { error: /id\s*=\s*"b_results"/i.test(html) ? 'results page had no parsable results' : 'not a results page (bot check or changed layout)' } } /** DuckDuckGo HTML results page → `{ sources }`, or `{ error }`. */ export function parseDuckDuckGo(html) { if (/anomaly-modal|bots use DuckDuckGo too|challenge-form/i.test(html)) return { error: 'bot check page' } const links = [...html.matchAll(/]*\bclass\s*=\s*"[^"]*\bresult__a\b[^"]*"[^>]*)>([\s\S]*?)<\/a>/gi)] const sources = links.map((link, index) => { const href = attrOf(link[1], 'href') ?? '' let target try { const parsed = new URL(href, 'https://duckduckgo.com') // `/y.js` links are ads. if (/(^|\.)duckduckgo\.com$/i.test(parsed.hostname)) target = parsed.pathname === '/l/' ? parsed.searchParams.get('uddg') : undefined else target = parsed.href } catch { target = undefined } const rest = html.slice(link.index + link[0].length, links[index + 1]?.index ?? link.index + 6000) const snippet = /<(?:a|div|td)\b[^>]*\bclass\s*=\s*"[^"]*\bresult__snippet\b[^"]*"[^>]*>([\s\S]*?)<\/(?:a|div|td)>/i.exec(rest) return sourceOf(target, textOf(link[2]), snippet ? textOf(snippet[1]) : undefined) }) const parsed = dedupe(sources) if (parsed.length > 0) return { sources: parsed } if (/\bresult--no-results\b|\bno-results\b/i.test(html)) return { sources: [] } return { error: 'not a results page (bot check or changed layout)' } } export const ENGINES = { // No market or language parameter: Bing picks them from the network, as it does for a browser. // `regional` is the hosts Bing itself may redirect to (mainland networks go to cn.bing.com). bing: { name: 'Bing', url: query => `https://www.bing.com/search?q=${encodeURIComponent(query)}`, parse: parseBing, regional: /^https:\/\/([a-z0-9-]+\.)*bing\.com$/i }, duckduckgo: { name: 'DuckDuckGo', url: query => `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`, parse: parseDuckDuckGo }, } // --------------------------------------------------------------------------- // The provider const aborted = signal => new WebError('web search aborted', 'WEB_ABORTED', { cause: signal?.reason }) // AbortSignal.timeout's wait is unref'd, so under Node 22's test runner a pending subtest // is cancelled as soon as nothing ref'd holds the loop — and a mocked fetch holds nothing. // A plain setTimeout keeps the loop honest; the abort reason stays a TimeoutError. const timeoutAfter = (ms) => { const timeout = new AbortController() const timer = setTimeout(() => timeout.abort(new DOMException('The operation timed out.', 'TimeoutError')), ms) return { timeout, timer } } function checkSearchApi(api, index) { const where = `web-search-auto: searchApis[${index}]` if (!isRecord(api)) throw new Error(`${where} must be an object`) if (str(api.id) === undefined) throw new Error(`${where} needs an id`) if (!Array.isArray(api.hosts) || api.hosts.length === 0 || !api.hosts.every(host => typeof host === 'string' && /^(\*\.)?[a-z0-9-]+(\.[a-z0-9-]+)*$/i.test(host))) { throw new Error(`${where}.hosts must be a list of host names; a leading *. matches subdomains`) } if (typeof api.path !== 'string' || !/^\/[^/]/.test(api.path)) throw new Error(`${where}.path must be a path on the route's host, starting with one /`) if (!isRecord(api.body)) throw new Error(`${where}.body must be an object`) if (api.auth !== undefined && (!Array.isArray(api.auth) || api.auth.length === 0 || api.auth.some(style => !Object.hasOwn(AUTH_STYLES, style)))) { throw new Error(`${where}.auth must be a list of ${Object.keys(AUTH_STYLES).join(', ')}`) } for (const key of ['maxResults', 'maxQueryChars']) { if (api[key] !== undefined && !(Number.isInteger(api[key]) && api[key] >= 1)) throw new Error(`${where}.${key} must be a positive integer`) } if (api.name !== undefined && str(api.name) === undefined) throw new Error(`${where}.name must be text`) } /** * Normalize plugin config. Invalid entries fail loud at load rather than silently searching less. * Declared `searchApis` come before the built-in SEARCH_APIS, so a user entry wins on its host. */ export function resolveConfig(config = {}) { const resolved = { ...DEFAULTS, ...Object.fromEntries(Object.entries(config ?? {}).filter(([, value]) => value !== undefined)) } if (!Array.isArray(resolved.engines) || resolved.engines.some(engine => !Object.hasOwn(ENGINES, engine))) { throw new Error(`web-search-auto: engines must be a list of ${Object.keys(ENGINES).join(', ')}`) } for (const key of ['nativeTimeoutMs', 'fallbackTimeoutMs']) { if (!Number.isFinite(resolved[key]) || resolved[key] < 1000) throw new Error(`web-search-auto: ${key} must be at least 1000`) } if (!Array.isArray(resolved.searchApis)) throw new Error('web-search-auto: searchApis must be a list') resolved.searchApis.forEach(checkSearchApi) return { ...resolved, native: resolved.native !== false, fallback: resolved.fallback !== false, searchApis: [...resolved.searchApis, ...SEARCH_APIS], } } const errorDetail = (text) => { try { const parsed = JSON.parse(text) return str(typeof parsed?.error === 'string' ? parsed.error : parsed?.error?.message ?? parsed?.message ?? parsed?.msg) ?? '' } catch { return '' } } /** * Build the `auto` search provider over injected effects, so tests run it without a host. * @param deps.route - the asking Session's route: `{ provider, model, api, baseURL, apiKeyEnv, account, headers }`. * @param deps.resolveKey - credential reference → key, or undefined. * @param deps.resolveAccountToken - endpoint → DeepSeek account token, or undefined. * @param deps.fetch - the HTTP client for native searches. * @param deps.webFetch - `ctx.web.fetch`. */ export function createAutoSearchProvider(deps, config = {}) { const options = resolveConfig(config) const now = deps.now ?? Date.now // route + endpoint + model → `{ works: true }` or `{ until, reason }`, for this process. const memory = new Map() async function credentialFor(route, endpoint) { if (route.account) { let token try { token = str(await deps.resolveAccountToken?.(endpoint)) } catch (error) { return { reason: `the DeepSeek account token could not be read: ${reasonOf(error)}` } } return token === undefined ? { reason: 'it is not signed in to a DeepSeek account' } : { headers: () => ({ 'x-dsh-auth-token': token }), via: 'the DeepSeek account sign-in' } } if (route.apiKeyEnv === undefined) return { reason: 'it names no API key' } let key try { key = str(await deps.resolveKey(route.apiKeyEnv)) } catch (error) { return { reason: `the key ${route.apiKeyEnv} could not be read: ${reasonOf(error)}` } } return key === undefined ? { reason: `no API key is stored for ${route.apiKeyEnv}` } : { headers: candidate => candidate.auth(key), via: `the API key ${route.apiKeyEnv}` } } async function attempt(candidate, route, credential, request, signal, budget, known) { const call = candidate.request({ query: request.query, maxResults: request.maxResults }) const { timeout, timer } = timeoutAfter(budget) let response let text try { response = await deps.fetch(candidate.url, { method: 'POST', // Credential-bearing requests never follow redirects (deepseek-harness/packages/web/AGENTS.md). redirect: 'error', headers: { 'content-type': 'application/json', accept: 'application/json', 'user-agent': USER_AGENT, ...route.headers, ...call.headers, ...credential.headers(candidate) }, body: JSON.stringify(call.body), signal: signal === undefined ? timeout.signal : AbortSignal.any([signal, timeout.signal]), }) text = await response.text() } catch (error) { if (signal?.aborted) throw aborted(signal) return timeout.signal.aborted ? { reason: `timed out after ${seconds(budget)}`, timedOut: true } : { reason: `request failed: ${reasonOf(error)}` } } finally { clearTimeout(timer) } if (!response.ok) { const detail = errorDetail(text) return { reason: `HTTP ${response.status}${detail ? `: ${clip(detail, 200)}` : ''}`, skip: STATUS_OUTCOMES[response.status] } } let body try { body = JSON.parse(text) } catch { return { reason: 'answered with a body that is not JSON', skip: 'ignored' } } const read = candidate.read(body) if (read.sources.length > 0) return { sources: read.sources } if (read.error !== undefined) return { reason: read.error, final: true } if (read.ran || known) return { reason: 'returned no results', final: true } // The cap says nothing about support, so a cut-off answer is not remembered. if (read.cut) return { reason: 'stopped at the output cap before searching' } return { reason: 'answered without running a search, so it does not seem to support this one', skip: 'ignored' } } async function runNative(request, signal) { if (!options.native) return { reason: 'native search is turned off (web-search-auto native: false)' } let route try { route = deps.route() } catch (error) { return { reason: `the conversation's model route could not be read: ${reasonOf(error)}` } } if (route === undefined) return { reason: 'no conversation model route is known for this search' } const plan = planNative(route, options.searchApis) if (plan.candidates.length === 0) return { reason: plan.reason } const keyOf = candidate => [route.provider, candidate.url, candidate.model ?? ''].join('\n') const works = candidate => memory.get(keyOf(candidate))?.works === true const ordered = [...plan.candidates.filter(works), ...plan.candidates.filter(candidate => !works(candidate))] let deadline = now() + options.nativeTimeoutMs const notes = [] let credential for (const candidate of ordered) { const where = `${candidate.name} (${new URL(candidate.url).host}${new URL(candidate.url).pathname})` const memo = memory.get(keyOf(candidate)) if (memo?.until !== undefined && memo.until > now()) { notes.push(`${where}: ${memo.reason}; not retried for ${Math.ceil((memo.until - now()) / MINUTE)} min`) continue } credential ??= await credentialFor(route, candidate.url) if (signal?.aborted) throw aborted(signal) if (credential.reason !== undefined) return { reason: `route "${route.provider}" could not search natively: ${credential.reason}` } const remaining = deadline - now() if (remaining <= 0) { notes.push(`${where}: skipped, out of time`); continue } const outcome = await attempt(candidate, route, credential, request, signal, remaining, memo?.works === true) if (outcome.sources !== undefined) { memory.set(keyOf(candidate), { works: true }) return { result: { content: `Search backend: native search of this conversation's model route "${route.provider}": ${where}, billed to ${credential.via}.`, sources: outcome.sources, truncated: false, }, } } notes.push(`${where}: ${outcome.reason}`) if (outcome.skip !== undefined) memory.set(keyOf(candidate), { until: now() + SKIP_FOR[outcome.skip], reason: outcome.reason }) if (outcome.final) break // A timed-out attempt had all the time that was left, but the timer and now() use // different clocks, so now() can still read a millisecond before the deadline. if (outcome.timedOut) deadline = now() } return { reason: `route "${route.provider}": ${notes.join('; ')}` } } async function fetchPage(engine, url, signal) { try { return { page: await deps.webFetch({ url }, signal), url } } catch (error) { // The fetch provider refuses cross-origin redirects and names the target; follow one hop // when it is the engine's own regional host. const origin = error?.code === 'WEB_REDIRECT_BLOCKED' ? /https?:\/\/[a-z0-9.-]+(:\d+)?/i.exec(error.message ?? '')?.[0] : undefined if (origin === undefined || engine.regional?.test(origin) !== true || origin === new URL(url).origin) throw error const { pathname, search } = new URL(url) const regional = `${origin}${pathname}${search}` return { page: await deps.webFetch({ url: regional }, signal), url: regional } } } async function runFallback(request, signal) { const attempts = [] let empty const deadline = now() + options.fallbackTimeoutMs for (const [index, id] of options.engines.entries()) { const engine = ENGINES[id] const url = engine.url(request.query) const host = new URL(url).host const remaining = deadline - now() if (remaining <= 0) { attempts.push(`${host}: skipped, out of time`); continue } // Leave later engines a share of the budget when an earlier one hangs. const { timeout, timer } = timeoutAfter(index === options.engines.length - 1 ? remaining : Math.min(remaining, Math.ceil(options.fallbackTimeoutMs / 2))) let fetched try { fetched = await fetchPage(engine, url, signal === undefined ? timeout.signal : AbortSignal.any([signal, timeout.signal])) } catch (error) { if (signal?.aborted) throw aborted(signal) attempts.push(`${host}: ${timeout.signal.aborted ? 'timed out' : `${error?.code ? `${error.code} ` : ''}${reasonOf(error)}`}`) continue } finally { clearTimeout(timer) } const at = new URL(fetched.url).host if (fetched.page.statusCode !== 200) { attempts.push(`${at}: HTTP ${fetched.page.statusCode}`); continue } const parsed = engine.parse(fetched.page.body?.content ?? '') if (parsed.error !== undefined) { attempts.push(`${at}: ${parsed.error}`); continue } if (parsed.sources.length === 0) { empty ??= { engine, at }; attempts.push(`${at}: no results`); continue } return { result: { engine, at, sources: parsed.sources } } } return empty === undefined ? { attempts } : { result: { ...empty, sources: [] } } } return { id: AUTO_PROVIDER_ID, available: () => options.native || options.fallback, async search(request, signal) { if (signal?.aborted) throw aborted(signal) const native = await runNative(request, signal) if (native.result !== undefined) return native.result let fallbackReason = 'turned off (web-search-auto fallback: false)' if (options.fallback) { const fallback = await runFallback(request, signal) if (fallback.result !== undefined) { return { content: `Search backend: keyless fallback, parsed from the public ${fallback.result.engine.name} results page at ${fallback.result.at} (no API key, no charge; less reliable than an API search). Native search was not used: ${native.reason}.`, sources: fallback.result.sources, truncated: false, } } fallbackReason = fallback.attempts.join('; ') } throw new WebError([ 'Web search is unavailable for this query.', `Native search: ${native.reason}.`, `Keyless fallback: ${fallbackReason}.`, 'Do not guess URLs or present pages you remember as search results. If you have a browser tool, open a search engine in it; otherwise tell the user that web search is unavailable and why.', ].join('\n'), 'WEB_SEARCH_UNAVAILABLE') }, } } // --------------------------------------------------------------------------- // Host wiring: read the asking Session's route through public dsh services only. const stringRecord = value => (isRecord(value) ? Object.fromEntries(Object.entries(value).filter(([, item]) => typeof item === 'string')) : {}) // OpenCode's gateways turn away a request that does not name its conversation. DSH's // adapter names it by the Session id, so a search for that Session does the same, // unless the provider settings name one. const OPENCODE_SESSION = 'x-opencode-session' const OPENCODE_ROUTES = new Set(['opencode', 'opencode-go']) const withConversation = (provider, headers, sessionId) => (!OPENCODE_ROUTES.has(provider) || sessionId === undefined || Object.keys(headers).some(name => name.toLowerCase() === OPENCODE_SESSION) ? headers : { ...headers, [OPENCODE_SESSION]: sessionId }) /** The route of the Session that started the current tool call: protocol, endpoint, key name, and headers. */ export function routeOf(ctx) { const session = ctx.get('agents')?.currentInitiator()?.session const request = session?.requestContext() const provider = str(request?.provider) if (provider === undefined) return undefined let section try { const entry = ctx.get('llm')?.listConfigurableProviders().find(item => item.provider === provider) if (entry !== undefined) { const descriptor = ctx.get('settings')?.describe().find(item => item.ns === entry.settingsNs) section = entry.settingsPath.reduce((value, key) => value?.[key], descriptor?.value) } } catch { section = undefined } const [catalogApi, catalogBase, catalogKey] = CATALOG[provider] ?? [] const account = provider === 'deepseek-account' return { provider, model: str(request.model), api: str(section?.api) ?? catalogApi, baseURL: str(section?.baseURL) ?? (DSH_DEEPSEEK_ROUTES.has(provider) ? str(envOf(ctx)('DEEPSEEK_BASE_URL')) : undefined) ?? catalogBase, apiKeyEnv: account ? undefined : str(section?.apiKeyEnv) ?? catalogKey, account, headers: withConversation(provider, stringRecord(section?.headers), str(session.id)), } } const envOf = ctx => name => ctx.get('launchEnvironment')?.get(name)?.value ?? process.env[name] export function apply(ctx, config = {}) { const env = envOf(ctx) ctx.web.registerSearchProvider(createAutoSearchProvider({ route: () => routeOf(ctx), resolveKey: async (ref) => { const credentials = ctx.get('credentials') // Without the seam the environment is the whole credential plane, as in dsh's own adapters. return credentials === undefined ? env(ref) : (await credentials.resolve(ref))?.value }, resolveAccountToken: endpoint => ctx.get('deepseekAccount')?.resolveToken(endpoint), fetch: (url, init) => fetch(url, init), webFetch: (request, signal) => ctx.web.fetch(request, signal), }, config)) }