]*\bclass\s*=\s*"[^"]*\bb_caption\b/i)))
const paragraph = /
]*>([\s\S]*?)<\/p>/i.exec(caption)
sources.push(sourceOf(decodeBingUrl(href), textOf(heading[2]), paragraph ? textOf(paragraph[1]) : undefined))
})
const parsed = dedupe(sources)
if (parsed.length > 0) return { sources: parsed }
if (/\bclass\s*=\s*"[^"]*\bb_no\b/i.test(html)) return { sources: [] }
return { error: /id\s*=\s*"b_results"/i.test(html) ? 'results page had no parsable results' : 'not a results page (bot check or changed layout)' }
}
/** DuckDuckGo HTML results page → `{ sources }`, or `{ error }`. */
export function parseDuckDuckGo(html) {
if (/anomaly-modal|bots use DuckDuckGo too|challenge-form/i.test(html)) return { error: 'bot check page' }
const links = [...html.matchAll(/]*\bclass\s*=\s*"[^"]*\bresult__a\b[^"]*"[^>]*)>([\s\S]*?)<\/a>/gi)]
const sources = links.map((link, index) => {
const href = attrOf(link[1], 'href') ?? ''
let target
try {
const parsed = new URL(href, 'https://duckduckgo.com')
// `/y.js` links are ads.
if (/(^|\.)duckduckgo\.com$/i.test(parsed.hostname)) target = parsed.pathname === '/l/' ? parsed.searchParams.get('uddg') : undefined
else target = parsed.href
} catch { target = undefined }
const rest = html.slice(link.index + link[0].length, links[index + 1]?.index ?? link.index + 6000)
const snippet = /<(?:a|div|td)\b[^>]*\bclass\s*=\s*"[^"]*\bresult__snippet\b[^"]*"[^>]*>([\s\S]*?)<\/(?:a|div|td)>/i.exec(rest)
return sourceOf(target, textOf(link[2]), snippet ? textOf(snippet[1]) : undefined)
})
const parsed = dedupe(sources)
if (parsed.length > 0) return { sources: parsed }
if (/\bresult--no-results\b|\bno-results\b/i.test(html)) return { sources: [] }
return { error: 'not a results page (bot check or changed layout)' }
}
export const ENGINES = {
// No market or language parameter: Bing picks them from the network, as it does for a browser.
// `regional` is the hosts Bing itself may redirect to (mainland networks go to cn.bing.com).
bing: { name: 'Bing', url: query => `https://www.bing.com/search?q=${encodeURIComponent(query)}`, parse: parseBing, regional: /^https:\/\/([a-z0-9-]+\.)*bing\.com$/i },
duckduckgo: { name: 'DuckDuckGo', url: query => `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`, parse: parseDuckDuckGo },
}
// ---------------------------------------------------------------------------
// The provider
const aborted = signal => new WebError('web search aborted', 'WEB_ABORTED', { cause: signal?.reason })
// AbortSignal.timeout's wait is unref'd, so under Node 22's test runner a pending subtest
// is cancelled as soon as nothing ref'd holds the loop — and a mocked fetch holds nothing.
// A plain setTimeout keeps the loop honest; the abort reason stays a TimeoutError.
const timeoutAfter = (ms) => {
const timeout = new AbortController()
const timer = setTimeout(() => timeout.abort(new DOMException('The operation timed out.', 'TimeoutError')), ms)
return { timeout, timer }
}
function checkSearchApi(api, index) {
const where = `web-search-auto: searchApis[${index}]`
if (!isRecord(api)) throw new Error(`${where} must be an object`)
if (str(api.id) === undefined) throw new Error(`${where} needs an id`)
if (!Array.isArray(api.hosts) || api.hosts.length === 0 || !api.hosts.every(host => typeof host === 'string' && /^(\*\.)?[a-z0-9-]+(\.[a-z0-9-]+)*$/i.test(host))) {
throw new Error(`${where}.hosts must be a list of host names; a leading *. matches subdomains`)
}
if (typeof api.path !== 'string' || !/^\/[^/]/.test(api.path)) throw new Error(`${where}.path must be a path on the route's host, starting with one /`)
if (!isRecord(api.body)) throw new Error(`${where}.body must be an object`)
if (api.auth !== undefined && (!Array.isArray(api.auth) || api.auth.length === 0 || api.auth.some(style => !Object.hasOwn(AUTH_STYLES, style)))) {
throw new Error(`${where}.auth must be a list of ${Object.keys(AUTH_STYLES).join(', ')}`)
}
for (const key of ['maxResults', 'maxQueryChars']) {
if (api[key] !== undefined && !(Number.isInteger(api[key]) && api[key] >= 1)) throw new Error(`${where}.${key} must be a positive integer`)
}
if (api.name !== undefined && str(api.name) === undefined) throw new Error(`${where}.name must be text`)
}
/**
* Normalize plugin config. Invalid entries fail loud at load rather than silently searching less.
* Declared `searchApis` come before the built-in SEARCH_APIS, so a user entry wins on its host.
*/
export function resolveConfig(config = {}) {
const resolved = { ...DEFAULTS, ...Object.fromEntries(Object.entries(config ?? {}).filter(([, value]) => value !== undefined)) }
if (!Array.isArray(resolved.engines) || resolved.engines.some(engine => !Object.hasOwn(ENGINES, engine))) {
throw new Error(`web-search-auto: engines must be a list of ${Object.keys(ENGINES).join(', ')}`)
}
for (const key of ['nativeTimeoutMs', 'fallbackTimeoutMs']) {
if (!Number.isFinite(resolved[key]) || resolved[key] < 1000) throw new Error(`web-search-auto: ${key} must be at least 1000`)
}
if (!Array.isArray(resolved.searchApis)) throw new Error('web-search-auto: searchApis must be a list')
resolved.searchApis.forEach(checkSearchApi)
return {
...resolved,
native: resolved.native !== false,
fallback: resolved.fallback !== false,
searchApis: [...resolved.searchApis, ...SEARCH_APIS],
}
}
const errorDetail = (text) => {
try {
const parsed = JSON.parse(text)
return str(typeof parsed?.error === 'string' ? parsed.error : parsed?.error?.message ?? parsed?.message ?? parsed?.msg) ?? ''
} catch { return '' }
}
/**
* Build the `auto` search provider over injected effects, so tests run it without a host.
* @param deps.route - the asking Session's route: `{ provider, model, api, baseURL, apiKeyEnv, account, headers }`.
* @param deps.resolveKey - credential reference → key, or undefined.
* @param deps.resolveAccountToken - endpoint → DeepSeek account token, or undefined.
* @param deps.fetch - the HTTP client for native searches.
* @param deps.webFetch - `ctx.web.fetch`.
*/
export function createAutoSearchProvider(deps, config = {}) {
const options = resolveConfig(config)
const now = deps.now ?? Date.now
// route + endpoint + model → `{ works: true }` or `{ until, reason }`, for this process.
const memory = new Map()
async function credentialFor(route, endpoint) {
if (route.account) {
let token
try { token = str(await deps.resolveAccountToken?.(endpoint)) } catch (error) { return { reason: `the DeepSeek account token could not be read: ${reasonOf(error)}` } }
return token === undefined
? { reason: 'it is not signed in to a DeepSeek account' }
: { headers: () => ({ 'x-dsh-auth-token': token }), via: 'the DeepSeek account sign-in' }
}
if (route.apiKeyEnv === undefined) return { reason: 'it names no API key' }
let key
try { key = str(await deps.resolveKey(route.apiKeyEnv)) } catch (error) { return { reason: `the key ${route.apiKeyEnv} could not be read: ${reasonOf(error)}` } }
return key === undefined
? { reason: `no API key is stored for ${route.apiKeyEnv}` }
: { headers: candidate => candidate.auth(key), via: `the API key ${route.apiKeyEnv}` }
}
async function attempt(candidate, route, credential, request, signal, budget, known) {
const call = candidate.request({ query: request.query, maxResults: request.maxResults })
const { timeout, timer } = timeoutAfter(budget)
let response
let text
try {
response = await deps.fetch(candidate.url, {
method: 'POST',
// Credential-bearing requests never follow redirects (deepseek-harness/packages/web/AGENTS.md).
redirect: 'error',
headers: { 'content-type': 'application/json', accept: 'application/json', 'user-agent': USER_AGENT, ...route.headers, ...call.headers, ...credential.headers(candidate) },
body: JSON.stringify(call.body),
signal: signal === undefined ? timeout.signal : AbortSignal.any([signal, timeout.signal]),
})
text = await response.text()
} catch (error) {
if (signal?.aborted) throw aborted(signal)
return timeout.signal.aborted ? { reason: `timed out after ${seconds(budget)}`, timedOut: true } : { reason: `request failed: ${reasonOf(error)}` }
} finally {
clearTimeout(timer)
}
if (!response.ok) {
const detail = errorDetail(text)
return { reason: `HTTP ${response.status}${detail ? `: ${clip(detail, 200)}` : ''}`, skip: STATUS_OUTCOMES[response.status] }
}
let body
try { body = JSON.parse(text) } catch { return { reason: 'answered with a body that is not JSON', skip: 'ignored' } }
const read = candidate.read(body)
if (read.sources.length > 0) return { sources: read.sources }
if (read.error !== undefined) return { reason: read.error, final: true }
if (read.ran || known) return { reason: 'returned no results', final: true }
// The cap says nothing about support, so a cut-off answer is not remembered.
if (read.cut) return { reason: 'stopped at the output cap before searching' }
return { reason: 'answered without running a search, so it does not seem to support this one', skip: 'ignored' }
}
async function runNative(request, signal) {
if (!options.native) return { reason: 'native search is turned off (web-search-auto native: false)' }
let route
try { route = deps.route() } catch (error) { return { reason: `the conversation's model route could not be read: ${reasonOf(error)}` } }
if (route === undefined) return { reason: 'no conversation model route is known for this search' }
const plan = planNative(route, options.searchApis)
if (plan.candidates.length === 0) return { reason: plan.reason }
const keyOf = candidate => [route.provider, candidate.url, candidate.model ?? ''].join('\n')
const works = candidate => memory.get(keyOf(candidate))?.works === true
const ordered = [...plan.candidates.filter(works), ...plan.candidates.filter(candidate => !works(candidate))]
let deadline = now() + options.nativeTimeoutMs
const notes = []
let credential
for (const candidate of ordered) {
const where = `${candidate.name} (${new URL(candidate.url).host}${new URL(candidate.url).pathname})`
const memo = memory.get(keyOf(candidate))
if (memo?.until !== undefined && memo.until > now()) {
notes.push(`${where}: ${memo.reason}; not retried for ${Math.ceil((memo.until - now()) / MINUTE)} min`)
continue
}
credential ??= await credentialFor(route, candidate.url)
if (signal?.aborted) throw aborted(signal)
if (credential.reason !== undefined) return { reason: `route "${route.provider}" could not search natively: ${credential.reason}` }
const remaining = deadline - now()
if (remaining <= 0) { notes.push(`${where}: skipped, out of time`); continue }
const outcome = await attempt(candidate, route, credential, request, signal, remaining, memo?.works === true)
if (outcome.sources !== undefined) {
memory.set(keyOf(candidate), { works: true })
return {
result: {
content: `Search backend: native search of this conversation's model route "${route.provider}": ${where}, billed to ${credential.via}.`,
sources: outcome.sources,
truncated: false,
},
}
}
notes.push(`${where}: ${outcome.reason}`)
if (outcome.skip !== undefined) memory.set(keyOf(candidate), { until: now() + SKIP_FOR[outcome.skip], reason: outcome.reason })
if (outcome.final) break
// A timed-out attempt had all the time that was left, but the timer and now() use
// different clocks, so now() can still read a millisecond before the deadline.
if (outcome.timedOut) deadline = now()
}
return { reason: `route "${route.provider}": ${notes.join('; ')}` }
}
async function fetchPage(engine, url, signal) {
try {
return { page: await deps.webFetch({ url }, signal), url }
} catch (error) {
// The fetch provider refuses cross-origin redirects and names the target; follow one hop
// when it is the engine's own regional host.
const origin = error?.code === 'WEB_REDIRECT_BLOCKED' ? /https?:\/\/[a-z0-9.-]+(:\d+)?/i.exec(error.message ?? '')?.[0] : undefined
if (origin === undefined || engine.regional?.test(origin) !== true || origin === new URL(url).origin) throw error
const { pathname, search } = new URL(url)
const regional = `${origin}${pathname}${search}`
return { page: await deps.webFetch({ url: regional }, signal), url: regional }
}
}
async function runFallback(request, signal) {
const attempts = []
let empty
const deadline = now() + options.fallbackTimeoutMs
for (const [index, id] of options.engines.entries()) {
const engine = ENGINES[id]
const url = engine.url(request.query)
const host = new URL(url).host
const remaining = deadline - now()
if (remaining <= 0) { attempts.push(`${host}: skipped, out of time`); continue }
// Leave later engines a share of the budget when an earlier one hangs.
const { timeout, timer } = timeoutAfter(index === options.engines.length - 1 ? remaining : Math.min(remaining, Math.ceil(options.fallbackTimeoutMs / 2)))
let fetched
try {
fetched = await fetchPage(engine, url, signal === undefined ? timeout.signal : AbortSignal.any([signal, timeout.signal]))
} catch (error) {
if (signal?.aborted) throw aborted(signal)
attempts.push(`${host}: ${timeout.signal.aborted ? 'timed out' : `${error?.code ? `${error.code} ` : ''}${reasonOf(error)}`}`)
continue
} finally {
clearTimeout(timer)
}
const at = new URL(fetched.url).host
if (fetched.page.statusCode !== 200) { attempts.push(`${at}: HTTP ${fetched.page.statusCode}`); continue }
const parsed = engine.parse(fetched.page.body?.content ?? '')
if (parsed.error !== undefined) { attempts.push(`${at}: ${parsed.error}`); continue }
if (parsed.sources.length === 0) { empty ??= { engine, at }; attempts.push(`${at}: no results`); continue }
return { result: { engine, at, sources: parsed.sources } }
}
return empty === undefined ? { attempts } : { result: { ...empty, sources: [] } }
}
return {
id: AUTO_PROVIDER_ID,
available: () => options.native || options.fallback,
async search(request, signal) {
if (signal?.aborted) throw aborted(signal)
const native = await runNative(request, signal)
if (native.result !== undefined) return native.result
let fallbackReason = 'turned off (web-search-auto fallback: false)'
if (options.fallback) {
const fallback = await runFallback(request, signal)
if (fallback.result !== undefined) {
return {
content: `Search backend: keyless fallback, parsed from the public ${fallback.result.engine.name} results page at ${fallback.result.at} (no API key, no charge; less reliable than an API search). Native search was not used: ${native.reason}.`,
sources: fallback.result.sources,
truncated: false,
}
}
fallbackReason = fallback.attempts.join('; ')
}
throw new WebError([
'Web search is unavailable for this query.',
`Native search: ${native.reason}.`,
`Keyless fallback: ${fallbackReason}.`,
'Do not guess URLs or present pages you remember as search results. If you have a browser tool, open a search engine in it; otherwise tell the user that web search is unavailable and why.',
].join('\n'), 'WEB_SEARCH_UNAVAILABLE')
},
}
}
// ---------------------------------------------------------------------------
// Host wiring: read the asking Session's route through public dsh services only.
const stringRecord = value => (isRecord(value) ? Object.fromEntries(Object.entries(value).filter(([, item]) => typeof item === 'string')) : {})
// OpenCode's gateways turn away a request that does not name its conversation. DSH's
// adapter names it by the Session id, so a search for that Session does the same,
// unless the provider settings name one.
const OPENCODE_SESSION = 'x-opencode-session'
const OPENCODE_ROUTES = new Set(['opencode', 'opencode-go'])
const withConversation = (provider, headers, sessionId) => (!OPENCODE_ROUTES.has(provider) || sessionId === undefined
|| Object.keys(headers).some(name => name.toLowerCase() === OPENCODE_SESSION)
? headers
: { ...headers, [OPENCODE_SESSION]: sessionId })
/** The route of the Session that started the current tool call: protocol, endpoint, key name, and headers. */
export function routeOf(ctx) {
const session = ctx.get('agents')?.currentInitiator()?.session
const request = session?.requestContext()
const provider = str(request?.provider)
if (provider === undefined) return undefined
let section
try {
const entry = ctx.get('llm')?.listConfigurableProviders().find(item => item.provider === provider)
if (entry !== undefined) {
const descriptor = ctx.get('settings')?.describe().find(item => item.ns === entry.settingsNs)
section = entry.settingsPath.reduce((value, key) => value?.[key], descriptor?.value)
}
} catch { section = undefined }
const [catalogApi, catalogBase, catalogKey] = CATALOG[provider] ?? []
const account = provider === 'deepseek-account'
return {
provider,
model: str(request.model),
api: str(section?.api) ?? catalogApi,
baseURL: str(section?.baseURL) ?? (DSH_DEEPSEEK_ROUTES.has(provider) ? str(envOf(ctx)('DEEPSEEK_BASE_URL')) : undefined) ?? catalogBase,
apiKeyEnv: account ? undefined : str(section?.apiKeyEnv) ?? catalogKey,
account,
headers: withConversation(provider, stringRecord(section?.headers), str(session.id)),
}
}
const envOf = ctx => name => ctx.get('launchEnvironment')?.get(name)?.value ?? process.env[name]
export function apply(ctx, config = {}) {
const env = envOf(ctx)
ctx.web.registerSearchProvider(createAutoSearchProvider({
route: () => routeOf(ctx),
resolveKey: async (ref) => {
const credentials = ctx.get('credentials')
// Without the seam the environment is the whole credential plane, as in dsh's own adapters.
return credentials === undefined ? env(ref) : (await credentials.resolve(ref))?.value
},
resolveAccountToken: endpoint => ctx.get('deepseekAccount')?.resolveToken(endpoint),
fetch: (url, init) => fetch(url, init),
webFetch: (request, signal) => ctx.web.fetch(request, signal),
}, config))
}