1 line
51 KiB
JSON
1 line
51 KiB
JSON
{"repo": "berkay123001/osint-agent", "files": {"src/agents/baseAgent.ts": {"eb9e6c774b2718b76f4c7853313605002b616f3f": {"t": 1785596056.006314, "n": 50194, "h": "ddc2f82547fde4e6", "c": "import OpenAI from 'openai';\nimport type { ChatCompletion } from 'openai/resources/chat/completions';\nimport { normalizeAssistantMessage, normalizeToolContent } from '../lib/chatHistory.js';\nimport { buildProviderMessages as buildDurableProviderMessages } from '../lib/providerContextBuilder.js';\nimport { buildToolCacheKey, rebuildAgentSession } from '../lib/agentSession.js';\nimport { logger } from '../lib/logger.js';\nimport { emitProgress, emitSessionGraphDirty, emitTelemetry, emitToolDetail } from '../lib/progressEmitter.js';\nimport { buildLLMTelemetryEvent, persistLLMTelemetryEvent } from '../lib/llmTelemetry.js';\nimport { readFile } from 'fs/promises';\nimport path from 'path';\nimport { fileURLToPath } from 'url';\nimport type { Message, AgentConfig, AgentResult } from './types.js';\nimport { BloomFilter } from '../lib/bloomFilter.js';\n\nconst __filename = fileURLToPath(import.meta.url);\nconst __dirname = path.dirname(__filename);\n\nconst DEFAULT_MODEL = 'kwaipilot/kat-coder-pro-v2';\nconst SUPERVISOR_MODEL = 'qwen/qwen3.6-plus';\nexport { DEFAULT_MODEL, SUPERVISOR_MODEL };\n\n// Fallback models — tried sequentially on content filter (PII) or rate limit errors\nconst FALLBACK_MODELS = [\n 'qwen/qwen3.6-plus',\n 'minimax/minimax-m2.5',\n 'google/gemini-2.0-flash-001',\n 'deepseek/deepseek-chat-v3-0324',\n];\n\nconst DEFAULT_MAX_TOOL_CALLS = 30;\nconst DEFAULT_MAX_TOKENS = 32768; // Generous budget for Qwen3 thinking tokens\n\nconst client = new OpenAI({\n apiKey: process.env.OPENROUTER_API_KEY,\n baseURL: 'https://openrouter.ai/api/v1',\n});\n\n/**\n * Qwen3 models perform reasoning between <think/> tags.\n * This function strips those tags, leaving only the content to be returned to the user.\n */\nfunction stripThinkingTokens(text: string): string {\n return text.replace(/<think[\\s\\S]*?<\\/think>/gi, '').trim();\n}\n\nfunction isInternalControlPrompt(content: string): boolean {\n return content.startsWith('ALL TOOLS COMPLETED')\n || content.startsWith('TOOL_CALL_DISABLED')\n || content.startsWith('The previous tool call had invalid JSON format.')\n || content.startsWith('Do not call any tools. Present all collected data directly as text.')\n || content.startsWith('IMPORTANT: Using the tool results above, write a comprehensive Markdown report NOW.')\n || content.startsWith('Stop calling tools. Summarize ALL the information you have collected above as PLAIN TEXT ONLY.')\n || content.startsWith('[STRATEGY REVIEW — Correction Required]')\n || content.startsWith('[STRATEGY REVIEW — Final Report Required]')\n || content.startsWith('⚠️ STAGNATION DETECTED')\n || content.startsWith('⚠️ TOOL DIVERSITY REQUIREMENT');\n}\n\nexport async function runAgentLoop(\n history: Message[],\n config: AgentConfig,\n _clientOverride?: OpenAI,\n _retryDelayMs?: number\n): Promise<AgentResult> {\n const _client = _clientOverride ?? client;\n const _delay = (ms: number) => new Promise<void>(resolve => setTimeout(resolve, _retryDelayMs ?? ms));\n let session = rebuildAgentSession(config.name, history);\n let toolCallCount = 0;\n let toolCallCountFloor = 0;\n let emptyRetries = 0;\n let correctionRetries = 0;\n let totalCorrectionAttempts = 0; // Global cap — prevents infinite loops\n let forceTextRetries = 0; // Text forcing retry counter\n let toolsDisabledMessageSent = false; // Summary request sent only once when tool budget is exhausted\n let toolsUsed: Record<string, number> = {};\n const pushHistory = (message: Message): void => {\n history.push(message);\n emitSessionGraphDirty();\n };\n const maxToolCalls = config.maxToolCalls ?? DEFAULT_MAX_TOOL_CALLS;\n // Tool call deduplication: return from cache if the same tool+args combination was called before\n let callCache = new Map<string, string>();\n // Per-tool hard limit: prevents a single tool from consuming the entire budget\n const PER_TOOL_LIMITS: Record<string, number> = {\n search_academic_papers: 8,\n search_web: 20,\n search_web_multi: 8,\n search_researcher_papers: 8,\n };\n let perToolCount: Record<string, number> = {};\n // Stagnation detection: track consecutive low-yield calls\n let lowYieldStreak = 0;\n const LOW_YIELD_THRESHOLD = 5; // After this many consecutive low-yield calls, force synthesis\n let seenUrls = new Set<string>(); // Track unique URLs found across all search results (exact, for session persistence)\n let urlBloom = new BloomFilter(10_000, 0.01); // O(1) URL membership — Bloom Filter deduplication\n let completionAttempt = 0;\n\n function refreshRuntimeState(): void {\n session = rebuildAgentSession(config.name, history);\n toolCallCount = Math.max(toolCallCountFloor, session.runtime.toolCallCount);\n toolsUsed = { ...session.runtime.toolsUsed };\n callCache = new Map(Object.entries(session.runtime.duplicateToolCache));\n perToolCount = { ...session.runtime.perToolCount };\n lowYieldStreak = session.runtime.lowYieldStreak;\n seenUrls = new Set(session.runtime.seenUrls);\n urlBloom = new BloomFilter(10_000, 0.01);\n session.runtime.seenUrls.forEach(url => urlBloom.add(url));\n }\n\n function buildRequestMessages(modelName: string): Message[] {\n const prepared = buildDurableProviderMessages(history, {\n agentName: config.name,\n modelName,\n session,\n });\n session = prepared.session;\n return prepared.messages;\n }\n\n // Tools that return free-form page content — broad error words like \"error analysis\",\n // \"failure modes\", \"failed state\" are benign in their output and must not suppress success.\n const broadContentTools = new Set(['web_fetch', 'nitter_profile', 'auto_visual_intel']);\n\n function isSuccessfulVerificationResult(toolName: string, result: string): boolean {\n const normalized = result\n .trim()\n .toLocaleLowerCase('tr-TR')\n .replace(/ı/g, 'i')\n .normalize('NFKD')\n .replace(/[\\u0300-\\u036f]/g, '');\n // Patterns that unambiguously mean the tool found nothing or hit a gate —\n // safe to apply to ALL tools including broad-content ones.\n const emptyVerificationPatterns = [\n /\\bno (profiles?|results?|matches?|identifiers?|evidence|data|findings?)\\b/,\n /\\bnothing found\\b/,\n /\\bcould not verify\\b/,\n /\\bnot found\\b/,\n /\\b0 profiles?\\b/,\n /\\b0 matches?\\b/,\n /\\bsign in to continue\\b/,\n /\\blogin required\\b/,\n /\\bcreate an account\\b/,\n /\\bagree & join\\b/,\n /\\bsign up\\b/,\n /\\byetersiz kanit\\b/,\n /\\bdogrulanamadi\\b/,\n /\\bhicbir dogrulanmis tanimlayici bulunamadi\\b/,\n /\\bgrafta dogrulanacak profil yok\\b/,\n /\\btimeout\\b/,\n /\\btimed out\\b/,\n /\\bunavailable\\b/,\n /\\bunreachable\\b/,\n /\\bconnection refused\\b/,\n /\\brate limit\\b/,\n /\\bbaglanti kurulamadi\\b/,\n /\\blogin wall\\b/,\n /\\bforbidden\\b/,\n /\\bdenied\\b/,\n /\\bcaptcha\\b/,\n ];\n // Broad single-word patterns that are ambiguous in free-form page content (e.g. \"error\n // analysis\", \"failed state\", \"failure mode\"). Only applied to narrow-output tools.\n const strictErrorPatterns = [\n /\\berror\\b/,\n /\\bfailed\\b/,\n /\\bfailure\\b/,\n ];\n const isBroadContentTool = broadContentTools.has(toolName);\n const activePatterns = isBroadContentTool\n ? emptyVerificationPatterns\n : [...emptyVerificationPatterns, ...strictErrorPatterns];\n const explicitPositivePatternsByTool: Partial<Record<string, RegExp[]>> = {\n cross_reference: [\n /\\bverified\\b/,\n /\\bconfirmed\\b/,\n /\\bindependent evidence\\b/,\n /\\bcorroborated\\b/,\n /\\bmatch(?:ed)?\\b/,\n /\\bsame (?:email|avatar|organization|location)\\b/,\n /\\bcapraz dogrulama sonuclari\\b/,\n /\\bdogrulanmis tanimlayicilar\\b/,\n ],\n verify_claim: [\n /\\bverified\\b/,\n /\\bconfirmed\\b/,\n /\\bsupported\\b/,\n /\\bdogrulandi\\b/,\n /\\bonaylandi\\b/,\n /\\btrue\\b/,\n /\\biddia dogrulama\\b[\\s\\S]{0,40}\\bguven:\\s*(yuksek|orta|dusuk)\\b/,\n ],\n };\n\n const explicitPositivePatterns = explicitPositivePatternsByTool[toolName];\n\n if (toolName === 'verify_profiles') {\n const verifiedCountMatch = normalized.match(/(?:\\bsonuc\\b:?\\s*)?(\\d+)\\s+(?:dogrulandi|verified)\\b/);\n if (verifiedCountMatch) {\n return Number.parseInt(verifiedCountMatch[1], 10) > 0;\n }\n\n const explicitPositiveVerificationPatterns = [\n /^\\s*[✅✔]/m,\n /\\bverified\\b/,\n /\\bdogrulandi\\b/,\n /\\bconfirmed\\b/,\n /\\bmatch\\b/,\n ];\n\n return isUsableRecoveryToolResult(result)\n && !activePatterns.some(pattern => pattern.test(normalized))\n && explicitPositiveVerificationPatterns.some(pattern => pattern.test(normalized));\n }\n\n return isUsableRecoveryToolResult(result)\n && !activePatterns.some(pattern => pattern.test(normalized))\n && (!explicitPositivePatterns || explicitPositivePatterns.some(pattern => pattern.test(normalized)));\n }\n\n function normalizeToolMessageContent(content: unknown): string {\n return typeof content === 'string'\n ? content\n : Array.isArray(content)\n ? content.map((part: { text?: string }) => part.text ?? '').join('')\n : '';\n }\n\n function collectRecoveryToolEvidenceBlocks(): string[] {\n const toolMetadataById = new Map<string, { name: string; args: string }>();\n const evidenceBlocks: string[] = [];\n\n for (const message of history) {\n const assistantMessage = message as { role: string; tool_calls?: Array<{ id: string; type: string; function: { name: string; arguments: string } }> };\n if (assistantMessage.role === 'assistant' && assistantMessage.tool_calls) {\n for (const toolCall of assistantMessage.tool_calls) {\n if (toolCall.type === 'function') {\n toolMetadataById.set(toolCall.id, {\n name: toolCall.function.name,\n args: toolCall.function.arguments,\n });\n }\n }\n continue;\n }\n\n const toolMessage = message as { role: string; tool_call_id?: string; content?: unknown };\n if (toolMessage.role !== 'tool') {\n continue;\n }\n\n const result = normalizeToolMessageContent(toolMessage.content);\n if (!isUsableRecoveryToolResult(result)) {\n continue;\n }\n\n const metadata = toolMessage.tool_call_id ? toolMetadataById.get(toolMessage.tool_call_id) : undefined;\n if (!metadata) {\n evidenceBlocks.push(result);\n continue;\n }\n\n evidenceBlocks.push(\n `Tool: ${metadata.name}\\nArgs: ${metadata.args.slice(0, 400)}\\nResult:\\n${result}`\n );\n }\n\n return evidenceBlocks;\n }\n\n function isUsableRecoveryToolResult(result: string): boolean {\n const normalized = result.trim().toLowerCase();\n\n if (normalized.length === 0) {\n return false;\n }\n\n if (result.startsWith('Tool error (')\n || result.startsWith('[TOOL_LIMIT]')\n || result.startsWith('[DUPLICATE_CALL]')\n || result.startsWith('[TOOL_ARGS_INVALID]')\n || normalized === 'tool produced no output.') {\n return false;\n }\n\n return true;\n }\n\n function countSuccessfulVerificationCalls(verificationTools: string[]): number {\n const toolNameById = new Map<string, string>();\n let successfulVerificationCount = 0;\n\n for (const message of history) {\n const assistantMessage = message as { role: string; tool_calls?: Array<{ id: string; type: string; function: { name: string } }> };\n if (assistantMessage.role === 'assistant' && assistantMessage.tool_calls) {\n for (const toolCall of assistantMessage.tool_calls) {\n if (toolCall.type === 'function') {\n toolNameById.set(toolCall.id, toolCall.function.name);\n }\n }\n continue;\n }\n\n const toolMessage = message as { role: string; tool_call_id?: string; content?: unknown };\n if (toolMessage.role !== 'tool' || !toolMessage.tool_call_id) {\n continue;\n }\n\n const toolName = toolNameById.get(toolMessage.tool_call_id);\n if (!toolName || !verificationTools.includes(toolName)) {\n continue;\n }\n\n const content = normalizeToolMessageContent(toolMessage.content);\n\n if (isSuccessfulVerificationResult(toolName, content)) {\n successfulVerificationCount++;\n }\n }\n\n return successfulVerificationCount;\n }\n\n async function createTrackedCompletion(\n request: {\n model: string\n messages: Message[]\n tools?: OpenAI.Chat.ChatCompletionTool[]\n tool_choice?: 'auto' | 'none'\n max_tokens: number\n },\n meta: {\n reason: string\n phase?: string\n }\n ): Promise<ChatCompletion> {\n const startedAt = Date.now();\n const attempt = ++completionAttempt;\n\n try {\n const completion = (await _client.chat.completions.create(request) as ChatCompletion);\n const telemetry = buildLLMTelemetryEvent({\n agent: config.name,\n phase: meta.phase,\n reason: meta.reason,\n attempt,\n requestedModel: request.model,\n actualModel: completion.model,\n responseId: completion.id,\n status: 'success',\n latencyMs: Date.now() - startedAt,\n messages: request.messages,\n usage: completion.usage as any,\n });\n emitTelemetry(telemetry);\n void persistLLMTelemetryEvent(telemetry).catch((error) => {\n emitProgress(`⚠️ [${config.name}] Telemetry persist failed: ${(error as Error).message}`);\n });\n return completion;\n } catch (error) {\n const telemetry = buildLLMTelemetryEvent({\n agent: config.name,\n phase: meta.phase,\n reason: meta.reason,\n attempt,\n requestedModel: request.model,\n status: 'error',\n latencyMs: Date.now() - startedAt,\n messages: request.messages,\n errorMessage: error instanceof Error ? error.message : String(error),\n });\n emitTelemetry(telemetry);\n void persistLLMTelemetryEvent(telemetry).catch((persistError) => {\n emitProgress(`⚠️ [${config.name}] Telemetry persist failed: ${(persistError as Error).message}`);\n });\n throw error;\n }\n }\n\n while (true) {\n refreshRuntimeState();\n const toolsDisabledByBudget = toolCallCount >= maxToolCalls;\n const toolsDisabled = toolsDisabledByBudget || session.runtime.toolsDisabled;\n const toolChoice: 'auto' | 'none' = toolsDisabled ? 'none' : 'auto';\n if (toolsDisabledByBudget && !toolsDisabledMessageSent) {\n toolsDisabledMessageSent = true;\n logger.warn('AGENT', `[${config.name}] Maximum tool calls exceeded, requesting final report...`);\n // Explicit guidance to prevent the model from losing context and returning a greeting\n pushHistory({\n role: 'user',\n content:\n 'ALL TOOLS COMPLETED. Using the results from the tools executed above, ' +\n 'write the final Markdown report NOW. ' +\n 'Do not call any new tools — present a complete and detailed final report from existing findings only. ' +\n 'Do NOT write a greeting or \"how can I help you\".',\n });\n }\n\n logger.agentThinking(config.name);\n\n // OpenAI SDK: create() method has 2 overloads:\n // 1) stream:false → returns ChatCompletion\n // 2) stream:true → returns Stream<ChatCompletionChunk>\n // TypeScript infers a union type covering both: Stream | ChatCompletion\n // Since we don't use stream:true, we always get ChatCompletion,\n // but TS can't know this → .choices access causes a compile error (Stream has no .choices).\n // Solution: skip TS type checking with `any` type + cast as ChatCompletion on each call.\n let response: ChatCompletion | undefined;\n try {\n response = await createTrackedCompletion({\n model: config.model ?? DEFAULT_MODEL,\n messages: buildRequestMessages(config.model ?? DEFAULT_MODEL),\n tools: config.tools.length > 0 ? config.tools : undefined,\n tool_choice: config.tools.length > 0 ? toolChoice : undefined,\n max_tokens: config.maxTokens ?? DEFAULT_MAX_TOKENS,\n }, {\n reason: 'initial',\n });\n } catch (apiError: unknown) {\n const msg = apiError instanceof Error ? apiError.message : String(apiError);\n const currentModel = config.model ?? DEFAULT_MODEL;\n\n // Alibaba content filter — switch to fallback model when PII content is detected\n if (msg.includes('DataInspectionFailed') || msg.includes('inappropriate content')) {\n const fallbackModel = FALLBACK_MODELS[0]; // Gemini\n logger.warn('AGENT', `[${config.name}] Alibaba content filter → switching to ${fallbackModel} model...`);\n try {\n response = await createTrackedCompletion({\n model: fallbackModel,\n messages: buildRequestMessages(fallbackModel),\n tools: config.tools.length > 0 ? config.tools : undefined,\n tool_choice: config.tools.length > 0 ? toolChoice : undefined,\n max_tokens: config.maxTokens ?? DEFAULT_MAX_TOKENS,\n }, {\n reason: 'content-filter-fallback',\n });\n } catch (fallbackErr) {\n // If fallback model also refuses → graceful degradation, don't crash\n logger.warn('AGENT', `[${config.name}] Fallback model also failed — content filter cannot be bypassed.`);\n return {\n finalResponse: '⚠️ This query was caught by the content filter. Try rephrasing the question or changing the topic.',\n toolsUsed,\n toolCallCount,\n history,\n };\n }\n }\n // OpenRouter rate limit — exponential backoff + fallback model chain\n else if (msg.includes('429') || msg.toLowerCase().includes('rate limit') || msg.toLowerCase().includes('too many requests')) {\n // 1st attempt: same model, wait 5s\n const waitMs = 5000;\n logger.warn('AGENT', `[${config.name}] Rate limit (429) — waiting ${waitMs / 1000}s before retrying...`);\n await _delay(waitMs);\n try {\n response = await createTrackedCompletion({\n model: currentModel,\n messages: buildRequestMessages(currentModel),\n tools: config.tools.length > 0 ? config.tools : undefined,\n tool_choice: config.tools.length > 0 ? toolChoice : undefined,\n max_tokens: config.maxTokens ?? DEFAULT_MAX_TOKENS,\n }, {\n reason: 'rate-limit-retry',\n });\n } catch (retryErr) {\n // 2nd attempt: try fallback models sequentially\n const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr);\n if (retryMsg.includes('429') || retryMsg.toLowerCase().includes('rate limit')) {\n for (const fbModel of FALLBACK_MODELS) {\n if (fbModel === currentModel) continue; // skip same model\n logger.warn('AGENT', `[${config.name}] 429 persists — trying fallback: ${fbModel}`);\n await _delay(2000);\n try {\n response = await createTrackedCompletion({\n model: fbModel,\n messages: buildRequestMessages(fbModel),\n tools: config.tools.length > 0 ? config.tools : undefined,\n tool_choice: config.tools.length > 0 ? toolChoice : undefined,\n max_tokens: config.maxTokens ?? DEFAULT_MAX_TOKENS,\n }, {\n reason: 'rate-limit-fallback',\n });\n logger.info('AGENT', `[${config.name}] Fallback successful: ${fbModel}`);\n break; // success → exit loop\n } catch {\n // this fallback also failed → try next\n continue;\n }\n }\n if (!response?.choices?.[0]) {\n throw new Error(`All models rate limited (429) — wait a moment and try again.`);\n }\n } else {\n throw retryErr;\n }\n }\n }\n // Other transient errors (502, 504, 529) — retry with same model\n else if (msg.includes('502') || msg.includes('504') || msg.includes('529') || msg.includes('InternalError') || msg.toLowerCase().includes('aborted') || msg.toLowerCase().includes('timeout')) {\n logger.warn('AGENT', `[${config.name}] API transient error (${msg.slice(0, 60)}...), waiting 3s before retrying...`);\n await _delay(3000);\n response = await createTrackedCompletion({\n model: currentModel,\n messages: buildRequestMessages(currentModel),\n tools: config.tools.length > 0 ? config.tools : undefined,\n tool_choice: config.tools.length > 0 ? toolChoice : undefined,\n max_tokens: config.maxTokens ?? DEFAULT_MAX_TOKENS,\n }, {\n reason: 'transient-retry',\n });\n }\n // 400 Provider errors — try fallback models\n else if (msg.includes('400') || msg.includes('Provider returned error')) {\n logger.warn('AGENT', `[${config.name}] 400 Provider error (${msg.slice(0, 80)}...) — trying fallback models...`);\n let lastErr = apiError;\n for (const fbModel of FALLBACK_MODELS) {\n if (fbModel === currentModel) continue;\n try {\n logger.info('AGENT', `[${config.name}] 400 fallback → trying ${fbModel}`);\n response = await createTrackedCompletion({\n model: fbModel,\n messages: buildRequestMessages(fbModel),\n tools: config.tools.length > 0 ? config.tools : undefined,\n tool_choice: config.tools.length > 0 ? toolChoice : undefined,\n max_tokens: config.maxTokens ?? DEFAULT_MAX_TOKENS,\n }, {\n reason: '400-provider-fallback',\n });\n logger.info('AGENT', `[${config.name}] 400 fallback successful: ${fbModel}`);\n break;\n } catch (fbErr) {\n lastErr = fbErr;\n // try next fallback\n }\n }\n if (!response) throw lastErr;\n }\n else {\n throw apiError;\n }\n }\n\n // OpenRouter sometimes returns an object without choices on error\n const respAny = response as unknown as Record<string, unknown>;\n if (!response?.choices?.[0]) {\n if (respAny['error']) {\n const upstreamErr = JSON.stringify(respAny['error']);\n // Log actual error content — to understand what broke\n logger.error('AGENT', `[${config.name}] OpenRouter upstream error: ${upstreamErr.slice(0, 500)}`);\n\n // 429 rate limit → try fallback model chain (429 received in response body)\n if (upstreamErr.includes('429') || upstreamErr.toLowerCase().includes('rate limit') || upstreamErr.toLowerCase().includes('too many requests')) {\n const currentModel = config.model ?? DEFAULT_MODEL;\n for (const fbModel of FALLBACK_MODELS) {\n if (fbModel === currentModel) continue;\n logger.warn('AGENT', `[${config.name}] Response-body 429 → trying fallback: ${fbModel}`);\n await _delay(2000);\n try {\n response = await createTrackedCompletion({\n model: fbModel,\n messages: buildRequestMessages(fbModel),\n tools: config.tools.length > 0 ? config.tools : undefined,\n tool_choice: config.tools.length > 0 ? toolChoice : undefined,\n max_tokens: config.maxTokens ?? DEFAULT_MAX_TOKENS,\n }, {\n reason: 'response-body-rate-limit-fallback',\n });\n if (response?.choices?.[0]) {\n logger.info('AGENT', `[${config.name}] Fallback successful: ${fbModel}`);\n break;\n }\n } catch {\n continue;\n }\n }\n if (!response?.choices?.[0]) {\n throw new Error(`All models rate limited (429) — wait a moment and try again.`);\n }\n }\n // Alibaba content filter → switch to fallback model\n else if (upstreamErr.includes('DataInspectionFailed')) {\n const fallbackModel = FALLBACK_MODELS[0];\n logger.warn('AGENT', `[${config.name}] Alibaba DataInspection → trying ${fallbackModel} fallback...`);\n response = await createTrackedCompletion({\n model: fallbackModel,\n messages: buildRequestMessages(fallbackModel),\n tools: config.tools.length > 0 ? config.tools : undefined,\n tool_choice: config.tools.length > 0 ? toolChoice : undefined,\n max_tokens: config.maxTokens ?? DEFAULT_MAX_TOKENS,\n }, {\n reason: 'response-body-content-filter-fallback',\n });\n // Check again after fallback\n if (!response?.choices?.[0]) {\n throw new Error(`Model error (fallback also failed): ${upstreamErr.slice(0, 200)}`);\n }\n }\n // 502 / tokenization error (TextEncodeInput) → switch to fallback model\n // DeepSeek V4 Flash crashes with \"TextEncodeInput must be Union...\" when context is large\n else if (upstreamErr.includes('TextEncodeInput')) {\n const currentModel502 = config.model ?? DEFAULT_MODEL;\n logger.warn('AGENT', `[${config.name}] 502/tokenization error → trying fallback model chain...`);\n let recovered = false;\n for (const fbModel of FALLBACK_MODELS) {\n if (fbModel === currentModel502) continue;\n await _delay(2000);\n try {\n response = await createTrackedCompletion({\n model: fbModel,\n messages: buildRequestMessages(fbModel),\n tools: config.tools.length > 0 ? config.tools : undefined,\n tool_choice: config.tools.length > 0 ? toolChoice : undefined,\n max_tokens: config.maxTokens ?? DEFAULT_MAX_TOKENS,\n }, {\n reason: 'response-body-502-fallback',\n });\n if (response?.choices?.[0]) {\n logger.info('AGENT', `[${config.name}] 502 fallback successful: ${fbModel}`);\n recovered = true;\n break;\n }\n } catch { continue; }\n }\n if (!recovered || !response?.choices?.[0]) {\n // Last resort: force text response with truncated context\n const toolResults = collectRecoveryToolEvidenceBlocks().join('\\n---\\n');\n if (toolResults.trim().length === 0) {\n throw new Error(`502/tokenization error — all fallbacks failed: ${upstreamErr.slice(0, 200)}`);\n }\n logger.warn('AGENT', `[${config.name}] All 502 fallbacks failed — forcing synthesis with truncated context.`);\n const truncatedData = toolResults.length > 10000 ? toolResults.slice(0, 10000) + '\\n[...truncated]' : toolResults;\n const recoverySystemPrompt = history.find(message => message.role === 'system' && typeof message.content === 'string');\n const recoveryUserPrompt = [...history].reverse().find(\n message => message.role === 'user'\n && typeof message.content === 'string'\n && !isInternalControlPrompt(message.content as string)\n );\n try {\n const cleanResp = await createTrackedCompletion({\n model: FALLBACK_MODELS[0],\n messages: [\n {\n role: 'system',\n content: `${typeof recoverySystemPrompt?.content === 'string' ? recoverySystemPrompt.content.slice(0, 3000) + '\\n\\n' : ''}Summarize the research data in Markdown. Preserve the original task filters, evidence rules, and limitation handling. Write only the report.`,\n },\n {\n role: 'user',\n content: `Original task:\\n${typeof recoveryUserPrompt?.content === 'string' ? recoveryUserPrompt.content : 'Unknown'}\\n\\nResearch data:\\n${truncatedData}`,\n },\n ],\n max_tokens: config.maxTokens ?? DEFAULT_MAX_TOKENS,\n }, {\n reason: '502-truncated-recovery',\n phase: 'recovery',\n });\n const text = cleanResp.choices?.[0]?.message?.content?.trim() ?? '';\n const cleanedText = stripThinkingTokens(text);\n if (cleanedText.length > 50) {\n pushHistory({ role: 'assistant', content: cleanedText });\n return { finalResponse: cleanedText, toolCallCount, toolsUsed, session: rebuildAgentSession(config.name, history), history };\n }\n } catch { /* clean call also failed */ }\n throw new Error(`502/tokenization error — all fallbacks failed: ${upstreamErr.slice(0, 200)}`);\n }\n }\n // Invalid JSON argument error → give the model a chance to correct (max 3 attempts)\n // 504 upstream timeout / operation aborted → try fallback model chain\n else if (upstreamErr.includes('504') || upstreamErr.toLowerCase().includes('aborted') || upstreamErr.toLowerCase().includes('timeout')) {\n const currentModel504 = config.model ?? DEFAULT_MODEL;\n logger.warn('AGENT', `[${config.name}] 504/timeout upstream error → trying fallback model chain...`);\n let recovered = false;\n for (const fbModel of FALLBACK_MODELS) {\n if (fbModel === currentModel504) continue;\n await _delay(3000);\n try {\n response = await createTrackedCompletion({\n model: fbModel,\n messages: buildRequestMessages(fbModel),\n tools: config.tools.length > 0 ? config.tools : undefined,\n tool_choice: config.tools.length > 0 ? toolChoice : undefined,\n max_tokens: config.maxTokens ?? DEFAULT_MAX_TOKENS,\n }, {\n reason: 'response-body-timeout-fallback',\n });\n if (response?.choices?.[0]) {\n logger.info('AGENT', `[${config.name}] Fallback successful after 504: ${fbModel}`);\n recovered = true;\n break;\n }\n } catch { continue; }\n }\n if (!recovered || !response?.choices?.[0]) {\n throw new Error(`504 upstream timeout — all fallback models failed. Wait a moment and try again.`);\n }\n }\n else if (upstreamErr.includes('function.arguments') || upstreamErr.includes('InvalidParameter')) {\n // When tools are disabled or model still tries to call tools after 3 corrections\n if (toolsDisabled || totalCorrectionAttempts >= 3) {\n forceTextRetries++\n if (forceTextRetries > 1) {\n // Still failing on 2nd attempt → make a clean API call (with rich context from session files)\n logger.warn('AGENT', `[${config.name}] Model insists on calling tools — trying clean text response with context from session files.`)\n\n // Read rich context from session files (instead of 2000 char limit)\n let contextSnippet = ''\n const sessionDir = path.resolve(__dirname, '../../.osint-sessions')\n const sessionFiles = [\n 'academic-last-session.md',\n 'identity-last-session.md',\n 'media-last-session.md',\n 'academic-knowledge.md',\n 'identity-knowledge.md',\n 'media-knowledge.md',\n ]\n for (const f of sessionFiles) {\n try {\n const content = await readFile(path.join(sessionDir, f), 'utf-8')\n if (content.length > 100) {\n // Take at most 3000 chars from each session file, max 8000 total\n const slice = content.length > 3000 ? content.slice(0, 3000) + '\\n[...truncated]' : content\n contextSnippet += `\\n\\n--- ${f} ---\\n${slice}`\n if (contextSnippet.length > 8000) break\n }\n } catch { /* skip if file doesn't exist */ }\n }\n\n // If session files are empty, get context from history\n if (contextSnippet.length < 200) {\n const toolResults = history\n .filter(m => m.role === 'tool' && typeof m.content === 'string')\n .map(m => (m.content as string))\n contextSnippet = toolResults.join('\\n---\\n')\n if (contextSnippet.length > 6000) {\n contextSnippet = contextSnippet.slice(0, 6000) + '\\n[...truncated]'\n }\n }\n\n const lastUserMsg = [...history].reverse().find(m => m.role === 'user' && typeof m.content === 'string' && !(m.content as string).startsWith('TOOL_CALL_DISABLED') && !(m.content as string).startsWith('JSON'))\n const userQuestion = lastUserMsg && typeof lastUserMsg.content === 'string' ? lastUserMsg.content : 'User question'\n\n try {\n const cleanResponse = await createTrackedCompletion({\n model: config.model ?? DEFAULT_MODEL,\n messages: [\n { role: 'system', content: 'You are an OSINT research assistant. Analyze the provided research data and summarize it in Markdown format. Do not call any tools — only write plain text.' },\n { role: 'user', content: `The user asked: \"${userQuestion}\"\\n\\nBelow is the research data. Create a detailed Markdown report using this data:\\n\\n${contextSnippet}` },\n ],\n // no tools parameter — model cannot call tools\n max_tokens: config.maxTokens ?? DEFAULT_MAX_TOKENS,\n }, {\n reason: 'clean-text-recovery',\n phase: 'recovery',\n })\n const text = cleanResponse.choices?.[0]?.message?.content?.trim()\n if (text) {\n const cleaned = stripThinkingTokens(text)\n if (cleaned.length > 50) {\n pushHistory({ role: 'assistant', content: cleaned });\n return { finalResponse: cleaned, toolCallCount, toolsUsed, session: rebuildAgentSession(config.name, history) }\n }\n }\n } catch (cleanErr) {\n logger.error('AGENT', `[${config.name}] Clean API call also failed: ${(cleanErr as Error).message}`)\n }\n const fallbackResponse = 'Model could not generate a response. Collected data has been saved to session files — check the `.osint-sessions/` directory.';\n pushHistory({ role: 'assistant', content: fallbackResponse });\n return {\n finalResponse: fallbackResponse,\n toolCallCount,\n toolsUsed,\n session: rebuildAgentSession(config.name, history),\n history,\n }\n }\n logger.warn('AGENT', `[${config.name}] JSON error + tools disabled — forcing text response (${forceTextRetries}/1).`)\n // Fix broken history: add assistant placeholder after tool message\n const lastMsgForce = history[history.length - 1];\n if (lastMsgForce && (lastMsgForce.role === 'tool' || lastMsgForce.role === 'user')) {\n if (lastMsgForce.role === 'tool') {\n pushHistory({ role: 'assistant', content: 'Tool call failed, providing text response.' });\n }\n pushHistory({ role: 'user', content: 'Do not call any tools. Present all collected data directly as text.' });\n }\n config = { ...config, tools: [] }\n continue\n }\n\n correctionRetries++;\n totalCorrectionAttempts++;\n logger.warn('AGENT', `[${config.name}] Model produced invalid JSON, requesting correction... (attempt ${correctionRetries}/3, total: ${totalCorrectionAttempts})`);\n\n // CRITICAL: If the last message in history is 'tool', add a placeholder assistant message\n\t // before the correction user message. Otherwise:\n // tool → user (INVALID FORMAT — OpenRouter/model produces broken response)\n // tool → assistant → user (CORRECT FORMAT)\n const lastMsg = history[history.length - 1];\n if (lastMsg && lastMsg.role === 'tool') {\n pushHistory({\n role: 'assistant',\n content: 'Tool call produced invalid JSON, correcting.',\n });\n }\n\n // Global cap: after 3 total attempts, force text response\n if (totalCorrectionAttempts >= 3) {\n logger.error('AGENT', `[${config.name}] JSON correction failed 3 times — disabling tool calls.`);\n pushHistory({\n role: 'user',\n content:\n 'TOOL_CALL_DISABLED. Respond directly in Markdown text using all the information you have collected. ' +\n 'Do not call any tools — write text only.',\n });\n toolCallCountFloor = maxToolCalls + 1;\n correctionRetries = 0;\n continue;\n }\n\n if (correctionRetries >= 2) {\n correctionRetries = 0;\n pushHistory({\n role: 'user',\n content:\n 'TOOL_CALL_DISABLED FAILED. Stop calling tools and summarize the collected data directly in Markdown text. ' +\n 'Do not call any tools — not save_finding, not generate_report, NOTHING. Write plain text only.',\n });\n } else {\n pushHistory({\n role: 'user',\n content: 'The previous tool call had invalid JSON format. ' +\n 'Call a single tool and keep the arguments very short. ' +\n 'If you are trying to call too many tools, STOP and write the results as text.',\n });\n }\n continue;\n }\n if (!response?.choices?.[0]) {\n throw new Error(`Upstream API error: ${upstreamErr}`);\n }\n }\n if (!response?.choices?.[0]) {\n throw new Error(`Invalid API response: no choices. Response: ${JSON.stringify(response)}`);\n }\n }\n\n const message = response.choices[0].message;\n\n // Sanitize tool_calls with invalid JSON arguments BEFORE pushing to history.\n // A broken function.arguments in an assistant message causes Alibaba to reject\n // every subsequent API call with InvalidParameter (502). By replacing bad args\n // with a sentinel object here we keep the history structurally valid so the\n // next request succeeds and the model receives a clear error result.\n const sanitizedToolCalls = message.tool_calls?.map(tc => {\n if (tc.type !== 'function') return tc;\n try {\n JSON.parse(tc.function.arguments);\n return tc; // valid JSON — no change\n } catch {\n logger.warn('AGENT',\n `[${config.name}] Tool call '${tc.function.name}' has malformed JSON args (likely content too long) — sanitizing`);\n return {\n ...tc,\n function: { ...tc.function, arguments: JSON.stringify({ _sanitized: true }) },\n };\n }\n });\n\n const messageForHistory = sanitizedToolCalls\n ? ({ ...message, tool_calls: sanitizedToolCalls } as typeof message)\n : message;\n pushHistory(normalizeAssistantMessage(messageForHistory));\n\n if (!message.tool_calls || message.tool_calls.length === 0) {\n const rawContent = typeof message.content === 'string' ? message.content : '';\n const cleanContent = stripThinkingTokens(rawContent);\n\n const refusalText = typeof message.refusal === 'string' && message.refusal.trim().length > 0\n ? message.refusal\n : '';\n\n // Thinking models may write everything inside <think>...</think> and leave the final response empty.\n // stripThinkingTokens() removes those tags → cleanContent = \"\".\n // Solution: 3 retries with increasing aggressiveness:\n // 1. Politely request a report\n // 2. Disable tools, request text only\n // 3. Make a new clean API call (system prompt + only the last tool results)\n if (cleanContent.length === 0 && emptyRetries < 3) {\n emptyRetries++;\n logger.warn('AGENT', `[${config.name}] Model returned empty response (attempt ${emptyRetries}/3)...`);\n\n if (emptyRetries === 1) {\n pushHistory({\n role: 'user',\n content: 'IMPORTANT: Using the tool results above, write a comprehensive Markdown report NOW. ' +\n 'Write your answer DIRECTLY OUTSIDE of any <think> tag. I want the final report, not a thinking block.'\n });\n } else if (emptyRetries === 2) {\n // Disable tools completely, request text-only response\n config = { ...config, tools: [] };\n pushHistory({\n role: 'user',\n content: 'Stop calling tools. Summarize ALL the information you have collected above as PLAIN TEXT ONLY. ' +\n 'Use Markdown headings and lists. Do NOT use <think> tags — write only the final answer.'\n });\n } else {\n // 3rd attempt: collect tool results from history and make a clean API call\n const toolResults = history\n .filter(m => m.role === 'tool' && typeof m.content === 'string')\n .map(m => m.content as string)\n .join('\\n---\\n')\n .slice(0, 8000);\n try {\n const cleanResp = await createTrackedCompletion({\n model: config.model ?? DEFAULT_MODEL,\n messages: [\n { role: 'system', content: 'You are an assistant that summarizes research data in Markdown format. Write only the final report, do not use <think>.' },\n { role: 'user', content: `Research data:\\n${toolResults}\\n\\nCreate a detailed Markdown report using this data.` },\n ],\n max_tokens: config.maxTokens ?? DEFAULT_MAX_TOKENS,\n }, {\n reason: 'empty-response-clean-recovery',\n phase: 'recovery',\n });\n const text = cleanResp.choices?.[0]?.message?.content?.trim() ?? '';\n const cleaned = stripThinkingTokens(text);\n if (cleaned.length > 50) {\n pushHistory({ role: 'assistant', content: cleaned });\n return { finalResponse: cleaned, toolCallCount, toolsUsed, session: rebuildAgentSession(config.name, history) };\n }\n } catch { /* clean call also failed → return fallback message below */ }\n }\n continue;\n }\n\n const finalText = cleanContent.length > 0\n ? cleanContent\n : (refusalText || 'Tools completed but the model returned an empty response.');\n\n return {\n finalResponse: finalText,\n toolCallCount,\n toolsUsed,\n session: rebuildAgentSession(config.name, history),\n history,\n };\n }\n\n const pendingFollowUpUserMessages: string[] = [];\n const verificationTools = ['run_sherlock', 'run_maigret', 'cross_reference', 'verify_profiles',\n 'run_github_osint', 'check_email_registrations', 'scrape_profile', 'verify_claim',\n 'nitter_profile', 'web_fetch', 'auto_visual_intel'];\n const searchCountBeforeBatch = (toolsUsed['search_web'] ?? 0) + (toolsUsed['search_web_multi'] ?? 0);\n const verificationCountBeforeBatch = countSuccessfulVerificationCalls(verificationTools);\n let batchSearchCount = 0;\n let batchVerificationCount = 0;\n for (const toolCall of (sanitizedToolCalls ?? message.tool_calls ?? [])) {\n if (toolCall.type !== 'function') continue;\n let result = '';\n let followUpUserMessage: string | null = null;\n const toolName = toolCall.function.name;\n try {\n const args = JSON.parse(toolCall.function.arguments) as Record<string, string>;\n if (toolName === 'search_web' || toolName === 'search_web_multi') {\n batchSearchCount++;\n }\n\n // Detect calls that were sanitized due to invalid JSON arguments.\n // Give the model a clear error instead of trying to execute with empty/wrong args.\n if ('_sanitized' in args) {\n result = `[TOOL_ARGS_INVALID] The arguments for '${toolName}' contained invalid JSON ` +\n `(the content was too long or had unescaped special characters). ` +\n `Do NOT call obsidian tools with large content. ` +\n `Write the report text directly in your response instead of saving it to Obsidian.`;\n emitProgress(` ❌ ${toolName} → Args had invalid JSON (too long/unescaped) — skipped`);\n pushHistory({\n role: 'tool',\n tool_call_id: toolCall.id,\n content: result,\n });\n toolsUsed[toolName] = (toolsUsed[toolName] ?? 0) + 1;\n toolCallCount++;\n toolCallCountFloor = toolCallCount;\n continue;\n }\n\n const cacheKey = buildToolCacheKey(toolName, args);\n\n // Per-tool hard limit check\n perToolCount[toolName] = (perToolCount[toolName] ?? 0) + 1;\n const toolLimit = PER_TOOL_LIMITS[toolName];\n if (toolLimit && perToolCount[toolName] > toolLimit) {\n logger.warn('AGENT', `[${config.name}] ${toolName} hard limit exceeded (${toolLimit}). Skipping.`);\n result = `[TOOL_LIMIT] ${toolName} has been called ${toolLimit} times in this session — limit reached. Continue with existing data and write a report.`;\n } else if (callCache.has(cacheKey)) {\n logger.warn('AGENT', `[${config.name}] Duplicate tool call blocked: ${toolName} (same arguments)`);\n result = `[DUPLICATE_CALL] This query was called before and the result is already in history. Try a different query or move to the next phase.\\n\\n[cached: ${(callCache.get(cacheKey) ?? '').slice(0, 500)}...]`;\n } else {\n // Build a short arg summary (written to log panel)\n const argSummary = Object.entries(args)\n .slice(0, 2)\n .map(([k, v]) => `${k}=${String(v).slice(0, 40)}`)\n .join(', ');\n emitProgress(` 🔧 ${toolName}(${argSummary})`);\n result = await config.executeTool(toolName, args);\n callCache.set(cacheKey, result);\n\n // Stagnation detection: count new URLs in search results\n if (toolName === 'search_web' || toolName === 'search_web_multi') {\n const urlMatches = result.match(/https?:\\/\\/[^\\s|\\]]+/g) || [];\n const newUrls = urlMatches.filter(u => !urlBloom.mightContain(u));\n newUrls.forEach(u => { urlBloom.add(u); seenUrls.add(u); });\n if (newUrls.length <= 2) {\n lowYieldStreak++;\n } else {\n lowYieldStreak = 0;\n }\n if (lowYieldStreak >= LOW_YIELD_THRESHOLD) {\n logger.warn('AGENT', `[${config.name}] Stagnation detected: ${lowYieldStreak} consecutive low-yield searches. Forcing synthesis.`);\n emitProgress(`⚠️ [${config.name}] Stagnation: ${lowYieldStreak} searches with no new results — forcing report synthesis.`);\n followUpUserMessage =\n '⚠️ STAGNATION DETECTED: Your recent searches are returning no new information. ' +\n 'STOP searching immediately. Synthesize a final report from the data you already have. ' +\n 'If you cannot fully answer the query, state what you found and what is missing.';\n lowYieldStreak = 0; // Reset to prevent repeated injections\n }\n }\n // TUI: short preview (first line, 80 chars)\n const resultPreview = result.split('\\n')[0].slice(0, 80);\n emitProgress(` ✓ ${toolName} → ${resultPreview}`);\n // Web log: full output (untruncated)\n emitToolDetail(toolName, result, toolCall.id);\n }\n } catch (error) {\n result = `Tool error (${toolName}): ${(error as Error).message}`;\n emitProgress(` ❌ ${toolName} → ${(error as Error).message.slice(0, 80)}`);\n }\n pushHistory({\n role: 'tool',\n tool_call_id: toolCall.id,\n content: typeof result === 'string' && result.trim().length > 0\n ? result\n : normalizeToolContent(result),\n });\n if (followUpUserMessage) {\n if (!pendingFollowUpUserMessages.includes(followUpUserMessage)) {\n pendingFollowUpUserMessages.push(followUpUserMessage);\n }\n }\n if (verificationTools.includes(toolName) && isSuccessfulVerificationResult(toolName, result)) {\n batchVerificationCount++;\n }\n toolsUsed[toolName] = (toolsUsed[toolName] ?? 0) + 1;\n toolCallCount++;\n toolCallCountFloor = toolCallCount;\n }\n\n const totalSearchCount = searchCountBeforeBatch + batchSearchCount;\n const totalVerificationCount = verificationCountBeforeBatch + batchVerificationCount;\n const crossedDiversityThreshold = Array.from({ length: batchSearchCount }, (_, index) => searchCountBeforeBatch + index + 1)\n .some(count => count >= 4 && count % 4 === 0);\n if (batchSearchCount > 0 && crossedDiversityThreshold && totalVerificationCount === 0) {\n const availableTools = verificationTools.filter(toolName =>\n config.tools.some((tool: any) => tool.type === 'function' && tool.function.name === toolName)\n );\n if (availableTools.length > 0) {\n const diversityMessage =\n '⚠️ TOOL DIVERSITY REQUIREMENT: You have made ' + totalSearchCount + ' search calls but used ZERO verification/deep-dive tools. ' +\n 'Before making any more searches, you MUST call at least one of: ' + availableTools.slice(0, 5).join(', ') + '. ' +\n 'Search-only investigations produce unreliable results. Verify your findings NOW.';\n if (!pendingFollowUpUserMessages.includes(diversityMessage)) {\n pendingFollowUpUserMessages.push(diversityMessage);\n }\n logger.warn('AGENT', `[${config.name}] Search-only loop detected (${totalSearchCount} searches, 0 verification). Forcing diversity.`);\n }\n }\n\n for (const pendingMessage of pendingFollowUpUserMessages) {\n pushHistory({\n role: 'user',\n content: pendingMessage,\n });\n }\n }\n}\n"}}}} |