{"repo": "anas-asghar4831/api-keys-fetcher", "files": {"functions/scraper-scheduled/src/main.js": {"33e272febb96c53c61dd3e1597897ede96c24373": {"t": 1785747478.9062183, "n": 10568, "h": "5d17434f0487713f", "c": "import { Client, Databases, Query } from 'node-appwrite';\nimport { Octokit } from '@octokit/rest';\n\n// Constants\nconst DATABASE_ID = 'main';\nconst COLLECTIONS = {\n  API_KEYS: 'api_keys',\n  REPO_REFERENCES: 'repo_references',\n  SEARCH_QUERIES: 'search_queries',\n  SEARCH_PROVIDER_TOKENS: 'search_provider_tokens',\n};\n\nconst MAX_CONCURRENT_QUERIES = 3;\nconst MAX_CONCURRENT_FILES = 10;\nconst MAX_FILES_PER_QUERY = 30;\nconst GITHUB_MAX_RESULTS_PER_PAGE = 100;\nconst GITHUB_MAX_PAGES = 10;\nconst GITHUB_PAGE_DELAY_MS = 6000;\n\n// Provider patterns for key extraction\nconst PROVIDER_PATTERNS = [\n  { name: 'OpenAI', pattern: /sk-[a-zA-Z0-9]{20,}T3BlbkFJ[a-zA-Z0-9]{20,}/g, apiType: 'OpenAI' },\n  { name: 'OpenAI-Project', pattern: /sk-proj-[a-zA-Z0-9_-]{80,}/g, apiType: 'OpenAI' },\n  { name: 'OpenAI-Service', pattern: /sk-svcacct-[a-zA-Z0-9_-]{80,}/g, apiType: 'OpenAI' },\n  { name: 'Anthropic', pattern: /sk-ant-api\\d{2}-[a-zA-Z0-9_-]{93}/g, apiType: 'Anthropic' },\n  { name: 'Google', pattern: /AIzaSy[a-zA-Z0-9_-]{33}/g, apiType: 'Google' },\n  { name: 'OpenRouter', pattern: /sk-or-v1-[a-f0-9]{64}/g, apiType: 'OpenRouter' },\n  { name: 'Groq', pattern: /gsk_[a-zA-Z0-9]{52}/g, apiType: 'Groq' },\n  { name: 'Mistral', pattern: /[a-zA-Z0-9]{32}/g, apiType: 'Mistral' },\n  { name: 'Cohere', pattern: /[a-zA-Z0-9]{40}/g, apiType: 'Cohere' },\n  { name: 'HuggingFace', pattern: /hf_[a-zA-Z0-9]{34}/g, apiType: 'HuggingFace' },\n  { name: 'Replicate', pattern: /r8_[a-zA-Z0-9]{37}/g, apiType: 'Replicate' },\n  { name: 'Fireworks', pattern: /fw_[a-zA-Z0-9]{42}/g, apiType: 'Fireworks' },\n  { name: 'Together', pattern: /[a-f0-9]{64}/g, apiType: 'Together' },\n  { name: 'Perplexity', pattern: /pplx-[a-f0-9]{48}/g, apiType: 'Perplexity' },\n  { name: 'DeepSeek', pattern: /sk-[a-f0-9]{32}/g, apiType: 'DeepSeek' },\n  { name: 'xAI', pattern: /xai-[a-zA-Z0-9]{48}/g, apiType: 'xAI' },\n];\n\n// Sleep helper\nconst sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));\n\n// Run tasks with concurrency limit\nasync function runWithConcurrency(items, limit, fn) {\n  const results = [];\n  const executing = [];\n\n  for (let i = 0; i < items.length; i++) {\n    const promise = fn(items[i], i).then((result) => {\n      results[i] = result;\n    });\n\n    const e = promise.then(() => {\n      executing.splice(executing.indexOf(e), 1);\n    });\n    executing.push(e);\n\n    if (executing.length >= limit) {\n      await Promise.race(executing);\n    }\n  }\n\n  await Promise.all(executing);\n  return results;\n}\n\n// Extract keys from text\nfunction extractKeysFromText(text) {\n  const keys = [];\n  for (const provider of PROVIDER_PATTERNS) {\n    const matches = text.match(provider.pattern) || [];\n    for (const key of matches) {\n      // Skip short matches (likely false positives)\n      if (key.length < 20) continue;\n      keys.push({ key, provider: provider.name, apiType: provider.apiType });\n    }\n  }\n  return keys;\n}\n\n// Main function\nconst handler = async ({ res, log, error }) => {\n  const startTime = Date.now();\n  log('Scheduled scraper starting...');\n\n  try {\n    // Initialize Appwrite client\n    const client = new Client()\n      .setEndpoint(process.env.APPWRITE_FUNCTION_API_ENDPOINT)\n      .setProject(process.env.APPWRITE_FUNCTION_PROJECT_ID)\n      .setKey(process.env.APPWRITE_FUNCTION_API_KEY);\n\n    const databases = new Databases(client);\n\n    // Get GitHub token\n    log('Fetching GitHub token...');\n    const tokenResult = await databases.listDocuments(\n      DATABASE_ID,\n      COLLECTIONS.SEARCH_PROVIDER_TOKENS,\n      [Query.equal('provider', 'GitHub'), Query.limit(1)]\n    );\n\n    if (!tokenResult.documents.length || !tokenResult.documents[0].token) {\n      error('No GitHub token configured');\n      return res.json({ success: false, error: 'No GitHub token configured' });\n    }\n\n    const githubToken = tokenResult.documents[0].token;\n    const tokenDocId = tokenResult.documents[0].$id;\n\n    // Get enabled search queries\n    log('Fetching search queries...');\n    const queriesResult = await databases.listDocuments(\n      DATABASE_ID,\n      COLLECTIONS.SEARCH_QUERIES,\n      [Query.equal('enabled', true), Query.limit(100)]\n    );\n\n    const queries = queriesResult.documents;\n    if (!queries.length) {\n      log('No enabled search queries');\n      return res.json({ success: true, message: 'No enabled search queries' });\n    }\n\n    log(`Processing ${queries.length} queries (max ${MAX_CONCURRENT_QUERIES} concurrent)`);\n\n    // Initialize Octokit\n    const octokit = new Octokit({ auth: githubToken });\n\n    // Stats\n    let totalNewKeys = 0;\n    let totalDuplicates = 0;\n    let totalErrors = 0;\n    let totalFiles = 0;\n\n    // Process queries in parallel\n    await runWithConcurrency(queries, MAX_CONCURRENT_QUERIES, async (query) => {\n      try {\n        log(`Processing query: \"${query.query}\"`);\n\n        // Update query timestamp\n        await databases.updateDocument(\n          DATABASE_ID,\n          COLLECTIONS.SEARCH_QUERIES,\n          query.$id,\n          { lastSearchUtc: new Date().toISOString() }\n        );\n\n        // Search GitHub\n        const searchResults = [];\n        let page = 1;\n        let totalCount = 0;\n\n        while (page <= GITHUB_MAX_PAGES) {\n          try {\n            const response = await octokit.search.code({\n              q: query.query,\n              per_page: GITHUB_MAX_RESULTS_PER_PAGE,\n              page,\n            });\n\n            if (page === 1) {\n              totalCount = response.data.total_count;\n              log(`Query \"${query.query}\": ${totalCount} total results`);\n            }\n\n            if (!response.data.items.length) break;\n\n            for (const item of response.data.items) {\n              searchResults.push({\n                repoOwner: item.repository.owner.login,\n                repoName: item.repository.name,\n                repoUrl: item.repository.html_url,\n                repoDescription: item.repository.description,\n                fileName: item.name,\n                filePath: item.path,\n                fileUrl: item.html_url,\n                fileSha: item.sha,\n                branch: item.repository.default_branch,\n              });\n            }\n\n            if (searchResults.length >= MAX_FILES_PER_QUERY) break;\n            if (response.data.items.length < GITHUB_MAX_RESULTS_PER_PAGE) break;\n\n            page++;\n            await sleep(GITHUB_PAGE_DELAY_MS);\n          } catch (err) {\n            if (err.status === 403) {\n              error(`Rate limited on query \"${query.query}\"`);\n              break;\n            }\n            throw err;\n          }\n        }\n\n        // Update query results count\n        await databases.updateDocument(\n          DATABASE_ID,\n          COLLECTIONS.SEARCH_QUERIES,\n          query.$id,\n          { searchResultsCount: totalCount }\n        );\n\n        // Limit files\n        const filesToProcess = searchResults.slice(0, MAX_FILES_PER_QUERY);\n        totalFiles += filesToProcess.length;\n\n        // Process files\n        await runWithConcurrency(filesToProcess, MAX_CONCURRENT_FILES, async (ref) => {\n          try {\n            // Fetch file content\n            const contentResponse = await octokit.repos.getContent({\n              owner: ref.repoOwner,\n              repo: ref.repoName,\n              path: ref.filePath,\n            });\n\n            if (contentResponse.data.type !== 'file' || !contentResponse.data.content) {\n              return;\n            }\n\n            const content = Buffer.from(contentResponse.data.content, 'base64').toString('utf-8');\n\n            // Extract keys\n            const extractedKeys = extractKeysFromText(content);\n\n            for (const { key, provider, apiType } of extractedKeys) {\n              // Check if key exists\n              const existingKeys = await databases.listDocuments(\n                DATABASE_ID,\n                COLLECTIONS.API_KEYS,\n                [Query.equal('apiKey', key), Query.limit(1)]\n              );\n\n              if (existingKeys.documents.length > 0) {\n                totalDuplicates++;\n                continue;\n              }\n\n              // Create new key\n              const now = new Date().toISOString();\n              const createdKey = await databases.createDocument(\n                DATABASE_ID,\n                COLLECTIONS.API_KEYS,\n                'unique()',\n                {\n                  apiKey: key,\n                  status: 'Unverified',\n                  apiType,\n                  searchProvider: 'GitHub',\n                  firstFoundUtc: now,\n                  lastFoundUtc: now,\n                  timesDisplayed: 0,\n                  errorCount: 0,\n                }\n              );\n\n              // Create repo reference\n              await databases.createDocument(\n                DATABASE_ID,\n                COLLECTIONS.REPO_REFERENCES,\n                'unique()',\n                {\n                  apiKeyId: createdKey.$id,\n                  searchQueryId: query.$id,\n                  repoUrl: ref.repoUrl || '',\n                  repoOwner: ref.repoOwner,\n                  repoName: ref.repoName,\n                  repoDescription: ref.repoDescription || '',\n                  fileUrl: ref.fileUrl || '',\n                  fileName: ref.fileName,\n                  filePath: ref.filePath,\n                  fileSha: ref.fileSha,\n                  branch: ref.branch,\n                  provider: 'GitHub',\n                  foundUtc: now,\n                }\n              );\n\n              totalNewKeys++;\n              log(`New ${provider} key found in ${ref.repoOwner}/${ref.repoName}`);\n            }\n          } catch {\n            totalErrors++;\n          }\n        });\n\n        log(`Query \"${query.query}\" complete`);\n      } catch (err) {\n        totalErrors++;\n        error(`Query \"${query.query}\" failed: ${err.message}`);\n      }\n    });\n\n    // Update token last used\n    await databases.updateDocument(\n      DATABASE_ID,\n      COLLECTIONS.SEARCH_PROVIDER_TOKENS,\n      tokenDocId,\n      { lastUsedUtc: new Date().toISOString() }\n    );\n\n    const duration = ((Date.now() - startTime) / 1000).toFixed(1);\n    const summary = {\n      success: true,\n      duration: `${duration}s`,\n      queries: queries.length,\n      files: totalFiles,\n      newKeys: totalNewKeys,\n      duplicates: totalDuplicates,\n      errors: totalErrors,\n    };\n\n    log(`Scraper complete: ${JSON.stringify(summary)}`);\n    return res.json(summary);\n\n  } catch (err) {\n    error(`Scraper failed: ${err.message}`);\n    return res.json({ success: false, error: err.message });\n  }\n};\n\nexport default handler;\n"}, "a32a63fb331bfd22f3179f21892b4bd51322f326": {"t": 1785773529.7417266, "n": 10568, "h": "5d17434f0487713f", "c": "import { Client, Databases, Query } from 'node-appwrite';\nimport { Octokit } from '@octokit/rest';\n\n// Constants\nconst DATABASE_ID = 'main';\nconst COLLECTIONS = {\n  API_KEYS: 'api_keys',\n  REPO_REFERENCES: 'repo_references',\n  SEARCH_QUERIES: 'search_queries',\n  SEARCH_PROVIDER_TOKENS: 'search_provider_tokens',\n};\n\nconst MAX_CONCURRENT_QUERIES = 3;\nconst MAX_CONCURRENT_FILES = 10;\nconst MAX_FILES_PER_QUERY = 30;\nconst GITHUB_MAX_RESULTS_PER_PAGE = 100;\nconst GITHUB_MAX_PAGES = 10;\nconst GITHUB_PAGE_DELAY_MS = 6000;\n\n// Provider patterns for key extraction\nconst PROVIDER_PATTERNS = [\n  { name: 'OpenAI', pattern: /sk-[a-zA-Z0-9]{20,}T3BlbkFJ[a-zA-Z0-9]{20,}/g, apiType: 'OpenAI' },\n  { name: 'OpenAI-Project', pattern: /sk-proj-[a-zA-Z0-9_-]{80,}/g, apiType: 'OpenAI' },\n  { name: 'OpenAI-Service', pattern: /sk-svcacct-[a-zA-Z0-9_-]{80,}/g, apiType: 'OpenAI' },\n  { name: 'Anthropic', pattern: /sk-ant-api\\d{2}-[a-zA-Z0-9_-]{93}/g, apiType: 'Anthropic' },\n  { name: 'Google', pattern: /AIzaSy[a-zA-Z0-9_-]{33}/g, apiType: 'Google' },\n  { name: 'OpenRouter', pattern: /sk-or-v1-[a-f0-9]{64}/g, apiType: 'OpenRouter' },\n  { name: 'Groq', pattern: /gsk_[a-zA-Z0-9]{52}/g, apiType: 'Groq' },\n  { name: 'Mistral', pattern: /[a-zA-Z0-9]{32}/g, apiType: 'Mistral' },\n  { name: 'Cohere', pattern: /[a-zA-Z0-9]{40}/g, apiType: 'Cohere' },\n  { name: 'HuggingFace', pattern: /hf_[a-zA-Z0-9]{34}/g, apiType: 'HuggingFace' },\n  { name: 'Replicate', pattern: /r8_[a-zA-Z0-9]{37}/g, apiType: 'Replicate' },\n  { name: 'Fireworks', pattern: /fw_[a-zA-Z0-9]{42}/g, apiType: 'Fireworks' },\n  { name: 'Together', pattern: /[a-f0-9]{64}/g, apiType: 'Together' },\n  { name: 'Perplexity', pattern: /pplx-[a-f0-9]{48}/g, apiType: 'Perplexity' },\n  { name: 'DeepSeek', pattern: /sk-[a-f0-9]{32}/g, apiType: 'DeepSeek' },\n  { name: 'xAI', pattern: /xai-[a-zA-Z0-9]{48}/g, apiType: 'xAI' },\n];\n\n// Sleep helper\nconst sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));\n\n// Run tasks with concurrency limit\nasync function runWithConcurrency(items, limit, fn) {\n  const results = [];\n  const executing = [];\n\n  for (let i = 0; i < items.length; i++) {\n    const promise = fn(items[i], i).then((result) => {\n      results[i] = result;\n    });\n\n    const e = promise.then(() => {\n      executing.splice(executing.indexOf(e), 1);\n    });\n    executing.push(e);\n\n    if (executing.length >= limit) {\n      await Promise.race(executing);\n    }\n  }\n\n  await Promise.all(executing);\n  return results;\n}\n\n// Extract keys from text\nfunction extractKeysFromText(text) {\n  const keys = [];\n  for (const provider of PROVIDER_PATTERNS) {\n    const matches = text.match(provider.pattern) || [];\n    for (const key of matches) {\n      // Skip short matches (likely false positives)\n      if (key.length < 20) continue;\n      keys.push({ key, provider: provider.name, apiType: provider.apiType });\n    }\n  }\n  return keys;\n}\n\n// Main function\nconst handler = async ({ res, log, error }) => {\n  const startTime = Date.now();\n  log('Scheduled scraper starting...');\n\n  try {\n    // Initialize Appwrite client\n    const client = new Client()\n      .setEndpoint(process.env.APPWRITE_FUNCTION_API_ENDPOINT)\n      .setProject(process.env.APPWRITE_FUNCTION_PROJECT_ID)\n      .setKey(process.env.APPWRITE_FUNCTION_API_KEY);\n\n    const databases = new Databases(client);\n\n    // Get GitHub token\n    log('Fetching GitHub token...');\n    const tokenResult = await databases.listDocuments(\n      DATABASE_ID,\n      COLLECTIONS.SEARCH_PROVIDER_TOKENS,\n      [Query.equal('provider', 'GitHub'), Query.limit(1)]\n    );\n\n    if (!tokenResult.documents.length || !tokenResult.documents[0].token) {\n      error('No GitHub token configured');\n      return res.json({ success: false, error: 'No GitHub token configured' });\n    }\n\n    const githubToken = tokenResult.documents[0].token;\n    const tokenDocId = tokenResult.documents[0].$id;\n\n    // Get enabled search queries\n    log('Fetching search queries...');\n    const queriesResult = await databases.listDocuments(\n      DATABASE_ID,\n      COLLECTIONS.SEARCH_QUERIES,\n      [Query.equal('enabled', true), Query.limit(100)]\n    );\n\n    const queries = queriesResult.documents;\n    if (!queries.length) {\n      log('No enabled search queries');\n      return res.json({ success: true, message: 'No enabled search queries' });\n    }\n\n    log(`Processing ${queries.length} queries (max ${MAX_CONCURRENT_QUERIES} concurrent)`);\n\n    // Initialize Octokit\n    const octokit = new Octokit({ auth: githubToken });\n\n    // Stats\n    let totalNewKeys = 0;\n    let totalDuplicates = 0;\n    let totalErrors = 0;\n    let totalFiles = 0;\n\n    // Process queries in parallel\n    await runWithConcurrency(queries, MAX_CONCURRENT_QUERIES, async (query) => {\n      try {\n        log(`Processing query: \"${query.query}\"`);\n\n        // Update query timestamp\n        await databases.updateDocument(\n          DATABASE_ID,\n          COLLECTIONS.SEARCH_QUERIES,\n          query.$id,\n          { lastSearchUtc: new Date().toISOString() }\n        );\n\n        // Search GitHub\n        const searchResults = [];\n        let page = 1;\n        let totalCount = 0;\n\n        while (page <= GITHUB_MAX_PAGES) {\n          try {\n            const response = await octokit.search.code({\n              q: query.query,\n              per_page: GITHUB_MAX_RESULTS_PER_PAGE,\n              page,\n            });\n\n            if (page === 1) {\n              totalCount = response.data.total_count;\n              log(`Query \"${query.query}\": ${totalCount} total results`);\n            }\n\n            if (!response.data.items.length) break;\n\n            for (const item of response.data.items) {\n              searchResults.push({\n                repoOwner: item.repository.owner.login,\n                repoName: item.repository.name,\n                repoUrl: item.repository.html_url,\n                repoDescription: item.repository.description,\n                fileName: item.name,\n                filePath: item.path,\n                fileUrl: item.html_url,\n                fileSha: item.sha,\n                branch: item.repository.default_branch,\n              });\n            }\n\n            if (searchResults.length >= MAX_FILES_PER_QUERY) break;\n            if (response.data.items.length < GITHUB_MAX_RESULTS_PER_PAGE) break;\n\n            page++;\n            await sleep(GITHUB_PAGE_DELAY_MS);\n          } catch (err) {\n            if (err.status === 403) {\n              error(`Rate limited on query \"${query.query}\"`);\n              break;\n            }\n            throw err;\n          }\n        }\n\n        // Update query results count\n        await databases.updateDocument(\n          DATABASE_ID,\n          COLLECTIONS.SEARCH_QUERIES,\n          query.$id,\n          { searchResultsCount: totalCount }\n        );\n\n        // Limit files\n        const filesToProcess = searchResults.slice(0, MAX_FILES_PER_QUERY);\n        totalFiles += filesToProcess.length;\n\n        // Process files\n        await runWithConcurrency(filesToProcess, MAX_CONCURRENT_FILES, async (ref) => {\n          try {\n            // Fetch file content\n            const contentResponse = await octokit.repos.getContent({\n              owner: ref.repoOwner,\n              repo: ref.repoName,\n              path: ref.filePath,\n            });\n\n            if (contentResponse.data.type !== 'file' || !contentResponse.data.content) {\n              return;\n            }\n\n            const content = Buffer.from(contentResponse.data.content, 'base64').toString('utf-8');\n\n            // Extract keys\n            const extractedKeys = extractKeysFromText(content);\n\n            for (const { key, provider, apiType } of extractedKeys) {\n              // Check if key exists\n              const existingKeys = await databases.listDocuments(\n                DATABASE_ID,\n                COLLECTIONS.API_KEYS,\n                [Query.equal('apiKey', key), Query.limit(1)]\n              );\n\n              if (existingKeys.documents.length > 0) {\n                totalDuplicates++;\n                continue;\n              }\n\n              // Create new key\n              const now = new Date().toISOString();\n              const createdKey = await databases.createDocument(\n                DATABASE_ID,\n                COLLECTIONS.API_KEYS,\n                'unique()',\n                {\n                  apiKey: key,\n                  status: 'Unverified',\n                  apiType,\n                  searchProvider: 'GitHub',\n                  firstFoundUtc: now,\n                  lastFoundUtc: now,\n                  timesDisplayed: 0,\n                  errorCount: 0,\n                }\n              );\n\n              // Create repo reference\n              await databases.createDocument(\n                DATABASE_ID,\n                COLLECTIONS.REPO_REFERENCES,\n                'unique()',\n                {\n                  apiKeyId: createdKey.$id,\n                  searchQueryId: query.$id,\n                  repoUrl: ref.repoUrl || '',\n                  repoOwner: ref.repoOwner,\n                  repoName: ref.repoName,\n                  repoDescription: ref.repoDescription || '',\n                  fileUrl: ref.fileUrl || '',\n                  fileName: ref.fileName,\n                  filePath: ref.filePath,\n                  fileSha: ref.fileSha,\n                  branch: ref.branch,\n                  provider: 'GitHub',\n                  foundUtc: now,\n                }\n              );\n\n              totalNewKeys++;\n              log(`New ${provider} key found in ${ref.repoOwner}/${ref.repoName}`);\n            }\n          } catch {\n            totalErrors++;\n          }\n        });\n\n        log(`Query \"${query.query}\" complete`);\n      } catch (err) {\n        totalErrors++;\n        error(`Query \"${query.query}\" failed: ${err.message}`);\n      }\n    });\n\n    // Update token last used\n    await databases.updateDocument(\n      DATABASE_ID,\n      COLLECTIONS.SEARCH_PROVIDER_TOKENS,\n      tokenDocId,\n      { lastUsedUtc: new Date().toISOString() }\n    );\n\n    const duration = ((Date.now() - startTime) / 1000).toFixed(1);\n    const summary = {\n      success: true,\n      duration: `${duration}s`,\n      queries: queries.length,\n      files: totalFiles,\n      newKeys: totalNewKeys,\n      duplicates: totalDuplicates,\n      errors: totalErrors,\n    };\n\n    log(`Scraper complete: ${JSON.stringify(summary)}`);\n    return res.json(summary);\n\n  } catch (err) {\n    error(`Scraper failed: ${err.message}`);\n    return res.json({ success: false, error: err.message });\n  }\n};\n\nexport default handler;\n"}, "deb69c696c6f94b2123e074f914a16e730bd31bb": {"t": 1785773531.9701002, "n": 10553, "h": "d73b937148ead728", "c": "import { Client, Databases, Query } from 'node-appwrite';\nimport { Octokit } from '@octokit/rest';\n\n// Constants\nconst DATABASE_ID = 'main';\nconst COLLECTIONS = {\n  API_KEYS: 'api_keys',\n  REPO_REFERENCES: 'repo_references',\n  SEARCH_QUERIES: 'search_queries',\n  SEARCH_PROVIDER_TOKENS: 'search_provider_tokens',\n};\n\nconst MAX_CONCURRENT_QUERIES = 3;\nconst MAX_CONCURRENT_FILES = 10;\nconst MAX_FILES_PER_QUERY = 30;\nconst GITHUB_MAX_RESULTS_PER_PAGE = 100;\nconst GITHUB_MAX_PAGES = 10;\nconst GITHUB_PAGE_DELAY_MS = 6000;\n\n// Provider patterns for key extraction\nconst PROVIDER_PATTERNS = [\n  { name: 'OpenAI', pattern: /sk-[a-zA-Z0-9]{20,}T3BlbkFJ[a-zA-Z0-9]{20,}/g, apiType: 'OpenAI' },\n  { name: 'OpenAI-Project', pattern: /sk-proj-[a-zA-Z0-9_-]{80,}/g, apiType: 'OpenAI' },\n  { name: 'OpenAI-Service', pattern: /sk-svcacct-[a-zA-Z0-9_-]{80,}/g, apiType: 'OpenAI' },\n  { name: 'Anthropic', pattern: /sk-ant-api\\d{2}-[a-zA-Z0-9_-]{93}/g, apiType: 'Anthropic' },\n  { name: 'Google', pattern: /AIzaSy[a-zA-Z0-9_-]{33}/g, apiType: 'Google' },\n  { name: 'OpenRouter', pattern: /sk-or-v1-[a-f0-9]{64}/g, apiType: 'OpenRouter' },\n  { name: 'Groq', pattern: /gsk_[a-zA-Z0-9]{52}/g, apiType: 'Groq' },\n  { name: 'Mistral', pattern: /[a-zA-Z0-9]{32}/g, apiType: 'Mistral' },\n  { name: 'Cohere', pattern: /[a-zA-Z0-9]{40}/g, apiType: 'Cohere' },\n  { name: 'HuggingFace', pattern: /hf_[a-zA-Z0-9]{34}/g, apiType: 'HuggingFace' },\n  { name: 'Replicate', pattern: /r8_[a-zA-Z0-9]{37}/g, apiType: 'Replicate' },\n  { name: 'Fireworks', pattern: /fw_[a-zA-Z0-9]{42}/g, apiType: 'Fireworks' },\n  { name: 'Together', pattern: /[a-f0-9]{64}/g, apiType: 'Together' },\n  { name: 'Perplexity', pattern: /pplx-[a-f0-9]{48}/g, apiType: 'Perplexity' },\n  { name: 'DeepSeek', pattern: /sk-[a-f0-9]{32}/g, apiType: 'DeepSeek' },\n  { name: 'xAI', pattern: /xai-[a-zA-Z0-9]{48}/g, apiType: 'xAI' },\n];\n\n// Sleep helper\nconst sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms));\n\n// Run tasks with concurrency limit\nasync function runWithConcurrency(items, limit, fn) {\n  const results = [];\n  const executing = [];\n\n  for (let i = 0; i < items.length; i++) {\n    const promise = fn(items[i], i).then((result) => {\n      results[i] = result;\n    });\n\n    const e = promise.then(() => {\n      executing.splice(executing.indexOf(e), 1);\n    });\n    executing.push(e);\n\n    if (executing.length >= limit) {\n      await Promise.race(executing);\n    }\n  }\n\n  await Promise.all(executing);\n  return results;\n}\n\n// Extract keys from text\nfunction extractKeysFromText(text) {\n  const keys = [];\n  for (const provider of PROVIDER_PATTERNS) {\n    const matches = text.match(provider.pattern) || [];\n    for (const key of matches) {\n      // Skip short matches (likely false positives)\n      if (key.length < 20) continue;\n      keys.push({ key, provider: provider.name, apiType: provider.apiType });\n    }\n  }\n  return keys;\n}\n\n// Main function\nexport default async ({ req, res, log, error }) => {\n  const startTime = Date.now();\n  log('Scheduled scraper starting...');\n\n  try {\n    // Initialize Appwrite client\n    const client = new Client()\n      .setEndpoint(process.env.APPWRITE_FUNCTION_API_ENDPOINT)\n      .setProject(process.env.APPWRITE_FUNCTION_PROJECT_ID)\n      .setKey(process.env.APPWRITE_FUNCTION_API_KEY);\n\n    const databases = new Databases(client);\n\n    // Get GitHub token\n    log('Fetching GitHub token...');\n    const tokenResult = await databases.listDocuments(\n      DATABASE_ID,\n      COLLECTIONS.SEARCH_PROVIDER_TOKENS,\n      [Query.equal('provider', 'GitHub'), Query.limit(1)]\n    );\n\n    if (!tokenResult.documents.length || !tokenResult.documents[0].token) {\n      error('No GitHub token configured');\n      return res.json({ success: false, error: 'No GitHub token configured' });\n    }\n\n    const githubToken = tokenResult.documents[0].token;\n    const tokenDocId = tokenResult.documents[0].$id;\n\n    // Get enabled search queries\n    log('Fetching search queries...');\n    const queriesResult = await databases.listDocuments(\n      DATABASE_ID,\n      COLLECTIONS.SEARCH_QUERIES,\n      [Query.equal('enabled', true), Query.limit(100)]\n    );\n\n    const queries = queriesResult.documents;\n    if (!queries.length) {\n      log('No enabled search queries');\n      return res.json({ success: true, message: 'No enabled search queries' });\n    }\n\n    log(`Processing ${queries.length} queries (max ${MAX_CONCURRENT_QUERIES} concurrent)`);\n\n    // Initialize Octokit\n    const octokit = new Octokit({ auth: githubToken });\n\n    // Stats\n    let totalNewKeys = 0;\n    let totalDuplicates = 0;\n    let totalErrors = 0;\n    let totalFiles = 0;\n\n    // Process queries in parallel\n    await runWithConcurrency(queries, MAX_CONCURRENT_QUERIES, async (query) => {\n      try {\n        log(`Processing query: \"${query.query}\"`);\n\n        // Update query timestamp\n        await databases.updateDocument(\n          DATABASE_ID,\n          COLLECTIONS.SEARCH_QUERIES,\n          query.$id,\n          { lastSearchUtc: new Date().toISOString() }\n        );\n\n        // Search GitHub\n        const searchResults = [];\n        let page = 1;\n        let totalCount = 0;\n\n        while (page <= GITHUB_MAX_PAGES) {\n          try {\n            const response = await octokit.search.code({\n              q: query.query,\n              per_page: GITHUB_MAX_RESULTS_PER_PAGE,\n              page,\n            });\n\n            if (page === 1) {\n              totalCount = response.data.total_count;\n              log(`Query \"${query.query}\": ${totalCount} total results`);\n            }\n\n            if (!response.data.items.length) break;\n\n            for (const item of response.data.items) {\n              searchResults.push({\n                repoOwner: item.repository.owner.login,\n                repoName: item.repository.name,\n                repoUrl: item.repository.html_url,\n                repoDescription: item.repository.description,\n                fileName: item.name,\n                filePath: item.path,\n                fileUrl: item.html_url,\n                fileSha: item.sha,\n                branch: item.repository.default_branch,\n              });\n            }\n\n            if (searchResults.length >= MAX_FILES_PER_QUERY) break;\n            if (response.data.items.length < GITHUB_MAX_RESULTS_PER_PAGE) break;\n\n            page++;\n            await sleep(GITHUB_PAGE_DELAY_MS);\n          } catch (err) {\n            if (err.status === 403) {\n              error(`Rate limited on query \"${query.query}\"`);\n              break;\n            }\n            throw err;\n          }\n        }\n\n        // Update query results count\n        await databases.updateDocument(\n          DATABASE_ID,\n          COLLECTIONS.SEARCH_QUERIES,\n          query.$id,\n          { searchResultsCount: totalCount }\n        );\n\n        // Limit files\n        const filesToProcess = searchResults.slice(0, MAX_FILES_PER_QUERY);\n        totalFiles += filesToProcess.length;\n\n        // Process files\n        await runWithConcurrency(filesToProcess, MAX_CONCURRENT_FILES, async (ref) => {\n          try {\n            // Fetch file content\n            const contentResponse = await octokit.repos.getContent({\n              owner: ref.repoOwner,\n              repo: ref.repoName,\n              path: ref.filePath,\n            });\n\n            if (contentResponse.data.type !== 'file' || !contentResponse.data.content) {\n              return;\n            }\n\n            const content = Buffer.from(contentResponse.data.content, 'base64').toString('utf-8');\n\n            // Extract keys\n            const extractedKeys = extractKeysFromText(content);\n\n            for (const { key, provider, apiType } of extractedKeys) {\n              // Check if key exists\n              const existingKeys = await databases.listDocuments(\n                DATABASE_ID,\n                COLLECTIONS.API_KEYS,\n                [Query.equal('apiKey', key), Query.limit(1)]\n              );\n\n              if (existingKeys.documents.length > 0) {\n                totalDuplicates++;\n                continue;\n              }\n\n              // Create new key\n              const now = new Date().toISOString();\n              const createdKey = await databases.createDocument(\n                DATABASE_ID,\n                COLLECTIONS.API_KEYS,\n                'unique()',\n                {\n                  apiKey: key,\n                  status: 'Unverified',\n                  apiType,\n                  searchProvider: 'GitHub',\n                  firstFoundUtc: now,\n                  lastFoundUtc: now,\n                  timesDisplayed: 0,\n                  errorCount: 0,\n                }\n              );\n\n              // Create repo reference\n              await databases.createDocument(\n                DATABASE_ID,\n                COLLECTIONS.REPO_REFERENCES,\n                'unique()',\n                {\n                  apiKeyId: createdKey.$id,\n                  searchQueryId: query.$id,\n                  repoUrl: ref.repoUrl || '',\n                  repoOwner: ref.repoOwner,\n                  repoName: ref.repoName,\n                  repoDescription: ref.repoDescription || '',\n                  fileUrl: ref.fileUrl || '',\n                  fileName: ref.fileName,\n                  filePath: ref.filePath,\n                  fileSha: ref.fileSha,\n                  branch: ref.branch,\n                  provider: 'GitHub',\n                  foundUtc: now,\n                }\n              );\n\n              totalNewKeys++;\n              log(`New ${provider} key found in ${ref.repoOwner}/${ref.repoName}`);\n            }\n          } catch (err) {\n            totalErrors++;\n          }\n        });\n\n        log(`Query \"${query.query}\" complete`);\n      } catch (err) {\n        totalErrors++;\n        error(`Query \"${query.query}\" failed: ${err.message}`);\n      }\n    });\n\n    // Update token last used\n    await databases.updateDocument(\n      DATABASE_ID,\n      COLLECTIONS.SEARCH_PROVIDER_TOKENS,\n      tokenDocId,\n      { lastUsedUtc: new Date().toISOString() }\n    );\n\n    const duration = ((Date.now() - startTime) / 1000).toFixed(1);\n    const summary = {\n      success: true,\n      duration: `${duration}s`,\n      queries: queries.length,\n      files: totalFiles,\n      newKeys: totalNewKeys,\n      duplicates: totalDuplicates,\n      errors: totalErrors,\n    };\n\n    log(`Scraper complete: ${JSON.stringify(summary)}`);\n    return res.json(summary);\n\n  } catch (err) {\n    error(`Scraper failed: ${err.message}`);\n    return res.json({ success: false, error: err.message });\n  }\n};\n"}}}}