diff --git a/anify-backend/.env.example b/anify-backend/.env.example index b350620..7360771 100644 --- a/anify-backend/.env.example +++ b/anify-backend/.env.example @@ -62,4 +62,6 @@ PROXY_CRON_ENABLED=false # NovelUpdates login credentials # Optional -NOVELUPDATES_LOGIN= \ No newline at end of file +NOVELUPDATES_LOGIN= + +PROXIES_API_KEY= \ No newline at end of file diff --git a/anify-backend/.gitignore b/anify-backend/.gitignore index e242f33..f4d9272 100644 --- a/anify-backend/.gitignore +++ b/anify-backend/.gitignore @@ -53,4 +53,5 @@ METAProxies.json proxy-config.json configs .wrangler -test.ts \ No newline at end of file +test.ts +proxies.json \ No newline at end of file diff --git a/anify-backend/package.json b/anify-backend/package.json index 0c55265..f94bf41 100644 --- a/anify-backend/package.json +++ b/anify-backend/package.json @@ -11,7 +11,8 @@ "prettier": "bunx prettier --write .", "lint": "bun run prettier && bunx tsc --noEmit && bunx eslint --fix .", "scripts": "bun src/scripts/index.ts", - "test": "bun src/tests/index.ts" + "test": "bun src/tests/index.ts", + "extract-proxies": "bun run src/proxies/impl/proxies.ts" }, "dependencies": { "@extractus/article-extractor": "^8.0.18", diff --git a/anify-backend/src/mappings/impl/base/impl/anilist.ts b/anify-backend/src/mappings/impl/base/impl/anilist.ts index 15882bd..50aa8eb 100644 --- a/anify-backend/src/mappings/impl/base/impl/anilist.ts +++ b/anify-backend/src/mappings/impl/base/impl/anilist.ts @@ -1352,3 +1352,5 @@ interface IMediaEntry { type: MediaType; format: MediaFormat; } + +// console.dir(await (new AniListBase).search("Alya sometimes hides her feelings in russian", MediaType.ANIME, [MediaFormat.TV], 1, 20)) diff --git a/anify-backend/src/proxies/impl/cloudflare/index.js b/anify-backend/src/proxies/impl/cloudflare/index.js deleted file mode 100644 index d51c089..0000000 --- a/anify-backend/src/proxies/impl/cloudflare/index.js +++ /dev/null @@ -1,90 +0,0 @@ -export default { - async fetch(request) { - // Clone the request to be able to modify it (if needed later) - const url = new URL(request.url); - console.log(`[LOG] Received request for: ${request.url}`); - - // Basic security: Ensure we are proxying requests for a specific domain or path if necessary. - // For now, we'll proxy the exact URL passed in the request but remove the worker's path. - // Example: if worker is at proxy.example.com and request is proxy.example.com/http://target.com - // we want to fetch http://target.com - - // A simple way to get the target URL is to expect it as a query parameter - // e.g. https://your-worker.your-account.workers.dev/?target=https://example.com/image.png - const targetUrl = url.searchParams.get("target"); - console.log(`[LOG] Target URL specified: ${targetUrl}`); - - if (!targetUrl) { - console.error("[ERROR] Missing target URL in request."); - return new Response('Missing target URL. Please provide a target URL via the "target" query parameter.', { - status: 400, - headers: { "Content-Type": "text/plain" }, - }); - } - - try { - new URL(targetUrl); // Validate if the targetUrl is a valid URL - } catch { - console.error(`[ERROR] Invalid target URL provided: ${targetUrl}`); - return new Response("Invalid target URL provided.", { - status: 400, - headers: { "Content-Type": "text/plain" }, - }); - } - - // Create a new Request object for the target URL, copying method, headers, and body - // We don't want to send Cloudflare-specific headers to the target - const newHeaders = new Headers(request.headers); - // Essential headers like Content-Type (which dictates body format), Accept, Authorization, etc., - // are copied from the original request unless explicitly modified or deleted below. - - // Remove Cloudflare-specific and common proxy headers to avoid leaking information - // or causing issues with the target server. - newHeaders.delete("cf-connecting-ip"); - newHeaders.delete("cf-ipcountry"); - newHeaders.delete("cf-ray"); - newHeaders.delete("cf-visitor"); - newHeaders.delete("cf-worker"); - newHeaders.delete("x-forwarded-proto"); - newHeaders.delete("x-real-ip"); - - // Log the headers being sent to the target (be mindful of sensitive data in production) - // Convert Headers object to a plain object for logging if needed, or iterate - let headersForLog = {}; - for (let [key, value] of newHeaders.entries()) { - headersForLog[key] = value; - } - console.log("[LOG] Headers being sent to target:", JSON.stringify(headersForLog)); - - // You might want to add your own headers or modify existing ones - // newHeaders.set('X-My-Proxy', 'Anify-Cloudflare-Worker'); - - const proxyRequest = new Request(targetUrl, { - method: request.method, // Preserves the method (GET, POST, PUT, DELETE, etc.) from the original request. - headers: newHeaders, // Uses the processed headers, including Content-Type and other important client headers. - body: request.body, // Forwards the original request body, supporting various formats (JSON, form data, binary, etc.). - redirect: "manual", // Handle redirects manually if needed, or 'follow' - }); - - try { - console.log(`[LOG] Attempting to fetch target: ${targetUrl}`); - const response = await fetch(proxyRequest); - console.log(`[LOG] Successfully fetched target: ${targetUrl}, status: ${response.status}`); - // You might want to modify the response headers here as well - // const newResponseHeaders = new Headers(response.headers); - // newResponseHeaders.set('Access-Control-Allow-Origin', '*'); // Example: Add CORS header - - return new Response(response.body, { - status: response.status, - statusText: response.statusText, - headers: response.headers, // or newResponseHeaders if modified - }); - } catch (error) { - console.error(`[ERROR] Error fetching target URL '${targetUrl}':`, error.message, error.stack); - return new Response(`Error fetching the target URL: ${error.message}`, { - status: 500, - headers: { "Content-Type": "text/plain" }, - }); - } - }, -}; diff --git a/anify-backend/src/proxies/impl/cloudflare/package.json b/anify-backend/src/proxies/impl/cloudflare/package.json deleted file mode 100644 index 6eb02b1..0000000 --- a/anify-backend/src/proxies/impl/cloudflare/package.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "anify-proxy-worker", - "version": "0.1.0", - "private": true, - "scripts": { - "deploy": "wrangler deploy", - "dev": "wrangler dev" - }, - "devDependencies": { - "wrangler": "^3.0.0" - } -} \ No newline at end of file diff --git a/anify-backend/src/proxies/impl/cloudflare/wrangler.toml b/anify-backend/src/proxies/impl/cloudflare/wrangler.toml deleted file mode 100644 index e907b59..0000000 --- a/anify-backend/src/proxies/impl/cloudflare/wrangler.toml +++ /dev/null @@ -1,19 +0,0 @@ -name = "anify-proxy" # Replace with your desired worker name -main = "index.js" # Entry point for your worker -compatibility_date = "2023-10-30" # Use a recent compatibility date - -[vars] -# You can define environment variables here if needed -# EXAMPLE_VAR = "example_value" - -# Add other configurations as needed, for example: -# [build] -# command = "npm install && npm run build" # If you have a build step - -# [env.production] -# name = "anify-proxy-prod" -# Add production-specific settings here if any - -[observability] -enabled = true -head_sampling_rate = 1 \ No newline at end of file diff --git a/anify-backend/src/proxies/impl/helper/loadJSON.ts b/anify-backend/src/proxies/impl/helper/loadJSON.ts deleted file mode 100644 index 4fb587a..0000000 --- a/anify-backend/src/proxies/impl/helper/loadJSON.ts +++ /dev/null @@ -1,12 +0,0 @@ -export async function loadJSON(filePath: string): Promise { - try { - const file = Bun.file(filePath); - if (!(await file.exists())) { - return [] as unknown as T; - } - return (await file.json()) as T; - } catch (err) { - console.error(err); - return [] as unknown as T; - } -} diff --git a/anify-backend/src/proxies/impl/helper/saveJSON.ts b/anify-backend/src/proxies/impl/helper/saveJSON.ts deleted file mode 100644 index 43cb4be..0000000 --- a/anify-backend/src/proxies/impl/helper/saveJSON.ts +++ /dev/null @@ -1,3 +0,0 @@ -export async function saveJSON(filePath: string, data: T): Promise { - await Bun.write(filePath, JSON.stringify(data, null, 2)); -} diff --git a/anify-backend/src/proxies/impl/manager.ts b/anify-backend/src/proxies/impl/manager.ts new file mode 100644 index 0000000..454ac1d --- /dev/null +++ b/anify-backend/src/proxies/impl/manager.ts @@ -0,0 +1,64 @@ +// 🌸 proxies/impl/manager.ts + +import fs from "fs"; +import path from "path"; +import { ProviderType } from "../../types"; + +// 🗂️ Map each ProviderType to its proxy file +const PROXY_FILES: Record = { + [ProviderType.BASE]: "BASEProxies.json", + [ProviderType.INFORMATION]: "INFORMATIONProxies.json", + [ProviderType.MANGA]: "MANGAProxies.json", + [ProviderType.META]: "METAProxies.json", + [ProviderType.ANIME]: "", +}; + +function loadProxyFile(filename: string): string[] { + if (!filename) return []; + try { + const filepath = path.resolve(process.cwd(), filename); + const raw = fs.readFileSync(filepath, "utf-8"); + return JSON.parse(raw) as string[]; + } catch { + return []; + } +} + +// 🗂️ Lazily loaded proxy pool +const PROXY_POOL: Partial> = {}; + +function getPool(providerType: ProviderType): string[] { + if (!PROXY_POOL[providerType]) { + PROXY_POOL[providerType] = loadProxyFile(PROXY_FILES[providerType]); + } + return PROXY_POOL[providerType]!; +} + +// 🔄 Round-robin index per provider instance +const roundRobinIndex: Map = new Map(); + +/** + * 🎯 Selects a proxy from the pool for the given provider type. + * Uses round-robin per providerId to spread load evenly~ + */ +export function selectProxy(providerType: ProviderType, providerId: string): string | null { + const pool = getPool(providerType); + if (pool.length === 0) return null; + + const key = `${providerType}:${providerId}`; + const current = roundRobinIndex.get(key) ?? 0; + const proxy = pool[current % pool.length]; + + roundRobinIndex.set(key, current + 1); + + return proxy ?? null; +} + +/** + * 🔗 Converts a proxy string ("ip:port") to a full CORS Anywhere URL. + * Returns null if no proxy provided~ + */ +export function proxyToUrl(proxy: string | null): string | null { + if (!proxy) return null; + return `http://${proxy}`; +} diff --git a/anify-backend/src/proxies/impl/manager/impl/check/impl/runProxyChecks.ts b/anify-backend/src/proxies/impl/manager/impl/check/impl/runProxyChecks.ts deleted file mode 100644 index 714c081..0000000 --- a/anify-backend/src/proxies/impl/manager/impl/check/impl/runProxyChecks.ts +++ /dev/null @@ -1,204 +0,0 @@ -import { proxyCache } from "../../.."; -import { env } from "../../../../../../env"; -import { MediaProvider } from "../../../../../../types/impl/mappings/impl/mediaProvider"; -import type { IProxy, IProxyChecksConfig } from "../../../../../../types/impl/proxies"; -import colors from "colors"; -import { saveProviderProxies } from "../../file/saveProviderProxies"; -import pLimit from "p-limit"; -import fs from "fs"; -import path from "path"; - -/** - * Small helper to throttle how often we update stdout with progress text - */ -let lastUpdateTime = 0; -const THROTTLE_INTERVAL = 100; // in milliseconds, adjust to be more or less aggressive -function throttledProgressUpdate(output: string) { - const now = Date.now(); - if (now - lastUpdateTime > THROTTLE_INTERVAL) { - process.stdout.clearLine(0); - process.stdout.cursorTo(0); - process.stdout.write(output); - lastUpdateTime = now; - } -} - -// Check if enough time has passed since last check (2.5 hours) -function shouldCheckProvider(lastChecked: string | undefined): boolean { - if (!lastChecked) return true; - - const lastCheckTime = new Date(lastChecked).getTime(); - const currentTime = new Date().getTime(); - const hoursDiff = (currentTime - lastCheckTime) / (1000 * 60 * 60); - - return hoursDiff >= 2.5; // Check if 2.5 hours or more have passed -} - -export const runProxyChecks = async (providers: MediaProvider[], verbose: boolean = false) => { - const totalProxies = proxyCache.proxies.length; - - // Load config file - const configPath = path.join(process.cwd(), "proxy-config.json"); - let config: IProxyChecksConfig = { proxyChecks: {} }; - try { - config = JSON.parse(fs.readFileSync(configPath, "utf-8")) as IProxyChecksConfig; - } catch { - // If file doesn't exist or is invalid, we'll use default empty config - } - - // Limit how many proxies are checked in parallel: - // Increase/decrease as suits your environment: - const concurrencyLimit = 50; - const limit = pLimit(concurrencyLimit); - - for (const provider of providers) { - if (!provider.needsProxy || provider.useGoogleTranslate) continue; - - // Get the provider's last check info - const lastCheckInfo = config.proxyChecks[provider.providerType]?.[provider.id]; - - // Skip if not enough time has passed since last check - if (!shouldCheckProvider(lastCheckInfo?.lastChecked)) { - if (env.DEBUG && verbose) { - console.log(colors.yellow(`Skipping ${provider.providerType} ${provider.id} - last checked ${lastCheckInfo?.lastChecked}`)); - } - continue; - } - - provider.isCheckingProxies = true; - - let checkedCount = 0; - let validCount = 0; - let invalidCount = 0; - - if (env.DEBUG) { - console.log(colors.yellow(`Checking proxies for ${provider.providerType} ${provider.id}...`)); - } - - // Reset the last updated time for each provider so the progress line is consistent: - lastUpdateTime = 0; - - // Get the starting index from last check or start from beginning - const startIndex = lastCheckInfo?.lastCheckedIndex ?? 0; - - // Get the slice of proxies to check, starting from where we left off - const proxiesToCheck = proxyCache.proxies.slice(startIndex); - - // Check proxies in parallel with concurrency limiting: - let checkResults: (IProxy | null)[] = []; - try { - const startTime = Date.now(); - checkResults = await Promise.all( - proxiesToCheck.map((proxy) => - limit(async () => { - // Add timeout for the entire proxy check process - const proxyCheckPromise = (async () => { - const url = `${proxy.protocol}://${proxy.username && proxy.password ? `${proxy.username}:${proxy.password}@` : ""}${proxy.ip}:${proxy.port}`; - let isValid = false; - - try { - // Directly use provider's proxyCheck - const providerCheckPromise = provider.proxyCheck(url); - const providerTimeoutPromise = new Promise((_, reject) => { - setTimeout(() => reject(new Error("Provider check timed out")), 15000); - }); - - try { - const check = await Promise.race([providerCheckPromise, providerTimeoutPromise]); - isValid = check ?? false; - } catch (error) { - if (env.DEBUG && verbose) { - console.error(`Provider check failed for ${url}: ${error instanceof Error ? error.message : String(error)}`); - } - isValid = false; - } - } catch (error) { - // Catch any unexpected errors during the setup or execution of provider.proxyCheck - if (env.DEBUG && verbose) { - console.error(`Error during proxy check for ${url}: ${error instanceof Error ? error.message : String(error)}`); - } - isValid = false; - } - - // Update progress counters - checkedCount += 1; - if (isValid) { - validCount += 1; - } else { - invalidCount += 1; - } - - if (env.DEBUG && verbose) { - throttledProgressUpdate( - colors.yellow(`Provider: ${provider.providerType} ${provider.id} | ` + `${checkedCount}/${proxiesToCheck.length} checked, ` + `${validCount} valid, ` + `${invalidCount} invalid | ` + `Time elapsed: ${((Date.now() - startTime) / 1000).toFixed(1)}s`), - ); - } - - return isValid ? proxy : null; - })(); - - // Global timeout for entire proxy check - const globalTimeout = new Promise((_, reject) => { - setTimeout(() => reject(new Error("Global proxy check timeout")), 30000); - }); - - return Promise.race([proxyCheckPromise, globalTimeout]); - }).catch(() => { - checkedCount += 1; - invalidCount += 1; - return null; - }), - ), - ); - } catch (error) { - if (env.DEBUG && verbose) { - console.error(`Error in proxy batch check: ${error instanceof Error ? error.message : String(error)}`); - } - continue; - } finally { - // Ensure we clear any remaining tasks in the limit queue - limit.clearQueue(); - } - - // After finishing all checks, print a new line to avoid overwriting - if (env.DEBUG && verbose) { - process.stdout.write("\n"); - } - - // Filter out null to get valid proxies - const validProxies = checkResults.filter((p): p is IProxy => p !== null); - - // If we found valid proxies, save them and update config - if (validProxies.length > 0) { - proxyCache.validProxies[provider.providerType][provider.id] = validProxies; - await saveProviderProxies(provider.providerType); - - // Update config with timestamp and last checked index - if (!config.proxyChecks[provider.providerType]) { - config.proxyChecks[provider.providerType] = {}; - } - - // If we've checked all proxies, reset the index to 0, otherwise save current position - const newIndex = startIndex + proxiesToCheck.length >= totalProxies ? 0 : startIndex + proxiesToCheck.length; - - config.proxyChecks[provider.providerType]![provider.id] = { - lastChecked: new Date().toISOString(), - validProxiesFound: validProxies.length, - lastCheckedIndex: newIndex, - }; - - // Save updated config - fs.writeFileSync(configPath, JSON.stringify(config, null, 4)); - - if (env.DEBUG) { - console.log(colors.green(`Found ${validProxies.length} valid proxies for ${provider.providerType} ${provider.id}`)); - } - } else { - if (env.DEBUG) { - console.log(colors.red(`No valid proxies found for ${provider.providerType} ${provider.id}`)); - } - } - - provider.isCheckingProxies = false; - } -}; diff --git a/anify-backend/src/proxies/impl/manager/impl/check/index.ts b/anify-backend/src/proxies/impl/manager/impl/check/index.ts deleted file mode 100644 index 0cd4d36..0000000 --- a/anify-backend/src/proxies/impl/manager/impl/check/index.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { MediaProvider } from "../../../../../types/impl/mappings/impl/mediaProvider"; -import colors from "colors"; -import { runProxyChecks } from "./impl/runProxyChecks"; -import { saveProxies } from "../file/saveProxies"; -import { loadJSON } from "../../../helper/loadJSON"; -import { ProviderType } from "../../../../../types"; -import type { IProxy } from "../../../../../types/impl/proxies"; -import { env } from "../../../../../env"; -import { preloadProxies } from "../file/preloadProxies"; - -export const checkProxies = async (providers: MediaProvider[], verbose: boolean = false) => { - // Load existing proxies from proxies.json - let existingProxies: IProxy[] = []; - try { - existingProxies = await loadJSON("proxies.json"); - if (env.DEBUG && verbose) { - console.log(colors.green(`Loaded ${existingProxies.length} existing proxies from proxies.json`)); - } - } catch { - // If file doesn't exist or is invalid, continue with empty array - if (env.DEBUG && verbose) { - console.log(colors.yellow("No existing proxies found in proxies.json")); - } - } - - // Load and consolidate existing provider-specific proxies - const providerTypes = Object.values(ProviderType); - const allProviderProxies: IProxy[] = []; - - for (const providerType of providerTypes) { - try { - const fileName = `${providerType}Proxies.json`; - const typeProxies = await loadJSON(fileName); - allProviderProxies.push(...typeProxies); - } catch { - // Skip if file doesn't exist or is invalid - continue; - } - } - - // Merge existing proxies with provider-specific proxies, preserving metrics - for (const providerProxy of allProviderProxies) { - const existingProxy = existingProxies.find((p) => p.ip === providerProxy.ip && p.port === providerProxy.port); - if (existingProxy) { - // Merge provider metrics - Object.entries(providerProxy.providerMetrics).forEach(([providerId, metrics]) => { - existingProxy.providerMetrics[providerId] = { - ...existingProxy.providerMetrics[providerId], - ...metrics, - }; - }); - } else { - existingProxies.push(providerProxy); - } - } - - // Save consolidated proxies - if (existingProxies.length > 0) { - await saveProxies(existingProxies); - if (env.DEBUG && verbose) { - console.log(colors.green(`Saved ${existingProxies.length} consolidated proxies to proxies.json`)); - } - } - - // Preload the proxies into cache before running checks - await preloadProxies(); - - if (env.DEBUG && verbose) { - console.log(colors.green(`Checking ${providers.filter((p) => p.needsProxy).length} providers for proxy support.`)); - } - - await runProxyChecks(providers, verbose); -}; diff --git a/anify-backend/src/proxies/impl/manager/impl/file/clearProxies.ts b/anify-backend/src/proxies/impl/manager/impl/file/clearProxies.ts deleted file mode 100644 index b2ae232..0000000 --- a/anify-backend/src/proxies/impl/manager/impl/file/clearProxies.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { saveJSON } from "../../../helper/saveJSON"; -import { PROVIDERS } from "../../../../../mappings"; - -export async function clearProxies(clearProviderFiles: boolean = false): Promise { - // Clear main proxies file - await saveJSON("proxies.json", []); - - // Only clear provider files if explicitly requested - if (clearProviderFiles) { - const providers = await Promise.all((await PROVIDERS).map((p) => p)); - for (const provider of providers) { - const fileName = `${provider.providerType}Proxies.json`; - const emptyProviderProxies: Record = {}; - emptyProviderProxies[provider.id] = []; - await saveJSON(fileName, emptyProviderProxies); - } - } -} diff --git a/anify-backend/src/proxies/impl/manager/impl/file/preloadProxies.ts b/anify-backend/src/proxies/impl/manager/impl/file/preloadProxies.ts deleted file mode 100644 index ad3592e..0000000 --- a/anify-backend/src/proxies/impl/manager/impl/file/preloadProxies.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { proxyCache } from "../.."; -import { PROVIDERS } from "../../../../../mappings"; -import type { IProxy } from "../../../../../types/impl/proxies"; -import { loadJSON } from "../../../helper/loadJSON"; -import { emitter } from "../../../../../events"; -import { Events } from "../../../../../types/impl/events"; -import colors from "colors"; - -// Helper function to convert date strings to Date objects in proxy metrics -const convertDates = (proxy: IProxy): IProxy => { - if (!proxy.providerMetrics) { - proxy.providerMetrics = {}; - return proxy; - } - - Object.entries(proxy.providerMetrics).forEach(([providerId, providerMetrics]) => { - if (!providerMetrics) { - proxy.providerMetrics[providerId] = { - healthScore: 50, - consecutiveFailures: 0, - successRate: 0, - averageResponseTime: 0, - successfulRequests: 0, - totalRequests: 0, - successStreak: 0, - latencyScore: 50, - }; - return; - } - - // Convert timestamps to numbers if they're strings - if (providerMetrics.lastSuccessTime) { - providerMetrics.lastSuccessTime = typeof providerMetrics.lastSuccessTime === "string" ? new Date(providerMetrics.lastSuccessTime).getTime() : providerMetrics.lastSuccessTime; - } - if (providerMetrics.lastFailureTime) { - providerMetrics.lastFailureTime = typeof providerMetrics.lastFailureTime === "string" ? new Date(providerMetrics.lastFailureTime).getTime() : providerMetrics.lastFailureTime; - } - }); - return proxy; -}; - -export async function preloadProxies(): Promise { - try { - // Load the main proxy list - const proxies = await loadJSON("proxies.json"); - if (!Array.isArray(proxies)) { - console.error(colors.red("[Proxy Manager] Main proxies.json is not an array")); - return; - } - proxyCache.proxies = proxies.map(convertDates); - console.log(colors.green(`[Proxy Manager] Loaded ${proxyCache.proxies.length} main proxies`)); - - // Load provider-specific proxies and merge them into the main list - for (const prov of await PROVIDERS) { - const provider = await prov; - const fileName = `${provider.providerType}Proxies.json`; - - try { - const typeProxies = await loadJSON(fileName); - if (!Array.isArray(typeProxies)) { - console.error(colors.red(`[Proxy Manager] ${fileName} is not an array`)); - continue; - } - - // Ensure the provider type and ID are initialized - if (!proxyCache.validProxies[provider.providerType]) { - proxyCache.validProxies[provider.providerType] = {}; - } - if (!proxyCache.validProxies[provider.providerType][provider.id]) { - proxyCache.validProxies[provider.providerType][provider.id] = []; - } - - // Only include proxies that have metrics for this specific provider - const convertedProxies = typeProxies.map(convertDates).filter((proxy) => proxy.providerMetrics && proxy.providerMetrics[provider.id]); - - // Update the provider's proxy list - proxyCache.validProxies[provider.providerType][provider.id] = convertedProxies; - - // Update the main proxy list with any new proxies that have metrics for this provider - convertedProxies.forEach((typeProxy) => { - const existingProxy = proxyCache.proxies.find((p) => p.ip === typeProxy.ip && p.port === typeProxy.port); - if (existingProxy) { - existingProxy.providerMetrics = { - ...existingProxy.providerMetrics, - [provider.id]: typeProxy.providerMetrics[provider.id], - }; - } else { - // Only include the metrics for this specific provider but preserve all other properties - const newProxy = { - ...typeProxy, - providerMetrics: { - [provider.id]: typeProxy.providerMetrics[provider.id], - }, - }; - proxyCache.proxies.push(newProxy); - } - }); - } catch (error) { - console.error(colors.red(`[Proxy Manager] Error loading ${fileName}:`), error); - // Initialize empty array for this provider if file loading fails - if (!proxyCache.validProxies[provider.providerType]) { - proxyCache.validProxies[provider.providerType] = {}; - } - proxyCache.validProxies[provider.providerType][provider.id] = []; - } - } - - emitter.emit(Events.PROXIES_LOADED, proxyCache.proxies); - } catch (error) { - console.error(colors.red("[Proxy Manager] Error in preloadProxies:"), error); - throw error; - } -} diff --git a/anify-backend/src/proxies/impl/manager/impl/file/saveProviderProxies.ts b/anify-backend/src/proxies/impl/manager/impl/file/saveProviderProxies.ts deleted file mode 100644 index 00682fa..0000000 --- a/anify-backend/src/proxies/impl/manager/impl/file/saveProviderProxies.ts +++ /dev/null @@ -1,97 +0,0 @@ -import { proxyCache } from "../.."; -import { ProviderType } from "../../../../../types"; -import { saveJSON } from "../../../helper/saveJSON"; -import type { IProxy } from "../../../../../types/impl/proxies"; -import { env } from "../../../../../env"; -import colors from "colors"; - -export async function saveProviderProxies(providerType: ProviderType): Promise { - const fileName = `${providerType}Proxies.json`; - - // Create a map to store the final proxies with their metrics - const finalProxies: Record = {}; - - // Process all proxies for this provider type - for (const providerId in proxyCache.validProxies[providerType]) { - const proxies = proxyCache.validProxies[providerType][providerId] || []; - - for (const proxy of proxies) { - const key = `${proxy.ip}:${proxy.port}`; - - if (!finalProxies[key]) { - // If this is a new proxy, add it to our final list with proper type - finalProxies[key] = { - ...proxy, - providerMetrics: { - [providerId]: { - healthScore: 50, - consecutiveFailures: 0, - successRate: 0, - averageResponseTime: 0, - successfulRequests: 0, - totalRequests: 0, - successStreak: 0, - latencyScore: 50, - }, - }, - }; - - // Add the current provider's metrics - finalProxies[key].providerMetrics[providerId] = proxy.providerMetrics?.[providerId] || { - healthScore: 50, - consecutiveFailures: 0, - successRate: 0, - averageResponseTime: 0, - successfulRequests: 0, - totalRequests: 0, - successStreak: 0, - latencyScore: 50, - }; - } else { - finalProxies[key].providerMetrics[providerId] = proxy.providerMetrics?.[providerId] || { - healthScore: 50, - consecutiveFailures: 0, - successRate: 0, - averageResponseTime: 0, - successfulRequests: 0, - totalRequests: 0, - successStreak: 0, - latencyScore: 50, - }; - } - } - } - - // Convert the map to an array and save - const proxiesToSave = Object.values(finalProxies).filter((proxy) => { - // Check if any provider for this proxy has valid metrics - for (const providerId in proxyCache.validProxies[providerType]) { - const metrics = proxy.providerMetrics[providerId]; - if (metrics && metrics.healthScore >= 0 && metrics.consecutiveFailures < 3) { - return true; - } - } - return false; - }); - - if (env.DEBUG) { - console.log(colors.green(`Saving ${proxiesToSave.length} proxies for ${providerType}`)); - } - - await saveJSON(fileName, proxiesToSave); -} - -export async function removeProviderProxy(providerType: ProviderType, providerId: string, proxyUrl: string): Promise { - const proxies = proxyCache.validProxies[providerType][providerId] || []; - const [ip, port] = proxyUrl.replace(/^(https?|socks5):\/\//, "").split(":"); - - // Remove the proxy from the cache - proxyCache.validProxies[providerType][providerId] = proxies.filter((proxy) => !(proxy.ip === ip && proxy.port === Number(port))); - - // Save the updated proxy list - await saveProviderProxies(providerType); - - if (env.DEBUG) { - console.log(colors.red(`Removed proxy ${proxyUrl} from ${providerType} ${providerId}`)); - } -} diff --git a/anify-backend/src/proxies/impl/manager/impl/file/saveProxies.ts b/anify-backend/src/proxies/impl/manager/impl/file/saveProxies.ts deleted file mode 100644 index 4d88d29..0000000 --- a/anify-backend/src/proxies/impl/manager/impl/file/saveProxies.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { IProxy } from "../../../../../types/impl/proxies"; -import { loadJSON } from "../../../helper/loadJSON"; -import { saveJSON } from "../../../helper/saveJSON"; - -export async function saveProxies(proxies: IProxy[]): Promise { - const fileName = `proxies.json`; - const currentData = await loadJSON(fileName); - // Only save new proxies, remove duplicates - const newData = currentData.concat(proxies.filter((p) => !currentData.some((c) => c.ip === p.ip && c.port === p.port))); - await saveJSON(fileName, newData); -} diff --git a/anify-backend/src/proxies/impl/manager/index.ts b/anify-backend/src/proxies/impl/manager/index.ts deleted file mode 100644 index 1d6a41a..0000000 --- a/anify-backend/src/proxies/impl/manager/index.ts +++ /dev/null @@ -1,319 +0,0 @@ -import { ProviderType } from "../../../types"; -import type { IProxy, IProxyProviderMetrics } from "../../../types/impl/proxies"; -import fs from "fs"; -import path from "path"; - -const MIN_HEALTH_SCORE = 0; -const MAX_HEALTH_SCORE = 100; -const FAILURE_PENALTY = 3; -const SUCCESS_BONUS = 2; -const RESPONSE_TIME_WEIGHT = 0.25; -const SUCCESS_RATE_WEIGHT = 0.75; -const MAX_CONSECUTIVE_FAILURES = 5; -const MIN_COOLDOWN_MS = 60000; -const MAX_COOLDOWN_MS = 3600000; -const HEALTH_DECAY_RATE = 0.95; -const MIN_VIABLE_HEALTH = 20; - -// Add new constants for improved health scoring -const LATENCY_THRESHOLD_EXCELLENT = 300; -const LATENCY_THRESHOLD_GOOD = 800; -const LATENCY_THRESHOLD_FAIR = 1500; -const SUCCESS_STREAK_BONUS = 1.2; -const MIN_REQUESTS_FOR_RELIABILITY = 5; -const RELIABILITY_WEIGHT = 0.3; -const ADAPTIVE_WEIGHT_THRESHOLD = 50; - -export const proxyCache: { - proxies: IProxy[]; - validProxies: Record>; -} = { - proxies: [], - validProxies: Object.values(ProviderType).reduce( - (acc, type) => { - acc[type] = {}; - return acc; - }, - {} as Record>, - ), -}; - -// Helper function to convert proxy to URL string -export const proxyToUrl = (proxy: IProxy | null): string | null => { - if (!proxy) return null; - return `http://${proxy.ip}:${proxy.port}`; -}; - -// Helper function to get provider metrics, creating if doesn't exist -const getProviderMetrics = (proxy: IProxy, providerType: ProviderType, providerId: string): IProxyProviderMetrics => { - if (!proxy.providerMetrics) { - proxy.providerMetrics = {} as Record; - } - if (!proxy.providerMetrics[providerId]) { - proxy.providerMetrics[providerId] = { - healthScore: 50, - consecutiveFailures: 0, - successRate: 0, - averageResponseTime: 0, - successfulRequests: 0, - totalRequests: 0, - successStreak: 0, - latencyScore: 50, - }; - } - return proxy.providerMetrics[providerId]; -}; - -// Save proxies to provider-specific files -const saveProxiesToFile = (providerType: ProviderType) => { - // Convert providerType to uppercase for ANIME, MANGA, etc. - const filename = `${providerType.toUpperCase()}Proxies.json`; - const filePath = path.join(process.cwd(), filename); - - // Read existing proxies from file - let existingProxies: IProxy[] = []; - try { - if (fs.existsSync(filePath)) { - const fileContent = fs.readFileSync(filePath, "utf-8"); - existingProxies = JSON.parse(fileContent); - } - } catch (error) { - console.error(`Error reading existing proxies from ${filename}:`, error); - } - - // Create a map of existing proxies for easy lookup - const existingProxyMap = new Map(existingProxies.map((proxy) => [`${proxy.ip}:${proxy.port}`, proxy])); - - // Update or add proxies from cache - proxyCache.proxies.forEach((proxy) => { - const proxyKey = `${proxy.ip}:${proxy.port}`; - // Check if proxy has metrics for any provider ID in this provider type - const hasMetricsForType = Object.keys(proxyCache.validProxies[providerType]).some((providerId) => proxy.providerMetrics?.[providerId]); - - if (hasMetricsForType) { - // If proxy exists in file, merge the metrics - const existingProxy = existingProxyMap.get(proxyKey); - if (existingProxy) { - existingProxyMap.set(proxyKey, { - ...existingProxy, - providerMetrics: { - ...existingProxy.providerMetrics, - ...proxy.providerMetrics, // Merge all provider metrics - }, - }); - } else { - // Add new proxy - existingProxyMap.set(proxyKey, proxy); - } - } - }); - - // Convert map back to array - const updatedProxies = Array.from(existingProxyMap.values()); - - // Update validProxies cache for each provider - Object.keys(proxyCache.validProxies[providerType]).forEach((providerId) => { - proxyCache.validProxies[providerType][providerId] = updatedProxies.filter((proxy) => proxy.providerMetrics?.[providerId] && proxy.providerMetrics[providerId].healthScore > MIN_VIABLE_HEALTH); - }); - - // Save all proxies with their metrics - fs.writeFileSync(filePath, JSON.stringify(updatedProxies, null, 2)); -}; - -export const updateProxyHealth = (proxy: IProxy, success: boolean, providerType: ProviderType, providerId: string, responseTime?: number) => { - const now = Date.now(); - const metrics = getProviderMetrics(proxy, providerType, providerId); - - // Apply time-based health decay (only if not successful) - if (!success && metrics.lastSuccessTime) { - const timeSinceLastUpdate = (now - metrics.lastSuccessTime) / (24 * 60 * 60 * 1000); // Days since last success - metrics.healthScore *= Math.pow(HEALTH_DECAY_RATE, timeSinceLastUpdate); - } - - // Update basic metrics - metrics.totalRequests = (metrics.totalRequests || 0) + 1; - metrics.successfulRequests = (metrics.successfulRequests || 0) + (success ? 1 : 0); - - // Update success rate with exponential moving average for stability - const currentSuccessRate = (metrics.successfulRequests / metrics.totalRequests) * 100; - metrics.successRate = - metrics.successRate !== undefined - ? metrics.successRate * 0.75 + currentSuccessRate * 0.25 // 75% old rate, 25% new rate - : currentSuccessRate; - - if (success) { - metrics.lastSuccessTime = now; - metrics.successStreak = (metrics.successStreak || 0) + 1; - - // Calculate streak bonus with diminishing returns - const streakBonus = Math.min(SUCCESS_STREAK_BONUS * Math.log10(metrics.successStreak + 1), SUCCESS_BONUS * 2); - - // Calculate success bonus based on response time - let speedBonus = 0; - if (responseTime) { - if (responseTime < LATENCY_THRESHOLD_EXCELLENT) { - speedBonus = SUCCESS_BONUS * 0.5; // 50% bonus for excellent speed - } else if (responseTime < LATENCY_THRESHOLD_GOOD) { - speedBonus = SUCCESS_BONUS * 0.3; // 30% bonus for good speed - } - } - - // Apply success bonus with streak and speed consideration - const totalBonus = SUCCESS_BONUS + streakBonus + speedBonus; - metrics.healthScore = Math.min(MAX_HEALTH_SCORE, metrics.healthScore + totalBonus); - metrics.consecutiveFailures = 0; - } else { - metrics.lastFailureTime = now; - metrics.consecutiveFailures = (metrics.consecutiveFailures || 0) + 1; - metrics.successStreak = 0; - - // Progressive penalty based on consecutive failures and response time - let failurePenalty = FAILURE_PENALTY * Math.min(2, Math.pow(1.2, metrics.consecutiveFailures)); - - // Additional penalty for slow response times that led to failure - if (responseTime && responseTime > LATENCY_THRESHOLD_FAIR) { - failurePenalty *= 1.5; // 50% more penalty for slow failures - } - - metrics.healthScore = Math.max(MIN_HEALTH_SCORE, metrics.healthScore - failurePenalty); - } - - // Update response time metrics with exponential moving average - if (responseTime) { - // Use different weights based on success/failure - const oldWeight = success ? 0.75 : 0.25; // Weight history more on success - const newWeight = 1 - oldWeight; - - metrics.averageResponseTime = metrics.averageResponseTime ? metrics.averageResponseTime * oldWeight + responseTime * newWeight : responseTime; - - // Calculate latency score (0-100) with more granular scaling - let latencyScore = 100; - if (responseTime > LATENCY_THRESHOLD_FAIR) { - latencyScore = 50; - } else if (responseTime > LATENCY_THRESHOLD_GOOD) { - latencyScore = 75; - } else if (responseTime > LATENCY_THRESHOLD_EXCELLENT) { - latencyScore = 90; - } - - // Update latency score with more weight on recent performance - metrics.latencyScore = metrics.latencyScore - ? metrics.latencyScore * 0.6 + latencyScore * 0.4 // More weight on recent performance - : latencyScore; - } - - // Calculate reliability score based on total requests with minimum threshold - const reliabilityScore = metrics.totalRequests >= MIN_REQUESTS_FOR_RELIABILITY ? (metrics.successfulRequests / metrics.totalRequests) * 100 : Math.min(50 + metrics.totalRequests * 5, 100); // Gradually increase base reliability - - // Adaptive weights based on request volume and performance - let adaptiveSuccessWeight = SUCCESS_RATE_WEIGHT; - let adaptiveResponseWeight = RESPONSE_TIME_WEIGHT; - let adaptiveReliabilityWeight = RELIABILITY_WEIGHT; - - if (metrics.totalRequests > ADAPTIVE_WEIGHT_THRESHOLD) { - // Adjust weights based on performance patterns - const successRate = metrics.successfulRequests / metrics.totalRequests; - if (successRate > 0.9) { - adaptiveResponseWeight *= 1.2; // Prioritize speed for reliable proxies - adaptiveSuccessWeight *= 0.9; - } else if (successRate < 0.7) { - adaptiveSuccessWeight *= 1.2; // Prioritize success rate for unreliable proxies - adaptiveResponseWeight *= 0.9; - } - - // Normalize weights - const totalWeight = adaptiveSuccessWeight + adaptiveResponseWeight + adaptiveReliabilityWeight; - adaptiveSuccessWeight /= totalWeight; - adaptiveResponseWeight /= totalWeight; - adaptiveReliabilityWeight /= totalWeight; - } - - // Calculate final health score components - const successRateScore = metrics.successRate; - const responseTimeScore = metrics.latencyScore || 50; - - // Weighted health score calculation with reliability - const targetScore = successRateScore * adaptiveSuccessWeight + responseTimeScore * adaptiveResponseWeight + reliabilityScore * adaptiveReliabilityWeight; - - // Smooth transition to target score with adaptive rate - const adaptationRate = metrics.totalRequests < MIN_REQUESTS_FOR_RELIABILITY ? 0.2 : 0.1; - metrics.healthScore += (targetScore - metrics.healthScore) * adaptationRate; - - // Ensure health score stays within bounds - metrics.healthScore = Math.min(MAX_HEALTH_SCORE, Math.max(MIN_HEALTH_SCORE, metrics.healthScore)); - - // Save updated proxies to file - saveProxiesToFile(providerType); -}; - -// Add rotation tracking -const proxyUsageMap = new Map(); - -export const selectProxy = (providerType: ProviderType, providerId: string): IProxy | null => { - const validProxies = proxyCache.validProxies[providerType][providerId] || []; - - if (validProxies.length === 0) return null; - - // Filter proxies by health and usage - const viableProxies = validProxies.filter((proxy) => { - const metrics = proxy.providerMetrics?.[providerId]; - if (!metrics) return false; - - // Check health score - if (metrics.healthScore < MIN_VIABLE_HEALTH) return false; - - // Check consecutive failures - if (metrics.consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) return false; - - // Check cooldown if proxy has been used recently - const proxyKey = `${proxy.ip}:${proxy.port}`; - const lastUsed = proxyUsageMap.get(proxyKey) || 0; - const timeSinceLastUse = Date.now() - lastUsed; - - return timeSinceLastUse >= MIN_COOLDOWN_MS; - }); - - if (viableProxies.length === 0) return null; - - // First try to find any unused proxies - const unusedProxies = viableProxies.filter((proxy) => { - const proxyKey = `${proxy.ip}:${proxy.port}`; - return !proxyUsageMap.has(proxyKey); - }); - - // If we have unused proxies, randomly select one to distribute load - if (unusedProxies.length > 0) { - const selectedProxy = unusedProxies[Math.floor(Math.random() * unusedProxies.length)]; - const proxyKey = `${selectedProxy.ip}:${selectedProxy.port}`; - proxyUsageMap.set(proxyKey, Date.now()); - return selectedProxy; - } - - // If no unused proxies, sort by health score and usage time - const sortedProxies = viableProxies.sort((a, b) => { - const aMetrics = a.providerMetrics[providerId]; - const bMetrics = b.providerMetrics[providerId]; - - // Calculate score including usage time - const aKey = `${a.ip}:${a.port}`; - const bKey = `${b.ip}:${b.port}`; - const aLastUsed = proxyUsageMap.get(aKey) || 0; - const bLastUsed = proxyUsageMap.get(bKey) || 0; - - const aTimeFactor = Math.min(1, (Date.now() - aLastUsed) / MAX_COOLDOWN_MS); - const bTimeFactor = Math.min(1, (Date.now() - bLastUsed) / MAX_COOLDOWN_MS); - - // Heavily weight health score for subsequent requests - const aScore = aMetrics.healthScore * (0.8 + 0.2 * aTimeFactor); - const bScore = bMetrics.healthScore * (0.8 + 0.2 * bTimeFactor); - - return bScore - aScore; - }); - - // Select proxy and update usage time - const selectedProxy = sortedProxies[0]; - const proxyKey = `${selectedProxy.ip}:${selectedProxy.port}`; - proxyUsageMap.set(proxyKey, Date.now()); - - return selectedProxy; -}; diff --git a/anify-backend/src/proxies/impl/proxies.ts b/anify-backend/src/proxies/impl/proxies.ts new file mode 100644 index 0000000..3c907a0 --- /dev/null +++ b/anify-backend/src/proxies/impl/proxies.ts @@ -0,0 +1,33 @@ +import { writeFile } from "fs/promises"; + +const API_URL = "https://jubilant-lamp-pink.vercel.app/proxies"; +const OUTPUT_FILE = "proxies.json"; + +async function fetchProxies(): Promise { + const apiKey = process.env.PROXIES_API_KEY; + + if (!apiKey) { + throw new Error("💔 PROXIES_API_KEY is not set in environment variables!"); + } + + const response = await fetch(API_URL, { + headers: { + Authorization: apiKey, + }, + }); + + if (!response.ok) { + throw new Error(`🚨 Request failed with status ${response.status}: ${response.statusText}`); + } + + const data: unknown = await response.json(); + + await writeFile(OUTPUT_FILE, JSON.stringify(data, null, 2), "utf-8"); + + console.log(`✨ Proxies saved to ${OUTPUT_FILE}!`); +} + +fetchProxies().catch((err: Error) => { + console.error("💥 Oops!", err.message); + process.exit(1); +}); diff --git a/anify-backend/src/proxies/impl/request.ts b/anify-backend/src/proxies/impl/request.ts new file mode 100644 index 0000000..873a47f --- /dev/null +++ b/anify-backend/src/proxies/impl/request.ts @@ -0,0 +1,60 @@ +// 🌸 proxies/impl/request.ts + +import type { IRequestConfig } from "../../types/impl/proxies"; + +const CORS_ORIGIN = "https://anify.to"; + +function wrapWithProxy(targetUrl: string, proxyBase: string): string { + const base = proxyBase.endsWith("/") ? proxyBase : `${proxyBase}/`; + return `${proxyBase.startsWith("http://") ? "" : "http://"}${base}${targetUrl}`; +} + +export async function customRequest(url: string, config: IRequestConfig = {}): Promise { + const { _proxyURL, timeout = 15000, signal, headers: extraHeaders = {}, proxy, ...restConfig } = config; + + const proxyUrl = _proxyURL ?? proxy; + const finalUrl = proxyUrl ? wrapWithProxy(url, proxyUrl) : url; + + const timeoutController = new AbortController(); + const timeoutId = setTimeout(() => timeoutController.abort(), timeout); + + const mergedSignal = signal ? mergeSignals(signal, timeoutController.signal) : timeoutController.signal; + + const headers: Record = { + Origin: CORS_ORIGIN, + "X-Requested-With": "XMLHttpRequest", + ...(extraHeaders as Record), + }; + + try { + const response = await fetch(finalUrl, { + ...restConfig, + headers, + signal: mergedSignal, + }); + + return response; + } catch (err) { + if (err instanceof Error && err.name === "AbortError") { + return null; + } + console.error(`❌ Request failed [${finalUrl}]:`, err); + return null; + } finally { + clearTimeout(timeoutId); + } +} + +function mergeSignals(...signals: AbortSignal[]): AbortSignal { + const controller = new AbortController(); + + for (const signal of signals) { + if (signal.aborted) { + controller.abort(); + break; + } + signal.addEventListener("abort", () => controller.abort(), { once: true }); + } + + return controller.signal; +} diff --git a/anify-backend/src/proxies/impl/request/impl/googleTranslate.ts b/anify-backend/src/proxies/impl/request/impl/googleTranslate.ts deleted file mode 100644 index 52ee773..0000000 --- a/anify-backend/src/proxies/impl/request/impl/googleTranslate.ts +++ /dev/null @@ -1,25 +0,0 @@ -import type { IRequestConfig } from "../../../../types/impl/proxies"; - -const isGoogleTranslate = (options: IRequestConfig = {}): boolean => { - return options.useGoogleTranslate ?? false; -}; - -const googleTranslate = async (url: string, options: IRequestConfig = {}): Promise => { - const { useGoogleTranslate, validateResponse } = options; - - if (useGoogleTranslate) { - const translatedUrl = `http://translate.google.com/translate?sl=ja&tl=en&u=${encodeURIComponent(url)}`; - const response = await fetch(translatedUrl, options); - - if (response && (!validateResponse || (await validateResponse(response.clone() as Response)))) { - return response; - } - } - - return null; -}; - -export default { - googleTranslate, - isGoogleTranslate, -}; diff --git a/anify-backend/src/proxies/impl/request/impl/proxy.ts b/anify-backend/src/proxies/impl/request/impl/proxy.ts deleted file mode 100644 index e8acd5e..0000000 --- a/anify-backend/src/proxies/impl/request/impl/proxy.ts +++ /dev/null @@ -1,91 +0,0 @@ -import type { IRequestConfig } from "../../../../types/impl/proxies"; -import { proxyToUrl, selectProxy } from "../../manager"; -import { ProxyAgent } from "undici"; - -const MAX_PARALLEL_RETRIES = 3; - -const isJsonParseError = (error: unknown): boolean => { - return error instanceof SyntaxError && error.message.includes("Unexpected end of JSON input"); -}; - -const isProxy = (options: IRequestConfig = {}, attempt?: number): boolean => { - const { isChecking, proxy, useGoogleTranslate, providerType, providerId } = options; - - const proxyURL = isChecking ? proxy : useGoogleTranslate ? null : proxy && attempt === 1 ? proxy : providerType && providerId ? proxyToUrl(selectProxy(providerType, providerId)) : null; - if (proxyURL) { - return true; - } - - return false; -}; - -const attemptRequest = async (url: string, options: IRequestConfig = {}, attempt: number): Promise => { - try { - const proxyResponse = await proxy(url, options, attempt); - if (options.signal?.aborted) return null; - - if (proxyResponse) { - // Try to validate JSON if there's a validateResponse function - if (options.validateResponse) { - try { - const clonedResponse = proxyResponse.clone(); - await options.validateResponse(clonedResponse as Response); - return proxyResponse; - } catch (error) { - if (!isJsonParseError(error)) { - return proxyResponse; - } - // If JSON parsing fails, return null to trigger retry - return null; - } - } - return proxyResponse; - } - } catch (error) { - if (error instanceof Error && error.name === "AbortError") { - return null; - } - console.error(`Proxy attempt ${attempt} failed:`, error); - } - return null; -}; - -const proxyWithRetries = async (url: string, options: IRequestConfig = {}): Promise => { - // Try parallel CORS requests - const proxyPromises = Array.from({ length: MAX_PARALLEL_RETRIES }, (_, i) => attemptRequest(url, options, i + 1)); - - const responses = await Promise.all(proxyPromises); - return responses.find((response) => response !== null) ?? null; -}; - -const proxy = async (url: string, options: IRequestConfig = {}, attempt?: number): Promise => { - const { isChecking, proxy, useGoogleTranslate, providerType, providerId, validateResponse } = options; - - const proxyURL = isChecking ? proxy : useGoogleTranslate ? null : proxy && attempt === 1 ? proxy : providerType && providerId ? proxyToUrl(selectProxy(providerType, providerId)) : null; - - if (proxyURL) { - const requestOptions = { - ...options, - signal: options.signal, - } as RequestInit; - - const proxyAgent = new ProxyAgent(proxyURL); - Object.assign(requestOptions, { - dispatcher: proxyAgent - }); - - const response = await fetch(url, requestOptions); - - if (response && (!validateResponse || (await validateResponse(response.clone() as Response)))) { - return response; - } - } - - return null; -}; - -export default { - proxy, - proxyWithRetries, - isProxy, -}; diff --git a/anify-backend/src/proxies/impl/request/index.ts b/anify-backend/src/proxies/impl/request/index.ts deleted file mode 100644 index 1de1347..0000000 --- a/anify-backend/src/proxies/impl/request/index.ts +++ /dev/null @@ -1,27 +0,0 @@ -import type { IRequestConfig } from "../../../types/impl/proxies"; -import proxy from "./impl/proxy"; -import googleTranslate from "./impl/googleTranslate"; - -export const customRequest = async (url: string, options: IRequestConfig = {}) => { - // First attempt: Try CORS proxy or Google Translate based on options - if (proxy.isProxy(options)) { - const proxyResponse = await proxy.proxyWithRetries(url, options); - if (proxyResponse) return proxyResponse; - } else if (googleTranslate.isGoogleTranslate(options)) { - const translateResponse = await googleTranslate.googleTranslate(url, options); - if (options.signal?.aborted) return null; - if (translateResponse) return translateResponse; - } - - // Second attempt: Try normal request with AbortSignal - if (options.signal) { - const response = await fetch(url, { ...options, signal: options.signal }); - if (response) return response; - } else { - const response = await fetch(url, options); - if (response) return response; - } - - // If all attempts fail, return null - return null; -}; diff --git a/anify-backend/src/proxies/impl/scrape/webshare.ts b/anify-backend/src/proxies/impl/scrape/webshare.ts deleted file mode 100644 index 16791fc..0000000 --- a/anify-backend/src/proxies/impl/scrape/webshare.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { env } from "../../../env"; -import type { IProxy } from "../../../types/impl/proxies"; -import { join } from "path"; -import type { IWebshareListResponse, IWebshareProxy } from "../../../types/impl/proxies/impl/webshare"; - -/** - * Scraper for fetching proxies from Webshare.io - */ -export class WebshareScraper { - private apiKey: string; - private apiURL = "https://proxy.webshare.io/api/v2/"; // Default or example API URL - - constructor(apiKey?: string) { - this.apiKey = apiKey || env.WEBSHARE_API_KEY || ""; - if (!this.apiKey) { - console.warn("Webshare API key not provided. Please set the WEBSHARE_API_KEY environment variable."); - // Optionally throw an error if the key is absolutely required - // throw new Error("Webshare API key is required."); - } - } - - /** - * Fetches proxies from the Webshare API. - * @returns A promise that resolves to an array of IProxy objects. - */ - async scrapeProxies(): Promise { - if (!this.apiKey) { - console.error("Cannot scrape Webshare proxies without an API key."); - return []; - } - - try { - // Example: Fetching proxy list - Adjust endpoint and parameters as needed - const endpoint = `${this.apiURL}proxy/list/?mode=direct&page=1&page_size=100`; // Example endpoint - const response = await fetch(endpoint, { - headers: { - Authorization: `Token ${this.apiKey}`, - }, - }); - - if (!response.ok) { - throw new Error(`Webshare API request failed with status ${response.status}: ${await response.text()}`); - } - - const data = (await response.json()) as IWebshareListResponse; - - // TODO: Add more robust validation and error handling for the API response structure - if (!data || !data.results || !Array.isArray(data.results)) { - console.error("Invalid response format from Webshare API:", data); - throw new Error("Invalid response format from Webshare API"); - } - - // Map the Webshare response format to the IProxy format - const proxies: IProxy[] = data.results - .map((proxy: IWebshareProxy): IProxy | null => { - // TODO: Add more validation based on specific needs (e.g., https_support) - if (!proxy || typeof proxy.proxy_address !== "string" || typeof proxy.port !== "number" || !proxy.valid) { - console.warn("Skipping invalid proxy entry from Webshare:", proxy); - return null; // Skip invalid entries - } - return { - id: crypto.randomUUID(), - ip: proxy.proxy_address, - port: proxy.port, - // Assume http/https based on Webshare documentation or proxy_type field if available - // Defaulting to 'http' here, adjust as necessary. - protocol: proxy.proxy_type === "socks5" ? "socks5" : "http", - country: proxy.country_code || "Unknown", - source: "webshare", - username: proxy.username, // Include authentication if provided - password: proxy.password, // Include authentication if provided - providerMetrics: {}, // Initialize empty provider metrics - }; - }) - .filter((p: IProxy | null): p is IProxy => p !== null); // Filter out null values from invalid entries - - await this.writeProxiesToFile(proxies); - - return proxies; - } catch (error) { - console.error("Error scraping proxies from Webshare:", error); - return []; // Return empty array on error - } - } - - private async writeProxiesToFile(proxies: IProxy[]) { - const filePath = join(process.cwd(), "proxies.json"); - const file = Bun.file(filePath); - await file.write(JSON.stringify(proxies, null, 2)); - } - - // Optional: Add methods for health checks or other specific Webshare interactions - async checkHealth(): Promise { - if (!this.apiKey) return false; - try { - // Example: Fetch account info or perform a simple request to check API key validity - const endpoint = `${this.apiURL}profile/`; - const response = await fetch(endpoint, { - headers: { - Authorization: `Token ${this.apiKey}`, - }, - }); - return response.ok; - } catch (error) { - console.error("Webshare health check failed:", error); - return false; - } - } -} diff --git a/anify-backend/src/types/impl/mappings/impl/mediaProvider.ts b/anify-backend/src/types/impl/mappings/impl/mediaProvider.ts index 75fc411..c381b1f 100644 --- a/anify-backend/src/types/impl/mappings/impl/mediaProvider.ts +++ b/anify-backend/src/types/impl/mappings/impl/mediaProvider.ts @@ -1,9 +1,35 @@ +import fs from "fs"; +import path from "path"; import Bottleneck from "bottleneck"; import { ProviderType } from "../../.."; import type { IRequestConfig } from "../../proxies"; -import { selectProxy, proxyToUrl } from "../../../../proxies/impl/manager"; import { customRequest } from "../../../../proxies/impl/request"; +interface Proxy { + ip: string; + port: number; +} + +// 🌸 Load proxies once at startup, round-robin per provider +const ALL_PROXIES: Proxy[] = (() => { + try { + const raw = fs.readFileSync(path.resolve(process.cwd(), "proxies.json"), "utf-8"); + return JSON.parse(raw) as Proxy[]; + } catch { + return []; + } +})(); + +const roundRobinIndex: Map = new Map(); + +function pickProxy(providerId: string): string | undefined { + if (ALL_PROXIES.length === 0) return undefined; + const current = roundRobinIndex.get(providerId) ?? 0; + const proxy = ALL_PROXIES[current % ALL_PROXIES.length]; + roundRobinIndex.set(providerId, current + 1); + return proxy ? `${proxy.ip}:${proxy.port}` : undefined; +} + export class RequestError extends Error { constructor( message: string, @@ -32,7 +58,6 @@ export abstract class MediaProvider { abstract proxyCheck(proxyUrl: string): Promise; - // Add method to abort all pending requests for this provider public abortRequests(): void { if (this.controller) { this.controller.abort(); @@ -40,21 +65,15 @@ export abstract class MediaProvider { } } - /** - * Queued request function that respects this.rateLimit (seconds/10). - * Returns Response if successful, throws RequestError for handled failures. - * Throws other errors only for unexpected failures that should halt execution. - */ async request(url: string, config: IRequestConfig = {}, proxyRequest: boolean = false): Promise { - // Create a new controller for this request chain this.controller = new AbortController(); if (!MediaProvider.limiterMap.has(this.id)) { const bottleneck = new Bottleneck({ minTime: this.rateLimit, - reservoir: 10, // Add a reservoir to prevent too many requests + reservoir: 10, reservoirRefreshAmount: 10, - reservoirRefreshInterval: this.rateLimit * 1000, // Refresh interval in ms + reservoirRefreshInterval: this.rateLimit * 1000, }); if (this.maxConcurrentRequests > 0) { @@ -70,15 +89,14 @@ export abstract class MediaProvider { try { const response = await limiter.schedule(async () => { - // Check if request has been aborted if (this.controller?.signal.aborted) { throw new RequestError("Request aborted", url, this.id, this.providerType); } - const selectedProxy = selectProxy(this.providerType, this.id); - const proxyURL = proxyToUrl(selectedProxy); const useProxy = (config.proxy && config.proxy.length > 0) || proxyRequest || this.needsProxy; + const resolvedProxy = useProxy && !this.useGoogleTranslate ? (config.proxy && config.proxy.length > 0 ? config.proxy : pickProxy(this.id)) : undefined; + const finalConfig: IRequestConfig = { ...config, providerId: this.id, @@ -86,13 +104,13 @@ export abstract class MediaProvider { isChecking: this.isCheckingProxies || config.isChecking, useGoogleTranslate: this.useGoogleTranslate, signal: this.controller?.signal, - timeout: config.timeout || 15000, // Ensure timeout is set - _proxyURL: useProxy ? (this.useGoogleTranslate ? undefined : config.proxy && config.proxy.length > 0 ? config.proxy : (proxyURL ?? undefined)) : undefined, + timeout: config.timeout ?? 15000, + _proxyURL: resolvedProxy, }; const result = await customRequest(url, finalConfig); if (!result) { - this.abortRequests(); // Abort on failure + this.abortRequests(); throw new RequestError("Request failed after all retry attempts", url, this.id, this.providerType); } return result; @@ -101,10 +119,8 @@ export abstract class MediaProvider { return response; } catch (error) { if (error instanceof RequestError) { - // Abort any remaining requests this.abortRequests(); - // Return an empty 204 response return new Response(null, { status: 204, statusText: "No Content - Request Failed",