1  import fetch from "node-fetch";
   2  import { HttpsProxyAgent } from "https-proxy-agent";
   3  import WebSocket from "ws";
   4  import { logger } from "@/middleware/logging.js";
   5  import config from "@/config.js";
   6  import { SocksProxyAgent } from "socks-proxy-agent";
   7  import crypto from "crypto";
   8  import https from "https";
   9  import net from "net";
  10  import { getRandomUserAgent } from "@/utils/userAgentGenerator.js";
  11  
  12  class IpRotatorService {
  13    constructor() {
  14      this.tor = {
  15        available: false,
  16        controlAvailable: false,
  17        agent: null,
  18        currentExit: null,
  19        failures: 0,
  20        maliciousExits: new Set(),
  21      };
  22      this.proxyPool = [];
  23      this.deadProxies = new Map();
  24      this.proxyStats = new Map();
  25      this.realIp = null;
  26      this.blacklistedProxies = new Set();
  27      this.proxyFailureHistory = new Map();
  28      this.domainRateLimits = new Map();
  29      this.stats = {
  30        totalRequests: 0,
  31        successfulRequests: 0,
  32        failedRequests: 0,
  33        uniqueIps: new Set(),
  34        torUsed: 0,
  35        proxyUsed: 0,
  36        directUsed: 0,
  37        avgLatency: 0,
  38        poolHealth: 0,
  39      };
  40      this.isInitialized = false;
  41      this.refreshInterval = null;
  42      this.cleanupInterval = null;
  43      this.healthCheckInterval = null;
  44      this.monitoringInterval = null;
  45      this.lastHealthCheck = 0;
  46      this.requestCounts = new Map();
  47      this.sessionProxies = new Map();
  48      this.requestTimestamps = [];
  49      this.ipCheckUrls = config.ipRotator.ipCheckUrls;
  50      this.proxySources = config.ipRotator.proxySources;
  51      this.blacklistTimeout = config.ipRotator.blacklistTimeout;
  52      this.generalRateLimit = config.security.rateLimit.limit || 100;
  53      this.httpsAgent = new https.Agent({
  54        rejectUnauthorized: true,
  55        keepAlive: true,
  56        timeout: config.timeouts.medium,
  57      });
  58    }
  59  
  60    async initialize() {
  61      if (this.isInitialized) {
  62        return;
  63      }
  64  
  65      try {
  66        await this.detectRealIp();
  67        await this.initializeTor();
  68  
  69        this.refreshProxyPool().catch((error) => {
  70          logger.error("Error refreshing proxy pool:", error);
  71        });
  72        this.startBackgroundTasks();
  73        this.isInitialized = true;
  74  
  75        logger.info("IP Rotator Service initialized successfully", {
  76          torAvailable: this.tor.available,
  77          proxyCount: this.proxyPool.length,
  78          realIp: this.realIp ? "detected" : "unknown",
  79        });
  80      } catch (error) {
  81        logger.error("Failed to initialize IP Rotator Service:", error);
  82  
  83      }
  84    }
  85  
  86    async detectRealIp() {
  87      try {
  88        const response = await fetch(this.ipCheckUrls[1], { timeout: config.timeouts.short });
  89        if (!response.ok) {
  90          throw new Error(`HTTP ${response.status}`);
  91        }
  92        const data = await response.json();
  93        this.realIp = data.ip;
  94        logger.debug(`Real IP detected: ${this.realIp}`);
  95      } catch (error) {
  96        logger.warn("Could not detect real IP address:", error.message);
  97      }
  98    }
  99  
 100    async initializeTor() {
 101      try {
 102        this.tor.agent = new SocksProxyAgent("socks5h://127.0.0.1:9050");
 103  
 104        const testResponse = await fetch(this.ipCheckUrls[1], {
 105          agent: this.tor.agent,
 106          timeout: config.timeouts.short,
 107        });
 108  
 109        if (testResponse.ok) {
 110          this.tor.available = true;
 111          logger.info("Tor connection established");
 112  
 113          setInterval(async () => {
 114            try {
 115              await this.refreshTorExit();
 116            } catch (error) {
 117              logger.error("Failed to refresh Tor exit:", error);
 118            }
 119          }, config.timeouts.veryLong);
 120        }
 121      } catch (error) {
 122        this.tor.available = false;
 123        logger.error("Tor not available - will use proxies only", error);
 124      }
 125    }
 126  
 127    async scrapeProxies() {
 128      const sources = this.proxySources;
 129  
 130      const allProxies = new Set();
 131      const fetchPromises = sources.map(async (url) => {
 132        try {
 133          const controller = new AbortController();
 134          const timeoutId = setTimeout(() => controller.abort(), config.timeouts.short);
 135  
 136          const response = await fetch(url, {
 137            signal: controller.signal,
 138            timeout: config.timeouts.short,
 139            headers: {
 140              "User-Agent": this.getRandomUserAgent(),
 141            },
 142          });
 143  
 144          clearTimeout(timeoutId);
 145  
 146          if (!response.ok) {
 147            return;
 148          }
 149  
 150          const text = await response.text();
 151          const lines = text.split("\n");
 152  
 153          for (const line of lines) {
 154            const trimmed = line.trim();
 155            if (trimmed && trimmed.includes(":") && !trimmed.startsWith("#")) {
 156              const parts = trimmed.split(":");
 157              if (parts.length >= 2) {
 158                const ip = parts[0];
 159                const port = parts[1];
 160                if (/^(\d{1,3}\.){3}\d{1,3}$/.test(ip) && !isNaN(port) && parseInt(port) > 0 && parseInt(port) <= 65535) { // eslint-disable-line security/detect-unsafe-regex
 161                  const proxy = trimmed.startsWith("http") ? trimmed : `http://${trimmed}`;
 162                  allProxies.add(proxy);
 163                }
 164              }
 165            }
 166          }
 167        } catch (error) {
 168          logger.debug(`Failed to fetch proxies from ${url}:`, error.message);
 169        }
 170      });
 171  
 172      await Promise.allSettled(fetchPromises);
 173      logger.important(`Scraped ${allProxies.size} unique proxies from sources`);
 174      return Array.from(allProxies);
 175    }
 176  
 177    async testProxy(proxy, timeout = null) {
 178      const timeoutMs = timeout ?? config.timeouts.medium;
 179      const startTime = Date.now();
 180      try {
 181        let agent;
 182  
 183        if (proxy.startsWith("socks5://")) {
 184          agent = new SocksProxyAgent(proxy);
 185        } else if (proxy.startsWith("http://") || proxy.startsWith("https://")) {
 186          agent = new HttpsProxyAgent(proxy, {
 187            rejectUnauthorized: true,
 188            timeout: timeoutMs / 2,
 189          });
 190        } else {
 191          agent = new HttpsProxyAgent(`http://${proxy}`, {
 192            rejectUnauthorized: true,
 193            timeout: timeoutMs / 2,
 194          });
 195        }
 196  
 197        const controller = new AbortController();
 198        const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
 199  
 200        const testUrls = this.ipCheckUrls;
 201  
 202        let response, data, ip;
 203        for (const url of testUrls) {
 204          try {
 205            response = await fetch(url, {
 206              agent,
 207              signal: controller.signal,
 208              timeout: timeoutMs / 2,
 209              headers: {
 210                "User-Agent": this.getRandomUserAgent(),
 211              },
 212            });
 213  
 214            if (response.ok) {
 215              const text = await response.text();
 216  
 217              try {
 218                data = JSON.parse(text);
 219              } catch (parseError) {
 220                logger.debug(`Invalid JSON from ${url}:`, parseError.message);
 221                continue;
 222              }
 223  
 224              ip = data.ip || data.origin;
 225              if (ip && typeof ip === "string" && ip.length > 0) {
 226                break;
 227              }
 228            }
 229          } catch (e) {
 230            logger.debug(`Proxy ${proxy} failed for ${url}:`, e.message);
 231            continue;
 232          }
 233        }
 234  
 235        clearTimeout(timeoutId);
 236  
 237        if (!ip) {
 238          logger.debug(`Proxy ${proxy}: No IP returned from test endpoints`);
 239          return null;
 240        }
 241  
 242        const latency = Date.now() - startTime;
 243        if (this.realIp && ip.includes(this.realIp)) {
 244          logger.warn(`Proxy ${proxy} leaked real IP - blacklisting`);
 245          return null;
 246        }
 247  
 248        const category = this.categorizeProxy(proxy, ip);
 249  
 250        if (this.blacklistedProxies.has(proxy)) {
 251          logger.debug(`Proxy ${proxy} is blacklisted - skipping`);
 252          return null;
 253        }
 254  
 255        return {
 256          proxy,
 257          ip,
 258          lastUsed: Date.now(),
 259          failures: 0,
 260          latency,
 261          category,
 262          successRate: 100,
 263          lastHealthCheck: Date.now(),
 264          rpcFriendly: true,
 265        };
 266      } catch (error) {
 267        if (error.name === "AbortError") {
 268          logger.debug(`Proxy ${proxy} timed out`);
 269        } else {
 270          logger.debug(`Proxy ${proxy} failed:`, error.message);
 271        }
 272        return null;
 273      }
 274    }
 275  
 276    async validateProxies(proxies, targetCount = 200) {
 277      const batchSize = 50;
 278      const working = [];
 279  
 280      for (let i = 0; i < proxies.length && working.length < targetCount; i += batchSize) {
 281        const batch = proxies.slice(i, i + batchSize);
 282        const testPromises = batch.map((proxy) => this.testProxy(proxy));
 283        const results = await Promise.allSettled(testPromises);
 284  
 285        for (const result of results) {
 286          if (result.status === "fulfilled" && result.value) {
 287            working.push(result.value);
 288            if (working.length >= targetCount) {
 289              break;
 290            }
 291          }
 292        }
 293  
 294        await new Promise(resolve => setTimeout(resolve, 100));
 295      }
 296  
 297      working.sort((a, b) => a.latency - b.latency);
 298      return working;
 299    }
 300  
 301    async refreshProxyPool() {
 302      try {
 303        const scraped = await this.scrapeProxies();
 304  
 305        if (scraped.length === 0) {
 306          logger.warn("No proxies scraped from sources");
 307          return;
 308        }
 309  
 310        const targetPoolSize = 500;
 311        const validated = await this.validateProxies(scraped, targetPoolSize);
 312  
 313        if (validated.length > 0) {
 314          const newProxies = validated.filter(p =>
 315            !this.proxyPool.some(existing => existing.proxy === p.proxy),
 316          );
 317  
 318          this.proxyPool = [...this.proxyPool, ...newProxies];
 319          this.proxyPool.sort((a, b) => a.latency - b.latency);
 320  
 321          this.proxyPool = this.proxyPool.slice(-500);
 322  
 323          logger.info(`Proxy pool refreshed: +${newProxies.length} new, total: ${this.proxyPool.length}`);
 324  
 325          newProxies.forEach(p => {
 326            this.proxyStats.set(p.proxy, {
 327              successes: 0,
 328              failures: 0,
 329              totalRequests: 0,
 330              avgLatency: p.latency,
 331              lastUsed: Date.now(),
 332            });
 333          });
 334        } else {
 335          logger.warn("No working proxies found during refresh");
 336        }
 337      } catch (error) {
 338        logger.error("Error refreshing proxy pool:", error);
 339      }
 340    }
 341  
 342    cleanupRequestCounts() {
 343      const now = Date.now();
 344      const staleTimeout = config.timeouts.veryLong;
 345  
 346      for (const [proxy, count] of this.requestCounts.entries()) {
 347        if (count === 0 || (now - (this.proxyStats.get(proxy)?.lastUsed || 0)) > staleTimeout) {
 348          this.requestCounts.delete(proxy);
 349        }
 350      }
 351  
 352      if (this.requestCounts.size > 1000) {
 353        logger.warn(`RequestCounts map size is ${this.requestCounts.size}, forcing cleanup`);
 354        this.requestCounts.clear();
 355      }
 356    }
 357  
 358    cleanupDeadProxies() {
 359      const now = Date.now();
 360      const baseDeadTimeout = config.timeouts.extraLong;
 361      const baseQuarantineTimeout = config.timeouts.extraLong / 2;
 362  
 363      const failureRate = this.stats.totalRequests > 0 ?
 364        (this.stats.failedRequests / this.stats.totalRequests) : 0;
 365  
 366      const adaptiveMultiplier = Math.max(0.5, Math.min(2.0, 1 + failureRate));
 367      const deadTimeout = baseDeadTimeout * adaptiveMultiplier;
 368      const quarantineTimeout = baseQuarantineTimeout * adaptiveMultiplier;
 369  
 370      for (const [proxy, data] of this.deadProxies.entries()) {
 371        const age = now - data.markedAt;
 372        const consecutiveFailures = data.consecutiveFailures || 0;
 373        const failureMultiplier = Math.max(1, consecutiveFailures * 0.5);
 374        const adjustedDeadTimeout = deadTimeout * failureMultiplier;
 375  
 376        if (age > adjustedDeadTimeout) {
 377          this.deadProxies.delete(proxy);
 378          this.proxyStats.delete(proxy);
 379          this.requestCounts.delete(proxy);
 380          logger.debug(`Proxy ${proxy} removed from dead list after ${Math.round(age / 1000)}s`);
 381        } else if (age > quarantineTimeout && data.attempts < 3) {
 382          this.deadProxies.delete(proxy);
 383          logger.info(`Proxy ${proxy} released from quarantine for retest`);
 384        }
 385      }
 386  
 387      this.proxyPool = this.proxyPool.filter((proxyData) => {
 388        const stats = this.proxyStats.get(proxyData.proxy);
 389        if (!stats) {
 390          return true;
 391        }
 392  
 393        const successRate = stats.totalRequests > 0 ? (stats.successes / stats.totalRequests) * 100 : 100;
 394        const failureThreshold = Math.min(5, Math.max(3, Math.floor(5 * (1 + failureRate))));
 395        const successThreshold = Math.max(70, 80 - (failureRate * 10));
 396  
 397        if (proxyData.failures >= failureThreshold || successRate < successThreshold) {
 398          const deadData = this.deadProxies.get(proxyData.proxy);
 399          if (deadData) {
 400            deadData.attempts++;
 401            deadData.consecutiveFailures = (deadData.consecutiveFailures || 0) + 1;
 402            if (deadData.attempts >= 3) {
 403              logger.warn(`Proxy ${proxyData.proxy} permanently removed (success rate: ${successRate.toFixed(1)}%, failures: ${proxyData.failures})`);
 404              this.proxyStats.delete(proxyData.proxy);
 405            }
 406          } else {
 407            this.deadProxies.set(proxyData.proxy, {
 408              markedAt: now,
 409              attempts: 1,
 410              consecutiveFailures: 1,
 411              lastError: successRate < successThreshold ? "Low success rate" : "Multiple failures",
 412            });
 413          }
 414          return false;
 415        }
 416        return true;
 417      });
 418  
 419      this.updatePoolHealth();
 420    }
 421  
 422    startBackgroundTasks() {
 423      this.refreshInterval = setInterval(() => {
 424        this.refreshProxyPool();
 425      }, (config.timeouts.veryLong || 300000));
 426  
 427      this.cleanupInterval = setInterval(() => {
 428        this.cleanupDeadProxies();
 429        this.cleanupRequestCounts();
 430        this.cleanupBlacklistedProxies();
 431      }, config.timeouts.extraLong || 600000);
 432  
 433      this.healthCheckInterval = setInterval(() => {
 434        this.performHealthChecks();
 435      }, config.timeouts.veryLong || 600000);
 436  
 437      this.monitoringInterval = setInterval(() => {
 438        this.logPoolHealth();
 439      }, config.timeouts.extraLong || 600000);
 440    }
 441  
 442    stopBackgroundTasks() {
 443      if (this.refreshInterval) {
 444        clearInterval(this.refreshInterval);
 445        this.refreshInterval = null;
 446      }
 447      if (this.cleanupInterval) {
 448        clearInterval(this.cleanupInterval);
 449        this.cleanupInterval = null;
 450      }
 451      if (this.healthCheckInterval) {
 452        clearInterval(this.healthCheckInterval);
 453        this.healthCheckInterval = null;
 454      }
 455      if (this.monitoringInterval) {
 456        clearInterval(this.monitoringInterval);
 457        this.monitoringInterval = null;
 458      }
 459      logger.info("Background tasks stopped");
 460    }
 461  
 462    selectProxy(sessionId = null) {
 463      if (this.proxyPool.length === 0) {
 464        return null;
 465      }
 466  
 467      if (sessionId && this.sessionProxies.has(sessionId)) {
 468        const sessionProxy = this.sessionProxies.get(sessionId);
 469        if (this.proxyPool.some(p => p.proxy === sessionProxy.proxy)) {
 470          return sessionProxy;
 471        } else {
 472          this.sessionProxies.delete(sessionId);
 473        }
 474      }
 475  
 476      const availableProxies = this.proxyPool.filter(p => {
 477        const stats = this.proxyStats.get(p.proxy);
 478        if (!stats) {
 479          return true;
 480        }
 481  
 482        const successRate = stats.totalRequests > 0 ? (stats.successes / stats.totalRequests) * 100 : 100;
 483        const recentRequests = this.requestCounts.get(p.proxy) || 0;
 484  
 485        return successRate >= 80 && recentRequests < 10;
 486      });
 487  
 488      if (availableProxies.length === 0) {
 489        logger.warn("No healthy proxies available, using least failed proxy");
 490        return this.proxyPool.reduce((best, current) =>
 491          current.failures < best.failures ? current : best,
 492        );
 493      }
 494  
 495      availableProxies.sort((a, b) => {
 496        const statsA = this.proxyStats.get(a.proxy) || {};
 497        const statsB = this.proxyStats.get(b.proxy) || {};
 498  
 499        const successRateA = statsA.totalRequests > 0 ? (statsA.successes / statsA.totalRequests) * 100 : 100;
 500        const successRateB = statsB.totalRequests > 0 ? (statsB.successes / statsB.totalRequests) * 100 : 100;
 501  
 502        if (Math.abs(successRateA - successRateB) > 5) {
 503          return successRateB - successRateA;
 504        }
 505  
 506        return a.latency - b.latency;
 507      });
 508  
 509      const topCandidates = availableProxies.slice(0, Math.min(10, availableProxies.length));
 510      const selected = topCandidates[Math.floor(Math.random() * topCandidates.length)];
 511  
 512      if (sessionId) {
 513        this.sessionProxies.set(sessionId, selected);
 514      }
 515  
 516      return selected;
 517    }
 518  
 519    async makeRequest(url, options = {}) {
 520      if (!this.isInitialized) {
 521        throw new Error("IP Rotator Service not initialized. Call initialize() first.");
 522      }
 523  
 524      const now = Date.now();
 525      this.requestTimestamps = this.requestTimestamps.filter(t => now - t < config.timeouts.long || 60000);
 526  
 527      if (this.requestTimestamps.length >= this.generalRateLimit) {
 528        logger.warn("Request rate limit approached", {
 529          current: this.requestTimestamps.length,
 530          limit: this.generalRateLimit,
 531        });
 532      }
 533  
 534      this.requestTimestamps.push(now);
 535      this.stats.totalRequests++;
 536      const maxRetries = options.maxRetries || 5;
 537      const timeout = options.timeout || config.timeouts.medium || 10000;
 538      const method = options.method || "GET";
 539      const sessionId = options.sessionId || this.generateSessionId();
 540      const prioritizeTor = options.prioritizeTor !== false;
 541  
 542      let lastError = null;
 543      let attempt = 0;
 544      const useTor = !options.sensitive && this.tor.available && this.tor.failures < 3;
 545      const tryTorFirst = prioritizeTor && useTor;
 546      const tryTorSecond = !prioritizeTor && useTor;
 547  
 548      const attemptRequest = async (useTorMethod) => {
 549        let agent = null;
 550        let method_used = "direct";
 551        let proxyData = null;
 552  
 553        if (useTorMethod) {
 554          agent = this.tor.agent;
 555          method_used = "tor";
 556        } else {
 557          proxyData = this.selectProxy(sessionId);
 558          if (proxyData) {
 559            if (proxyData.proxy.startsWith("socks5://")) {
 560              agent = new SocksProxyAgent(proxyData.proxy);
 561            } else {
 562              agent = new HttpsProxyAgent(proxyData.proxy);
 563            }
 564            method_used = "proxy";
 565          }
 566        }
 567  
 568        if (!agent) {
 569          return { success: false, proxyData: null, method_used };
 570        }
 571  
 572        const startTime = Date.now();
 573        const controller = new AbortController();
 574        const timeoutId = setTimeout(() => controller.abort(), timeout);
 575  
 576        const fetchOptions = {
 577          ...options,
 578          method,
 579          timeout,
 580          signal: controller.signal,
 581          agent: this.httpsAgent,
 582          headers: {
 583            ...options.headers,
 584            "User-Agent": this.getRandomUserAgent(),
 585            "Accept": options.headers?.Accept || "application/json, text/plain, */*",
 586            "Accept-Language": "en-US,en;q=0.9",
 587            "Accept-Encoding": "gzip, deflate, br",
 588            "DNT": "1",
 589            "Connection": "keep-alive",
 590            "Upgrade-Insecure-Requests": "1",
 591          },
 592        };
 593  
 594        if (agent) {
 595          fetchOptions.agent = agent;
 596        }
 597  
 598        try {
 599          const response = await fetch(url, fetchOptions);
 600          clearTimeout(timeoutId);
 601  
 602          const latency = Date.now() - startTime;
 603  
 604          if (!response.ok && response.status >= 400 && response.status < 600) {
 605            if (response.status === 429) {
 606              const domain = new URL(url).hostname;
 607              this.handleDomainRateLimit(domain);
 608              
 609              if (proxyData) {
 610                logger.debug(`Proxy ${proxyData.proxy} hit rate limit, trying different proxy`);
 611              }
 612              
 613              throw new Error("Rate limited (429)");
 614            } else if (response.status >= 500) {
 615              throw new Error(`Server error ${response.status}`);
 616            } else {
 617              throw new Error(`HTTP ${response.status}: ${response.statusText}`);
 618            }
 619          }
 620  
 621        if (proxyData) {
 622          proxyData.lastUsed = Date.now();
 623          proxyData.failures = 0;
 624  
 625          const stats = this.proxyStats.get(proxyData.proxy);
 626          if (stats) {
 627            stats.successes++;
 628            stats.totalRequests++;
 629            stats.avgLatency = (stats.avgLatency + latency) / 2;
 630            stats.lastUsed = Date.now();
 631          }
 632        }
 633  
 634          this.stats.successfulRequests++;
 635          this.stats[`${method_used}Used`]++;
 636          this.stats.avgLatency = (this.stats.avgLatency + latency) / 2;
 637  
 638          const result = {
 639            response,
 640            method: method_used,
 641            proxy: proxyData?.proxy,
 642            attempt,
 643            latency,
 644          };
 645  
 646          try {
 647            const text = await response.clone().text();
 648            let testData;
 649  
 650            try {
 651              testData = JSON.parse(text);
 652            } catch (parseError) {
 653              logger.debug("Invalid JSON in response:", parseError.message);
 654              return { success: true, result, proxyData, method_used };
 655            }
 656  
 657            if (testData.origin || testData.ip) {
 658              const ip = (testData.origin || testData.ip).split(",")[0].trim();
 659  
 660              if (this.realIp && ip === this.realIp) {
 661                logger.warn(`Real IP exposed via ${method_used} - retrying with different method`);
 662                if (proxyData) {
 663                  proxyData.failures += 10;
 664                } else if (method_used === "tor") {
 665                  this.tor.failures++;
 666                  await this.refreshTorExit();
 667                }
 668                throw new Error("Real IP exposed");
 669              }
 670  
 671              this.stats.uniqueIps.add(ip);
 672  
 673              if (method_used === "tor") {
 674                this.tor.currentExit = ip;
 675              }
 676            }
 677          } catch { }
 678  
 679          const currentCount = this.requestCounts.get(proxyData?.proxy || "direct") || 0;
 680          this.requestCounts.set(proxyData?.proxy || "direct", currentCount + 1);
 681  
 682          setTimeout(() => {
 683            const count = this.requestCounts.get(proxyData?.proxy || "direct") || 0;
 684            if (count > 0) {
 685              this.requestCounts.set(proxyData?.proxy || "direct", count - 1);
 686            }
 687          }, config.timeouts.long || 60000);
 688  
 689          return { success: true, result, proxyData, method_used };
 690        } catch (error) {
 691          clearTimeout(timeoutId);
 692          return { success: false, error, proxyData, method_used };
 693        }
 694      };
 695  
 696      if (tryTorFirst) {
 697        for (let retry = 0; retry < maxRetries; retry++) {
 698          attempt++;
 699          const response = await attemptRequest(true);
 700          if (response.success) {
 701            return response.result;
 702          }
 703          
 704          lastError = response.error;
 705          this.tor.failures++;
 706          if (this.tor.failures >= 3) {
 707            logger.error("Tor failed multiple times, switching to proxies");
 708          }
 709          if (lastError?.message.includes("429")) {
 710            logger.debug(`Rate limit detected on Tor, switching to proxies`);
 711            break;
 712          }
 713          if (retry < maxRetries - 1) {
 714            const baseDelay = Math.min(1000 * Math.pow(2, retry), 5000);
 715            const jitter = Math.random() * 1000;
 716            await new Promise((resolve) => setTimeout(resolve, baseDelay + jitter));
 717          }
 718        }
 719      }
 720  
 721      for (let retry = 0; retry < maxRetries; retry++) {
 722        attempt++;
 723        const response = await attemptRequest(false);
 724        if (response.success) {
 725          return response.result;
 726        }
 727        
 728        lastError = response.error;
 729        const proxyData = response.proxyData;
 730        
 731        if (proxyData) {
 732          proxyData.failures++;
 733  
 734          const stats = this.proxyStats.get(proxyData.proxy);
 735          if (stats) {
 736            stats.failures++;
 737            stats.totalRequests++;
 738          }
 739  
 740          this.trackProxyFailure(proxyData.proxy, lastError?.message || "Unknown error");
 741  
 742          if (this.shouldBlacklistProxy(proxyData.proxy)) {
 743            this.blacklistProxy(proxyData.proxy, "Auto-blacklisted due to poor performance");
 744          }
 745  
 746          if (lastError?.message.includes("429") || lastError?.message.includes("rate limit")) {
 747            logger.debug(`Proxy ${proxyData.proxy} failed (failures: ${proxyData.failures}):`, lastError?.message);
 748            continue;
 749          }
 750  
 751          logger.debug(`Proxy ${proxyData.proxy} failed (failures: ${proxyData.failures}):`, lastError?.message);
 752        }
 753        
 754        if (retry < maxRetries - 1) {
 755          const baseDelay = Math.min(1000 * Math.pow(2, retry), 5000);
 756          const jitter = Math.random() * 1000;
 757          await new Promise((resolve) => setTimeout(resolve, baseDelay + jitter));
 758        }
 759      }
 760  
 761      if (tryTorSecond) {
 762        for (let retry = 0; retry < maxRetries; retry++) {
 763          attempt++;
 764          const response = await attemptRequest(true);
 765          if (response.success) {
 766            return response.result;
 767          }
 768          
 769          lastError = response.error;
 770          this.tor.failures++;
 771          if (this.tor.failures >= 3) {
 772            logger.error("Tor failed multiple times");
 773          }
 774          if (retry < maxRetries - 1) {
 775            const baseDelay = Math.min(1000 * Math.pow(2, retry), 5000);
 776            const jitter = Math.random() * 1000;
 777            await new Promise((resolve) => setTimeout(resolve, baseDelay + jitter));
 778          }
 779        }
 780      }
 781  
 782      if (!useTor && this.proxyPool.length === 0) {
 783        logger.error("All proxy methods failed, would fall back to direct connection - blocked for security");
 784        throw new Error("No anonymous connection methods available");
 785      }
 786  
 787      this.stats.failedRequests++;
 788  
 789      if (lastError?.message.includes("429") || lastError?.message.includes("Too Many Requests")) {
 790        logger.warn(`Rate limited on all endpoints after ${attempt} attempts`, {
 791          url,
 792          error: lastError?.message,
 793        });
 794      } else {
 795        logger.error(`Request failed after ${attempt} attempts:`, {
 796          url,
 797          error: lastError?.message,
 798        });
 799      }
 800  
 801      throw lastError || new Error("Request failed after all retries");
 802    }
 803  
 804    async get(url, options = {}) {
 805      return this.makeRequest(url, { ...options, method: "GET", prioritizeTor: true });
 806    }
 807  
 808    async post(url, options = {}) {
 809      return this.makeRequest(url, { ...options, method: "POST", prioritizeTor: true });
 810    }
 811  
 812    getStats() {
 813      const healthyProxies = this.proxyPool.filter(p => {
 814        const stats = this.proxyStats.get(p.proxy);
 815        return stats ? stats.successes / stats.totalRequests >= 0.8 : true;
 816      }).length;
 817  
 818      return {
 819        ...this.stats,
 820        uniqueIps: this.stats.uniqueIps.size,
 821        successRate: this.stats.totalRequests > 0
 822          ? ((this.stats.successfulRequests / this.stats.totalRequests) * 100).toFixed(2) + "%"
 823          : "0%",
 824        proxyPoolSize: this.proxyPool.length,
 825        healthyProxies,
 826        deadProxiesCount: this.deadProxies.size,
 827        blacklistedProxiesCount: this.blacklistedProxies.size,
 828        torAvailable: this.tor.available,
 829        torFailures: this.tor.failures,
 830        currentTorExit: this.tor.currentExit,
 831        anonymityProtection: this.realIp ? "ACTIVE" : "UNKNOWN",
 832        poolHealth: this.stats.poolHealth.toFixed(1) + "%",
 833        avgLatency: Math.round(this.stats.avgLatency) + "ms",
 834      };
 835    }
 836  
 837    logStats() {
 838      const stats = this.getStats();
 839      logger.info("IP Rotator Statistics:", stats);
 840    }
 841  
 842    async getWsAgent(preferTor = false, sessionId = null) {
 843      if (preferTor && this.tor.available && this.tor.failures < 3) {
 844        return this.tor.agent;
 845      }
 846  
 847      const proxyData = this.selectProxy(sessionId);
 848      if (proxyData) {
 849        let agent;
 850        if (proxyData.proxy.startsWith("socks5://")) {
 851          agent = new SocksProxyAgent(proxyData.proxy);
 852        } else {
 853          agent = new HttpsProxyAgent(proxyData.proxy);
 854        }
 855        agent.proxy = proxyData.proxy;
 856        return agent;
 857      }
 858  
 859      return null;
 860    }
 861  
 862    createProxiedWebSocket(url, options = {}) {
 863      const { preferTor = false, agent: customAgent, sessionId = null } = options;
 864      let agent = customAgent;
 865  
 866      if (!agent) {
 867        agent = this.getWsAgent(preferTor, sessionId);
 868      }
 869  
 870      const wsOptions = {
 871        handshakeTimeout: config.timeouts.medium,
 872        perMessageDeflate: false,
 873        headers: {
 874          "User-Agent": this.getRandomUserAgent(),
 875          "Origin": "https://google.com",
 876        },
 877      };
 878  
 879      if (agent) {
 880        wsOptions.agent = agent;
 881      }
 882  
 883      const ws = new WebSocket(url, wsOptions);
 884  
 885      if (agent) {
 886        ws.on("error", (error) => {
 887          logger.debug("WebSocket error through proxy:", error.message);
 888          if (agent === this.tor.agent) {
 889            this.tor.failures++;
 890            this.stats.torUsed++;
 891          } else {
 892            const proxyData = this.proxyPool.find(p => p.proxy === agent.proxy);
 893            if (proxyData) {
 894              proxyData.failures++;
 895              const stats = this.proxyStats.get(proxyData.proxy);
 896              if (stats) {
 897                stats.failures++;
 898                stats.totalRequests++;
 899              }
 900            }
 901            this.stats.proxyUsed++;
 902          }
 903        });
 904  
 905        ws.on("close", (code, reason) => {
 906          if (code !== 1000) {
 907            logger.debug(`WebSocket closed with code ${code}:`, reason?.toString());
 908          }
 909        });
 910      } else {
 911        this.stats.directUsed++;
 912        logger.warn("WebSocket connection without proxy - anonymity risk");
 913      }
 914  
 915      return ws;
 916    }
 917  
 918    async createProxiedWebSocketAsync(url, options = {}) {
 919      const { preferTor = false, agent: customAgent, sessionId = null } = options;
 920      let agent = customAgent;
 921  
 922      if (!agent) {
 923        agent = this.getWsAgent(preferTor, sessionId);
 924      }
 925  
 926      const wsOptions = {
 927        handshakeTimeout: config.timeouts.medium,
 928        perMessageDeflate: false,
 929        headers: {
 930          "User-Agent": this.getRandomUserAgent(),
 931          "Origin": "https://google.com",
 932        },
 933      };
 934  
 935      if (agent) {
 936        wsOptions.agent = agent;
 937      }
 938  
 939      const ws = new WebSocket(url, wsOptions);
 940  
 941      if (agent) {
 942        ws.on("error", (error) => {
 943          logger.debug("WebSocket error through proxy:", error.message);
 944          if (agent === this.tor.agent) {
 945            this.tor.failures++;
 946            this.stats.torUsed++;
 947          } else {
 948            const proxyData = this.proxyPool.find(p => p.proxy === agent.proxy);
 949            if (proxyData) {
 950              proxyData.failures++;
 951              const stats = this.proxyStats.get(proxyData.proxy);
 952              if (stats) {
 953                stats.failures++;
 954                stats.totalRequests++;
 955              }
 956            }
 957            this.stats.proxyUsed++;
 958          }
 959        });
 960  
 961        ws.on("close", (code, reason) => {
 962          if (code !== 1000) {
 963            logger.debug(`WebSocket closed with code ${code}:`, reason?.toString());
 964          }
 965        });
 966      } else {
 967        this.stats.directUsed++;
 968        logger.warn("WebSocket connection without proxy - anonymity risk");
 969      }
 970  
 971      return ws;
 972    }
 973  
 974    async testWebSocketProxy(url, timeout = null) {
 975      const timeoutMs = timeout ?? config.timeouts.medium;
 976      return new Promise((resolve) => {
 977        const ws = this.createProxiedWebSocket(url, { preferTor: true });
 978        let resolved = false;
 979  
 980        const timer = setTimeout(() => {
 981          if (!resolved) {
 982            resolved = true;
 983            ws.terminate();
 984            resolve(false);
 985          }
 986        }, timeoutMs);
 987  
 988        ws.on("open", () => {
 989          if (!resolved) {
 990            resolved = true;
 991            clearTimeout(timer);
 992            ws.close();
 993            resolve(true);
 994          }
 995        });
 996  
 997        ws.on("error", () => {
 998          if (!resolved) {
 999            resolved = true;
1000            clearTimeout(timer);
1001            resolve(false);
1002          }
1003        });
1004      });
1005    }
1006  
1007    categorizeProxy(proxy, ip) {
1008      if (proxy.includes("tor") || proxy.includes("socks5")) {
1009        return "premium";
1010      }
1011  
1012      const datacenterRanges = [
1013        /^\d+\.\d+\.\d+\.(1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31)/,
1014        /^10\./,
1015        /^172\.(1[6-9]|2[0-9]|3[0-1])\./,
1016        /^192\.168\./,
1017      ];
1018  
1019      const isDatacenter = datacenterRanges.some(range => range.test(ip));
1020  
1021      if (isDatacenter) {
1022        return "datacenter";
1023      }
1024  
1025      return "residential";
1026    }
1027  
1028    getRandomUserAgent() {
1029      return getRandomUserAgent();
1030    }
1031  
1032    generateSessionId() {
1033      return crypto.randomBytes(16).toString("hex");
1034    }
1035  
1036    async sendTorControlCommand(command) {
1037      return new Promise((resolve, reject) => {
1038        const socket = net.connect({ host: "127.0.0.1", port: 9051 }, () => {
1039          socket.write(`AUTHENTICATE ""\r\n`);
1040        });
1041  
1042        let authenticated = false;
1043        let buffer = "";
1044  
1045        socket.on("data", (data) => {
1046          buffer += data.toString();
1047          const lines = buffer.split("\r\n");
1048          buffer = lines.pop();
1049  
1050          for (const line of lines) {
1051            if (line.startsWith("250")) {
1052              if (!authenticated) {
1053                authenticated = true;
1054                socket.write(`${command}\r\n`);
1055              } else {
1056                socket.end();
1057                this.tor.controlAvailable = true;
1058                resolve(true);
1059                return;
1060              }
1061            } else if (line.startsWith("515")) {
1062              socket.end();
1063              this.tor.controlAvailable = false;
1064              reject(new Error("Tor authentication failed"));
1065              return;
1066            } else if (line.startsWith("5")) {
1067              socket.end();
1068              this.tor.controlAvailable = false;
1069              reject(new Error(`Tor control error: ${line}`));
1070              return;
1071            }
1072          }
1073        });
1074  
1075        socket.on("error", (error) => {
1076          if (error.code === "ECONNREFUSED") {
1077            this.tor.controlAvailable = false;
1078          }
1079          reject(error);
1080        });
1081  
1082        socket.setTimeout(5000, () => {
1083          socket.destroy();
1084          this.tor.controlAvailable = false;
1085          reject(new Error("Tor control connection timeout"));
1086        });
1087      });
1088    }
1089  
1090    async refreshTorExit() {
1091      try {
1092        if (!this.tor.agent) {
1093          return;
1094        }
1095  
1096        if (!this.tor.controlAvailable && this.tor.controlAvailable !== undefined) {
1097          logger.debug("Tor control port not available, skipping exit node refresh");
1098          return;
1099        }
1100  
1101        try {
1102          await this.sendTorControlCommand("SIGNAL NEWNYM");
1103          this.tor.currentExit = null;
1104          logger.debug("Requested new Tor exit node");
1105  
1106          await new Promise(resolve => setTimeout(resolve, 5000));
1107  
1108          const testResponse = await fetch(this.ipCheckUrls[1], {
1109            agent: this.tor.agent,
1110            timeout: config.timeouts.short,
1111          });
1112  
1113          if (testResponse.ok) {
1114            const data = await testResponse.json();
1115            const newIp = data.ip;
1116  
1117            if (this.tor.maliciousExits.has(newIp)) {
1118              logger.warn(`New Tor exit ${newIp} is known to be malicious, refreshing again`);
1119              await this.refreshTorExit();
1120            } else {
1121              this.tor.currentExit = newIp;
1122              logger.debug(`Switched to new Tor exit: ${newIp}`);
1123            }
1124          }
1125        } catch (error) {
1126          if (error.code === "ECONNREFUSED" || error.message.includes("connection")) {
1127            this.tor.controlAvailable = false;
1128            logger.warn("Tor control port not available - exit node refresh disabled. Tor connections will still work but exit nodes won't be refreshed automatically.");
1129          } else {
1130            throw error;
1131          }
1132        }
1133      } catch (error) {
1134        logger.error("Failed to refresh Tor exit node:", error);
1135      }
1136    }
1137  
1138    async performHealthChecks() {
1139      if (Date.now() - this.lastHealthCheck < config.timeouts.veryLong) {
1140        return;
1141      }
1142  
1143      this.lastHealthCheck = Date.now();
1144      const sampleSize = Math.min(20, this.proxyPool.length);
1145      const sample = this.proxyPool
1146        .sort(() => Math.random() - 0.5)
1147        .slice(0, sampleSize);
1148  
1149      logger.info(`Performing health checks on ${sample.length} proxies`);
1150  
1151      for (const proxy of sample) {
1152        const isHealthy = await this.testProxy(proxy.proxy, 3000);
1153        if (!isHealthy) {
1154          proxy.failures++;
1155          logger.debug(`Proxy ${proxy.proxy} failed health check`);
1156        } else {
1157          proxy.lastHealthCheck = Date.now();
1158        }
1159  
1160        await new Promise(resolve => setTimeout(resolve, 200));
1161      }
1162  
1163      this.updatePoolHealth();
1164    }
1165  
1166    updatePoolHealth() {
1167      if (this.proxyPool.length === 0) {
1168        this.stats.poolHealth = 0;
1169        return;
1170      }
1171  
1172      const healthyCount = this.proxyPool.filter(p => {
1173        const stats = this.proxyStats.get(p.proxy);
1174        if (!stats) {
1175          return true;
1176        }
1177        return stats.totalRequests === 0 || (stats.successes / stats.totalRequests) >= 0.8;
1178      }).length;
1179  
1180      this.stats.poolHealth = (healthyCount / this.proxyPool.length) * 100;
1181    }
1182  
1183    trackProxyFailure(proxy, errorMessage) {
1184      if (!this.proxyFailureHistory.has(proxy)) {
1185        this.proxyFailureHistory.set(proxy, {
1186          failures: [],
1187          firstFailure: Date.now(),
1188          blacklisted: false,
1189        });
1190      }
1191  
1192      const history = this.proxyFailureHistory.get(proxy);
1193      history.failures.push({
1194        timestamp: Date.now(),
1195        error: errorMessage,
1196      });
1197  
1198      const oneHourAgo = Date.now() - (360000);
1199      history.failures = history.failures.filter(f => f.timestamp > oneHourAgo);
1200    }
1201  
1202    shouldBlacklistProxy(proxy) {
1203      const history = this.proxyFailureHistory.get(proxy);
1204      if (!history || history.blacklisted) {
1205        return false;
1206      }
1207  
1208      const recentFailures = history.failures.length;
1209      const timeSinceFirst = Date.now() - history.firstFailure;
1210      const oneHour = 360000;
1211  
1212      if (timeSinceFirst > oneHour) {
1213        history.firstFailure = Date.now();
1214        history.failures = history.failures.filter(f => Date.now() - f.timestamp < oneHour);
1215      }
1216  
1217      if (recentFailures >= 10) {
1218        return true;
1219      } else if (recentFailures >= 5 && timeSinceFirst < config.timeouts.extraLong) {
1220        return true;
1221      }
1222  
1223      const stats = this.proxyStats.get(proxy);
1224      if (stats && stats.totalRequests >= 20) {
1225        const successRate = stats.successes / stats.totalRequests;
1226        if (successRate < 0.3) {
1227          return true;
1228        }
1229      }
1230  
1231      const errorTypes = new Set(history.failures.map(f => f.error));
1232      if (errorTypes.has("Connection refused") || errorTypes.has("Connection timeout") || errorTypes.has("Socket timeout")) {
1233        if (recentFailures >= 3) {
1234          return true;
1235        }
1236      }
1237  
1238      return false;
1239    }
1240  
1241    blacklistProxy(proxy, reason = "Poor performance") {
1242      this.blacklistedProxies.add(proxy);
1243  
1244      const history = this.proxyFailureHistory.get(proxy);
1245      if (history) {
1246        history.blacklisted = true;
1247      }
1248  
1249      this.proxyPool = this.proxyPool.filter(p => p.proxy !== proxy);
1250      this.deadProxies.set(proxy, {
1251        markedAt: Date.now(),
1252        attempts: 3,
1253        lastError: reason,
1254        blacklisted: true,
1255      });
1256  
1257      this.proxyStats.delete(proxy);
1258      this.requestCounts.delete(proxy);
1259  
1260      logger.warn(`Proxy ${proxy} blacklisted: ${reason}`);
1261    }
1262  
1263    cleanupBlacklistedProxies() {
1264      const now = Date.now();
1265      const blacklistTimeout = this.blacklistTimeout;
1266  
1267      for (const proxy of this.blacklistedProxies) {
1268        const deadData = this.deadProxies.get(proxy);
1269        if (deadData && now - deadData.markedAt > blacklistTimeout) {
1270          this.blacklistedProxies.delete(proxy);
1271          this.deadProxies.delete(proxy);
1272          this.proxyFailureHistory.delete(proxy);
1273          logger.info(`Proxy ${proxy} removed from blacklist after timeout`);
1274        }
1275      }
1276    }
1277  
1278    logPoolHealth() {
1279      const stats = this.getStats();
1280  
1281      logger.info("Proxy Pool Health Report:", {
1282        totalProxies: stats.proxyPoolSize,
1283        healthyProxies: stats.healthyProxies,
1284        poolHealth: stats.poolHealth,
1285        torAvailable: stats.torAvailable,
1286        torFailures: stats.torFailures,
1287        avgLatency: stats.avgLatency,
1288        successRate: stats.successRate,
1289      });
1290  
1291      if (stats.poolHealth < 50) {
1292        logger.warn("Proxy pool health is below 50% - consider adding more sources");
1293      }
1294  
1295      if (this.tor.failures >= 3) {
1296        logger.error("Tor has failed multiple times - check Tor configuration");
1297      }
1298    }
1299  
1300    handleDomainRateLimit(domain) {
1301      const now = Date.now();
1302      if (!this.domainRateLimits.has(domain)) {
1303        this.domainRateLimits.set(domain, {
1304          firstHit: now,
1305          lastHit: now,
1306          hitCount: 1,
1307          backoffUntil: 0,
1308        });
1309      } else {
1310        const limit = this.domainRateLimits.get(domain);
1311        limit.lastHit = now;
1312        limit.hitCount++;
1313  
1314        const timeSinceFirst = now - limit.firstHit;
1315        if (timeSinceFirst < 60000) {
1316          limit.backoffUntil = now + Math.min(300000, Math.pow(2, limit.hitCount) * 1000);
1317          logger.warn(`Domain ${domain} hit rate limit ${limit.hitCount} times, backing off until ${new Date(limit.backoffUntil)}`);
1318        } else if (timeSinceFirst > 300000) {
1319          limit.hitCount = 1;
1320          limit.firstHit = now;
1321          limit.backoffUntil = 0;
1322        }
1323      }
1324    }
1325  
1326    getDomainRateLimitDelay(domain) {
1327      const limit = this.domainRateLimits.get(domain);
1328      if (!limit || limit.backoffUntil === 0) {
1329        return 0;
1330      }
1331  
1332      const now = Date.now();
1333      if (now < limit.backoffUntil) {
1334        return limit.backoffUntil - now;
1335      }
1336  
1337      return 0;
1338    }
1339  }
1340  
1341  export const ipRotatorService = new IpRotatorService();