feat(resolver): deterministic IPv6 ext-IP discovery via per-family probes (#220)

proxybroker/api.py

@@ -243,7 +243,7 @@ async def find(
             Added: :attr:`post`, :attr:`strict`, :attr:`dnsbl`.
             Changed: :attr:`types` is required.
         """
-        ip = await self._resolver.get_real_ext_ip()
+        ips = await self._resolver.get_real_ext_ips()
         types = _update_types(types)
 
         if not types:
@@ -254,7 +254,7 @@ async def find(
             timeout=self._timeout,
             verify_ssl=self._verify_ssl,
             max_tries=self._max_tries,
-            real_ext_ip=ip,
+            real_ext_ips=ips,
             types=types,
             post=post,
             strict=strict,

proxybroker/checker.py

@@ -41,12 +41,34 @@ def __init__(
         types=None,
         post=False,
         loop=None,
+        *,
+        real_ext_ips=None,
     ):
         Judge.clear()
         self._judges = get_judges(judges, timeout, verify_ssl)
         self._method = "POST" if post else "GET"
         self._max_tries = max_tries
-        self._real_ext_ip = real_ext_ip
+        # Set-aware ext-IP storage (#220). On dual-stack hosts the
+        # discovery returns BOTH families so judge response comparison
+        # passes regardless of which family the judge connection used.
+        # Backward-compat: legacy `real_ext_ip` (single string) is
+        # accepted and wrapped into the set; if both args are passed the
+        # newer plural argument wins.
+        # `real_ext_ips` is keyword-only so existing positional callers
+        # like `Checker(judges, 3, 8, False, False, None, ip, types)`
+        # don't get their `types`-and-after args silently shifted.
+        if real_ext_ips is None and real_ext_ip is not None:
+            real_ext_ips = (real_ext_ip,)
+        # Defensive: a caller passing a single str (e.g. misreading the
+        # plural arg name) gets it treated as one IP, not iterated into
+        # a set of individual characters.
+        if isinstance(real_ext_ips, str):
+            real_ext_ips = (real_ext_ips,)
+        self._real_ext_ips = frozenset(real_ext_ips or ())
+        # Legacy single-string accessor preserved for any external code.
+        self._real_ext_ip = (
+            next(iter(self._real_ext_ips)) if self._real_ext_ips else None
+        )
         self._strict = strict
         self._dnsbl = dnsbl or []
         self._types = types or {}
@@ -70,7 +92,7 @@ async def check_judges(self):
         log.debug("Start check judges")
         stime = time.time()
         await asyncio.gather(
-            *[j.check(real_ext_ip=self._real_ext_ip) for j in self._judges]
+            *[j.check(real_ext_ips=self._real_ext_ips) for j in self._judges]
         )
 
         self._judges = [j for j in self._judges if j.is_working]
@@ -231,7 +253,7 @@ async def _check(self, proxy, proto):
                 if result:
                     if proxy.ngtr.check_anon_lvl:
                         lvl = _get_anonymity_lvl(
-                            self._real_ext_ip, proxy, judge, content
+                            self._real_ext_ips, proxy, judge, content
                         )
                     else:
                         lvl = None
@@ -320,20 +342,35 @@ def _check_test_response(proxy, headers, content, rv):
         return False
 
 
-def _get_anonymity_lvl(real_ext_ip, proxy, judge, content):
+def _get_anonymity_lvl(real_ext_ips, proxy, judge, content):
+    """Classify a proxy as Transparent / Anonymous / High.
+
+    `real_ext_ips` accepts either an iterable of canonical IP strings
+    (the SOTA set-aware path used by Checker) or a single string (the
+    legacy single-ext-IP API). Empty/None means no real IP known —
+    Transparent classification disabled.
+
+    Set semantics: Transparent if ANY of the host's real ext-IPs (v4
+    OR v6) appear in the page. Critical for dual-stack hosts where a
+    judge response may echo whichever family the connection used.
+    """
     content = content.lower()
     foundIP = get_all_ip(content)
-    # Defense in depth: canonicalise the real IP even if the caller already
-    # passed canonical form. Comparison must be canonical-vs-canonical so
-    # IPv6 textual encodings (case, leading zeros, compression) compare
-    # equal. Falls back to the raw value only if canonicalisation fails.
-    real_canonical = canonicalize_ip(real_ext_ip) or real_ext_ip
+
+    # Normalise input to a frozenset of canonical strings.
+    if real_ext_ips is None:
+        real_canonicals = frozenset()
+    elif isinstance(real_ext_ips, str):
+        # Legacy single-string contract.
+        real_canonicals = frozenset({canonicalize_ip(real_ext_ips) or real_ext_ips})
+    else:
+        real_canonicals = frozenset(canonicalize_ip(ip) or ip for ip in real_ext_ips)
 
     via = (content.count("via") > judge.marks["via"]) or (
         content.count("proxy") > judge.marks["proxy"]
     )
 
-    if real_canonical in foundIP:
+    if real_canonicals & foundIP:
         lvl = "Transparent"
     elif via:
         lvl = "Anonymous"

proxybroker/judge.py

@@ -70,8 +70,27 @@ def clear(cls):
         cls.ev["HTTPS"].clear()
         cls.ev["SMTP"].clear()
 
-    async def check(self, real_ext_ip):
+    async def check(self, real_ext_ips=None, real_ext_ip=None):
+        """Probe judge endpoint and verify it echoes a known real ext-IP.
+
+        ``real_ext_ips`` (set/iterable, preferred) accepts the FULL set
+        of host external IPs from ``Resolver.get_real_ext_ips()`` so the
+        comparison passes whichever family the judge connection used.
+        ``real_ext_ip`` (single string, legacy) is kept for backward
+        compatibility; if both are passed, ``real_ext_ips`` wins.
+        """
         # TODO: need refactoring
+        # Normalise legacy single-string input into the set-aware path.
+        if real_ext_ips is None and real_ext_ip is not None:
+            real_ext_ips = (real_ext_ip,)
+        # Defensive: a caller passing a single str (e.g. via the OLD
+        # positional API `judge.check("203.0.113.5")` where the string
+        # now binds to `real_ext_ips`) gets it treated as one IP, not
+        # iterated into a set of individual characters.
+        if isinstance(real_ext_ips, str):
+            real_ext_ips = (real_ext_ips,)
+        real_ext_ips = frozenset(real_ext_ips or ())
+
         try:
             self.ip = await self._resolver.resolve(self.host)
         except ResolveError:
@@ -107,14 +126,13 @@ async def check(self, real_ext_ip):
             return
 
         page = page.lower()
-        # Use canonical-form set membership instead of substring `in page`
-        # so we correctly recognise IPv6 echoes regardless of how the
-        # judge formatted them (uppercase, expanded, compressed). For
-        # IPv4, canonical form equals the raw form, so behavior is
-        # preserved for the legacy v4 path.
+        # Canonical-form set membership: judges may echo whichever family
+        # the connection used, and the host may have v4 OR v6 reachable
+        # (or both on dual-stack). Pass if ANY of the host's real ext-IPs
+        # appears in the page.
         page_ips = get_all_ip(page)
-        real_canonical = canonicalize_ip(real_ext_ip) or real_ext_ip
-        real_ip_visible = real_canonical in page_ips
+        real_canonicals = frozenset(canonicalize_ip(ip) or ip for ip in real_ext_ips)
+        real_ip_visible = bool(real_canonicals & page_ips)
 
         if resp.status == 200 and real_ip_visible and rv in page:
             self.marks["via"] = page.count("via")