prevent unintended redirect

ohaoz · Jul 24, 2024 · 1123235 · 1123235
1 parent e89d241
commit 1123235
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 0 deletions.
diff --git a/subscribe/crawl.py b/subscribe/crawl.py
@@ -1598,6 +1598,13 @@ def extract_real_url(domain: str, retry: int = 2) -> str:
                 request = urllib.request.Request(url=url, headers=utils.DEFAULT_HTTP_HEADERS, method="GET")
                 response = urllib.request.urlopen(request, timeout=6, context=utils.CTX)
 
+                # do not redirect
+                # opener = urllib.request.build_opener(utils.NoRedirect)
+                # response = opener.open(request, timeout=6)
+
+                if not utils.trim(response.geturl()).endswith("/env.js"):
+                    return domain
+
                 content = response.read()
                 try:
                     content = str(content, encoding="utf8")

diff --git a/subscribe/utils.py b/subscribe/utils.py
@@ -362,6 +362,17 @@ def http_error_302(
     ) -> typing.IO[bytes]:
         return fp
 
+    def redirect_request(
+        self,
+        req: urllib.request.Request,
+        fp: typing.IO[bytes],
+        code: int,
+        msg: str,
+        headers: HTTPMessage,
+        newurl: str,
+    ):
+        return None
+
 
 def http_post(
     url: str,