[update] 🐜 add proxy attr: region

handler/configHandler.py

@@ -73,6 +73,10 @@ def maxFailCount(self):
     def poolSizeMin(self):
         return int(os.getenv("POOL_SIZE_MIN", setting.POOL_SIZE_MIN))
 
+    @LazyProperty
+    def proxyRegion(self):
+        return bool(os.getenv("PROXY_REGION", setting.PROXY_REGION))
+
     @LazyProperty
     def timezone(self):
         return os.getenv("TIMEZONE", setting.TIMEZONE)

helper/check.py

@@ -9,16 +9,15 @@
    Change Activity:
                    2019/08/06: 执行代理校验
                    2021/05/25: 分别校验http和https
+                   2022/08/16: 获取代理Region信息
 -------------------------------------------------
 """
 __author__ = 'JHao'
 
-import requests
-import json
-
 from util.six import Empty
 from threading import Thread
 from datetime import datetime
+from util.webRequest import WebRequest
 from handler.logHandler import LogHandler
 from helper.validator import ProxyValidator
 from handler.proxyHandler import ProxyHandler
@@ -28,12 +27,15 @@
 class DoValidator(object):
     """ 执行校验 """
 
+    conf = ConfigHandler()
+
     @classmethod
-    def validator(cls, proxy):
+    def validator(cls, proxy, work_type):
         """
         校验入口
         Args:
             proxy: Proxy Object
+            work_type: raw/use
         Returns:
             Proxy Object
         """
@@ -43,11 +45,12 @@ def validator(cls, proxy):
         proxy.check_count += 1
         proxy.last_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
         proxy.last_status = True if http_r else False
-        proxy._region = cls.region_get(proxy.proxy)
         if http_r:
             if proxy.fail_count > 0:
                 proxy.fail_count -= 1
             proxy.https = True if https_r else False
+            if work_type == "raw":
+                proxy.region = cls.regionGetter(proxy) if cls.conf.proxyRegion else ""
         else:
             proxy.fail_count += 1
         return proxy
@@ -74,12 +77,13 @@ def preValidator(cls, proxy):
         return True
 
     @classmethod
-    def region_get(cls, proxy):
+    def regionGetter(cls, proxy):
         try:
-            r = requests.get(url='https://searchplugin.csdn.net/api/v1/ip/get', params={'ip': proxy.split(':')[0]})
-            return json.loads(r.text)['data']['address']
+            url = 'https://searchplugin.csdn.net/api/v1/ip/get?ip=%s' % proxy.proxy.split(':')[0]
+            r = WebRequest().get(url=url, retry_time=1, timeout=2).json
+            return r['data']['address']
         except:
-            return '未知或请求失败'
+            return 'error'
 
 
 class _ThreadChecker(Thread):
@@ -101,7 +105,7 @@ def run(self):
             except Empty:
                 self.log.info("{}ProxyCheck - {}: complete".format(self.work_type.title(), self.name))
                 break
-            proxy = DoValidator.validator(proxy)
+            proxy = DoValidator.validator(proxy, self.work_type)
             if self.work_type == "raw":
                 self.__ifRaw(proxy)
             else:

helper/proxy.py

@@ -126,6 +126,10 @@ def last_time(self, value):
     def https(self, value):
         self._https = value
 
+    @region.setter
+    def region(self, value):
+        self._region = value
+
     def add_source(self, source_str):
         if source_str:
             self._source.append(source_str)

setting.py

@@ -75,6 +75,10 @@
 # proxyCheck时代理数量少于POOL_SIZE_MIN触发抓取
 POOL_SIZE_MIN = 20
 
+# ############# proxy attributes #################
+# 是否启用代理地域属性
+PROXY_REGION = True
+
 # ############# scheduler config #################
 
 # Set the timezone for the scheduler forcely (optional)