Merge pull request jhao104#695 from jhao104/Jerry12228-region_add

Jerry12228 region add

handler/configHandler.py

@@ -73,6 +73,10 @@ def maxFailCount(self):
     def poolSizeMin(self):
         return int(os.getenv("POOL_SIZE_MIN", setting.POOL_SIZE_MIN))
 
+    @LazyProperty
+    def proxyRegion(self):
+        return bool(os.getenv("PROXY_REGION", setting.PROXY_REGION))
+
     @LazyProperty
     def timezone(self):
         return os.getenv("TIMEZONE", setting.TIMEZONE)

helper/check.py

@@ -9,13 +9,15 @@
    Change Activity:
                    2019/08/06: 执行代理校验
                    2021/05/25: 分别校验http和https
+                   2022/08/16: 获取代理Region信息
 -------------------------------------------------
 """
 __author__ = 'JHao'
 
 from util.six import Empty
 from threading import Thread
 from datetime import datetime
+from util.webRequest import WebRequest
 from handler.logHandler import LogHandler
 from helper.validator import ProxyValidator
 from handler.proxyHandler import ProxyHandler
@@ -25,12 +27,15 @@
 class DoValidator(object):
     """ 执行校验 """
 
+    conf = ConfigHandler()
+
     @classmethod
-    def validator(cls, proxy):
+    def validator(cls, proxy, work_type):
         """
         校验入口
         Args:
             proxy: Proxy Object
+            work_type: raw/use
         Returns:
             Proxy Object
         """
@@ -44,6 +49,8 @@ def validator(cls, proxy):
             if proxy.fail_count > 0:
                 proxy.fail_count -= 1
             proxy.https = True if https_r else False
+            if work_type == "raw":
+                proxy.region = cls.regionGetter(proxy) if cls.conf.proxyRegion else ""
         else:
             proxy.fail_count += 1
         return proxy
@@ -69,6 +76,15 @@ def preValidator(cls, proxy):
                 return False
         return True
 
+    @classmethod
+    def regionGetter(cls, proxy):
+        try:
+            url = 'https://searchplugin.csdn.net/api/v1/ip/get?ip=%s' % proxy.proxy.split(':')[0]
+            r = WebRequest().get(url=url, retry_time=1, timeout=2).json
+            return r['data']['address']
+        except:
+            return 'error'
+
 
 class _ThreadChecker(Thread):
     """ 多线程检测 """
@@ -89,7 +105,7 @@ def run(self):
             except Empty:
                 self.log.info("{}ProxyCheck - {}: complete".format(self.work_type.title(), self.name))
                 break
-            proxy = DoValidator.validator(proxy)
+            proxy = DoValidator.validator(proxy, self.work_type)
             if self.work_type == "raw":
                 self.__ifRaw(proxy)
             else:

helper/proxy.py

@@ -126,6 +126,10 @@ def last_time(self, value):
     def https(self, value):
         self._https = value
 
+    @region.setter
+    def region(self, value):
+        self._region = value
+
     def add_source(self, source_str):
         if source_str:
             self._source.append(source_str)

setting.py

@@ -75,6 +75,10 @@
 # proxyCheck时代理数量少于POOL_SIZE_MIN触发抓取
 POOL_SIZE_MIN = 20
 
+# ############# proxy attributes #################
+# 是否启用代理地域属性
+PROXY_REGION = True
+
 # ############# scheduler config #################
 
 # Set the timezone for the scheduler forcely (optional)