-
Notifications
You must be signed in to change notification settings - Fork 4
/
Copy pathsetting.py
142 lines (116 loc) · 3.98 KB
/
setting.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
# -*- coding: utf-8 -*-
"""
-------------------------------------------------
File Name： setting.py
Description : 配置文件
Author : 007x
date： 2019/2/15
-------------------------------------------------
Change Activity:
2024/11/27:
-------------------------------------------------
"""
BANNER = r"""
****************************************************************
*** ______ ********************* ______ *********** _ ********
*** | ___ \_ ******************** | ___ \ ********* | | ********
*** | |_/ / \__ __ __ _ __ _ | |_/ /___ * ___ | | ********
*** | __/| _// _ \ \ \/ /| | | || __// _ \ / _ \ | | ********
*** | | | | | (_) | > < \ |_| || | | (_) | (_) || |___ ****
*** \_| |_| \___/ /_/\_\ \__ |\_| \___/ \___/ \_____/ ****
**** __ / / *****
************************* /___ / *******************************
************************* ********************************
****************************************************************
"""
VERSION = "2.8.0"
# ############### server config ###############
HOST = "0.0.0.0"
PORT = 5010
# ############### database config ###################
# db connection uri
# example:
# Redis: redis://:password@ip:port/db
# Ssdb: ssdb://:password@ip:port
DB_CONN = 'redis://:[email protected]:6379/0'
# proxy table name
TABLE_NAME = 'use_proxy'
""" #no usefull
"freeProxy01",
"freeProxy02",
"freeProxy03",
"freeProxy04",
"freeProxy05",
"freeProxy06",
"freeProxy07",
"freeProxy08",
"freeProxy09",
"freeProxy10",
#usefull
"freeProxy11",
"freeProxy12",
"freeProxy13",
"freeProxy96",
"freeProxy97",
"freeProxy98",
"freeProxy99",
"freeProxy100",
"freeProxy101"
"""
# ###### config the proxy fetch function ######
PROXY_FETCHER = [
"freeProxy11",
"freeProxy12",
"freeProxy13",
"freeProxy96",
"freeProxy97",
"freeProxy98",
"freeProxy99",
"freeProxy100",
"freeProxy101"
]
# ############# proxy validator #################
# 代理验证目标网站
HTTP_URL = "http://ifconfig.me/ip"
HTTPS_URL = "https://ifconfig.me/ip"
# 代理验证时超时时间
VERIFY_TIMEOUT = 5
# 近PROXY_CHECK_COUNT次校验中允许的最大失败次数,超过则剔除代理
MAX_FAIL_COUNT = 0
# 近PROXY_CHECK_COUNT次校验中允许的最大失败率,超过则剔除代理
MAX_FAIL_RATE = 0.3
# proxyCheck时代理数量少于POOL_SIZE_MIN触发抓取
POOL_SIZE_MIN = 2000
# getCount时，check_count_xxx_proxies 过滤条件下限，筛选出来质量比较高的代理
MIN_AVAIL_LIMIT = 20
# 解决代理源需fanqiang访问问题
API_PROXY_CONFIG = "http://127.0.0.1:1080"
# 每个proxy采集进程启动的线程数量
RAW_THREADS_NUM = 50
# 每个proxy检查进程启动的线程数量
USE_THREADS_NUM = 10
# proxy采集任务每间隔多少分钟执行一次
RAW_INTERVAL_MIN = 4
# proxy检查任务每间隔多少分钟执行一次
USE_INTERVAL_MIN = 2
# 控制 执行器（如线程池或进程池）中 最大并发执行任务的数量。
THREADPOOL_WORKS_NUM = 60
# 一个进程池（ProcessPoolExecutor），最多可以同时运行多少个进程
PROCESSPOOL_WORKS_NUM = 5
# 控制 同一个任务 在调度器中的最大实例数量
JOB_INSTANCES_NUM = 30
# ############# proxy attributes #################
# 是否启用代理地域属性
PROXY_REGION = True
SERVER_PIDFILE = '/tmp/proxyPoolServer.pid'
SERVER_ACCESS_LOGFILE = './log/access.log'
SERVER_ERROR_LOGFILE = './log/error.log'
SCHEDULER_PIDFILE = "/tmp/proxyPoolScheduler.pid"
# ############# scheduler config #################
# Set the timezone for the scheduler forcely (optional)
# If it is running on a VM, and
# "ValueError: Timezone offset does not match system offset"
# was raised during scheduling.
# Please uncomment the following line and set a timezone for the scheduler.
# Otherwise it will detect the timezone from the system automatically.
TIMEZONE = "Asia/Shanghai"