Skip to content

Commit

Permalink
uniformly use subconverter conversion to save
Browse files Browse the repository at this point in the history
  • Loading branch information
@wzdnzd
wzdnzd committed Jul 16, 2024
1 parent 98e03e8 commit b9d4428
Show file tree
Hide file tree
Showing 6 changed files with 52 additions and 102 deletions.
20 changes: 6 additions & 14 deletions subscribe/airport.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -440,13 +440,11 @@ def parse(
rate: float,
bin_name: str,
tag: str,
allow_insecure: bool = False,
disable_insecure: bool = False,
udp: bool = True,
ignore_exclude: bool = False,
chatgpt: dict = None,
special_protocols: bool = False,
emoji_patterns: dict = None,
remained: bool = False,
) -> list:
if "" == self.sub:
logger.error(f"[ParseError] cannot found any proxies because subscribe url is empty, domain: {self.ref}")
Expand Down Expand Up @@ -553,11 +551,6 @@ def parse(
f"rename error, name: {name},\trename: {self.rename}\tseparator: {RENAME_SEPARATOR}\tchatgpt: {pattern}\tdomain: {self.ref}"
)

# 是否添加 emoji
indexers = emoji_patterns
if remained and not re.search(r"^[\U0001F1E6-\U0001F1FF]{2}", name):
indexers = None

name = re.sub(
r"\[[^\[]*\]|[（\(][^（\(]*[\)）]|{[^{]*}|<[^<]*>|【[^【]*】|「[^「]*」|[^a-zA-Z0-9\u4e00-\u9fa5_×\.\-|\s]",
" ",
Expand Down Expand Up @@ -586,10 +579,6 @@ def parse(
name = f"{name[:i].strip()}-{abbreviation}-{name[-j:].strip()}"

name = re.sub(r"\s+(\d+)[\s_\-\|]+([A-Za-z])\b", r"-\1\2", name)
if indexers:
emoji = utils.get_emoji(text=name, patterns=indexers, default="🇺🇸")
name = f"{emoji} {name}" if emoji else name

item["name"] = re.sub(r"(-\d+[A-Za-z])+$", "", name).upper()

if "" != tag.strip():
Expand All @@ -599,8 +588,11 @@ def parse(
item["sub"] = self.sub
item["liveness"] = self.liveness

if allow_insecure:
item["skip-cert-verify"] = allow_insecure
if disable_insecure:
if "skip-cert-verify" in item:
item["skip-cert-verify"] = False
if "tls" in item:
item["tls"] = True

if udp and "udp" not in item:
item["udp"] = True
Expand Down
2 changes: 1 addition & 1 deletion subscribe/config/config.default.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
"rate": 2.5,
"count": 2,
"coupon": "",
"insecure": false,
"secure": false,
"renew": {
"account": [
{
Expand Down
12 changes: 2 additions & 10 deletions subscribe/location.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ def load_mmdb(directory: str, filename: str, update: bool = False) -> database.R
return database.Reader(filepath)


def rename(proxy: dict, reader: database.Reader, emoji_patterns: dict = None) -> dict:
def rename(proxy: dict, reader: database.Reader) -> dict:
if not proxy or not isinstance(proxy, dict):
return None

Expand All @@ -127,10 +127,6 @@ def rename(proxy: dict, reader: database.Reader, emoji_patterns: dict = None) ->
else:
name = country

if emoji_patterns and isinstance(emoji_patterns, dict):
emoji = utils.get_emoji(text=name, patterns=emoji_patterns, default="🇺🇸")
name = f"{emoji} {name}"

proxy["name"] = name
except Exception:
pass
Expand All @@ -142,7 +138,6 @@ def regularize(
proxies: list[dict],
directory: str = "",
update: bool = True,
emoji: bool = True,
num_threads: int = 0,
show_progress: bool = True,
locate: bool = False,
Expand All @@ -159,10 +154,7 @@ def regularize(
# load mmdb
reader = load_mmdb(directory=directory, filename="GeoLite2-Country.mmdb", update=update)
if reader:
# load emoji patterns
emoji_patterns = utils.load_emoji_pattern() if emoji else None

tasks = [[p, reader, emoji_patterns] for p in proxies if p and isinstance(p, dict)]
tasks = [[p, reader] for p in proxies if p and isinstance(p, dict)]
proxies = utils.multi_thread_run(rename, tasks, num_threads, show_progress, "")
else:
logger.error("skip rename proxies due to cannot load mmdb: GeoLite2-Country.mmdb")
Expand Down
97 changes: 35 additions & 62 deletions subscribe/process.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,10 +241,6 @@ def assign(
# 是否允许特殊协议
special_protocols = AirPort.enable_special_protocols()

# 解析 emoji 配置
emoji_conf = os.path.join(PATH, "subconverter", "snippets", "emoji.txt")
emoji_patterns = utils.load_emoji_pattern(filepath=emoji_conf)

for site in sites:
if not site:
continue
Expand Down Expand Up @@ -296,8 +292,8 @@ def assign(
# 是否对节点测活
liveness = site.get("liveness", True)

# 跳过证书验证
allow_insecure = site.get("insecure", False)
# 拒绝跳过证书验证
disable_insecure = site.get("secure", False)

# 优惠码
coupon = utils.trim(site.get("coupon", ""))
Expand All @@ -308,9 +304,6 @@ def assign(
# 覆盖subconverter默认exclude规则
ignoreder = site.get("ignorede", False)

# 是否添加国旗 emoji
emoji = site.get("emoji", False)

# 需要人机验证时是否直接放弃
chuck = site.get("chuck", False)

Expand Down Expand Up @@ -359,12 +352,11 @@ def assign(
chatgpt=chatgpt,
liveness=liveness,
coupon=coupon,
allow_insecure=allow_insecure,
disable_insecure=disable_insecure,
ignorede=ignoreder,
rigid=rigid,
chuck=chuck,
special_protocols=special_protocols,
emoji_patterns=emoji_patterns if emoji else None,
invite_code=invite_code,
)
found = workflow.exists(tasks=tasks, task=task)
Expand Down Expand Up @@ -401,9 +393,7 @@ def assign(
index=-1,
retry=retry,
bin_name=bin_name,
remained=True,
special_protocols=special_protocols,
emoji_patterns=emoji_patterns,
)
)
taskids.append(globalid)
Expand Down Expand Up @@ -541,6 +531,7 @@ def aggregate(args: argparse.Namespace) -> None:

push_conf = group_configs.get(k, {})
target = utils.trim(push_conf.get("target", "")) or "clash"

mixed = target in ["v2ray", "mixed"]
emoji = push_conf.get("emoji", True)

Expand All @@ -561,61 +552,48 @@ def aggregate(args: argparse.Namespace) -> None:
)

data = {"proxies": nochecks}
persisted, content, source_file = False, "", "config.yaml"

# compress if data is too large
compress = False if mixed else len(nochecks) >= 300
filepath = os.path.join(PATH, "subconverter", source_file)
with open(filepath, "w+", encoding="utf8") as f:
yaml.dump(data, f, allow_unicode=True)

persisted, content = False, ""
# convert
artifact, dest_file = "convert", "subscribe.txt"

if mixed or compress:
source_file = "config.yaml"
filepath = os.path.join(PATH, "subconverter", source_file)
with open(filepath, "w+", encoding="utf8") as f:
yaml.dump(data, f, allow_unicode=True)
if os.path.exists(generate_conf) and os.path.isfile(generate_conf):
os.remove(generate_conf)

# convert
dest_file = "subscribe.txt"
artifact = "convert"
success = subconverter.generate_conf(generate_conf, artifact, source_file, dest_file, target, emoji)
if not success:
logger.error(f"cannot generate subconverter config file, group=[{k}]")
continue

if os.path.exists(generate_conf) and os.path.isfile(generate_conf):
os.remove(generate_conf)
if subconverter.convert(binname=subconverter_bin, artifact=artifact):
filepath = os.path.join(PATH, "subconverter", dest_file)

success = subconverter.generate_conf(generate_conf, artifact, source_file, dest_file, target, emoji)
if not success:
logger.error(f"cannot generate subconverter config file, group=[{k}]")
if not os.path.exists(filepath) or not os.path.isfile(filepath):
logger.error(f"converted file {filepath} not found, group: {k}")
continue

if subconverter.convert(binname=subconverter_bin, artifact=artifact):
filepath = os.path.join(PATH, "subconverter", dest_file)
content = " "
with open(filepath, "r", encoding="utf8") as f:
content = f.read()

if not os.path.exists(filepath) or not os.path.isfile(filepath):
logger.error(f"converted file {filepath} not found, group: {k}")
if mixed and not utils.isb64encode(content=content):
# base64 encode
try:
content = base64.b64encode(content.encode(encoding="UTF8")).decode(encoding="UTF8")
except Exception as e:
logger.error(f"base64 encode error, message: {str(e)}")
continue

content = " "
with open(filepath, "r", encoding="utf8") as f:
content = f.read()

if mixed and not utils.isb64encode(content=content):
# base64 encode
try:
content = base64.b64encode(content.encode(encoding="UTF8")).decode(encoding="UTF8")
except Exception as e:
logger.error(f"base64 encode error, message: {str(e)}")
continue

# save to remote server
persisted = pushtool.push_to(content=content, push_conf=push_conf, group=k)

# clean workspace
workflow.cleanup(
os.path.join(PATH, "subconverter"),
[source_file, dest_file, "generate.ini"],
)
else:
content = yaml.dump(data=data, allow_unicode=True)
# save to remote server
persisted = pushtool.push_to(content=content, push_conf=push_conf, group=k)

# clean workspace
workflow.cleanup(os.path.join(PATH, "subconverter"), [source_file, dest_file, "generate.ini"])

if content and not persisted:
filename = os.path.join(PATH, "data", f"{k}.txt")

Expand All @@ -625,14 +603,9 @@ def aggregate(args: argparse.Namespace) -> None:
cost = "{:.2f}s".format(time.time() - starttime)
logger.info(f"group [{k}] process finished, count: {len(nochecks)}, cost: {cost}")

config = {
"domains": sites,
"crawl": crawl_conf,
"groups": group_configs,
"update": update_conf,
}

config = {"domains": sites, "crawl": crawl_conf, "groups": group_configs, "update": update_conf}
skip_remark = utils.trim(os.environ.get("SKIP_REMARK", "false")).lower() in ["true", "1"]

workflow.refresh(config=config, push=pushtool, alives=dict(subscribes), skip_remark=skip_remark)


Expand Down
17 changes: 5 additions & 12 deletions subscribe/workflow.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ class TaskConfig:
retry: int = 3

# 最高允许倍率
rate: float = 3.0
rate: float = 20.0

# 标签
tag: str = ""
Expand All @@ -64,20 +64,15 @@ class TaskConfig:
# 是否检测节点存活状态
liveness: bool = True

# skip-cert-verify
allow_insecure: bool = False
# 是否强制开启 tls 及阻止跳过证书验证
disable_insecure: bool = False

# 覆盖subconverter默认exclude规则
ignorede: bool = False

# 是否允许特殊协议
special_protocols: bool = False

# emoji 匹配规则
emoji_patterns: dict = None

remained: bool = False

# 对于具有邮箱域名白名单且需要验证码的情况，是否使用 Gmail 别名邮箱尝试，为 True 时表示不使用
rigid: bool = True

Expand All @@ -89,7 +84,7 @@ class TaskConfig:


def execute(task_conf: TaskConfig) -> list:
if not task_conf:
if not task_conf or not isinstance(task_conf, TaskConfig):
return []

obj = AirPort(
Expand Down Expand Up @@ -126,12 +121,10 @@ def execute(task_conf: TaskConfig) -> list:
rate=task_conf.rate,
bin_name=task_conf.bin_name,
tag=task_conf.tag,
allow_insecure=task_conf.allow_insecure,
disable_insecure=task_conf.disable_insecure,
ignore_exclude=task_conf.ignorede,
chatgpt=task_conf.chatgpt,
special_protocols=task_conf.special_protocols,
emoji_patterns=task_conf.emoji_patterns,
remained=task_conf.remained,
)

logger.info(
Expand Down
6 changes: 3 additions & 3 deletions tools/clean.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ def main(args: argparse.Namespace) -> None:
name = re.sub(r"(\d+|(-\d+)?[A-Z])$", "", item.get("name", "")).strip()
item["name"] = name

if args.secret:
if args.secure:
if "tls" in item:
item["tls"] = True
if "skip-cert-verify" in item:
Expand Down Expand Up @@ -122,8 +122,8 @@ def main(args: argparse.Namespace) -> None:

parser.add_argument(
"-s",
"--secret",
dest="secret",
"--secure",
dest="secure",
action="store_true",
default=False,
help="Enforce TLS and reject skipping certificate validation",
Expand Down

0 comments on commit b9d4428

Please sign in to comment.