perf: 优化重定向链接获取逻辑

1. 更新 chunk 参数默认值
2. 更新配置文件参数说明

Closes JoeanAmier#138
Closes JoeanAmier#139

README.md

@@ -287,7 +287,7 @@ async def example():
 <td align="center">chunk</td>
 <td align="center">int</td>
 <td align="center">下载文件时，每次从服务器获取的数据块大小，单位：字节</td>
-<td align="center">1048576(1 MB)</td>
+<td align="center">2097152(2 MB)</td>
 </tr>
 <tr>
 <td align="center">max_retry</td>
@@ -332,6 +332,12 @@ async def example():
 <td align="center">false</td>
 </tr>
 <tr>
+<td align="center">download_record</td>
+<td align="center">bool</td>
+<td align="center">是否记录下载成功的作品 ID，如果开启，程序将会自动跳过下载存在记录的作品</td>
+<td align="center">true</td>
+</tr>
+<tr>
 <td align="center">language</td>
 <td align="center">str</td>
 <td align="center">设置程序语言，目前支持：<code>zh_CN</code>、<code>en_GB</code></td>

README_EN.md

@@ -291,7 +291,7 @@ async def example():
 <td align="center">chunk</td>
 <td align="center">int</td>
 <td align="center">Size of data chunk to fetch from the server each time when downloading files, in bytes</td>
-<td align="center">1048576(1 MB)</td>
+<td align="center">2097152(2 MB)</td>
 </tr>
 <tr>
 <td align="center">max_retry</td>
@@ -336,6 +336,12 @@ async def example():
 <td align="center">false</td>
 </tr>
 <tr>
+<td align="center">download_record</td>
+<td align="center">bool</td>
+<td align="center">Do record the ID of successfully downloaded works? If enabled, the program will automatically skip downloading works with records</td>
+<td align="center">true</td>
+</tr>
+<tr>
 <td align="center">language</td>
 <td align="center">str</td>
 <td align="center">Set program language. Currently supported: <code>zh_CN</code>, <code>en_GB</code></td>

source/application/request.py

@@ -24,14 +24,16 @@ async def request_url(
  log=None,
  **kwargs,
  ) -> str:
+ headers = self.select_headers(url, )
  try:
- response = await self.client.get(
- url,
- headers=self.select_headers(url, ),
- **kwargs,
- )
- response.raise_for_status()
- return response.text if content else str(response.url)
+ match content:
+ case True:
+ response = await self.__request_url_get(url, headers, **kwargs, )
+ response.raise_for_status()
+ return response.text
+ case False:
+ response = await self.__request_url_head(url, headers, **kwargs, )
+ return str(response.url)
  except HTTPError as error:
  logging(log, str(error), ERROR)
  logging(
@@ -43,4 +45,18 @@ def format_url(url: str) -> str:
  return bytes(url, "utf-8").decode("unicode_escape")
 
  def select_headers(self, url: str) -> dict:
- return self.blank_headers if "discovery/item" in url else self.headers
+ return self.headers if "explore" in url else self.blank_headers
+
+ async def __request_url_head(self, url: str, headers: dict, **kwargs, ):
+ return await self.client.head(
+ url,
+ headers=headers,
+ **kwargs,
+ )
+
+ async def __request_url_get(self, url: str, headers: dict, **kwargs, ):
+ return await self.client.get(
+ url,
+ headers=headers,
+ **kwargs,
+ )

source/module/settings.py

@@ -22,7 +22,7 @@ class Settings:
  "cookie": "",
  "proxy": None,
  "timeout": 10,
- "chunk": 1024 * 1024,
+ "chunk": 1024 * 1024 * 2,
  "max_retry": 5,
  "record_data": False,
  "image_format": "PNG",