Skip to content

Commit

Permalink
Skip unavailable files instead of requeuing indefinitely
Browse files Browse the repository at this point in the history
Skip files that 404 or 403
  • Loading branch information
Pyxia-Code committed Sep 12, 2021
1 parent 842df1c commit bdbc2b4
Showing 1 changed file with 7 additions and 0 deletions.
7 changes: 7 additions & 0 deletions subscripts/classes.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ class GDRIVAC_Exception(Exception): pass
class NAGDError(GDRIVAC_Exception): pass
class DoAError(GDRIVAC_Exception): pass
class InvalidURLError(GDRIVAC_Exception): pass
class FileUnavailableError(GDRIVAC_Exception): pass

#Singleton Class for checking cookie shapes
#TODO: Write functions to check different cookie shapes
Expand Down Expand Up @@ -110,6 +111,8 @@ def immunize(self,args,visit_queue, cookie_payload):
#print(r.text)
#CHECK IF non-200 status
if r.status_code < 200 or r.status_code >= 300:
if r.status_code in [404, 403]:
raise FileUnavailableError
with self.print_lock: log("\033[94mRequeuing...\033[0m")
self.visit_queue.put(URL)
continue
Expand All @@ -131,6 +134,10 @@ def immunize(self,args,visit_queue, cookie_payload):
except (requests.exceptions.MissingSchema, InvalidURLError):
with self.print_lock: log("\033[91mERROR: '{}' is not a properly formatted URL!\033[0m".format(URL))
#Do NOT add back into the queue
#If the status is 404 or 403
except FileUnavailableError:
with self.print_lock: log("\033[91mERROR: '{}' is not available!\033[0m".format(URL))
#Do NOT add back into the queue
#If the Connection is DEAD
except requests.exceptions.ConnectionError:
with self.print_lock: log("\033[91mERROR: No Connection\n\033[94mRequeuing {}\033[0m".format(URL))
Expand Down

0 comments on commit bdbc2b4

Please sign in to comment.