Skip to content

Commit

Permalink
New Webhooks
Browse files Browse the repository at this point in the history 
Added a webhook for auth success and failure.

You can override the global webhook by adding a link to "webhooks" under "succeeded" or "failed"

"hide_sensitive_info" currently hides your account username
  • Loading branch information
@UltimaHoarder
UltimaHoarder committed Apr 1, 2021
1 parent 6b51094 commit 6f4a65c
Show file tree
Hide file tree
Showing 3 changed files with 107 additions and 25 deletions.
44 changes: 42 additions & 2 deletions classes/make_settings.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,11 @@ def fix(config={}):
settings["proxies"] = fixed_socks5_proxies
global_user_agent = settings.pop(
"global_user_agent", None)
if isinstance(settings.get(
"webhooks", {}), list):
webhook = settings["webhooks"]
settings["webhooks"] = {}
settings["webhooks"]["global_webhooks"] = webhook
if key == "supported":
for key2, value2 in value.items():
temp_auth = value2.pop("auth", None)
Expand Down Expand Up @@ -129,13 +134,45 @@ def fix(config={}):
class config(object):
def __init__(self, settings={}, supported={}):
class Settings(object):
def __init__(self, auto_site_choice="", profile_directories=[".profiles"], export_type="json", max_threads=-1, min_drive_space=0, webhooks=[], exit_on_completion=False, infinite_loop=True, loop_timeout="0", proxies=[], cert="", random_string=""):
def __init__(self, auto_site_choice="", profile_directories=[".profiles"], export_type="json", max_threads=-1, min_drive_space=0, webhooks={}, exit_on_completion=False, infinite_loop=True, loop_timeout="0", proxies=[], cert="", random_string=""):
class webhooks_settings:
def __init__(self, option={}) -> None:
class webhook_template:
def __init__(self, option={}) -> None:
self.webhooks = option.get(
'webhooks', [])
self.status = option.get(
'status', None)
self.hide_sensitive_info = option.get(
'hide_sensitive_info', True)
print

class auth_webhook:
def __init__(self, option={}) -> None:
self.succeeded = webhook_template(
option.get('succeeded', {}))
self.failed = webhook_template(
option.get('failed', {}))

class download_webhook:
def __init__(self, option={}) -> None:
self.succeeded = webhook_template(
option.get('succeeded', {}))
self.global_webhooks = option.get(
'global_webhooks', [])
self.global_status = option.get(
'global_status', True)
self.auth_webhook = auth_webhook(
option.get('auth_webhook', {}))
self.download_webhook = download_webhook(
option.get('download_webhook', {}))
self.auto_site_choice = auto_site_choice
self.export_type = export_type
self.profile_directories = profile_directories
self.max_threads = max_threads
self.min_drive_space = min_drive_space
self.webhooks = webhooks
self.webhooks = webhooks_settings(settings.get(
'webhooks', webhooks))
self.exit_on_completion = exit_on_completion
self.infinite_loop = infinite_loop
self.loop_timeout = loop_timeout
Expand All @@ -160,10 +197,12 @@ def __init__(self, option={}) -> None:
'scrape_names', True)
self.scrape_paid_content = option.get(
'scrape_paid_content', True)

class browser:
def __init__(self, option={}) -> None:
self.auth = option.get(
'auth', True)

class database:
def __init__(self, option={}) -> None:
self.posts = option.get(
Expand Down Expand Up @@ -228,6 +267,7 @@ def __init__(self, option={}) -> None:
'scrape_names', True)
self.scrape_paid_content = option.get(
'scrape_paid_content', True)

class browser:
def __init__(self, option={}) -> None:
self.auth = option.get(
Expand Down
9 changes: 8 additions & 1 deletion datascraper/main_datascraper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@

def start_datascraper(json_config, site_name_lower, apis: list = [], webhooks=True):
json_settings = json_config["settings"]
json_webhooks = json_settings["webhooks"]
json_sites = json_config["supported"]
domain = json_settings["auto_site_choice"]
main_helper.assign_vars(json_config)
Expand Down Expand Up @@ -78,6 +79,9 @@ def start_datascraper(json_config, site_name_lower, apis: list = [], webhooks=Tr
setup, subscriptions = module.account_setup(
api, identifiers, jobs)
if not setup:
if webhooks:
x = main_helper.process_webhooks(
[api], "auth_webhook", "failed")
auth_details = {}
auth_details["auth"] = api.auth.auth_details.__dict__
profile_directory = api.auth.profile_directory
Expand All @@ -88,6 +92,8 @@ def start_datascraper(json_config, site_name_lower, apis: list = [], webhooks=Tr
auth_details, user_auth_filepath)
continue
subscription_array += subscriptions
x = main_helper.process_webhooks(
[api], "auth_webhook", "succeeded")
subscription_list = module.format_options(
subscription_array, "usernames")
if jobs["scrape_paid_content"]:
Expand All @@ -99,7 +105,8 @@ def start_datascraper(json_config, site_name_lower, apis: list = [], webhooks=Tr
module, subscription_list, auto_scrape_names, apis, json_config, site_name_lower, site_name)
x = main_helper.process_downloads(apis, module)
if webhooks:
x = main_helper.process_webhooks(apis)
x = main_helper.process_webhooks(
apis, "download_webhook", "succeeded")
elif site_name_lower == "starsavn":
site_name = "StarsAVN"
original_api = StarsAVN
Expand Down
79 changes: 57 additions & 22 deletions helpers/main_helper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -329,7 +329,8 @@ def reformat(prepared_format, unformatted):
value = "Paid"
directory = prepared_format.directory
path = unformatted.replace("{site_name}", prepared_format.site_name)
path = path.replace("{first_letter}", prepared_format.username[0].capitalize())
path = path.replace(
"{first_letter}", prepared_format.username[0].capitalize())
path = path.replace("{post_id}", post_id)
path = path.replace("{media_id}", media_id)
path = path.replace("{username}", prepared_format.username)
Expand Down Expand Up @@ -644,14 +645,26 @@ def process_downloads(apis, module):
download_info["base_directory"])


def process_webhooks(apis):
for api in apis:
subscriptions = api.get_subscriptions(refresh=False)
for subscription in subscriptions:
download_info = subscription.download_info
if download_info:
if download_info["webhook"]:
send_webhook(subscription)
def process_webhooks(apis: list, category, category2):
global_webhooks = webhooks["global_webhooks"]
global_status = webhooks["global_status"]
webhook = webhooks[category]
webhook_state = webhook[category2]
webhook_links = []
webhook_status = global_status
webhook_hide_sensitive_info = True
if webhook_state["status"] != None:
webhook_status = webhook_state["status"]
if global_webhooks:
webhook_links = global_webhooks
if webhook_state["webhooks"]:
webhook_links = webhook_state["webhooks"]
if webhook_status:
for api in apis:
send_webhook(api, webhook_hide_sensitive_info,
webhook_links, category, category2)
print
print


def is_me(user_api):
Expand Down Expand Up @@ -717,6 +730,7 @@ def ordinal(n): return "%d%s" % (
def id_generator(size=6, chars=string.ascii_uppercase + string.digits):
return ''.join(random.choice(chars) for _ in range(size))


def humansize(nbytes):
i = 0
suffixes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB']
Expand All @@ -726,19 +740,40 @@ def humansize(nbytes):
f = ('%.2f' % nbytes).rstrip('0').rstrip('.')
return '%s %s' % (f, suffixes[i])

def send_webhook(item):
for webhook_link in webhooks:
message = prepare_webhooks.discord()
embed = message.embed()
embed.title = f"Downloaded: {item.username}"
embed.add_field("username", item.username)
embed.add_field("post_count", item.postsCount)
embed.add_field("link", item.link)
embed.image.url = item.avatar
message.embeds.append(embed)
message = json.loads(json.dumps(
message, default=lambda o: o.__dict__))
x = requests.post(webhook_link, json=message)

def send_webhook(item, webhook_hide_sensitive_info, webhook_links, category, category2: str):
if category == "auth_webhook":
for webhook_link in webhook_links:
auth = item.auth
username = auth.username
if webhook_hide_sensitive_info:
username = "REDACTED"
message = prepare_webhooks.discord()
embed = message.embed()
embed.title = f"Auth {category2.capitalize()}"
embed.add_field("username", username)
message.embeds.append(embed)
message = json.loads(json.dumps(
message, default=lambda o: o.__dict__))
x = requests.post(webhook_link, json=message)
if category == "download_webhook":
subscriptions = item.get_subscriptions(refresh=False)
for subscription in subscriptions:
download_info = subscription.download_info
if download_info:
for webhook_link in webhook_links:
message = prepare_webhooks.discord()
embed = message.embed()
embed.title = f"Downloaded: {subscription.username}"
embed.add_field("username", subscription.username)
embed.add_field("post_count", subscription.postsCount)
embed.add_field("link", subscription.link)
embed.image.url = subscription.avatar
message.embeds.append(embed)
message = json.loads(json.dumps(
message, default=lambda o: o.__dict__))
x = requests.post(webhook_link, json=message)
print


def find_between(s, start, end):
Expand Down

0 comments on commit 6f4a65c

Please sign in to comment.