-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathavito.py
124 lines (115 loc) · 6.78 KB
/
avito.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
import records
import requests
from bs4 import BeautifulSoup
from fake_useragent import UserAgent
from tg import Tg
class Avito:
"""docstring"""
def __init__(self, task):
"""Constructor"""
self.__task = task # protected
# Инициализируем базу
self.__db = self.__prepare_db(task["database"])
# Инициализируем бота для Telegram
if "proxy" in task:
self.__tg = Tg(task["tgBotKey"], task["proxy"])
else:
self.__tg = Tg(task["tgBotKey"])
def __prepare_db(self, link):
db = records.Database(link)
conn = db.get_connection()
with conn:
conn.query(
'Create table if not exists AVITO (tag VARCHAR(255),id VARCHAR(255),link VARCHAR(255),price VARCHAR(20))')
return db
def __check_record(self, avito_record):
conn = self.__db.get_connection()
with conn:
rows = conn.query(
"select price from AVITO where tag='{}' and id='{}'".format(avito_record['tag'], avito_record['id']),
fetchall=True)
if len(rows) == 0:
# new item
conn.query("insert into AVITO (tag,id,link,price) values('{}','{}','{}','{}')"
.format(avito_record['tag'], avito_record['id'], avito_record['link'],
avito_record['price']))
res = {'message': "New", 'icon': '✅'}
elif int(rows[0]['price']) > int(avito_record['price']):
# item changed, price down
conn.query("update AVITO set price='{}' where tag='{}' and id='{}'".format(avito_record['price'],
avito_record['tag'],
avito_record['id']))
res = {'message': "Price down", 'icon': '👍', 'oldPrice': rows[0]['price']}
elif int(rows[0]['price']) < int(avito_record['price']):
# item changed, price up
conn.query("update AVITO set price='{}' where tag='{}' and id='{}'".format(avito_record['price'],
avito_record['tag'],
avito_record['id']))
res = {'message': "Price up", 'icon': '👎', 'oldPrice': rows[0]['price']}
else:
# price not changed
res = None
return res
def __getText(self,parent):
return ''.join(parent.find_all(text=True, recursive=False)).strip()
def grab(self):
ua = UserAgent()
header = {'User-Agent': str(ua.chrome), 'Accept-Encoding': 'utf-8'}
task = self.__task
tag = task["tag"]
base_url = "https://www.avito.ru"
search_url = task["search"]
result = requests.get(base_url + search_url, headers=header)
content = result.content
print(content)
soup = BeautifulSoup(content, "html.parser")
main_item_div_list = soup.find_all('div', "item")
count_all = 0
count_new = 0
count_change = 0
for mainItemDiv in main_item_div_list:
item_id = mainItemDiv['id']
href_tag = mainItemDiv.find('a', 'snippet-link')
if href_tag is not None:
price_tag = mainItemDiv.find('span', 'snippet-price')
if price_tag is not None:
item_name = href_tag.get('title')
item_price = self.__getText(price_tag) #price_tag.get('content')
# Нашли цену и url, можем писать в базу
count_all += 1
avito_record = dict()
avito_record['tag'] = tag
avito_record['id'] = item_id
item_link = base_url + href_tag.get('href')
avito_record['link'] = item_link
avito_record['price'] = item_price.replace(' ','')
message = self.__check_record(avito_record)
if message is not None:
if 'oldPrice' in message:
count_change += 1
formatted_message = "{}{} \n{} \n💰 {}₽ -> {} \n🔗[Перейти]({})".format(message['icon'],
message[
'message'],
item_name,
message[
'oldPrice'],
avito_record[
'price'],
avito_record[
'link'],
)
print(formatted_message)
self.__tg.send_message(task["tgChannelId"], formatted_message)
else:
count_new += 1
formatted_message = "{}{} \n{} \n💰 {}₽ \n🔗[Перейти]({})".format(message['icon'],
message['message'],
item_name,
avito_record['price'],
avito_record['link'],
)
print(formatted_message)
self.__tg.send_message(task["tgChannelId"], formatted_message)
else:
print("Empty link for {}".format(item_id))
print("Total: {} new: {} changed: {}".format(count_all, count_new, count_change))