Remove useless field

room-expander/room_expander/crawler/__init__.py

@@ -0,0 +1,18 @@
+# import asyncio
+# from datetime import datetime
+#
+# from telethon import TelegramClient
+#
+# from room_expander.conf.env import settings
+# api_id = 1024
+# api_hash = "b18441a1ff607e10a989891a5462e627"
+#
+# async def main():
+#     client = TelegramClient(settings.SESSION_FILE, api_id, api_hash)
+#     await client.start("1")
+#     messages =await client.get_messages("ipl_gl_king",
+#                                         search="https://t.me/",
+#                                         limit=100,reverse=True)
+#     print(messages)
+# if __name__ == '__main__':
+#   asyncio.run(main())

room-expander/room_expander/crawler/link_crawler.py

@@ -19,8 +19,8 @@
 
 
 async def sleep():
-    logger.info("sleep 10s")
-    await asyncio.sleep(10)
+    logger.info("sleep 5s")
+    await asyncio.sleep(5)
 
 
 def build_rooms(links):
@@ -94,8 +94,6 @@ async def run(self):
     async def fetch_and_save_rooms(self, db_sess, expand_batch, rooms):
         for room in rooms:
             room.expand_batch = expand_batch
-            room.last_expand_at = datetime.now()
-            logger.info(f"expand room {room.link}")
             links = await self.crawl_links(room)
             logger.info(f"room {room.link} has {len(links)} links, save to db")
 
@@ -115,12 +113,12 @@ async def crawl_links(self, room):
             async for message in self.client.iter_messages(
                 room_name,
                 search="https://t.me/",
-                offset_date=room.last_expand_at,
                 limit=10000000,
             ):
                 links = extract_telegram_links(message.text)
                 for link in links:
                     res.add(link.lower())
+
             return res
         except Exception as e:
             logger.error(f"crawl room {room.link} error {e}")

room-expander/room_expander/dao/models.py

@@ -24,6 +24,5 @@ class Room(Base):  # type: ignore
     extra = Column(Text)
     expand_batch = Column(Integer, default=0)
     web_crawl_batch = Column(Integer, default=0)
-    last_expand_at = Column(TIMESTAMP, default=func.current_timestamp())
     last_web_crawl_at = Column(TIMESTAMP, default=func.current_timestamp())
     created_at = Column(TIMESTAMP, default=func.current_timestamp())