Skip to content

Commit

Permalink
feat: Add option to return web scrape content instead of printing
Browse files Browse the repository at this point in the history
paul-gauthier committed Nov 8, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
1 parent 77cb649 commit 52c49fc
Showing 3 changed files with 5 additions and 3 deletions.
2 changes: 1 addition & 1 deletion aider/coders/base_coder.py
Original file line number Diff line number Diff line change
@@ -820,7 +820,7 @@ def check_for_urls(self, inp: str) -> List[str]:
"Add URL to the chat?", subject=url, group=group, allow_never=True
):
inp += "\n\n"
inp += self.commands.cmd_web(url)
inp += self.commands.cmd_web(url, return_content=True)
added_urls.append(url)
else:
self.rejected_urls.add(url)
4 changes: 3 additions & 1 deletion aider/commands.py
Original file line number Diff line number Diff line change
@@ -139,7 +139,7 @@ def cmd_models(self, args):
else:
self.io.tool_output("Please provide a partial model name to search for.")

def cmd_web(self, args):
def cmd_web(self, args, return_content=False):
"Scrape a webpage, convert to markdown and send in a message"

url = args.strip()
@@ -159,6 +159,8 @@ def cmd_web(self, args):

content = self.scraper.scrape(url) or ""
content = f"Here is the content of {url}:\n\n" + content
if return_content:
return content

self.io.tool_output("... added to chat.")

2 changes: 1 addition & 1 deletion tests/scrape/test_scrape.py
Original file line number Diff line number Diff line change
@@ -44,7 +44,7 @@ def test_cmd_web_imports_playwright(self):
self.commands.io.tool_error = mock_print_error

# Run the cmd_web command
result = self.commands.cmd_web("https://example.com")
result = self.commands.cmd_web("https://example.com", return_content=True)

# Assert that the result contains some content
self.assertIsNotNone(result)

0 comments on commit 52c49fc

Please sign in to comment.