optimize async task and remove hover (Skyvern-AI#1095)

skyvern/forge/agent.py

@@ -209,8 +209,8 @@ async def create_task(self, task_request: TaskRequest, organization_id: str | No
         )
         return task
 
-    def register_async_operations(self, organization: Organization, task: Task, page: Page) -> None:
-        operations = app.AGENT_FUNCTION.generate_async_operations(organization, task, page)
+    async def register_async_operations(self, organization: Organization, task: Task, page: Page) -> None:
+        operations = await app.AGENT_FUNCTION.generate_async_operations(organization, task, page)
         self.async_operation_pool.add_operations(task.task_id, operations)
 
     async def execute_step(
@@ -273,7 +273,7 @@ async def execute_step(
             ) = await self._initialize_execution_state(task, step, workflow_run)
 
             if page := await browser_state.get_working_page():
-                self.register_async_operations(organization, task, page)
+                await self.register_async_operations(organization, task, page)
 
             step, detailed_output = await self.agent_step(
                 task, step, browser_state, organization=organization, task_block=task_block

skyvern/forge/agent_functions.py

@@ -303,7 +303,7 @@ async def prepare_step_execution(
         """
         return
 
-    def generate_async_operations(
+    async def generate_async_operations(
         self,
         organization: Organization,
         task: Task,

skyvern/webeye/actions/handler.py

@@ -1074,15 +1074,6 @@ async def chain_click(
     # File choosers are impossible to close if you don't expect one. Instead of dealing with it, close it!
 
     locator = skyvern_element.locator
-    try:
-        await locator.hover(timeout=timeout)
-    except Exception:
-        LOG.warning(
-            "Failed to hover over element in chain_click",
-            action=action,
-            locator=locator,
-            exc_info=True,
-        )
     # TODO (suchintan): This should likely result in an ActionFailure -- we can figure out how to do this later!
     LOG.info("Chain click starts", action=action, locator=locator)
     file: list[str] | str = []
@@ -1171,7 +1162,6 @@ async def fc_func(fc: FileChooser) -> None:
             parent_javascript_triggered = await is_javascript_triggered(scraped_page, page, parent_locator)
             javascript_triggered = javascript_triggered or parent_javascript_triggered
 
-            await parent_locator.hover(timeout=timeout)
             await parent_locator.click(timeout=timeout)
 
             LOG.info(
@@ -2283,10 +2273,6 @@ async def click_sibling_of_input(
             input_id = await input_element.get_attribute("id")
             sibling_label_css = f'label[for="{input_id}"]'
             label_locator = parent_locator.locator(sibling_label_css)
-            try:
-                await locator.hover(timeout=timeout)
-            except Exception:
-                LOG.warning("Failed to hover over input element in click_sibling_of_input", exc_info=True)
             await label_locator.click(timeout=timeout)
             LOG.info(
                 "Successfully clicked sibling label of input element",

skyvern/webeye/utils/dom.py

@@ -456,6 +456,34 @@ async def input_fill(
     async def input_clear(self, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS) -> None:
         await self.get_locator().clear(timeout=timeout)
 
+    async def move_mouse_to_safe(
+        self,
+        page: Page,
+        task_id: str | None = None,
+        step_id: str | None = None,
+        timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS,
+    ) -> tuple[float, float] | tuple[None, None]:
+        element_id = self.get_id()
+        try:
+            return await self.move_mouse_to(page, timeout=timeout)
+        except NoElementBoudingBox:
+            LOG.warning(
+                "Failed to move mouse to the element - NoElementBoudingBox",
+                task_id=task_id,
+                step_id=step_id,
+                element_id=element_id,
+                exc_info=True,
+            )
+        except Exception:
+            LOG.warning(
+                "Failed to move mouse to the element - unexpectd exception",
+                task_id=task_id,
+                step_id=step_id,
+                element_id=element_id,
+                exc_info=True,
+            )
+        return None, None
+
     async def move_mouse_to(
         self, page: Page, timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS
     ) -> tuple[float, float]: