Skip to content

Commit

Permalink
add disable element id (Skyvern-AI#1052)
Browse files Browse the repository at this point in the history
  • Loading branch information
LawyZheng authored Oct 25, 2024
1 parent d2f4e06 commit c933588
Showing 5 changed files with 113 additions and 3 deletions.
7 changes: 7 additions & 0 deletions skyvern/exceptions.py
Original file line number Diff line number Diff line change
@@ -521,3 +521,10 @@ def __init__(self, expected_parameter_type: str, value: str, workflow_permanent_
message,
status_code=status.HTTP_400_BAD_REQUEST,
)


class InteractWithDisabledElement(SkyvernException):
def __init__(self, element_id: str):
super().__init__(
f"The element(id={element_id}) now is disabled, try to interact with it later when it's enabled."
)
47 changes: 47 additions & 0 deletions skyvern/webeye/actions/handler.py
Original file line number Diff line number Diff line change
@@ -25,6 +25,7 @@
FailToSelectByValue,
IllegitComplete,
ImaginaryFileUrl,
InteractWithDisabledElement,
InvalidElementForTextInput,
MissingElement,
MissingFileUrl,
@@ -347,6 +348,18 @@ async def handle_click_action(
dom = DomUtil(scraped_page=scraped_page, page=page)
skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)
await asyncio.sleep(0.3)

# dynamically validate the attr, since it could change into enabled after the previous actions
if await skyvern_element.is_disabled(dynamic=True):
LOG.warning(
"Try to click on a disabled element",
action_type=action.action_type,
task_id=task.task_id,
step_id=step.step_id,
element_id=skyvern_element.get_id(),
)
return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]

if action.download:
results = await handle_click_to_download_file_action(action, page, scraped_page, task)
else:
@@ -417,6 +430,17 @@ async def handle_input_text_action(
if text is None:
return [ActionFailure(FailedToFetchSecret())]

# dynamically validate the attr, since it could change into enabled after the previous actions
if await skyvern_element.is_disabled(dynamic=True):
LOG.warning(
"Try to input text on a disabled element",
action_type=action.action_type,
task_id=task.task_id,
step_id=step.step_id,
element_id=skyvern_element.get_id(),
)
return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]

incremental_element: list[dict] = []
# check if it's selectable
if skyvern_element.get_tag_name() == InteractiveElement.INPUT and not await skyvern_element.is_raw_input():
@@ -572,6 +596,18 @@ async def handle_upload_file_action(

dom = DomUtil(scraped_page=scraped_page, page=page)
skyvern_element = await dom.get_skyvern_element_by_id(action.element_id)

# dynamically validate the attr, since it could change into enabled after the previous actions
if await skyvern_element.is_disabled(dynamic=True):
LOG.warning(
"Try to upload file on a disabled element",
action_type=action.action_type,
task_id=task.task_id,
step_id=step.step_id,
element_id=skyvern_element.get_id(),
)
return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]

locator = skyvern_element.locator

file_path = await download_file(file_url)
@@ -717,6 +753,17 @@ async def handle_select_option_action(
)
return await handle_select_option_action(select_action, page, scraped_page, task, step)

# dynamically validate the attr, since it could change into enabled after the previous actions
if await skyvern_element.is_disabled(dynamic=True):
LOG.warning(
"Try to select on a disabled element",
action_type=action.action_type,
task_id=task.task_id,
step_id=step.step_id,
element_id=skyvern_element.get_id(),
)
return [ActionFailure(InteractWithDisabledElement(skyvern_element.get_id()))]

if tag_name == InteractiveElement.SELECT:
LOG.info(
"SelectOptionAction is on <select>",
3 changes: 2 additions & 1 deletion skyvern/webeye/scraper/domUtils.js
Original file line number Diff line number Diff line change
@@ -836,7 +836,8 @@ function buildElementObject(frame, element, interactable, purgeable = false) {
attr.name === "aria-selected" ||
attr.name === "readonly" ||
attr.name === "aria-readonly" ||
attr.name === "disabled"
attr.name === "disabled" ||
attr.name === "aria-disabled"
) {
if (attrValue && attrValue.toLowerCase() === "false") {
attrValue = false;
19 changes: 18 additions & 1 deletion skyvern/webeye/scraper/scraper.py
Original file line number Diff line number Diff line change
@@ -32,6 +32,7 @@
"data-original-title", # for bootstrap tooltip
"data-ui",
"disabled", # for button
"aria-disabled",
"for",
"href", # For a tags
"maxlength",
@@ -550,14 +551,30 @@ def build_html_tree(self, element_tree: list[dict] | None = None) -> str:
return "".join([json_to_html(element) for element in (element_tree or self.element_tree_trimmed)])


def _should_keep_unique_id(element: dict) -> bool:
# case where we shouldn't keep unique_id
# 1. not disable attr and no interactable
# 2. disable=false and intrecatable=false

attributes = element.get("attributes", {})
if "disabled" not in attributes and "aria-disabled" not in attributes:
return element.get("interactable", False)

disabled = attributes.get("disabled")
aria_disabled = attributes.get("aria-disabled")
if disabled or aria_disabled:
return True
return element.get("interactable", False)


def trim_element(element: dict) -> dict:
queue = [element]
while queue:
queue_ele = queue.pop(0)
if "frame" in queue_ele:
del queue_ele["frame"]

if "id" in queue_ele and not queue_ele.get("interactable"):
if "id" in queue_ele and not _should_keep_unique_id(queue_ele):
del queue_ele["id"]

if "attributes" in queue_ele:
40 changes: 39 additions & 1 deletion skyvern/webeye/utils/dom.py
Original file line number Diff line number Diff line change
@@ -207,6 +207,43 @@ async def is_spinbtn_input(self) -> bool:
def is_interactable(self) -> bool:
return self.__static_element.get("interactable", False)

async def is_disabled(self, dynamic: bool = False) -> bool:
# if attr not exist, return None
# if attr is like 'disabled', return empty string or True
# if attr is like `disabled=false`, return the value
disabled = False
aria_disabled = False

disabled_attr: bool | str | None = None
aria_disabled_attr: bool | str | None = None

try:
disabled_attr = await self.get_attr("disabled", dynamic=dynamic)
aria_disabled_attr = await self.get_attr("aria-disabled", dynamic=dynamic)
except Exception:
# FIXME: maybe it should be considered as "disabled" element if failed to get the attributes?
LOG.exception(
"Failed to get the disabled attribute",
element=self.__static_element,
element_id=self.get_id(),
)

if disabled_attr is not None:
# disabled_attr should be bool or str
if isinstance(disabled_attr, bool):
disabled = disabled_attr
if isinstance(disabled_attr, str):
disabled = disabled_attr.lower() != "false"

if aria_disabled_attr is not None:
# aria_disabled_attr should be bool or str
if isinstance(aria_disabled_attr, bool):
aria_disabled = aria_disabled_attr
if isinstance(aria_disabled_attr, str):
aria_disabled = aria_disabled_attr.lower() != "false"

return disabled or aria_disabled

async def is_selectable(self) -> bool:
return self.get_selectable() or self.get_tag_name() in SELECTABLE_ELEMENT

@@ -367,7 +404,8 @@ async def get_attr(
timeout: float = SettingsManager.get_settings().BROWSER_ACTION_TIMEOUT_MS,
) -> typing.Any:
if not dynamic:
if attr := self.get_attributes().get(attr_name):
attr = self.get_attributes().get(attr_name)
if attr is not None:
return attr

return await self.locator.get_attribute(attr_name, timeout=timeout)

0 comments on commit c933588

Please sign in to comment.