bug fix

Author: vyokky

ufo/agent/agent.py

@@ -114,7 +114,7 @@ def message_constructor(
         request_history: str,
         action_history: str,
         control_info: str,
-        plan: str,
+        plan: List[str],
         request: str,
         include_last_screenshot: bool,
     ) -> list:
@@ -127,7 +127,7 @@ def message_constructor(
         :param request_history: The request history.
         :param action_history: The action history.
         :param control_info: The control information.
-        :param plan: The plan.
+        :param plan: The plan list.
         :param request: The request.
         :param include_last_screenshot: The flag indicating whether to include the last screenshot.
         :return: The prompt message.
@@ -188,7 +188,7 @@ def print_response(self, response_dict: Dict) -> None:
         )
         utils.print_with_color("Status📊: {status}".format(status=status), "blue")
         utils.print_with_color(
-            "Next Plan📚: {plan}".format(plan=str(plan).replace("\\n", "\n")), "cyan"
+            "Next Plan📚: {plan}".format(plan="\n".join(plan)), "cyan"
         )
         utils.print_with_color("Comment💬: {comment}".format(comment=comment), "green")
 
@@ -447,7 +447,7 @@ def message_constructor(
         request_history: str,
         action_history: str,
         os_info: str,
-        plan: str,
+        plan: List[str],
         request: str,
     ) -> list:
         """
@@ -522,7 +522,7 @@ def print_response(self, response_dict: Dict) -> None:
         )
         utils.print_with_color("Status📊: {status}".format(status=status), "blue")
         utils.print_with_color(
-            "Next Plan📚: {plan}".format(plan=str(plan).replace("\\n", "\n")), "cyan"
+            "Next Plan📚: {plan}".format(plan="\n".join(plan)), "cyan"
         )
         utils.print_with_color("Comment💬: {comment}".format(comment=comment), "green")
 
@@ -641,7 +641,7 @@ def message_constructor(
         request_history: str,
         action_history: str,
         control_info: str,
-        plan: str,
+        plan: List[str],
         request: str,
         include_last_screenshot: bool,
     ) -> list:
@@ -688,7 +688,7 @@ def message_constructor(
         request_history: str,
         action_history: str,
         control_info: str,
-        plan: str,
+        plan: List[str],
         request: str,
         current_state: dict,
         state_diff: dict,

ufo/automator/ui_control/control_filter.py

@@ -56,7 +56,7 @@ def inplace_append_filtered_annotation_dict(
         return filtered_control_dict
 
     @staticmethod
-    def get_plans(plan, topk_plan):
+    def get_plans(plan: List[str], topk_plan: int) -> List[str]:
         """
         Parses the given plan and returns a list of plans up to the specified topk_plan.
 
@@ -67,8 +67,7 @@ def get_plans(plan, topk_plan):
         Returns:
             list: A list of plans up to the specified topk_plan.
         """
-        plans = str(plan).split("\n")[:topk_plan]
-        return plans
+        return plan[:topk_plan]
 
 
 class BasicControlFilter:
@@ -128,7 +127,7 @@ def control_filter(self, control_dicts, plans, **kwargs):
         pass
 
     @staticmethod
-    def plans_to_keywords(plans: list) -> list:
+    def plans_to_keywords(plans: List[str]) -> List[str]:
         """
         Gets keywords from the plan.
         We only consider the words in the plan that are alphabetic or Chinese characters.

ufo/config/config_dev.yaml

@@ -47,7 +47,7 @@ DEMONSTRATION_PROMPT: "ufo/prompts/demonstration/demonstration_summary.yaml"
 DEMONSTRATION_SAVED_PATH: "vectordb/demonstration/"
 
 API_PROMPT: "ufo/prompts/share/base/api.yaml"  # The prompt for the API
-CLICK_API: "click" # The click API
+CLICK_API: "click_input" # The click API
 INPUT_TEXT_API: "type_keys" # The input text API
 INPUT_TEXT_ENTER: False # whether to press enter after typing the text
 

ufo/module/basic.py

@@ -31,6 +31,7 @@
     ErrorState,
     MaxStepReachedState,
     NoneState,
+    Status,
     StatusToStateMapper,
 )
 
@@ -72,7 +73,7 @@ def __init__(
         self.app_agent = None
 
         # Status-related properties
-        self._status = "APP_SELECTION"
+        self._status = Status.APP_SELECTION
 
         # Application-related properties
         self.application = ""
@@ -267,7 +268,7 @@ def __init__(self, task: str) -> None:
         self.app_agent = None
 
         # Status and state-related properties
-        self._status = "APP_SELECTION"
+        self._status = Status.APP_SELECTION
         self._state = StatusToStateMapper().get_appropriate_state(self._status)
 
         # Application-related properties

ufo/module/processors/processor.py

@@ -131,7 +131,7 @@ def get_prompt_message(self) -> None:
         if agent_memory.length > 0:
             plan = agent_memory.get_latest_item().to_dict()["Plan"]
         else:
-            plan = ""
+            plan = []
 
         # Construct the prompt message for the host agent.
         self._prompt_message = self.host_agent.message_constructor(
@@ -438,7 +438,7 @@ def __init__(
         self._operation = None
         self._args = None
         self._image_url = []
-        self.prev_plan = ""
+        self.prev_plan = []
         self._control_reannotate = control_reannotate
         self.control_filter_factory = ControlFilterFactory()
         self.filtered_annotation_dict = None
@@ -790,7 +790,7 @@ def _safe_guard_judgement(self, action: str, control_text: str) -> bool:
             return False
 
         # Handle the PENDING_AND_FINISH case
-        elif Status.FINISH in self._plan:
+        elif len(self._plan) > 0 and Status.FINISH in self._plan[0]:
             self._status = Status.FINISH
         return True
 
@@ -810,9 +810,9 @@ def get_prev_plan(self) -> str:
         agent_memory = self.app_agent.memory
 
         if agent_memory.length > 0:
-            prev_plan = agent_memory.get_latest_item().to_dict()["Plan"].strip()
+            prev_plan = agent_memory.get_latest_item().to_dict()["Plan"]
         else:
-            prev_plan = ""
+            prev_plan = []
 
         return prev_plan
 
@@ -861,7 +861,7 @@ def get_filtered_annotation_dict(
         control_filter_type = configs["CONTROL_FILTER_TYPE"]
         topk_plan = configs["CONTROL_FILTER_TOP_K_PLAN"]
 
-        if len(control_filter_type) == 0 or self.prev_plan == "":
+        if len(control_filter_type) == 0 or self.prev_plan == []:
             return annotation_dict
 
         control_filter_type_lower = [

ufo/prompter/agent_prompter.py

@@ -62,7 +62,7 @@ def user_prompt_construction(
         request_history: List[str],
         action_history: List[str],
         control_item: List[str],
-        prev_plan: str,
+        prev_plan: List[str],
         user_request: str,
         retrieved_docs: str = "",
     ) -> str:
@@ -78,7 +78,7 @@ def user_prompt_construction(
             action_history=json.dumps(action_history),
             request_history=json.dumps(request_history),
             control_item=json.dumps(control_item),
-            prev_plan=prev_plan,
+            prev_plan=json.dumps(prev_plan),
             user_request=user_request,
             retrieved_docs=retrieved_docs,
         )
@@ -250,7 +250,7 @@ def user_prompt_construction(
         request_history: List[str],
         action_history: List[str],
         control_item: List[str],
-        prev_plan: str,
+        prev_plan: List[str],
         user_request: str,
         retrieved_docs: str = "",
     ) -> str:
@@ -267,7 +267,7 @@ def user_prompt_construction(
             action_history=json.dumps(action_history),
             request_history=json.dumps(request_history),
             control_item=json.dumps(control_item),
-            prev_plan=prev_plan,
+            prev_plan=json.dumps(prev_plan),
             user_request=user_request,
             retrieved_docs=retrieved_docs,
         )
@@ -280,7 +280,7 @@ def user_content_construction(
         request_history: List[str],
         action_history: List[str],
         control_item: List[str],
-        prev_plan: str,
+        prev_plan: List[str],
         user_request: str,
         retrieved_docs: str = "",
         include_last_screenshot: bool = True,
@@ -484,7 +484,7 @@ def user_prompt_construction(
         request_history: List[str],
         action_history: List[str],
         control_item: List[str],
-        prev_plan: str,
+        prev_plan: List[str],
         user_request: str,
         retrieved_docs: str = "",
         current_state: dict = {},
@@ -505,7 +505,7 @@ def user_prompt_construction(
             action_history=json.dumps(action_history),
             request_history=json.dumps(request_history),
             control_item=json.dumps(control_item),
-            prev_plan=prev_plan,
+            prev_plan=json.dumps(prev_plan),
             user_request=user_request,
             retrieved_docs=retrieved_docs,
             current_state=json.dumps(current_state),
@@ -520,7 +520,7 @@ def user_content_construction(
         request_history: List[str],
         action_history: List[str],
         control_item: List[str],
-        prev_plan: str,
+        prev_plan: List[str],
         user_request: str,
         retrieved_docs: str = "",
         current_state: dict = {},

ufo/prompts/examples/lite/nonvisual/app_agent_example.yaml

@@ -18,11 +18,11 @@ example1:
         {"button": "left", "double": false}
       Status: |-
         CONTINUE
-      Plan: |-
-        (1) Input the email address of the receiver.
-        (2) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'.
-        (3) Input the content of the email. I need to input 'Dear Jack,\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\nBest regards,\nZac'.
-        (4) Click the Send button to send the email.
+      Plan:
+        - (1) Input the email address of the receiver.
+        - (2) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'.
+        - (3) Input the content of the email. I need to input 'Dear Jack,\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\nBest regards,\nZac'.
+        - (4) Click the Send button to send the email.
       Comment: |-
         After I click the New Email button, the New Email window will be opened and available for composing the email.
   Tips: |-

ufo/prompts/examples/lite/nonvisual/host_agent_example.yaml

@@ -15,12 +15,12 @@ example1:
       Mail - Outlook - Zac
     Status: |-
       CONTINUE
-    Plan: |-
-      (1) Open the windows of outlook application.
-      (2) Input the email address of the receiver.
-      (3) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'.
-      (4) Input the content of the email. I need to input 'Dear Jack,\\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\\nBest regards,\\nZac'.
-      (5) Click the Send button to send the email. This action is sensitive and need to be confirmed by the user.
+    Plan:
+      - (1) Open the windows of outlook application.
+      - (2) Input the email address of the receiver.
+      - (3) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'.
+      - (4) Input the content of the email. I need to input 'Dear Jack,\\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\\nBest regards,\\nZac'.
+      - (5) Click the Send button to send the email. This action is sensitive and need to be confirmed by the user.
     Comment: |- 
       It is time to open the outlook application!
 

ufo/prompts/examples/lite/visual/app_agent_example.yaml

@@ -18,11 +18,11 @@ example1:
       {"button": "left", "double": false}
     Status: |-
       CONTINUE
-    Plan: |-
-      (1) Input the email address of the receiver.
-      (2) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'.
-      (3) Input the content of the email. I need to input 'Dear Jack,\\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\\nBest regards,\\nZac'.
-      (4) Click the Send button to send the email.
+    Plan:
+      - (1) Input the email address of the receiver.
+      - (2) Input the title of the email. I need to input 'Thanks for your contribution on the open source.'.
+      - (3) Input the content of the email. I need to input 'Dear Jack,\\nI hope this message finds you well. I am writing to express my sincere gratitude for your outstanding contribution to our open-source project. Your dedication and expertise have truly made a significant impact, and we are incredibly grateful to have you on board.\\nYour commitment to the open-source community has not gone unnoticed, and your recent contributions have been instrumental in enhancing the functionality and quality of our project. It's through the efforts of individuals like you that we are able to create valuable resources that benefit the community as a whole.\\nYour code reviews, bug fixes, and innovative ideas have not only improved the project but have also inspired others to contribute their best. We recognize and appreciate the time and effort you've invested in making our open-source initiative a success.\\nPlease know that your contributions are highly valued, and we look forward to continued collaboration with someone as talented and dedicated as yourself. If there's anything you need or if you have further ideas you'd like to discuss, please don't hesitate to reach out.\\nOnce again, thank you for your exceptional contributions. We are fortunate to have you as part of our open-source community.\\nBest regards,\\nZac'.
+      - (4) Click the Send button to send the email.
     Comment: |-
       After I click the New Email button, the New Email window will be opened and available for composing the email.
   Tips: |-