update brainstorming case. fix some bugs.

agentverse/agents/pipeline/critic.py

@@ -28,6 +28,7 @@ async def astep(
         preliminary_solution: str,
         advice: str = "No advice yet.",
         task_description: str = "",
+        **kwargs,
     ) -> CriticMessage:
         """Asynchronous version of step"""
         logger.debug("", self.name, Fore.MAGENTA)
@@ -36,6 +37,7 @@ async def astep(
             advice=advice,
             task_description=task_description,
             role_description=self.role_description,
+            **kwargs,
         )
         history = self.memory.to_messages(self.name, start_index=-self.max_history)
         parsed_response: Union[AgentCriticism, None] = None

agentverse/agents/pipeline/role_assigner.py

@@ -40,6 +40,12 @@ def step(
                     prepend_prompt, history, append_prompt
                 )
                 parsed_response = self.output_parser.parse(response)
+                if len(parsed_response) < cnt_critic_agents:
+                    logger.warn(
+                        f"Number of generate roles ({len(parsed_response)}) and number of group members ({cnt_critic_agents}) do not match."
+                    )
+                    logger.warn("Retrying...")
+                    continue
                 break
             except (KeyboardInterrupt, bdb.BdbQuit):
                 raise

agentverse/agents/pipeline/solver.py

@@ -24,10 +24,7 @@ class SolverAgent(BaseAgent):
     max_history: int = 3
 
     def step(
-        self,
-        former_solution: str,
-        advice: str,
-        task_description: str = "",
+        self, former_solution: str, advice: str, task_description: str = "", **kwargs
     ) -> SolverMessage:
         logger.debug("", self.name, Fore.MAGENTA)
         # prompt = self._fill_prompt_template(
@@ -38,6 +35,7 @@ def step(
             task_description=task_description,
             advice=advice,
             role_description=self.role_description,
+            **kwargs,
         )
         history = self.memory.to_messages(self.name, start_index=-self.max_history)
         parsed_response = None

agentverse/environments/decision_maker/__init__.py

@@ -8,3 +8,5 @@
 from .dynamic import DynamicDecisionMaker
 from .vertical_solver_first import VerticalSolverFirstDecisionMaker
 from .concurrent import ConcurrentDecisionMaker
+from .central import CentralDecisionMaker
+from .brainstorming import BrainstormingDecisionMaker

agentverse/environments/decision_maker/brainstorming.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+import asyncio
+from colorama import Fore
+
+from typing import TYPE_CHECKING, List
+
+from . import decision_maker_registry
+from .base import BaseDecisionMaker
+from agentverse.logging import logger
+
+from agentverse.message import Message
+
+if TYPE_CHECKING:
+    from agentverse.agents.base import BaseAgent
+    from agentverse.message import CriticMessage
+
+
+@decision_maker_registry.register("brainstorming")
+class BrainstormingDecisionMaker(BaseDecisionMaker):
+    """
+    Much like the horizontal decision maker, but with some twists:
+    (1) Solver acts as a summarizer, summarizing the discussion of this turn
+    (2) After summarizing, all the agents' memory are cleared, and replaced with
+    the summary (to avoid exceeding maximum context length of the model too fast)
+    """
+
+    name: str = "brainstorming"
+
+    async def astep(
+        self,
+        agents: List[BaseAgent],
+        task_description: str,
+        previous_plan: str = "No solution yet.",
+        advice: str = "No advice yet.",
+        *args,
+        **kwargs,
+    ) -> List[str]:
+        if advice != "No advice yet.":
+            self.broadcast_messages(
+                agents, [Message(content=advice, sender="Evaluator")]
+            )
+        for agent in agents[1:]:
+            review: CriticMessage = await agent.astep(
+                previous_plan, advice, task_description
+            )
+            if review.content != "":
+                self.broadcast_messages(agents, [review])
+
+            logger.info("", "Reviews:", Fore.YELLOW)
+            logger.info(
+                "",
+                f"[{review.sender}]: {review.content}",
+                Fore.YELLOW,
+            )
+
+        result = agents[0].step(previous_plan, advice, task_description)
+        for agent in agents:
+            agent.memory.reset()
+        self.broadcast_messages(
+            agents,
+            [
+                Message(
+                    content=result.content, sender="Summary From Previous Discussion"
+                )
+            ],
+        )
+        return [result]

agentverse/environments/decision_maker/central.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+import asyncio
+from colorama import Fore
+
+from typing import TYPE_CHECKING, List
+
+from . import decision_maker_registry
+from .base import BaseDecisionMaker
+from agentverse.logging import typewriter_log, logger
+from agentverse.message import Message
+
+if TYPE_CHECKING:
+    from agentverse.agents import BaseAgent, SolverAgent, CriticAgent
+    from agentverse.message import SolverMessage
+
+
+@decision_maker_registry.register("central")
+class CentralDecisionMaker(BaseDecisionMaker):
+    """
+    Discuss in a central manner.
+    """
+
+    name: str = "central"
+
+    async def astep(
+        self,
+        agents: List[BaseAgent],
+        task_description: str,
+        previous_plan: str = "No solution yet.",
+        advice: str = "No advice yet.",
+        *args,
+        **kwargs,
+    ) -> List[SolverMessage]:
+        if advice != "No advice yet.":
+            agents[1].add_message_to_memory(
+                [Message(content=advice, sender="Evaluator")]
+            )
+        result = await agents[1].astep(
+            previous_plan,
+            advice,
+            task_description,
+            roles=", ".join(
+                [
+                    agent.role_description[0].lower() + agent.role_description[1:]
+                    for agent in agents
+                ]
+            ),
+        )
+        agents[1].add_message_to_memory([result])
+        result = agents[0].step(
+            previous_plan, advice, task_description, chat_record=result.content
+        )
+        return [result]
+
+    def reset(self):
+        pass

agentverse/environments/decision_maker/horizontal.py

@@ -8,9 +8,11 @@
 from .base import BaseDecisionMaker
 from agentverse.logging import logger
 
+from agentverse.message import Message
+
 if TYPE_CHECKING:
     from agentverse.agents.base import BaseAgent
-    from agentverse.message import Message, CriticMessage
+    from agentverse.message import CriticMessage
 
 
 @decision_maker_registry.register("horizontal")
@@ -31,6 +33,10 @@ async def astep(
         *args,
         **kwargs,
     ) -> List[str]:
+        if advice != "No advice yet.":
+            self.broadcast_messages(
+                agents, [Message(content=advice, sender="Evaluator")]
+            )
         for agent in agents[1:]:
             review: CriticMessage = await agent.astep(
                 previous_plan, advice, task_description

agentverse/environments/evaluator/base.py

@@ -5,9 +5,10 @@
 
 from pydantic import BaseModel
 
+from agentverse.message import EvaluatorMessage
+
 if TYPE_CHECKING:
     from agentverse.agents import EvaluatorAgent
-    from agentverse.message import EvaluatorMessage
 
 from . import evaluator_registry
 
@@ -48,3 +49,19 @@ def step(
     ) -> EvaluatorMessage:
         result = EvaluatorMessage(score=0, advice=result)
         return result
+
+
+@evaluator_registry.register("dummy")
+class DummyEvaluator(BaseEvaluator):
+    def step(
+        self,
+        agent: EvaluatorAgent,
+        solution: List[str] | str,
+        result: List[str] | str,
+        task_description: str,
+        all_role_description: List[str],
+        *args,
+        **kwargs,
+    ) -> EvaluatorMessage:
+        result = EvaluatorMessage(score=1, advice="")
+        return result

agentverse/environments/executor/__init__.py

@@ -4,3 +4,4 @@
 
 from .base import BaseExecutor, NoneExecutor
 from .code_test import CodeTestExecutor
+from .coverage_test import CoverageTestExecutor

agentverse/environments/executor/base.py

@@ -48,3 +48,23 @@ def step(
 
     def reset(self):
         pass
+
+
+@executor_registry.register("dummy")
+class DummyExecutor(BaseExecutor):
+    """
+    The base class of execution.
+    """
+
+    def step(
+        self,
+        agent: ExecutorAgent,
+        task_description: str,
+        solution: List[str],
+        *args,
+        **kwargs,
+    ) -> Any:
+        return solution
+
+    def reset(self):
+        pass

agentverse/environments/executor/code_test.py

@@ -6,14 +6,16 @@
 from typing import TYPE_CHECKING, Any, List, Tuple
 
 from agentverse.agents import ExecutorAgent
+from agentverse.logging import logger
 
 from . import BaseExecutor, executor_registry
 
 
-def execute_command(command: str) -> str:
+def execute_command(command: str, result_list) -> str:
     # TODO: make it more secure
     result = subprocess.run(command, capture_output=True, shell=True, encoding="utf-8")
-    return f"STDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}"
+    result_list.append(f"STDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}")
+    # return f"STDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}"
 
 
 @executor_registry.register("code-test")
@@ -38,7 +40,7 @@ def step(
             self.write_to_file(response["file_path"], response["code"])
             self.has_test[task_description] = f"python {response['file_path']}"
             p = multiprocessing.Process(
-                target=execute_command, args=(f"python {response['file_path']}",)
+                target=execute_command, args=(f"python {response['file_path']}", result)
             )
             p.start()
             p.join(timeout=self.timeout + 1)
@@ -48,7 +50,7 @@ def step(
         else:
             # result = execute_command(self.has_test[task_description])
             p = multiprocessing.Process(
-                target=execute_command, args=(self.has_test[task_description],)
+                target=execute_command, args=(self.has_test[task_description], result)
             )
             p.start()
             p.join(timeout=self.timeout + 1)
@@ -60,5 +62,9 @@ def step(
 
     def write_to_file(self, file_name, file_content):
         # TODO: generalize this method to a common tool
-        with open(file_name, "w") as f:
-            f.write(file_content)
+        try:
+            with open(file_name, "w") as f:
+                f.write(file_content)
+                f.flush()
+        except:
+            logger.error(f"Failed to write to {file_name}")

agentverse/environments/executor/coverage_test.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import os
+import subprocess
+import multiprocessing
+from typing import TYPE_CHECKING, Any, List, Tuple
+
+from agentverse.agents import ExecutorAgent
+from agentverse.logging import logger
+
+from . import BaseExecutor, executor_registry
+
+
+def execute_command(command: str, result_list) -> str:
+    # TODO: make it more secure
+    result = subprocess.run(command, capture_output=True, shell=True, encoding="utf-8")
+    result_list.append(f"STDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}")
+    # return f"STDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}"
+
+
+@executor_registry.register("coverage-test")
+class CoverageTestExecutor(BaseExecutor):
+    def step(
+        self,
+        agent: ExecutorAgent,
+        task_description: str,
+        solution: List[str],
+        *args,
+        **kwargs,
+    ) -> Any:
+        from evaluate_commongen import scoring
+
+        coverage, missing_tokens = scoring([solution], [task_description])
+        if len(missing_tokens[0]) == 0:
+            missing_tokens = "No missing tokens."
+        else:
+            missing_tokens = ", ".join(missing_tokens[0])
+        result = f"Coverage: {coverage*100:.2f}%\nMissing Tokens: {missing_tokens}"
+        return result

agentverse/environments/role_assigner/base.py

@@ -7,6 +7,7 @@
 from pydantic import BaseModel
 
 from abc import abstractmethod
+from . import role_assigner_registry
 
 if TYPE_CHECKING:
     from agentverse.agents import RoleAssignerAgent, CriticAgent
@@ -31,3 +32,24 @@ def step(
 
     def reset(self):
         pass
+
+
+@role_assigner_registry.register("dummy")
+class DummyRoleAssigner(BaseRoleAssigner):
+    """
+    The base class of role assignment class.
+    """
+
+    def step(
+        self,
+        role_assigner: RoleAssignerAgent,
+        group_members: List[CriticAgent],
+        advice: str = "No advice yet.",
+        task_description: str = "",
+        *args,
+        **kwargs,
+    ) -> List[CriticAgent]:
+        return group_members
+
+    def reset(self):
+        pass