Skip to content

Commit

Permalink
refine record-system
Browse files Browse the repository at this point in the history
  • Loading branch information
@yeyn19
yeyn19 committed Oct 21, 2023
1 parent 6717fc4 commit e3cd1c0
Show file tree
Hide file tree
Showing 5 changed files with 81 additions and 40 deletions.
16 changes: 12 additions & 4 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,11 +87,19 @@ We do not test or recommend using `gpt-3.5-turbo` to run XAgent due to very limi
```bash
python run.py --task "put your task here" --model "gpt-4"
```
You can use argument `--upload_files` to select files you want to submit to XAgent.
The local workspace for your XAgent is in `local_workspace`, where you can find all the files generated by XAgent throughout the running process. Besides, in `running_records` you can find all the intermediate steps information, e.g. task statuses, LLM's input-output pairs, used tools, etc.
After execution, the full `workspace` in `ToolServerNode` will be copied to `running_records` for your convenience.
1. You can use argument `--upload_files` to select the initial files you want to submit to XAgent.

If you want to load from a existing record, set `record_dir` in config,default to `Null`. All the runs will set to a record automatically, We remove `api key` and other unsafe items, so you can share your run with other people by sharing the records.
2. The local workspace for your XAgent is in `local_workspace`, where you can find all the files generated by XAgent throughout the running process.

3. After execution, the full `workspace` in `ToolServerNode` will be copied to `running_records` for your convenience.

4. Besides, in `running_records` you can find all the intermediate steps information, e.g. task statuses, LLM's input-output pairs, used tools, etc.

5. You can load from a record to reproduce a former run, just by setting `record_dir` in config(default to `Null`). The Record is a system-level recording tied to the code-version of XAgent. All running-config、query、code execution statuses (including errors)、server behavior will be documented.

6. We have removed all sensitive information (including API keys) from the record, so you can safely share it with others. In the near future, we will introduce more granular sharing options highlighting the contributions of humans during execution.



- Run XAgent with GUI
```bash
Expand Down
1 change: 0 additions & 1 deletion XAgent/agent/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ def _chat_completion_request(messages, functions=None,function_call=None, model=

# Yujia: maybe temperature == 0 is more stable? Not rigrously tested.
# json_data.update({"temperature": 0.1})
logger.debug("from _chat_completion_request")
response = openai_chatcompletion_request(**json_data)

return response
Expand Down
11 changes: 8 additions & 3 deletions XAgent/running_recorder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@ def query_llm_inout(self, llm_query_id, messages, functions, function_call, mode
f"query-id={llm_query_id}"
)
return cache["output"]
# import pdb; pdb.set_trace()
assert False, f"{llm_query_id} didn't find output"


Expand All @@ -124,12 +125,13 @@ def regist_tool_call(self, tool_name, tool_input, tool_output, tool_status_code,

self.tool_call_id += 1

def regist_tool_server(self, url, payload, output):
def regist_tool_server(self, url, payload, tool_output, response_status_code):
with open(os.path.join(self.record_root_dir, "tool_server_pair", f"{self.tool_server_interface_id:05d}.json"),"w",encoding="utf-8",) as writer:
tool_record = {
"url": dump_common_things(url.split("/")[-1]),
"payload": dump_common_things(payload),
"tool_output": dump_common_things(output),
"response_status_code": dump_common_things(response_status_code),
"tool_output": dump_common_things(tool_output),
}
json.dump(tool_record, writer, indent=2, ensure_ascii=False)

Expand All @@ -147,7 +149,10 @@ def query_tool_server_cache(self, url, payload):
Fore.BLUE,
cache["url"],
)
return cache["tool_output"]
return {
"tool_output": cache["tool_output"],
"response_status_code": cache["response_status_code"]
}

assert False

Expand Down
89 changes: 59 additions & 30 deletions XAgent/tool_call_handle.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,17 +112,21 @@ def get_available_tools(self):
cache_output = recorder.query_tool_server_cache(url,payload)
try:
if cache_output != None:
# import pdb; pdb.set_trace()
response = cache_output

response = cache_output["tool_output"]
status_code = cache_output["response_status_code"]
else:
response = requests.post(url, json=payload, timeout=10, cookies=self.cookies)
status_code = response.status_code
response.raise_for_status()
response = response.json()
if not isinstance(response, dict):
response = json.loads(response)

recorder.regist_tool_server(url=url,
payload=payload,
output=response)
tool_output=response,
response_status_code=status_code)
return response
except Exception as e:
raise Exception(f"Error when fetching available tools: {e}")
Expand All @@ -137,15 +141,18 @@ def retrieve_rapidapi_tools(self, query: str, top_k:int = 10,):
cache_output = recorder.query_tool_server_cache(url,payload)
try:
if cache_output != None:
response = cache_output
response = cache_output["tool_output"]
status_code = cache_output["tool_output_status_code"]
else:
response = requests.post(url, json=payload, timeout=20, cookies=self.cookies)
status_code = response.status_code
response = response.json()
if not isinstance(response, dict):
response = json.loads(response)
recorder.regist_tool_server(url=url,
payload=payload,
output=response)
tool_output=response,
response_status_code=status_code)
retrieved_tools = response["retrieved_tools"]
tools_json = response["tools_json"]
for tool_json in tools_json:
Expand All @@ -169,9 +176,11 @@ def get_json_schema_for_tools(self, command_names,):
cache_output = recorder.query_tool_server_cache(url,payload)
try:
if cache_output != None:
response = cache_output
response = cache_output["tool_output"]
status_code=cache_output["tool_output_status_code"]
else:
response = requests.post(url, json=payload, timeout=10, cookies=self.cookies)
status_code = response.status_code
response = response.json()
if not isinstance(response, dict):
try:
Expand All @@ -180,7 +189,8 @@ def get_json_schema_for_tools(self, command_names,):
pass
recorder.regist_tool_server(url=url,
payload=payload,
output=response)
tool_output=response,
response_status_code=status_code)
function_manager.register_function(response)
return response

Expand Down Expand Up @@ -210,10 +220,12 @@ def execute_command_client(
}
cache_output = recorder.query_tool_server_cache(url,payload)
if cache_output != None:
command_result = cache_output
tool_output_status_code = ToolCallStatusCode.TOOL_CALL_SUCCESS
command_result = cache_output["tool_output"]
response_status_code = cache_output["response_status_code"]
else:
response = requests.post(url, json=payload, cookies=self.cookies)
response_status_code = response.status_code
# import pdb; pdb.set_trace()
# print(response.json())

if response.status_code == 200 or response.status_code == 450:
Expand All @@ -222,27 +234,29 @@ def execute_command_client(
else:
command_result = response.text

# setting tool_output_status_code according to status_code
if response.status_code == 200:
tool_output_status_code = ToolCallStatusCode.TOOL_CALL_SUCCESS
elif response.status_code == 404:
tool_output_status_code = ToolCallStatusCode.HALLUCINATE_NAME
elif response.status_code == 422:
tool_output_status_code = ToolCallStatusCode.FORMAT_ERROR
elif response.status_code == 450:
tool_output_status_code = ToolCallStatusCode.TIMEOUT_ERROR
elif response.status_code == 500:
tool_output_status_code = ToolCallStatusCode.TOOL_CALL_FAILED
elif response.status_code == 503:
tool_output_status_code = ToolCallStatusCode.SERVER_ERROR
raise Exception("Server Error: "+ command_result)
else:
tool_output_status_code = ToolCallStatusCode.OTHER_ERROR
recorder.regist_tool_server(url=url,
payload=payload,
tool_output=command_result,
response_status_code=response_status_code)

# setting tool_output_status_code according to status_code
if response_status_code == 200:
tool_output_status_code = ToolCallStatusCode.TOOL_CALL_SUCCESS
elif response_status_code == 404:
tool_output_status_code = ToolCallStatusCode.HALLUCINATE_NAME
elif response_status_code == 422:
tool_output_status_code = ToolCallStatusCode.FORMAT_ERROR
elif response_status_code == 450:
tool_output_status_code = ToolCallStatusCode.TIMEOUT_ERROR
elif response_status_code == 500:
tool_output_status_code = ToolCallStatusCode.TOOL_CALL_FAILED
elif response_status_code == 503:
tool_output_status_code = ToolCallStatusCode.SERVER_ERROR
raise Exception("Server Error: "+ command_result)
else:
tool_output_status_code = ToolCallStatusCode.OTHER_ERROR


recorder.regist_tool_server(url=url,
payload=payload,
output=command_result)

return command_result, tool_output_status_code

Expand Down Expand Up @@ -467,8 +481,23 @@ def handle_human_help(self, arguments):
Fore.RED,
"You must give some suggestions, please type in your feedback and then press 'Enter' to send and continue the loop"
)
human_suggestion = input()
command_result = json.dumps({"output":f"{human_suggestion}"}, ensure_ascii=False)
url = "ask_human"
payload = arguments
tool_cache = recorder.query_tool_server_cache(url=url,payload=payload)
if tool_cache != None:
command_result = tool_cache["tool_output"]
status_code = tool_cache["response_status_code"]
else:
human_suggestion = input()
command_result = json.dumps({"output":f"{human_suggestion}"}, ensure_ascii=False)
status_code = "human has no status :)"
recorder.regist_tool_server(
url=url,
payload=payload,
tool_output=command_result,
response_status_code=status_code,
)

plan_refine = False
return plan_refine, ToolCallStatusCode.TOOL_CALL_SUCCESS, command_result

Expand Down
4 changes: 2 additions & 2 deletions run.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ def parse_args():
parser.add_argument("--upload_files", nargs='+',
help="upload files")
parser.add_argument("--model", type=str, default=CONFIG.default_completion_kwargs['model'],)
parser.add_argument("--record_dir", type=str, default=CONFIG.record_dir,)
parser.add_argument("--record_dir", type=str, default=None)
parser.add_argument("--mode", type=str, default="auto",
help="mode, only support auto and manual, if you choose manual, you need to press enter to continue in each step")
parser.add_argument("--quiet", action="store_true",default=False)
Expand All @@ -39,7 +39,7 @@ def parse_args():
CONFIG.max_plan_tree_depth = args.max_plan_tree_depth
CONFIG.max_plan_tree_width = args.max_plan_tree_width
CONFIG.max_retry_times = args.max_retry_times
CONFIG.record_dir = args.record_dir



cmd = CommandLine(XAgentServerEnv)
Expand Down

0 comments on commit e3cd1c0

Please sign in to comment.