[Resolver][Bug]: Fix success list to str representation bug (#5351)

Co-authored-by: openhands <[email protected]>
Co-authored-by: Engel Nyst <[email protected]>

openhands/resolver/issue_definitions.py

@@ -755,4 +755,4 @@ def guess_success(
         # Return overall success (all must be true) and explanations
         if not success_list:
             return False, None, 'No feedback was processed'
-        return all(success_list), success_list, '\n'.join(explanation_list)
+        return all(success_list), success_list, json.dumps(explanation_list)

openhands/resolver/send_pull_request.py

@@ -187,6 +187,13 @@ def make_commit(repo_dir: str, issue: GithubIssue, issue_type: str) -> None:
 
 
 def branch_exists(base_url: str, branch_name: str, headers: dict) -> bool:
+    """Check if a branch exists in the GitHub repository.
+
+    Args:
+        base_url: The base URL of the GitHub repository API
+        branch_name: The name of the branch to check
+        headers: The HTTP headers to use for authentication
+    """
     print(f'Checking if branch {branch_name} exists...')
     response = requests.get(f'{base_url}/branches/{branch_name}', headers=headers)
     exists = response.status_code == 200
@@ -334,6 +341,33 @@ def reply_to_comment(github_token: str, comment_id: str, reply: str):
     response.raise_for_status()
 
 
+def send_comment_msg(base_url: str, issue_number: int, github_token: str, msg: str):
+    """Send a comment message to a GitHub issue or pull request.
+
+    Args:
+        base_url: The base URL of the GitHub repository API
+        issue_number: The issue or pull request number
+        github_token: The GitHub token to use for authentication
+        msg: The message content to post as a comment
+    """
+    # Set up headers for GitHub API
+    headers = {
+        'Authorization': f'token {github_token}',
+        'Accept': 'application/vnd.github.v3+json',
+    }
+
+    # Post a comment on the PR
+    comment_url = f'{base_url}/issues/{issue_number}/comments'
+    comment_data = {'body': msg}
+    comment_response = requests.post(comment_url, headers=headers, json=comment_data)
+    if comment_response.status_code != 201:
+        print(
+            f'Failed to post comment: {comment_response.status_code} {comment_response.text}'
+        )
+    else:
+        print(f'Comment added to the PR: {msg}')
+
+
 def update_existing_pull_request(
     github_issue: GithubIssue,
     github_token: str,
@@ -354,11 +388,7 @@ def update_existing_pull_request(
         comment_message: The main message to post as a comment on the PR.
         additional_message: The additional messages to post as a comment on the PR in json list format.
     """
-    # Set up headers and base URL for GitHub API
-    headers = {
-        'Authorization': f'token {github_token}',
-        'Accept': 'application/vnd.github.v3+json',
-    }
+    # Set up base URL for GitHub API
     base_url = f'https://api.github.com/repos/{github_issue.owner}/{github_issue.repo}'
     branch_name = github_issue.head_branch
 
@@ -412,24 +442,18 @@ def update_existing_pull_request(
 
     # Post a comment on the PR
     if comment_message:
-        comment_url = f'{base_url}/issues/{github_issue.number}/comments'
-        comment_data = {'body': comment_message}
-        comment_response = requests.post(
-            comment_url, headers=headers, json=comment_data
-        )
-        if comment_response.status_code != 201:
-            print(
-                f'Failed to post comment: {comment_response.status_code} {comment_response.text}'
-            )
-        else:
-            print(f'Comment added to the PR: {comment_message}')
+        send_comment_msg(base_url, github_issue.number, github_token, comment_message)
 
     # Reply to each unresolved comment thread
     if additional_message and github_issue.thread_ids:
-        explanations = json.loads(additional_message)
-        for count, reply_comment in enumerate(explanations):
-            comment_id = github_issue.thread_ids[count]
-            reply_to_comment(github_token, comment_id, reply_comment)
+        try:
+            explanations = json.loads(additional_message)
+            for count, reply_comment in enumerate(explanations):
+                comment_id = github_issue.thread_ids[count]
+                reply_to_comment(github_token, comment_id, reply_comment)
+        except (json.JSONDecodeError, TypeError):
+            msg = f'Error occured when replying to threads; success explanations {additional_message}'
+            send_comment_msg(base_url, github_issue.number, github_token, msg)
 
     return pr_url
 

tests/unit/resolver/test_guess_success.py

@@ -1,3 +1,4 @@
+import json
 from unittest.mock import MagicMock, patch
 
 from openhands.core.config import LLMConfig
@@ -107,6 +108,7 @@ def test_pr_handler_guess_success_with_thread_comments():
         assert success is True
         assert success_list == [True]
         assert 'successfully address' in explanation
+        assert len(json.loads(explanation)) == 1
 
 
 def test_pr_handler_guess_success_only_review_comments():
@@ -155,7 +157,9 @@ def test_pr_handler_guess_success_only_review_comments():
         # Verify the results
         assert success is True
         assert success_list == [True]
-        assert 'successfully address' in explanation
+        assert (
+            '["The changes successfully address the review comments."]' in explanation
+        )
 
 
 def test_pr_handler_guess_success_no_comments():

tests/unit/resolver/test_pr_handler_guess_success.py

@@ -117,6 +117,8 @@ def test_guess_success_review_threads_litellm_call():
         )
         assert 'Last message from AI agent:\n' + history[0].content in second_prompt
 
+        assert len(json.loads(explanation)) == 2
+
 
 def test_guess_success_thread_comments_litellm_call():
     """Test that the litellm.completion() call for thread comments contains the expected content."""
@@ -188,6 +190,8 @@ def test_guess_success_thread_comments_litellm_call():
         assert 'PR Thread Comments:\n' + '\n---\n'.join(issue.thread_comments) in prompt
         assert 'Last message from AI agent:\n' + history[0].content in prompt
 
+        assert len(json.loads(explanation)) == 1
+
 
 def test_check_feedback_with_llm():
     """Test the _check_feedback_with_llm helper function."""
@@ -456,3 +460,5 @@ def test_guess_success_review_comments_litellm_call():
         )
         assert 'PR Review Comments:\n' + '\n---\n'.join(issue.review_comments) in prompt
         assert 'Last message from AI agent:\n' + history[0].content in prompt
+
+        assert len(json.loads(explanation)) == 1