Show all actions in the message window (#5190)

Co-authored-by: openhands <[email protected]>
Co-authored-by: Graham Neubig <[email protected]>
Co-authored-by: amanape <[email protected]>

frontend/__tests__/components/chat/chat-interface.test.tsx

@@ -9,7 +9,7 @@ import { WsClientProviderStatus } from "#/context/ws-client-provider";
 import { ChatInterface } from "#/components/features/chat/chat-interface";
 
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
-const renderChatInterface = (messages: (Message | ErrorMessage)[]) =>
+const renderChatInterface = (messages: (Message)[]) =>
   renderWithProviders(<ChatInterface />);
 
 describe("Empty state", () => {
@@ -278,7 +278,7 @@ describe.skip("ChatInterface", () => {
   });
 
   it("should render inline errors", () => {
-    const messages: (Message | ErrorMessage)[] = [
+    const messages: (Message)[] = [
       {
         sender: "assistant",
         content: "Hello",
@@ -287,9 +287,10 @@ describe.skip("ChatInterface", () => {
         pending: true,
       },
       {
-        error: true,
-        id: "",
-        message: "Something went wrong",
+        type: "error",
+        content: "Something went wrong",
+        sender: "assistant",
+        timestamp: new Date().toISOString(),
       },
     ];
     renderChatInterface(messages);

frontend/src/components/features/chat/chat-message.tsx

@@ -47,7 +47,7 @@ export function ChatMessage({
         "rounded-xl relative",
         "flex flex-col gap-2",
         type === "user" && " max-w-[305px] p-4 bg-neutral-700 self-end",
-        type === "assistant" && "pb-4 max-w-full bg-tranparent",
+        type === "assistant" && "mt-6 max-w-full bg-tranparent",
       )}
     >
       <CopyToClipboardButton

frontend/src/components/features/chat/expandable-message.tsx

@@ -0,0 +1,80 @@
+import { useState, useEffect } from "react";
+import { useTranslation } from "react-i18next";
+import Markdown from "react-markdown";
+import remarkGfm from "remark-gfm";
+import { code } from "../markdown/code";
+import { ol, ul } from "../markdown/list";
+import ArrowUp from "#/icons/angle-up-solid.svg?react";
+import ArrowDown from "#/icons/angle-down-solid.svg?react";
+
+interface ExpandableMessageProps {
+  id?: string;
+  message: string;
+  type: string;
+}
+
+export function ExpandableMessage({
+  id,
+  message,
+  type,
+}: ExpandableMessageProps) {
+  const { t, i18n } = useTranslation();
+  const [showDetails, setShowDetails] = useState(true);
+  const [headline, setHeadline] = useState("");
+  const [details, setDetails] = useState(message);
+
+  useEffect(() => {
+    if (id && i18n.exists(id)) {
+      setHeadline(t(id));
+      setDetails(message);
+      setShowDetails(false);
+    }
+  }, [id, message, i18n.language]);
+
+  const border = type === "error" ? "border-danger" : "border-neutral-300";
+  const textColor = type === "error" ? "text-danger" : "text-neutral-300";
+  let arrowClasses = "h-4 w-4 ml-2 inline";
+  if (type === "error") {
+    arrowClasses += " fill-danger";
+  } else {
+    arrowClasses += " fill-neutral-300";
+  }
+
+  return (
+    <div
+      className={`flex gap-2 items-center justify-start border-l-2 pl-2 my-2 py-2 ${border}`}
+    >
+      <div className="text-sm leading-4 flex flex-col gap-2 max-w-full">
+        {headline && (
+          <p className={`${textColor} font-bold`}>
+            {headline}
+            <button
+              type="button"
+              onClick={() => setShowDetails(!showDetails)}
+              className="cursor-pointer text-left"
+            >
+              {showDetails ? (
+                <ArrowUp className={arrowClasses} />
+              ) : (
+                <ArrowDown className={arrowClasses} />
+              )}
+            </button>
+          </p>
+        )}
+        {showDetails && (
+          <Markdown
+            className="text-sm overflow-auto"
+            components={{
+              code,
+              ul,
+              ol,
+            }}
+            remarkPlugins={[remarkGfm]}
+          >
+            {details}
+          </Markdown>
+        )}
+      </div>
+    </div>
+  );
+}

frontend/src/components/features/chat/messages.tsx

@@ -1,33 +1,39 @@
 import { ChatMessage } from "#/components/features/chat/chat-message";
 import { ConfirmationButtons } from "#/components/shared/buttons/confirmation-buttons";
 import { ImageCarousel } from "../images/image-carousel";
-import { ErrorMessage } from "./error-message";
-
-const isErrorMessage = (
-  message: Message | ErrorMessage,
-): message is ErrorMessage => "error" in message;
+import { ExpandableMessage } from "./expandable-message";
 
 interface MessagesProps {
-  messages: (Message | ErrorMessage)[];
+  messages: Message[];
   isAwaitingUserConfirmation: boolean;
 }
 
 export function Messages({
   messages,
   isAwaitingUserConfirmation,
 }: MessagesProps) {
-  return messages.map((message, index) =>
-    isErrorMessage(message) ? (
-      <ErrorMessage key={index} id={message.id} message={message.message} />
-    ) : (
+  return messages.map((message, index) => {
+    if (message.type === "error" || message.type === "action") {
+      console.log("expando", message);
+      return (
+        <ExpandableMessage
+          key={index}
+          type={message.type}
+          id={message.translationID}
+          message={message.content}
+        />
+      );
+    }
+
+    return (
       <ChatMessage key={index} type={message.sender} message={message.content}>
-        {message.imageUrls.length > 0 && (
+        {message.imageUrls && message.imageUrls.length > 0 && (
           <ImageCarousel size="small" images={message.imageUrls} />
         )}
         {messages.length - 1 === index &&
           message.sender === "assistant" &&
           isAwaitingUserConfirmation && <ConfirmationButtons />}
       </ChatMessage>
-    ),
-  );
+    );
+  });
 }

frontend/src/i18n/translation.json

@@ -1782,14 +1782,6 @@
     "fr": "Privé",
     "tr": "Özel"
   },
-  "ERROR_MESSAGE$SHOW_DETAILS": {
-    "en": "Show details",
-    "es": "Mostrar detalles"
-  },
-  "ERROR_MESSAGE$HIDE_DETAILS": {
-    "en": "Hide details",
-    "es": "Ocultar detalles"
-  },
   "STATUS$STARTING_RUNTIME": {
     "en": "Starting Runtime...",
     "zh-CN": "启动运行时...",
@@ -2012,5 +2004,35 @@
   "PROJECT_MENU_CARD_CONTEXT_MENU$DOWNLOAD_AS_ZIP_LABEL": {
     "en": "Download as .zip",
     "es": "Descargar como .zip"
+  },
+  "ACTION_MESSAGE$RUN": {
+    "en": "Running a bash command"
+  },
+  "ACTION_MESSAGE$RUN_IPYTHON": {
+    "en": "Running a Jupyter command"
+  },
+  "ACTION_MESSAGE$READ": {
+    "en": "Reading the contents of a file"
+  },
+  "ACTION_MESSAGE$WRITE": {
+    "en": "Writing to a file"
+  },
+  "OBSERVATION_MESSAGE$RUN": {
+    "en": "Ran a bash command"
+  },
+  "OBSERVATION_MESSAGE$RUN_IPYTHON": {
+    "en": "Ran a Jupyter command"
+  },
+  "OBSERVATION_MESSAGE$READ": {
+    "en": "Read the contents of a file"
+  },
+  "OBSERVATION_MESSAGE$WRITE": {
+    "en": "Wrote to a file"
+  },
+  "EXPANDABLE_MESSAGE$SHOW_DETAILS": {
+    "en": "Show details"
+  },
+  "EXPANDABLE_MESSAGE$HIDE_DETAILS": {
+    "en": "Hide details"
   }
 }

frontend/src/message.d.ts

@@ -1,13 +1,10 @@
 type Message = {
   sender: "user" | "assistant";
   content: string;
-  imageUrls: string[];
   timestamp: string;
+  imageUrls?: string[];
+  type?: "thought" | "error" | "action";
   pending?: boolean;
-};
-
-type ErrorMessage = {
-  error: boolean;
-  id?: string;
-  message: string;
+  translationID?: string;
+  eventID?: number;
 };