Optimize MSDL Search Engine Configuration System (InternLM#262)

* feat: add msdl interface language selection

* feat: auto relaunch after select different msdl interaction language

* feat: add search engine selection

* fix: refactor(search-engine): improve API key management with dedicated environment variables

docker/msdl/__main__.py

@@ -1,14 +1,13 @@
 # msdl/__main__.py
 import signal
 import sys
+import argparse
+import os
 from pathlib import Path
 
-from InquirerPy import inquirer
 from msdl.config import (
     BACKEND_DOCKERFILE_DIR,
-    CLOUD_LLM_DOCKERFILE,
     FRONTEND_DOCKERFILE_DIR,
-    LOCAL_LLM_DOCKERFILE,
     PACKAGE_DIR,
     PROJECT_ROOT,
     REACT_DOCKERFILE,
@@ -21,16 +20,17 @@
     stop_and_remove_containers,
     update_docker_compose_paths,
 )
-from msdl.i18n import setup_i18n, t
+from msdl.i18n import (
+    setup_i18n,
+    t,
+)
 from msdl.utils import (
-    clean_api_key,
     copy_templates_to_temp,
-    get_existing_api_key,
-    get_model_formats,
+    copy_backend_dockerfile,
+    copy_frontend_dockerfile,
     modify_docker_compose,
-    save_api_key_to_env,
-    validate_api_key,
 )
+from msdl.user_interaction import get_user_choices
 
 
 def signal_handler(signum, frame):
@@ -39,140 +39,32 @@ def signal_handler(signum, frame):
     sys.exit(0)
 
 
-def copy_backend_dockerfile(choice):
-    source_file = Path(BACKEND_DOCKERFILE_DIR) / choice
-    dest_file = "backend.dockerfile"
-    source_path = PACKAGE_DIR / "templates" / source_file
-    dest_path = TEMP_DIR / dest_file
-
-    if not source_path.exists():
-        raise FileNotFoundError(t("FILE_NOT_FOUND", file=source_file))
-
-    dest_path.parent.mkdir(parents=True, exist_ok=True)
-    dest_path.write_text(source_path.read_text())
-    print(
-        t(
-            "BACKEND_DOCKERFILE_COPIED",
-            source_path=str(source_path),
-            dest_path=str(dest_path),
-        ))
-
-
-def copy_frontend_dockerfile():
-    source_file = Path(FRONTEND_DOCKERFILE_DIR) / REACT_DOCKERFILE
-    dest_file = "frontend.dockerfile"
-    source_path = PACKAGE_DIR / "templates" / source_file
-    dest_path = TEMP_DIR / dest_file
-
-    if not source_path.exists():
-        raise FileNotFoundError(t("FILE_NOT_FOUND", file=source_file))
-
-    dest_path.parent.mkdir(parents=True, exist_ok=True)
-    dest_path.write_text(source_path.read_text())
-    print(
-        t(
-            "FRONTEND_DOCKERFILE_COPIED",
-            source_path=str(source_path),
-            dest_path=str(dest_path),
-        ))
-
-
-def get_user_choices():
-    backend_language_choices = [
-        {
-            "name": t("CHINESE"),
-            "value": "cn"
-        },
-        {
-            "name": t("ENGLISH"),
-            "value": "en"
-        },
-    ]
-
-    model_deployment_type = [
-        {
-            "name": t("CLOUD_MODEL"),
-            "value": CLOUD_LLM_DOCKERFILE
-        },
-        {
-            "name": t("LOCAL_MODEL"),
-            "value": LOCAL_LLM_DOCKERFILE
-        },
-    ]
-
-    backend_language = inquirer.select(
-        message=t("BACKEND_LANGUAGE_CHOICE"),
-        choices=backend_language_choices,
-    ).execute()
-
-    model = inquirer.select(
-        message=t("MODEL_DEPLOYMENT_TYPE"),
-        choices=model_deployment_type,
-    ).execute()
-
-    model_formats = get_model_formats(model)
-    model_format = inquirer.select(
-        message=t("MODEL_FORMAT_CHOICE"),
-        choices=[{
-            "name": format,
-            "value": format
-        } for format in model_formats],
-    ).execute()
-
-    # If the model is cloud_llm, ask for the API key
-    if model == CLOUD_LLM_DOCKERFILE:
-        env_var_name = {
-            "internlm_silicon": "SILICON_API_KEY",
-            "gpt4": "OPENAI_API_KEY",
-            "qwen": "QWEN_API_KEY",
-        }.get(model_format)
-
-        existing_api_key = get_existing_api_key(env_var_name)
-
-        if existing_api_key:
-            use_existing = inquirer.confirm(
-                message=t(
-                    "CONFIRM_USE_EXISTING_API_KEY", ENV_VAR_NAME=env_var_name),
-                default=True,
-            ).execute()
-
-            if use_existing:
-                return backend_language, model, model_format
-            else:
-                print(
-                    t("CONFIRM_OVERWRITE_EXISTING_API_KEY",
-                      ENV_VAR_NAME=env_var_name))
-        else:
-            print(t("PLEASE_INPUT_NEW_API_KEY", ENV_VAR_NAME=env_var_name))
-
-        while True:
-            api_key = inquirer.secret(
-                message=t(
-                    "PLEASE_INPUT_NEW_API_KEY_FROM_ZERO",
-                    ENV_VAR_NAME=env_var_name)).execute()
-            cleaned_api_key = clean_api_key(api_key)
-
-            if validate_api_key(cleaned_api_key, env_var_name, t):
-                save_api_key_to_env(model_format, cleaned_api_key, t)
-                break
-            else:
-                print(t("INVALID_API_KEY_FORMAT"))
-                retry = inquirer.confirm(
-                    message=t("RETRY_API_KEY_INPUT"), default=True).execute()
-                if not retry:
-                    print(t("API_KEY_INPUT_CANCELLED"))
-                    sys.exit(1)
-
-    return backend_language, model, model_format
+def parse_args():
+    parser = argparse.ArgumentParser(description=t("CLI_DESCRIPTION"))
+    parser.add_argument('--language', '-l', 
+                      help=t("LANGUAGE_HELP"),
+                      choices=["en", "zh_CN"],
+                      default=None)
+    parser.add_argument('--config-language', action='store_true',
+                      help=t("CONFIG_LANGUAGE_HELP"))
+    return parser.parse_args()
 
 
 def main():
-    # Set the display language of the msdl launcher (based on the system language)
-    setup_i18n()
-
+    # Setup signal handler
     signal.signal(signal.SIGINT, signal_handler)
     signal.signal(signal.SIGTERM, signal_handler)
 
+    # Initialize i18n
+    setup_i18n()
+
+    # Parse command line arguments
+    args = parse_args()
+    if args.language:
+        # set_language(args.language)
+        # Reinitialize i18n with new language
+        setup_i18n()
+
     try:
         # Check if TEMP_DIR exists, if not, create it
         if not TEMP_DIR.exists():
@@ -181,22 +73,22 @@ def main():
 
         check_docker_install()
 
-        # Get user choices
-        backend_language, model_choice, model_format = get_user_choices()
+        # Get user choices using the new module
+        backend_language, model, model_format, search_engine = get_user_choices()
 
-        # Copy backend Dockerfile to temp directory
-        copy_backend_dockerfile(model_choice)
+        # Copy template files
+        copy_templates_to_temp(TEMPLATE_FILES)
 
-        # Copy frontend Dockerfile to temp directory
+        # Copy Dockerfiles
+        copy_backend_dockerfile(model)
         copy_frontend_dockerfile()
 
-        # Copy templates to temp directory
-        copy_templates_to_temp(TEMPLATE_FILES)
+        # Update paths in docker-compose.yml
+        update_docker_compose_paths()
 
-        # Modify docker-compose.yaml
-        modify_docker_compose(model_choice, backend_language, model_format)
+        # Modify docker-compose.yml based on user choices
+        modify_docker_compose(model, backend_language, model_format, search_engine)
 
-        update_docker_compose_paths()
         stop_and_remove_containers()
         run_docker_compose()
 

docker/msdl/docker_manager.py

@@ -142,18 +142,18 @@ def run_docker_compose():
         sys.exit(1)
 
 
-def update_docker_compose_paths():
+def update_docker_compose_paths(project_root=PROJECT_ROOT):
     docker_compose_path = os.path.join(TEMP_DIR, "docker-compose.yaml")
     with open(docker_compose_path, "r") as file:
         compose_data = yaml.safe_load(file)
     for service in compose_data["services"].values():
         if "build" in service:
             if "context" in service["build"]:
                 if service["build"]["context"] == "..":
-                    service["build"]["context"] = PROJECT_ROOT
+                    service["build"]["context"] = project_root
                 else:
                     service["build"]["context"] = os.path.join(
-                        PROJECT_ROOT, service["build"]["context"]
+                        project_root, service["build"]["context"]
                     )
             if "dockerfile" in service["build"]:
                 dockerfile_name = os.path.basename(service["build"]["dockerfile"])

docker/msdl/i18n.py

@@ -1,41 +1,63 @@
-# msdl/translations/i18n_setup.py
+# msdl/translations/i18n.py
 
 import os
 import i18n
 import locale
 from dotenv import load_dotenv, set_key, find_dotenv
 from msdl.config import TRANSLATIONS_DIR, ENV_FILE_PATH
+from pathlib import Path
 
+# Load environment variables at module level
+load_dotenv(ENV_FILE_PATH)
 
 def get_env_variable(var_name, default=None):
-    load_dotenv(ENV_FILE_PATH)
     return os.getenv(var_name, default)
 
-
 def set_env_variable(var_name, value):
     dotenv_file = find_dotenv(ENV_FILE_PATH)
     set_key(dotenv_file, var_name, value)
-
+    # Reload environment variables after setting
+    os.environ[var_name] = value
 
 def get_system_language():
     try:
         return locale.getlocale()[0].split("_")[0]
     except:
         return "en"
 
+def get_available_languages():
+    """Get list of available language codes from translation files"""
+    translations_path = Path(TRANSLATIONS_DIR)
+    if not translations_path.exists():
+        return ["en"]
+    return [f.stem for f in translations_path.glob("*.yaml")]
+
+def set_language(language_code):
+    """Set the interaction language and persist it to .env file"""
+    available_langs = get_available_languages()
+    if language_code not in available_langs:
+        print(f"Warning: Language '{language_code}' not available. Using 'en' instead.")
+        language_code = "en"
+
+    set_env_variable("LAUNCHER_INTERACTION_LANGUAGE", language_code)
+    i18n.set("locale", language_code)
+
 
 def setup_i18n():
+    # Initialize i18n settings
     i18n.load_path.append(TRANSLATIONS_DIR)
     i18n.set("filename_format", "{locale}.{format}")
     i18n.set("file_format", "yaml")
 
+    # Get language from environment
     env_language = get_env_variable("LAUNCHER_INTERACTION_LANGUAGE")
     if not env_language:
-        system_language = get_system_language()
-        set_env_variable("LAUNCHER_INTERACTION_LANGUAGE", system_language)
-        env_language = system_language
+        # If no language is set, use English as default without saving to .env
+        env_language = "en"
 
-    i18n.set("locale", env_language)
+    # Force reload translations
+    i18n.set("locale", None)  # Clear current locale
+    i18n.set("locale", env_language)  # Set new locale
 
 
 def t(key, **kwargs):

docker/msdl/translations/en.yaml

@@ -27,9 +27,9 @@ en:
   TERMINATION_SIGNAL: "Termination signal caught. Exiting gracefully..."
   KEYBOARD_INTERRUPT: "Keyboard interrupt caught. Exiting gracefully..."
   UNEXPECTED_ERROR: "An unexpected error occurred: %{error}"
-  BACKEND_LANGUAGE_CHOICE: "Select backend language (default is cn):"
-  CHINESE: "中文(cn)"
-  ENGLISH: "English(en)"
+  BACKEND_LANGUAGE_CHOICE: "Select MindSearch backend language (default is cn)"
+  CHINESE: "Chinese (cn)"
+  ENGLISH: "English (en)"
   MODEL_DEPLOYMENT_TYPE: "Select model deployment type:"
   CLOUD_MODEL: "Cloud model"
   LOCAL_MODEL: "Local model"
@@ -56,3 +56,22 @@ en:
   NETWORK_PRUNE_ERROR: "Error pruning corresponding Docker networks: %{error}"
   DOCKER_LIST_ERROR: "Error listing Docker containers: %{error}"
   CONTAINERS_STOPPED_AND_REMOVED: "Docker containers stopped and removed"
+  CLI_DESCRIPTION: "MindSearch Docker Launcher - A tool to manage MindSearch docker containers"
+  LANGUAGE_HELP: "Set the msdl tool interface language (e.g. en, zh_CN)"
+  CONFIG_LANGUAGE_HELP: "Show language configuration prompt"
+  LANGUAGE_NOT_AVAILABLE: "Warning: Language '%{lang}' not available. Using English instead."
+  SELECT_INTERFACE_LANGUAGE: "Select msdl tool interface language"
+  SELECT_BACKEND_LANGUAGE: "Select MindSearch backend language (default is cn)"
+  LANGUAGE_CHANGED_RESTARTING: "Language changed, restarting msdl..."
+  SELECT_SEARCH_ENGINE: "Select search engine:"
+  NO_API_KEY_NEEDED: "No API key needed"
+  API_KEY_REQUIRED: "API key required"
+  SEARCH_ENGINE_GOOGLE: "Google Search"
+  SEARCH_ENGINE_BING: "Bing Search"
+  SEARCH_ENGINE_DUCKDUCKGO: "DuckDuckGo Search"
+  SEARCH_ENGINE_BRAVE: "Brave Search"
+  SEARCH_ENGINE_TENCENT: "Tencent Search"
+  TENCENT_ID_REQUIRED: "Please enter your Tencent Search Secret ID"
+  TENCENT_KEY_REQUIRED: "Please enter your Tencent Search Secret Key"
+  WEB_SEARCH_KEY_REQUIRED: "Please enter your Web Search API Key"
+  SEARCH_ENGINE_CONFIGURED: "Search engine %{engine} configured successfully"