Merge pull request stacklok#124 from stacklok/vllm-provider

Vllm provider

Author: lukehinds

config.yaml.example

@@ -1,19 +1,42 @@
-# Example configuration file
-# Copy this file to config.yaml and modify as needed
+# Codegate Example Configuration
 
-# Server configuration
-port: 8989
-host: "localhost"
+# Network settings
+port: 8989           # Port to listen on (1-65535)
+host: "localhost"      # Host to bind to (use localhost for all interfaces)
 
 # Logging configuration
-log_level: "INFO"  # ERROR, WARNING, INFO, DEBUG
-log_format: "JSON" # JSON, TEXT
+log_level: "INFO"  # One of: ERROR, WARNING, INFO, DEBUG
 
-# Prompts configuration
-# Option 1: Define prompts directly in the config file
-prompts:
-  my_system_prompt: "Custom system prompt defined in config"
-  another_prompt: "Another custom prompt"
+# Note: This configuration can be overridden by:
+# 1. CLI arguments (--port, --host, --log-level)
+# 2. Environment variables (CODEGATE_APP_PORT, CODEGATE_APP_HOST, CODEGATE_APP_LOG_LEVEL)
 
-# Option 2: Reference a separate prompts file
-# prompts: "prompts.yaml"  # Path to prompts file (relative to config file or absolute)
+# Provider URLs
+provider_urls:
+  openai: "https://api.openai.com/v1"
+  anthropic: "https://api.anthropic.com/v1"
+  vllm: "http://localhost:8000"  # Base URL without /v1 path, it will be added automatically
+
+# Note: Provider URLs can be overridden by environment variables:
+# CODEGATE_PROVIDER_OPENAI_URL
+# CODEGATE_PROVIDER_ANTHROPIC_URL
+# CODEGATE_PROVIDER_VLLM_URL
+# Or by CLI flags:
+# --vllm-url
+# --openai-url
+# --anthropic-url
+
+# Embedding model configuration
+
+####
+# Inference model configuration
+##
+
+# Model to use for chatting
+chat_model_path: "./models/qwen2.5-coder-1.5b-instruct-q5_k_m.gguf"
+
+# Context length of the model
+chat_model_n_ctx: 32768
+
+# Number of layers to offload to GPU. If -1, all layers are offloaded.
+chat_model_n_gpu_layers: -1

docs/cli.md

@@ -46,6 +46,21 @@ codegate serve [OPTIONS]
   - Must be a valid YAML file
   - Overrides default prompts and configuration file prompts
 
+- `--vllm-url TEXT`: vLLM provider URL (default: http://localhost:8000)
+  - Optional
+  - Base URL for vLLM provider (/v1 path is added automatically)
+  - Overrides configuration file and environment variables
+
+- `--openai-url TEXT`: OpenAI provider URL (default: https://api.openai.com/v1)
+  - Optional
+  - Base URL for OpenAI provider
+  - Overrides configuration file and environment variables
+
+- `--anthropic-url TEXT`: Anthropic provider URL (default: https://api.anthropic.com/v1)
+  - Optional
+  - Base URL for Anthropic provider
+  - Overrides configuration file and environment variables
+
 ### show-prompts
 
 Display the loaded system prompts:
@@ -100,6 +115,11 @@ Start server with custom prompts:
 codegate serve --prompts my-prompts.yaml
 ```
 
+Start server with custom vLLM endpoint:
+```bash
+codegate serve --vllm-url https://vllm.example.com
+```
+
 Show default system prompts:
 ```bash
 codegate show-prompts

docs/configuration.md

@@ -16,6 +16,10 @@ The configuration system in Codegate is managed through the `Config` class in `c
 - Log Level: "INFO"
 - Log Format: "JSON"
 - Prompts: Default prompts from prompts/default.yaml
+- Provider URLs:
+  - vLLM: "http://localhost:8000"
+  - OpenAI: "https://api.openai.com/v1"
+  - Anthropic: "https://api.anthropic.com/v1"
 
 ## Configuration Methods
 
@@ -27,6 +31,18 @@ Load configuration from a YAML file:
 config = Config.from_file("config.yaml")
 ```
 
+Example config.yaml:
+```yaml
+port: 8989
+host: localhost
+log_level: INFO
+log_format: JSON
+provider_urls:
+  vllm: "https://vllm.example.com"
+  openai: "https://api.openai.com/v1"
+  anthropic: "https://api.anthropic.com/v1"
+```
+
 ### From Environment Variables
 
 Environment variables are automatically loaded with these mappings:
@@ -36,13 +52,42 @@ Environment variables are automatically loaded with these mappings:
 - `CODEGATE_APP_LOG_LEVEL`: Logging level
 - `CODEGATE_LOG_FORMAT`: Log format
 - `CODEGATE_PROMPTS_FILE`: Path to prompts YAML file
+- `CODEGATE_PROVIDER_VLLM_URL`: vLLM provider URL
+- `CODEGATE_PROVIDER_OPENAI_URL`: OpenAI provider URL
+- `CODEGATE_PROVIDER_ANTHROPIC_URL`: Anthropic provider URL
 
 ```python
 config = Config.from_env()
 ```
 
 ## Configuration Options
 
+### Provider URLs
+
+Provider URLs can be configured in several ways:
+
+1. In Configuration File:
+   ```yaml
+   provider_urls:
+     vllm: "https://vllm.example.com"  # /v1 path is added automatically
+     openai: "https://api.openai.com/v1"
+     anthropic: "https://api.anthropic.com/v1"
+   ```
+
+2. Via Environment Variables:
+   ```bash
+   export CODEGATE_PROVIDER_VLLM_URL=https://vllm.example.com
+   export CODEGATE_PROVIDER_OPENAI_URL=https://api.openai.com/v1
+   export CODEGATE_PROVIDER_ANTHROPIC_URL=https://api.anthropic.com/v1
+   ```
+
+3. Via CLI Flags:
+   ```bash
+   codegate serve --vllm-url https://vllm.example.com
+   ```
+
+Note: For the vLLM provider, the /v1 path is automatically appended to the base URL if not present.
+
 ### Log Levels
 
 Available log levels (case-insensitive):

docs/development.md

@@ -9,6 +9,7 @@ Codegate is a configurable Generative AI gateway designed to protect developers
 - Secure coding recommendations
 - Prevention of AI recommending deprecated/malicious libraries
 - Modular system prompts configuration
+- Multiple AI provider support with configurable endpoints
 
 ## Development Setup
 
@@ -53,7 +54,11 @@ codegate/
 │       ├── logging.py       # Logging setup
 │       ├── prompts.py       # Prompts management
 │       ├── server.py        # Main server implementation
-│       └── providers/*      # External service providers (anthropic, openai, etc.)
+│       └── providers/       # External service providers
+│           ├── anthropic/   # Anthropic provider implementation
+│           ├── openai/      # OpenAI provider implementation
+│           ├── vllm/        # vLLM provider implementation
+│           └── base.py      # Base provider interface
 ├── tests/           # Test files
 └── docs/            # Documentation
 ```
@@ -128,9 +133,87 @@ Codegate uses a hierarchical configuration system with the following priority (h
 - Log Level: Logging level (ERROR|WARNING|INFO|DEBUG)
 - Log Format: Log format (JSON|TEXT)
 - Prompts: System prompts configuration
+- Provider URLs: AI provider endpoint configuration
 
 See [Configuration Documentation](configuration.md) for detailed information.
 
+## Working with Providers
+
+Codegate supports multiple AI providers through a modular provider system.
+
+### Available Providers
+
+1. **vLLM Provider**
+   - Default URL: http://localhost:8000
+   - Supports OpenAI-compatible API
+   - Automatically adds /v1 path to base URL
+   - Model names are prefixed with "hosted_vllm/"
+
+2. **OpenAI Provider**
+   - Default URL: https://api.openai.com/v1
+   - Standard OpenAI API implementation
+
+3. **Anthropic Provider**
+   - Default URL: https://api.anthropic.com/v1
+   - Anthropic Claude API implementation
+
+### Configuring Providers
+
+Provider URLs can be configured through:
+
+1. Config file (config.yaml):
+   ```yaml
+   provider_urls:
+     vllm: "https://vllm.example.com"
+     openai: "https://api.openai.com/v1"
+     anthropic: "https://api.anthropic.com/v1"
+   ```
+
+2. Environment variables:
+   ```bash
+   export CODEGATE_PROVIDER_VLLM_URL=https://vllm.example.com
+   export CODEGATE_PROVIDER_OPENAI_URL=https://api.openai.com/v1
+   export CODEGATE_PROVIDER_ANTHROPIC_URL=https://api.anthropic.com/v1
+   ```
+
+3. CLI flags:
+   ```bash
+   codegate serve --vllm-url https://vllm.example.com
+   ```
+
+### Implementing New Providers
+
+To add a new provider:
+
+1. Create a new directory in `src/codegate/providers/`
+2. Implement required components:
+   - `provider.py`: Main provider class extending BaseProvider
+   - `adapter.py`: Input/output normalizers
+   - `__init__.py`: Export provider class
+
+Example structure:
+```python
+from codegate.providers.base import BaseProvider
+
+class NewProvider(BaseProvider):
+    def __init__(self, ...):
+        super().__init__(
+            InputNormalizer(),
+            OutputNormalizer(),
+            completion_handler,
+            pipeline_processor,
+            fim_pipeline_processor
+        )
+
+    @property
+    def provider_route_name(self) -> str:
+        return "provider_name"
+
+    def _setup_routes(self):
+        # Implement route setup
+        pass
+```
+
 ## Working with Prompts
 
 ### Default Prompts
@@ -188,8 +271,9 @@ codegate serve --port 8989 --host localhost --log-level DEBUG
 
 # Start with custom prompts
 codegate serve --prompts my-prompts.yaml
+
+# Start with custom provider URL
+codegate serve --vllm-url https://vllm.example.com
 ```
 
 See [CLI Documentation](cli.md) for detailed command information.
-
-[Rest of development.md content remains unchanged...]

src/codegate/cli.py

@@ -2,7 +2,7 @@
 
 import sys
 from pathlib import Path
-from typing import Optional
+from typing import Dict, Optional
 
 import click
 
@@ -88,17 +88,47 @@ def show_prompts(prompts: Optional[Path]) -> None:
     default=None,
     help="Path to YAML prompts file",
 )
+@click.option(
+    "--vllm-url",
+    type=str,
+    default=None,
+    help="vLLM provider URL (default: http://localhost:8000/v1)",
+)
+@click.option(
+    "--openai-url",
+    type=str,
+    default=None,
+    help="OpenAI provider URL (default: https://api.openai.com/v1)",
+)
+@click.option(
+    "--anthropic-url",
+    type=str,
+    default=None,
+    help="Anthropic provider URL (default: https://api.anthropic.com/v1)",
+)
 def serve(
     port: Optional[int],
     host: Optional[str],
     log_level: Optional[str],
     log_format: Optional[str],
     config: Optional[Path],
     prompts: Optional[Path],
+    vllm_url: Optional[str],
+    openai_url: Optional[str],
+    anthropic_url: Optional[str],
 ) -> None:
     """Start the codegate server."""
     logger = None
     try:
+        # Create provider URLs dict from CLI options
+        cli_provider_urls: Dict[str, str] = {}
+        if vllm_url:
+            cli_provider_urls["vllm"] = vllm_url
+        if openai_url:
+            cli_provider_urls["openai"] = openai_url
+        if anthropic_url:
+            cli_provider_urls["anthropic"] = anthropic_url
+
         # Load configuration with priority resolution
         cfg = Config.load(
             config_path=config,
@@ -107,6 +137,7 @@ def serve(
             cli_host=host,
             cli_log_level=log_level,
             cli_log_format=log_format,
+            cli_provider_urls=cli_provider_urls,
         )
 
         logger = setup_logging(cfg.log_level, cfg.log_format)
@@ -118,6 +149,7 @@ def serve(
                 "log_level": cfg.log_level.value,
                 "log_format": cfg.log_format.value,
                 "prompts_loaded": len(cfg.prompts.prompts),
+                "provider_urls": cfg.provider_urls,
             },
         )