Merge pull request #1525 from Titan-Node/add-livepeer-image-gen-1

feat: Add Livepeer Image Provider
troopdegen · Dec 28, 2024 · 911984a · 911984a
2 parents 438d736 + f8028c1
commit 911984a
Show file tree

Hide file tree

Showing 12 changed files with 123 additions and 5 deletions.
diff --git a/.env.example b/.env.example
@@ -36,6 +36,10 @@ SMALL_HYPERBOLIC_MODEL=         # Default: meta-llama/Llama-3.2-3B-Instruct
 MEDIUM_HYPERBOLIC_MODEL=        # Default: meta-llama/Meta-Llama-3.1-70B-Instruct
 LARGE_HYPERBOLIC_MODEL=         # Default: meta-llama/Meta-Llama-3.1-405-Instruct
 
+# Livepeer configuration
+LIVEPEER_GATEWAY_URL=           # Free inference gateways and docs: https://livepeer-eliza.com/
+LIVEPEER_IMAGE_MODEL=           # Default: ByteDance/SDXL-Lightning
+
 # Speech Synthesis
 ELEVENLABS_XI_API_KEY=          # API key from elevenlabs
 

diff --git a/agent/src/index.ts b/agent/src/index.ts
@@ -554,7 +554,8 @@ export async function createAgent(
             getSecret(character, "FAL_API_KEY") ||
             getSecret(character, "OPENAI_API_KEY") ||
             getSecret(character, "VENICE_API_KEY") ||
-            getSecret(character, "HEURIST_API_KEY")
+            getSecret(character, "HEURIST_API_KEY") ||
+            getSecret(character, "LIVEPEER_GATEWAY_URL")
                 ? imageGenerationPlugin
                 : null,
             getSecret(character, "FAL_API_KEY") ? ThreeDGenerationPlugin : null,

diff --git a/docs/api/enumerations/ModelProviderName.md b/docs/api/enumerations/ModelProviderName.md
@@ -233,3 +233,13 @@ Available model providers
 #### Defined in
 
 [packages/core/src/types.ts:240](https://github.com/elizaOS/eliza/blob/main/packages/core/src/types.ts#L240)
+
+***
+
+### LIVEPEER
+
+> **LIVEPEER**: `"livepeer"`
+
+#### Defined in
+
+[packages/core/src/types.ts:241](https://github.com/elizaOS/eliza/blob/main/packages/core/src/types.ts#L241)
diff --git a/docs/api/type-aliases/Models.md b/docs/api/type-aliases/Models.md
@@ -100,6 +100,10 @@ Model configurations by provider
 
 > **akash\_chat\_api**: [`Model`](Model.md)
 
+### livepeer
+
+> **livepeer**: [`Model`](Model.md)
+
 ## Defined in
 
 [packages/core/src/types.ts:188](https://github.com/elizaOS/eliza/blob/main/packages/core/src/types.ts#L188)
diff --git a/docs/docs/api/enumerations/ModelProviderName.md b/docs/docs/api/enumerations/ModelProviderName.md
@@ -119,3 +119,13 @@
 #### Defined in
 
 [packages/core/src/types.ts:132](https://github.com/elizaos/eliza/blob/4d1e66cbf7deea87a8a67525670a963cd00108bc/packages/core/src/types.ts#L132)
+
+---
+
+### LIVEPEER
+
+> **LIVEPEER**: `"livepeer"`
+
+#### Defined in
+
+[packages/core/src/types.ts:133](https://github.com/elizaos/eliza/blob/4d1e66cbf7deea87a8a67525670a963cd00108bc/packages/core/src/types.ts#L133)
diff --git a/docs/docs/api/type-aliases/Models.md b/docs/docs/api/type-aliases/Models.md
@@ -52,6 +52,10 @@
 
 > **heurist**: [`Model`](Model.md)
 
+### livepeer
+
+> **livepeer**: [`Model`](Model.md)
+
 ## Defined in
 
 [packages/core/src/types.ts:105](https://github.com/elizaos/eliza/blob/7fcf54e7fb2ba027d110afcc319c0b01b3f181dc/packages/core/src/types.ts#L105)
diff --git a/docs/docs/guides/configuration.md b/docs/docs/guides/configuration.md
@@ -72,6 +72,9 @@ TOGETHER_API_KEY=
 # Heurist Settings
 HEURIST_API_KEY=
 
+# Livepeer Settings
+LIVEPEER_GATEWAY_URL=
+
 # Local Model Settings
 XAI_MODEL=meta-llama/Llama-3.1-7b-instruct
 ```

diff --git a/docs/docs/quickstart.md b/docs/docs/quickstart.md
@@ -82,6 +82,7 @@ pnpm build
     OPENAI_API_KEY=        # OpenAI API key
     GROK_API_KEY=          # Grok API key
     ELEVENLABS_XI_API_KEY= # API key from elevenlabs (for voice)
+    LIVEPEER_GATEWAY_URL=  # Livepeer gateway URL
     ```
 
 ## Choose Your Model
@@ -94,6 +95,7 @@ Eliza supports multiple AI models:
 - **Llama**: Set `XAI_MODEL=meta-llama/Meta-Llama-3.1-70B-Instruct-Turbo`
 - **Grok**: Set `XAI_MODEL=grok-beta`
 - **OpenAI**: Set `XAI_MODEL=gpt-4o-mini` or `gpt-4o`
+- **Livepeer**: Set `LIVEPEER_IMAGE_MODEL` to your chosen Livepeer image model, available models [here](https://livepeer-eliza.com/)
 
 You set which model to use inside the character JSON file
 
@@ -216,8 +218,8 @@ pnpm start --characters="characters/trump.character.json,characters/tate.charact
    - Ensure Node.js 23.3.0 is installed
    - Use `node -v` to check version
    - Consider using [nvm](https://github.com/nvm-sh/nvm) to manage Node versions
-   
-   NOTE: pnpm may be bundled with a different node version, ignoring nvm. If this is the case, you can use 
+
+   NOTE: pnpm may be bundled with a different node version, ignoring nvm. If this is the case, you can use
    ```bash
    pnpm env use --global 23.3.0
    ```

diff --git a/packages/core/src/generation.ts b/packages/core/src/generation.ts
@@ -978,13 +978,16 @@ export const generateImage = async (
                     return runtime.getSetting("OPENAI_API_KEY");
                 case ModelProviderName.VENICE:
                     return runtime.getSetting("VENICE_API_KEY");
+                case ModelProviderName.LIVEPEER:
+                    return runtime.getSetting("LIVEPEER_GATEWAY_URL");
                 default:
                     // If no specific match, try the fallback chain
                     return (runtime.getSetting("HEURIST_API_KEY") ??
                            runtime.getSetting("TOGETHER_API_KEY") ??
                            runtime.getSetting("FAL_API_KEY") ??
                            runtime.getSetting("OPENAI_API_KEY") ??
-                           runtime.getSetting("VENICE_API_KEY"));
+                           runtime.getSetting("VENICE_API_KEY"))??
+                           runtime.getSetting("LIVEPEER_GATEWAY_URL");
             }
         })();
     try {
@@ -1176,6 +1179,62 @@ export const generateImage = async (
             });
 
             return { success: true, data: base64s };
+
+        } else if (runtime.imageModelProvider === ModelProviderName.LIVEPEER) {
+            if (!apiKey) {
+                throw new Error("Livepeer Gateway is not defined");
+            }
+            try {
+                const baseUrl = new URL(apiKey);
+                if (!baseUrl.protocol.startsWith('http')) {
+                    throw new Error("Invalid Livepeer Gateway URL protocol");
+                }
+                const response = await fetch(`${baseUrl.toString()}text-to-image`, {
+                    method: "POST",
+                    headers: {
+                        "Content-Type": "application/json"
+                    },
+                    body: JSON.stringify({
+                        model_id: data.modelId || "ByteDance/SDXL-Lightning",
+                        prompt: data.prompt,
+                        width: data.width || 1024,
+                        height: data.height || 1024
+                    })
+                });
+                const result = await response.json();
+                if (!result.images?.length) {
+                    throw new Error("No images generated");
+                }
+                const base64Images = await Promise.all(
+                    result.images.map(async (image) => {
+                        console.log("imageUrl console log", image.url);
+                        let imageUrl;
+                        if (image.url.includes("http")) {
+                            imageUrl = image.url;
+                        } else {
+                            imageUrl = `${apiKey}${image.url}`;
+                        }
+                        const imageResponse = await fetch(imageUrl);
+                        if (!imageResponse.ok) {
+                            throw new Error(
+                                `Failed to fetch image: ${imageResponse.statusText}`
+                            );
+                        }
+                        const blob = await imageResponse.blob();
+                        const arrayBuffer = await blob.arrayBuffer();
+                        const base64 = Buffer.from(arrayBuffer).toString("base64");
+                        return `data:image/jpeg;base64,${base64}`;
+                    })
+                );
+                return {
+                    success: true,
+                    data: base64Images
+                };
+            } catch (error) {
+                console.error(error);
+                return { success: false, error: error };
+            }
+
         } else {
             let targetSize = `${data.width}x${data.height}`;
             if (

diff --git a/packages/core/src/models.ts b/packages/core/src/models.ts
@@ -486,6 +486,23 @@ export const models: Models = {
                 "Meta-Llama-3-1-405B-Instruct-FP8",
         },
     },
+    [ModelProviderName.LIVEPEER]: {
+        settings: {
+            stop: [],
+            maxInputTokens: 128000,
+            maxOutputTokens: 8192,
+            repetition_penalty: 0.4,
+            temperature: 0.7,
+        },
+        // livepeer endpoint is handled from the sdk
+        model: {
+            [ModelClass.SMALL]: "",
+            [ModelClass.MEDIUM]: "",
+            [ModelClass.LARGE]: "",
+            [ModelClass.EMBEDDING]: "",
+            [ModelClass.IMAGE]: settings.LIVEPEER_IMAGE_MODEL || "ByteDance/SDXL-Lightning",
+        },
+    },
 };
 
 export function getModel(provider: ModelProviderName, type: ModelClass) {

diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts
@@ -209,6 +209,7 @@ export type Models = {
     [ModelProviderName.HYPERBOLIC]: Model;
     [ModelProviderName.VENICE]: Model;
     [ModelProviderName.AKASH_CHAT_API]: Model;
+    [ModelProviderName.LIVEPEER]: Model;
 };
 
 /**
@@ -238,6 +239,7 @@ export enum ModelProviderName {
     HYPERBOLIC = "hyperbolic",
     VENICE = "venice",
     AKASH_CHAT_API = "akash_chat_api",
+    LIVEPEER = "livepeer",
 }
 
 /**

diff --git a/packages/plugin-image-generation/src/index.ts b/packages/plugin-image-generation/src/index.ts
@@ -86,14 +86,16 @@ const imageGeneration: Action = {
         const falApiKeyOk = !!runtime.getSetting("FAL_API_KEY");
         const openAiApiKeyOk = !!runtime.getSetting("OPENAI_API_KEY");
         const veniceApiKeyOk = !!runtime.getSetting("VENICE_API_KEY");
+        const livepeerGatewayUrlOk = !!runtime.getSetting("LIVEPEER_GATEWAY_URL");
 
         return (
             anthropicApiKeyOk ||
             togetherApiKeyOk ||
             heuristApiKeyOk ||
             falApiKeyOk ||
             openAiApiKeyOk ||
-            veniceApiKeyOk
+            veniceApiKeyOk ||
+            livepeerGatewayUrlOk
         );
     },
     handler: async (