From b67275f3fa69390e4b1914e7929c34bb779f4273 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Wed, 31 Jul 2024 18:21:46 -0400 Subject: [PATCH 01/78] feat: add default model provider option Signed-off-by: Donnie Adams --- README.md | 3 ++- opts.go | 12 ++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 418d4bd..a51d067 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,8 @@ When creating a `GTPScript` instance, you can pass the following global options. - `APIKey`: Specify an OpenAI API key for authenticating requests - `BaseURL`: A base URL for an OpenAI compatible API (the default is `https://api.openai.com/v1`) -- `DefaultModel`: The default model to use for OpenAI requests +- `DefaultModel`: The default model to use for chat completion requests +- `DefaultModelProvider`: The default model provider to use for chat completion requests - `Env`: Supply the environment variables. Supplying anything here means that nothing from the environment is used. The default is `os.Environ()`. Supplying `Env` at the run/evaluate level will be treated as "additional." ## Run Options diff --git a/opts.go b/opts.go index ee1dac3..e962e2a 100644 --- a/opts.go +++ b/opts.go @@ -3,10 +3,11 @@ package gptscript // GlobalOptions allows specification of settings that are used for every call made. // These options can be overridden by the corresponding Options. type GlobalOptions struct { - OpenAIAPIKey string `json:"APIKey"` - OpenAIBaseURL string `json:"BaseURL"` - DefaultModel string `json:"DefaultModel"` - Env []string `json:"env"` + OpenAIAPIKey string `json:"APIKey"` + OpenAIBaseURL string `json:"BaseURL"` + DefaultModel string `json:"DefaultModel"` + DefaultModelProvider string `json:"DefaultModelProvider"` + Env []string `json:"env"` } func (g GlobalOptions) toEnv() []string { @@ -20,6 +21,9 @@ func (g GlobalOptions) toEnv() []string { if g.DefaultModel != "" { args = append(args, "GPTSCRIPT_SDKSERVER_DEFAULT_MODEL="+g.DefaultModel) } + if g.DefaultModelProvider != "" { + args = append(args, "GPTSCRIPT_SDKSERVER_DEFAULT_MODEL_PROVIDER="+g.DefaultModelProvider) + } return args } From 23709a695f3aa80b8c783118eca06b420c021741 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Thu, 1 Aug 2024 12:32:42 -0400 Subject: [PATCH 02/78] fix: pass the environment with the run when server is disabled If the server is disabled, then it is presumably running with some other environment. In this case, we need to ensure that the run passes the environment so all new environment variables (like prompt server) is properly passed with the run. Signed-off-by: Donnie Adams --- gptscript.go | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/gptscript.go b/gptscript.go index 8eb688c..753e278 100644 --- a/gptscript.go +++ b/gptscript.go @@ -25,7 +25,8 @@ var ( const relativeToBinaryPath = "" type GPTScript struct { - url string + url string + globalEnv []string } func NewGPTScript(opts GlobalOptions) (*GPTScript, error) { @@ -39,16 +40,18 @@ func NewGPTScript(opts GlobalOptions) (*GPTScript, error) { serverURL = os.Getenv("GPTSCRIPT_URL") } + if opts.Env == nil { + opts.Env = os.Environ() + } + + opts.Env = append(opts.Env, opts.toEnv()...) + if serverProcessCancel == nil && !disableServer { ctx, cancel := context.WithCancel(context.Background()) in, _ := io.Pipe() serverProcess = exec.CommandContext(ctx, getCommand(), "sys.sdkserver", "--listen-address", serverURL) - if opts.Env == nil { - opts.Env = os.Environ() - } - - serverProcess.Env = append(opts.Env[:], opts.toEnv()...) + serverProcess.Env = opts.Env[:] serverProcess.Stdin = in stdErr, err := serverProcess.StderrPipe() @@ -88,7 +91,15 @@ func NewGPTScript(opts GlobalOptions) (*GPTScript, error) { serverURL = strings.TrimSpace(serverURL) } - return &GPTScript{url: "http://" + serverURL}, nil + g := &GPTScript{ + url: "http://" + serverURL, + } + + if disableServer { + g.globalEnv = opts.Env[:] + } + + return g, nil } func readAddress(stdErr io.Reader) (string, error) { @@ -117,6 +128,7 @@ func (g *GPTScript) Close() { } func (g *GPTScript) Evaluate(ctx context.Context, opts Options, tools ...ToolDef) (*Run, error) { + opts.Env = append(g.globalEnv, opts.Env...) return (&Run{ url: g.url, requestPath: "evaluate", @@ -127,6 +139,7 @@ func (g *GPTScript) Evaluate(ctx context.Context, opts Options, tools ...ToolDef } func (g *GPTScript) Run(ctx context.Context, toolPath string, opts Options) (*Run, error) { + opts.Env = append(g.globalEnv, opts.Env...) return (&Run{ url: g.url, requestPath: "run", From 5fa255d352c1b98132ec8bfef8b2f01bafff1c97 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Thu, 1 Aug 2024 12:51:46 -0400 Subject: [PATCH 03/78] feat: make GlobalOptions option by using variadic approach Signed-off-by: Donnie Adams --- gptscript.go | 13 +++++++------ opts.go | 23 +++++++++++++++++++++++ 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/gptscript.go b/gptscript.go index 753e278..24bc35b 100644 --- a/gptscript.go +++ b/gptscript.go @@ -29,7 +29,8 @@ type GPTScript struct { globalEnv []string } -func NewGPTScript(opts GlobalOptions) (*GPTScript, error) { +func NewGPTScript(opts ...GlobalOptions) (*GPTScript, error) { + opt := completeGlobalOptions(opts...) lock.Lock() defer lock.Unlock() gptscriptCount++ @@ -40,18 +41,18 @@ func NewGPTScript(opts GlobalOptions) (*GPTScript, error) { serverURL = os.Getenv("GPTSCRIPT_URL") } - if opts.Env == nil { - opts.Env = os.Environ() + if opt.Env == nil { + opt.Env = os.Environ() } - opts.Env = append(opts.Env, opts.toEnv()...) + opt.Env = append(opt.Env, opt.toEnv()...) if serverProcessCancel == nil && !disableServer { ctx, cancel := context.WithCancel(context.Background()) in, _ := io.Pipe() serverProcess = exec.CommandContext(ctx, getCommand(), "sys.sdkserver", "--listen-address", serverURL) - serverProcess.Env = opts.Env[:] + serverProcess.Env = opt.Env[:] serverProcess.Stdin = in stdErr, err := serverProcess.StderrPipe() @@ -96,7 +97,7 @@ func NewGPTScript(opts GlobalOptions) (*GPTScript, error) { } if disableServer { - g.globalEnv = opts.Env[:] + g.globalEnv = opt.Env[:] } return g, nil diff --git a/opts.go b/opts.go index e962e2a..6c383b6 100644 --- a/opts.go +++ b/opts.go @@ -28,6 +28,29 @@ func (g GlobalOptions) toEnv() []string { return args } +func completeGlobalOptions(opts ...GlobalOptions) GlobalOptions { + result := GlobalOptions{} + for _, opt := range opts { + result.OpenAIAPIKey = firstSet(opt.OpenAIAPIKey, result.OpenAIAPIKey) + result.OpenAIBaseURL = firstSet(opt.OpenAIBaseURL, result.OpenAIBaseURL) + result.DefaultModel = firstSet(opt.DefaultModel, result.DefaultModel) + result.DefaultModelProvider = firstSet(opt.DefaultModelProvider, result.DefaultModelProvider) + result.Env = append(result.Env, opt.Env...) + } + return opts[0] +} + +func firstSet[T comparable](in ...T) T { + var result T + for _, i := range in { + if i != result { + return i + } + } + + return result +} + // Options represents options for the gptscript tool or file. type Options struct { GlobalOptions `json:",inline"` From 76b65016e4e798e44c246706cf104949c8a48d99 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Thu, 1 Aug 2024 13:03:23 -0400 Subject: [PATCH 04/78] fix: attempt to fix the confirm test flakiness Signed-off-by: Donnie Adams --- gptscript_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gptscript_test.go b/gptscript_test.go index 1d85ef3..4e7b74a 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -726,7 +726,7 @@ func TestToolWithGlobalTools(t *testing.T) { func TestConfirm(t *testing.T) { var eventContent string tools := ToolDef{ - Instructions: "List the files in the current directory", + Instructions: "List all the files in the current directory", Tools: []string{"sys.exec"}, } From 840b14393b174324f3d6076f89230a0096ff2a85 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Thu, 1 Aug 2024 16:34:34 -0400 Subject: [PATCH 05/78] fix: address typo in completeGlobalOptions Signed-off-by: Donnie Adams --- opts.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opts.go b/opts.go index 6c383b6..bf91422 100644 --- a/opts.go +++ b/opts.go @@ -29,7 +29,7 @@ func (g GlobalOptions) toEnv() []string { } func completeGlobalOptions(opts ...GlobalOptions) GlobalOptions { - result := GlobalOptions{} + var result GlobalOptions for _, opt := range opts { result.OpenAIAPIKey = firstSet(opt.OpenAIAPIKey, result.OpenAIAPIKey) result.OpenAIBaseURL = firstSet(opt.OpenAIBaseURL, result.OpenAIBaseURL) @@ -37,7 +37,7 @@ func completeGlobalOptions(opts ...GlobalOptions) GlobalOptions { result.DefaultModelProvider = firstSet(opt.DefaultModelProvider, result.DefaultModelProvider) result.Env = append(result.Env, opt.Env...) } - return opts[0] + return result } func firstSet[T comparable](in ...T) T { From ecc199e8e927e8838ae5da0ff8920737e15ff1dc Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Fri, 2 Aug 2024 08:14:54 -0400 Subject: [PATCH 06/78] fix: address another flake in TestConfirm Signed-off-by: Donnie Adams --- gptscript_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gptscript_test.go b/gptscript_test.go index 4e7b74a..05449da 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -726,7 +726,7 @@ func TestToolWithGlobalTools(t *testing.T) { func TestConfirm(t *testing.T) { var eventContent string tools := ToolDef{ - Instructions: "List all the files in the current directory", + Instructions: "List all the files in the current directory. Respond with the names of the files in only the current directory.", Tools: []string{"sys.exec"}, } From a32b0347c03d4256e667343deab314a4e60dc311 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Fri, 2 Aug 2024 14:01:09 -0700 Subject: [PATCH 07/78] chore: add getenv --- gptscript.go | 28 ++++++++++++++++++++++++++ gptscript_test.go | 51 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/gptscript.go b/gptscript.go index 24bc35b..d16199c 100644 --- a/gptscript.go +++ b/gptscript.go @@ -2,7 +2,10 @@ package gptscript import ( "bufio" + "bytes" + "compress/gzip" "context" + "encoding/base64" "encoding/json" "fmt" "io" @@ -290,3 +293,28 @@ func determineProperCommand(dir, bin string) string { slog.Debug("Using gptscript binary: " + bin) return bin } + +func GetEnv(key, def string) string { + v := os.Getenv(key) + if v == "" { + return def + } + + if strings.HasPrefix(v, `{"_gz":"`) && strings.HasSuffix(v, `"}`) { + data, err := base64.StdEncoding.DecodeString(v[8 : len(v)-2]) + if err != nil { + return v + } + gz, err := gzip.NewReader(bytes.NewBuffer(data)) + if err != nil { + return v + } + strBytes, err := io.ReadAll(gz) + if err != nil { + return v + } + return string(strBytes) + } + + return v +} diff --git a/gptscript_test.go b/gptscript_test.go index 05449da..0cf8c19 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -1018,3 +1018,54 @@ func TestGetCommand(t *testing.T) { }) } } + +func TestGetEnv(t *testing.T) { + // Cleaning up + defer func(currentEnvValue string) { + os.Setenv("testKey", currentEnvValue) + }(os.Getenv("testKey")) + + // Tests + testCases := []struct { + name string + key string + def string + envValue string + expectedResult string + }{ + { + name: "NoValueUseDefault", + key: "testKey", + def: "defaultValue", + envValue: "", + expectedResult: "defaultValue", + }, + { + name: "ValueExistsNoCompress", + key: "testKey", + def: "defaultValue", + envValue: "testValue", + expectedResult: "testValue", + }, + { + name: "ValueExistsCompressed", + key: "testKey", + def: "defaultValue", + envValue: `{"_gz":"H4sIAEosrGYC/ytJLS5RKEvMKU0FACtB3ewKAAAA"}`, + + expectedResult: "test value", + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + os.Setenv(test.key, test.envValue) + + result := GetEnv(test.key, test.def) + + if result != test.expectedResult { + t.Errorf("expected: %s, got: %s", test.expectedResult, result) + } + }) + } +} From 69b98b90e29eea824759f7b90e2a14ba3c2e245a Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Wed, 7 Aug 2024 17:03:05 -0400 Subject: [PATCH 08/78] feat: add disable cache option to parse Signed-off-by: Donnie Adams --- gptscript.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/gptscript.go b/gptscript.go index 24bc35b..ad932aa 100644 --- a/gptscript.go +++ b/gptscript.go @@ -150,9 +150,18 @@ func (g *GPTScript) Run(ctx context.Context, toolPath string, opts Options) (*Ru }).NextChat(ctx, opts.Input) } +type ParseOptions struct { + DisableCache bool +} + // Parse will parse the given file into an array of Nodes. -func (g *GPTScript) Parse(ctx context.Context, fileName string) ([]Node, error) { - out, err := g.runBasicCommand(ctx, "parse", map[string]any{"file": fileName}) +func (g *GPTScript) Parse(ctx context.Context, fileName string, opts ...ParseOptions) ([]Node, error) { + var disableCache bool + for _, opt := range opts { + disableCache = disableCache || opt.DisableCache + } + + out, err := g.runBasicCommand(ctx, "parse", map[string]any{"file": fileName, "disableCache": disableCache}) if err != nil { return nil, err } From 401b9471ce73638634238d858fc2d3de0a4717f1 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Thu, 8 Aug 2024 14:10:18 -0400 Subject: [PATCH 09/78] feat: add metadata and type fields to tools Signed-off-by: Donnie Adams --- gptscript_test.go | 57 ++++++++++++++++++++++++++++++++++++ test/parse-with-metadata.gpt | 12 ++++++++ tool.go | 2 ++ 3 files changed, 71 insertions(+) create mode 100644 test/parse-with-metadata.gpt diff --git a/gptscript_test.go b/gptscript_test.go index 0cf8c19..ec450e0 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -404,6 +404,42 @@ func TestParseSimpleFile(t *testing.T) { } } +func TestParseFileWithMetadata(t *testing.T) { + wd, err := os.Getwd() + if err != nil { + t.Fatalf("Error getting working directory: %v", err) + } + + tools, err := g.Parse(context.Background(), wd+"/test/parse-with-metadata.gpt") + if err != nil { + t.Errorf("Error parsing file: %v", err) + } + + if len(tools) != 2 { + t.Fatalf("Unexpected number of tools: %d", len(tools)) + } + + if tools[0].ToolNode == nil { + t.Fatalf("No tool node found") + } + + if !strings.Contains(tools[0].ToolNode.Tool.Instructions, "requests.get(") { + t.Errorf("Unexpected instructions: %s", tools[0].ToolNode.Tool.Instructions) + } + + if tools[0].ToolNode.Tool.MetaData["requirements.txt"] != "requests" { + t.Errorf("Unexpected metadata: %s", tools[0].ToolNode.Tool.MetaData["requirements.txt"]) + } + + if tools[1].TextNode == nil { + t.Fatalf("No text node found") + } + + if tools[1].TextNode.Fmt != "metadata:foo:requirements.txt" { + t.Errorf("Unexpected text: %s", tools[1].TextNode.Fmt) + } +} + func TestParseTool(t *testing.T) { tools, err := g.ParseTool(context.Background(), "echo hello") if err != nil { @@ -1069,3 +1105,24 @@ func TestGetEnv(t *testing.T) { }) } } + +func TestRunPythonWithMetadata(t *testing.T) { + wd, err := os.Getwd() + if err != nil { + t.Fatalf("Error getting working directory: %v", err) + } + + run, err := g.Run(context.Background(), wd+"/test/parse-with-metadata.gpt", Options{IncludeEvents: true}) + if err != nil { + t.Fatalf("Error executing file: %v", err) + } + + out, err := run.Text() + if err != nil { + t.Fatalf("Error reading output: %v", err) + } + + if out != "200" { + t.Errorf("Unexpected output: %s", out) + } +} diff --git a/test/parse-with-metadata.gpt b/test/parse-with-metadata.gpt new file mode 100644 index 0000000..cfcb965 --- /dev/null +++ b/test/parse-with-metadata.gpt @@ -0,0 +1,12 @@ +Name: foo + +#!/usr/bin/env python3 +import requests + + +resp = requests.get("https://google.com") +print(resp.status_code, end="") + +--- +!metadata:foo:requirements.txt +requests \ No newline at end of file diff --git a/tool.go b/tool.go index b682912..c9d53f8 100644 --- a/tool.go +++ b/tool.go @@ -29,6 +29,7 @@ type ToolDef struct { Agents []string `json:"agents,omitempty"` Credentials []string `json:"credentials,omitempty"` Instructions string `json:"instructions,omitempty"` + Type string `json:"type,omitempty"` } func ObjectSchema(kv ...string) *openapi3.Schema { @@ -86,6 +87,7 @@ type Tool struct { ID string `json:"id,omitempty"` Arguments *openapi3.Schema `json:"arguments,omitempty"` ToolMapping map[string][]ToolReference `json:"toolMapping,omitempty"` + MetaData map[string]string `json:"metadata,omitempty"` LocalTools map[string]string `json:"localTools,omitempty"` Source ToolSource `json:"source,omitempty"` WorkingDir string `json:"workingDir,omitempty"` From 4a254d9524b33ee3df7c57b108ec89bba0abdd45 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Thu, 8 Aug 2024 18:22:39 -0400 Subject: [PATCH 10/78] fix: put metadata filed on tool def This change allows for the following flow: parse tool with metadata from file -> run tool defs from this parse. Signed-off-by: Donnie Adams --- gptscript_test.go | 30 ++++++++++++++++++++++++++++-- tool.go | 44 ++++++++++++++++++++++---------------------- 2 files changed, 50 insertions(+), 24 deletions(-) diff --git a/gptscript_test.go b/gptscript_test.go index ec450e0..d8014a6 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -667,8 +667,8 @@ func TestFileChat(t *testing.T) { } inputs := []string{ "List the 3 largest of the Great Lakes by volume.", - "What is the volume of the second one in cubic miles?", - "What is the total area of the third one in square miles?", + "What is the volume of the second in the list in cubic miles?", + "What is the total area of the third in the list in square miles?", } expectedOutputs := []string{ @@ -1126,3 +1126,29 @@ func TestRunPythonWithMetadata(t *testing.T) { t.Errorf("Unexpected output: %s", out) } } + +func TestParseThenEvaluateWithMetadata(t *testing.T) { + wd, err := os.Getwd() + if err != nil { + t.Fatalf("Error getting working directory: %v", err) + } + + tools, err := g.Parse(context.Background(), wd+"/test/parse-with-metadata.gpt") + if err != nil { + t.Fatalf("Error parsing file: %v", err) + } + + run, err := g.Evaluate(context.Background(), Options{}, tools[0].ToolNode.Tool.ToolDef) + if err != nil { + t.Fatalf("Error executing file: %v", err) + } + + out, err := run.Text() + if err != nil { + t.Fatalf("Error reading output: %v", err) + } + + if out != "200" { + t.Errorf("Unexpected output: %s", out) + } +} diff --git a/tool.go b/tool.go index c9d53f8..c2360ac 100644 --- a/tool.go +++ b/tool.go @@ -9,27 +9,28 @@ import ( // ToolDef struct represents a tool with various configurations. type ToolDef struct { - Name string `json:"name,omitempty"` - Description string `json:"description,omitempty"` - MaxTokens int `json:"maxTokens,omitempty"` - ModelName string `json:"modelName,omitempty"` - ModelProvider bool `json:"modelProvider,omitempty"` - JSONResponse bool `json:"jsonResponse,omitempty"` - Chat bool `json:"chat,omitempty"` - Temperature *float32 `json:"temperature,omitempty"` - Cache *bool `json:"cache,omitempty"` - InternalPrompt *bool `json:"internalPrompt"` - Arguments *openapi3.Schema `json:"arguments,omitempty"` - Tools []string `json:"tools,omitempty"` - GlobalTools []string `json:"globalTools,omitempty"` - GlobalModelName string `json:"globalModelName,omitempty"` - Context []string `json:"context,omitempty"` - ExportContext []string `json:"exportContext,omitempty"` - Export []string `json:"export,omitempty"` - Agents []string `json:"agents,omitempty"` - Credentials []string `json:"credentials,omitempty"` - Instructions string `json:"instructions,omitempty"` - Type string `json:"type,omitempty"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + MaxTokens int `json:"maxTokens,omitempty"` + ModelName string `json:"modelName,omitempty"` + ModelProvider bool `json:"modelProvider,omitempty"` + JSONResponse bool `json:"jsonResponse,omitempty"` + Chat bool `json:"chat,omitempty"` + Temperature *float32 `json:"temperature,omitempty"` + Cache *bool `json:"cache,omitempty"` + InternalPrompt *bool `json:"internalPrompt"` + Arguments *openapi3.Schema `json:"arguments,omitempty"` + Tools []string `json:"tools,omitempty"` + GlobalTools []string `json:"globalTools,omitempty"` + GlobalModelName string `json:"globalModelName,omitempty"` + Context []string `json:"context,omitempty"` + ExportContext []string `json:"exportContext,omitempty"` + Export []string `json:"export,omitempty"` + Agents []string `json:"agents,omitempty"` + Credentials []string `json:"credentials,omitempty"` + Instructions string `json:"instructions,omitempty"` + Type string `json:"type,omitempty"` + MetaData map[string]string `json:"metadata,omitempty"` } func ObjectSchema(kv ...string) *openapi3.Schema { @@ -87,7 +88,6 @@ type Tool struct { ID string `json:"id,omitempty"` Arguments *openapi3.Schema `json:"arguments,omitempty"` ToolMapping map[string][]ToolReference `json:"toolMapping,omitempty"` - MetaData map[string]string `json:"metadata,omitempty"` LocalTools map[string]string `json:"localTools,omitempty"` Source ToolSource `json:"source,omitempty"` WorkingDir string `json:"workingDir,omitempty"` From 758ea991f806be25f0f97ee0ec3c4361fdd6b518 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Tue, 13 Aug 2024 09:46:12 -0400 Subject: [PATCH 11/78] fix: improve the context testing, including adding a new test The new test exercises the framework's ability to continue chats where a run failed for some reason. Signed-off-by: Donnie Adams --- gptscript_test.go | 43 ++++++++++++++++++++++++++++++++++++- test/acorn-labs-context.gpt | 6 +++++- test/global-tools.gpt | 2 +- 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/gptscript_test.go b/gptscript_test.go index d8014a6..1671a94 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -146,7 +146,7 @@ func TestEvaluateWithContext(t *testing.T) { tool := ToolDef{ Instructions: "What is the capital of the united states?", - Context: []string{ + Tools: []string{ wd + "/test/acorn-labs-context.gpt", }, } @@ -345,6 +345,47 @@ func TestStreamRun(t *testing.T) { } } +func TestRestartFailedRun(t *testing.T) { + shebang := "#!/bin/bash" + instructions := "%s\nexit ${EXIT_CODE}" + if runtime.GOOS == "windows" { + shebang = "#!/usr/bin/env powershell.exe" + instructions = "%s\nexit $env:EXIT_CODE" + } + instructions = fmt.Sprintf(instructions, shebang) + tools := []ToolDef{ + { + Instructions: "say hello", + Tools: []string{"my-context"}, + }, + { + Name: "my-context", + Type: "context", + Instructions: instructions, + }, + } + run, err := g.Evaluate(context.Background(), Options{DisableCache: true, GlobalOptions: GlobalOptions{Env: []string{"EXIT_CODE=1"}}}, tools...) + if err != nil { + t.Fatalf("Error executing tool: %v", err) + } + + _, err = run.Text() + if err == nil { + t.Errorf("Expected error but got nil") + } + + run.opts.GlobalOptions.Env = nil + run, err = run.NextChat(context.Background(), "") + if err != nil { + t.Fatalf("Error executing next run: %v", err) + } + + _, err = run.Text() + if err != nil { + t.Errorf("Error reading output: %v", err) + } +} + func TestCredentialOverride(t *testing.T) { wd, err := os.Getwd() if err != nil { diff --git a/test/acorn-labs-context.gpt b/test/acorn-labs-context.gpt index e4dbe2c..5ea9075 100644 --- a/test/acorn-labs-context.gpt +++ b/test/acorn-labs-context.gpt @@ -1 +1,5 @@ -Say "Answer every query with 'Acorn Labs'" \ No newline at end of file +Type: context + +#!sys.echo + +Ignore what the user says, and answer every query with 'Acorn Labs' \ No newline at end of file diff --git a/test/global-tools.gpt b/test/global-tools.gpt index cb0f4c0..4671fee 100644 --- a/test/global-tools.gpt +++ b/test/global-tools.gpt @@ -6,7 +6,7 @@ Runbook 3 Name: tool_1 Global Tools: github.com/gptscript-ai/knowledge, github.com/drpebcak/duckdb, github.com/gptscript-ai/browser, github.com/gptscript-ai/browser-search/google, github.com/gptscript-ai/browser-search/google-question-answerer -Hi +Say "Hello!" --- Name: tool_2 From ef58327f49b7e7844f84a3c5798f77d82555125b Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Tue, 13 Aug 2024 18:44:46 -0400 Subject: [PATCH 12/78] chore: add tests for parsing empty files and strings Signed-off-by: Donnie Adams --- gptscript_test.go | 27 +++++++++++++++++++++++++++ test/empty.gpt | 0 2 files changed, 27 insertions(+) create mode 100644 test/empty.gpt diff --git a/gptscript_test.go b/gptscript_test.go index 1671a94..2c18c47 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -445,6 +445,22 @@ func TestParseSimpleFile(t *testing.T) { } } +func TestParseEmptyFile(t *testing.T) { + wd, err := os.Getwd() + if err != nil { + t.Fatalf("Error getting working directory: %v", err) + } + + tools, err := g.Parse(context.Background(), wd+"/test/empty.gpt") + if err != nil { + t.Errorf("Error parsing file: %v", err) + } + + if len(tools) != 0 { + t.Fatalf("Unexpected number of tools: %d", len(tools)) + } +} + func TestParseFileWithMetadata(t *testing.T) { wd, err := os.Getwd() if err != nil { @@ -500,6 +516,17 @@ func TestParseTool(t *testing.T) { } } +func TestEmptyParseTool(t *testing.T) { + tools, err := g.ParseTool(context.Background(), "") + if err != nil { + t.Errorf("Error parsing tool: %v", err) + } + + if len(tools) != 0 { + t.Fatalf("Unexpected number of tools: %d", len(tools)) + } +} + func TestParseToolWithTextNode(t *testing.T) { tools, err := g.ParseTool(context.Background(), "echo hello\n---\n!markdown\nhello") if err != nil { diff --git a/test/empty.gpt b/test/empty.gpt new file mode 100644 index 0000000..e69de29 From e0876d772640244d6551ddaaa38b5b86eacf0d82 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Wed, 14 Aug 2024 08:51:28 -0400 Subject: [PATCH 13/78] feat: add load API This change also changes ParseTool to ParseContent for consistency. Signed-off-by: Donnie Adams --- gptscript.go | 51 ++++++++++++++++++- gptscript_test.go | 122 ++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 166 insertions(+), 7 deletions(-) diff --git a/gptscript.go b/gptscript.go index d32a616..cc0d620 100644 --- a/gptscript.go +++ b/gptscript.go @@ -181,8 +181,8 @@ func (g *GPTScript) Parse(ctx context.Context, fileName string, opts ...ParseOpt return doc.Nodes, nil } -// ParseTool will parse the given string into a tool. -func (g *GPTScript) ParseTool(ctx context.Context, toolDef string) ([]Node, error) { +// ParseContent will parse the given string into a tool. +func (g *GPTScript) ParseContent(ctx context.Context, toolDef string) ([]Node, error) { out, err := g.runBasicCommand(ctx, "parse", map[string]any{"content": toolDef}) if err != nil { return nil, err @@ -214,6 +214,53 @@ func (g *GPTScript) Fmt(ctx context.Context, nodes []Node) (string, error) { return out, nil } +type LoadOptions struct { + DisableCache bool + SubTool string +} + +// LoadFile will load the given file into a Program. +func (g *GPTScript) LoadFile(ctx context.Context, fileName string, opts ...LoadOptions) (*Program, error) { + return g.load(ctx, map[string]any{"file": fileName}, opts...) +} + +// LoadContent will load the given content into a Program. +func (g *GPTScript) LoadContent(ctx context.Context, content string, opts ...LoadOptions) (*Program, error) { + return g.load(ctx, map[string]any{"content": content}, opts...) +} + +// LoadTools will load the given tools into a Program. +func (g *GPTScript) LoadTools(ctx context.Context, toolDefs []ToolDef, opts ...LoadOptions) (*Program, error) { + return g.load(ctx, map[string]any{"toolDefs": toolDefs}, opts...) +} + +func (g *GPTScript) load(ctx context.Context, payload map[string]any, opts ...LoadOptions) (*Program, error) { + for _, opt := range opts { + if opt.DisableCache { + payload["disableCache"] = true + } + if opt.SubTool != "" { + payload["subTool"] = opt.SubTool + } + } + + out, err := g.runBasicCommand(ctx, "load", payload) + if err != nil { + return nil, err + } + + type loadResponse struct { + Program *Program `json:"program"` + } + + prg := new(loadResponse) + if err = json.Unmarshal([]byte(out), prg); err != nil { + return nil, err + } + + return prg.Program, nil +} + // Version will return the output of `gptscript --version` func (g *GPTScript) Version(ctx context.Context) (string, error) { out, err := g.runBasicCommand(ctx, "version", nil) diff --git a/gptscript_test.go b/gptscript_test.go index 2c18c47..f298cab 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -498,7 +498,7 @@ func TestParseFileWithMetadata(t *testing.T) { } func TestParseTool(t *testing.T) { - tools, err := g.ParseTool(context.Background(), "echo hello") + tools, err := g.ParseContent(context.Background(), "echo hello") if err != nil { t.Errorf("Error parsing tool: %v", err) } @@ -517,7 +517,7 @@ func TestParseTool(t *testing.T) { } func TestEmptyParseTool(t *testing.T) { - tools, err := g.ParseTool(context.Background(), "") + tools, err := g.ParseContent(context.Background(), "") if err != nil { t.Errorf("Error parsing tool: %v", err) } @@ -528,7 +528,7 @@ func TestEmptyParseTool(t *testing.T) { } func TestParseToolWithTextNode(t *testing.T) { - tools, err := g.ParseTool(context.Background(), "echo hello\n---\n!markdown\nhello") + tools, err := g.ParseContent(context.Background(), "echo hello\n---\n!markdown\nhello") if err != nil { t.Errorf("Error parsing tool: %v", err) } @@ -735,8 +735,8 @@ func TestFileChat(t *testing.T) { } inputs := []string{ "List the 3 largest of the Great Lakes by volume.", - "What is the volume of the second in the list in cubic miles?", - "What is the total area of the third in the list in square miles?", + "For the second one in the list: what is the volume cubic miles?", + "For the third one in the list: what is the total area in square miles?", } expectedOutputs := []string{ @@ -1220,3 +1220,115 @@ func TestParseThenEvaluateWithMetadata(t *testing.T) { t.Errorf("Unexpected output: %s", out) } } + +func TestLoadFile(t *testing.T) { + wd, err := os.Getwd() + if err != nil { + t.Fatalf("Error getting working directory: %v", err) + } + + prg, err := g.LoadFile(context.Background(), wd+"/test/global-tools.gpt") + if err != nil { + t.Fatalf("Error loading file: %v", err) + } + + if prg.EntryToolID == "" { + t.Errorf("Unexpected entry tool ID: %s", prg.EntryToolID) + } + + if len(prg.ToolSet) == 0 { + t.Errorf("Unexpected number of tools: %d", len(prg.ToolSet)) + } + + if prg.Name == "" { + t.Errorf("Unexpected name: %s", prg.Name) + } +} + +func TestLoadRemoteFile(t *testing.T) { + prg, err := g.LoadFile(context.Background(), "github.com/gptscript-ai/context/workspace") + if err != nil { + t.Fatalf("Error loading file: %v", err) + } + + if prg.EntryToolID == "" { + t.Errorf("Unexpected entry tool ID: %s", prg.EntryToolID) + } + + if len(prg.ToolSet) == 0 { + t.Errorf("Unexpected number of tools: %d", len(prg.ToolSet)) + } + + if prg.Name == "" { + t.Errorf("Unexpected name: %s", prg.Name) + } +} + +func TestLoadContent(t *testing.T) { + wd, err := os.Getwd() + if err != nil { + t.Fatalf("Error getting working directory: %v", err) + } + + content, err := os.ReadFile(wd + "/test/global-tools.gpt") + if err != nil { + t.Fatalf("Error reading file: %v", err) + } + + prg, err := g.LoadContent(context.Background(), string(content)) + if err != nil { + t.Fatalf("Error loading file: %v", err) + } + + if prg.EntryToolID == "" { + t.Errorf("Unexpected entry tool ID: %s", prg.EntryToolID) + } + + if len(prg.ToolSet) == 0 { + t.Errorf("Unexpected number of tools: %d", len(prg.ToolSet)) + } + + // Name won't be set in this case + if prg.Name != "" { + t.Errorf("Unexpected name: %s", prg.Name) + } +} + +func TestLoadTools(t *testing.T) { + tools := []ToolDef{ + { + Tools: []string{"echo"}, + Instructions: "echo 'hello there'", + }, + { + Name: "other", + Tools: []string{"echo"}, + Instructions: "echo 'hello somewhere else'", + }, + { + Name: "echo", + Tools: []string{"sys.exec"}, + Description: "Echoes the input", + Arguments: ObjectSchema("input", "The string input to echo"), + Instructions: "#!/bin/bash\n echo ${input}", + }, + } + + prg, err := g.LoadTools(context.Background(), tools) + if err != nil { + t.Fatalf("Error loading file: %v", err) + } + + if prg.EntryToolID == "" { + t.Errorf("Unexpected entry tool ID: %s", prg.EntryToolID) + } + + if len(prg.ToolSet) == 0 { + t.Errorf("Unexpected number of tools: %d", len(prg.ToolSet)) + } + + // Name won't be set in this case + if prg.Name != "" { + t.Errorf("Unexpected name: %s", prg.Name) + } +} From 0733d08e806e4aca23980b379b3bc120e4a15d84 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Wed, 14 Aug 2024 14:39:45 -0400 Subject: [PATCH 14/78] chore: bump Go version to 1.23.0 Signed-off-by: Donnie Adams --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index fc551c5..7094ab6 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/gptscript-ai/go-gptscript -go 1.22.2 +go 1.23.0 require github.com/getkin/kin-openapi v0.124.0 From 732c57036a0252a3eabcf8b905aa7ece77551ede Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Wed, 14 Aug 2024 14:46:06 -0400 Subject: [PATCH 15/78] chore: bump golangci-lint version for Go 1.23 Signed-off-by: Donnie Adams --- .golangci.yaml | 3 ++- Makefile | 2 +- run.go | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 62d0b9e..03be8a7 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -2,7 +2,8 @@ run: timeout: 5m output: - format: github-actions + formats: + - format: colored-line-number linters: disable-all: true diff --git a/Makefile b/Makefile index 701a2fc..ecb61c5 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ tidy: test: go test -v ./... -GOLANGCI_LINT_VERSION ?= v1.56.1 +GOLANGCI_LINT_VERSION ?= v1.60.1 lint: if ! command -v golangci-lint &> /dev/null; then \ echo "Could not find golangci-lint, installing version $(GOLANGCI_LINT_VERSION)."; \ diff --git a/run.go b/run.go index 6e6c5fd..5b716aa 100644 --- a/run.go +++ b/run.go @@ -371,7 +371,7 @@ func (r *Run) request(ctx context.Context, payload any) (err error) { r.callsLock.Unlock() } else if event.Run.Type == EventTypeRunFinish && event.Run.Error != "" { r.state = Error - r.err = fmt.Errorf(event.Run.Error) + r.err = fmt.Errorf("%s", event.Run.Error) } } From 48c714dd5bbfdb57f82884152933b03fd4d29b6e Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Fri, 16 Aug 2024 18:01:08 -0400 Subject: [PATCH 16/78] chore: update GitHub actions file for upcoming changes Signed-off-by: Donnie Adams --- .github/workflows/run_tests.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index 5ef5d78..8e80a83 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -21,7 +21,7 @@ jobs: - uses: actions/setup-go@v5 with: cache: false - go-version: "1.22" + go-version: "1.23" - name: Validate run: make validate - name: Install gptscript @@ -32,6 +32,7 @@ jobs: env: GPTSCRIPT_BIN: ./gptscriptexe OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} run: make test test-windows: @@ -44,16 +45,14 @@ jobs: - uses: actions/setup-go@v5 with: cache: false - go-version: "1.22" + go-version: "1.23" - name: Install gptscript run: | curl https://get.gptscript.ai/releases/default_windows_amd64_v1/gptscript.exe -o gptscript.exe - name: Create config file - run: | - echo '{"credsStore":"file"}' > config - name: Run Tests env: GPTSCRIPT_BIN: .\gptscript.exe OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - GPTSCRIPT_CONFIG_FILE: .\config + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} run: make test From 4f05179de3611fb4de9f514ff66e45ab88e51c56 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Fri, 16 Aug 2024 18:05:40 -0400 Subject: [PATCH 17/78] feat: add ability to list models from providers Signed-off-by: Donnie Adams --- .github/workflows/run_tests.yaml | 1 - gptscript.go | 40 ++++++++++++++------ gptscript_test.go | 63 +++++++++++++++++++++++++++++++- 3 files changed, 90 insertions(+), 14 deletions(-) diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index 8e80a83..c3b3123 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -49,7 +49,6 @@ jobs: - name: Install gptscript run: | curl https://get.gptscript.ai/releases/default_windows_amd64_v1/gptscript.exe -o gptscript.exe - - name: Create config file - name: Run Tests env: GPTSCRIPT_BIN: .\gptscript.exe diff --git a/gptscript.go b/gptscript.go index cc0d620..f3a9e44 100644 --- a/gptscript.go +++ b/gptscript.go @@ -28,8 +28,8 @@ var ( const relativeToBinaryPath = "" type GPTScript struct { - url string - globalEnv []string + url string + globalOpts GlobalOptions } func NewGPTScript(opts ...GlobalOptions) (*GPTScript, error) { @@ -40,7 +40,7 @@ func NewGPTScript(opts ...GlobalOptions) (*GPTScript, error) { disableServer := os.Getenv("GPTSCRIPT_DISABLE_SERVER") == "true" - if serverURL == "" && disableServer { + if serverURL == "" { serverURL = os.Getenv("GPTSCRIPT_URL") } @@ -96,11 +96,8 @@ func NewGPTScript(opts ...GlobalOptions) (*GPTScript, error) { serverURL = strings.TrimSpace(serverURL) } g := &GPTScript{ - url: "http://" + serverURL, - } - - if disableServer { - g.globalEnv = opt.Env[:] + url: "http://" + serverURL, + globalOpts: opt, } return g, nil @@ -132,7 +129,7 @@ func (g *GPTScript) Close() { } func (g *GPTScript) Evaluate(ctx context.Context, opts Options, tools ...ToolDef) (*Run, error) { - opts.Env = append(g.globalEnv, opts.Env...) + opts.GlobalOptions = completeGlobalOptions(g.globalOpts, opts.GlobalOptions) return (&Run{ url: g.url, requestPath: "evaluate", @@ -143,7 +140,7 @@ func (g *GPTScript) Evaluate(ctx context.Context, opts Options, tools ...ToolDef } func (g *GPTScript) Run(ctx context.Context, toolPath string, opts Options) (*Run, error) { - opts.Env = append(g.globalEnv, opts.Env...) + opts.GlobalOptions = completeGlobalOptions(g.globalOpts, opts.GlobalOptions) return (&Run{ url: g.url, requestPath: "run", @@ -281,9 +278,28 @@ func (g *GPTScript) ListTools(ctx context.Context) (string, error) { return out, nil } +type ListModelsOptions struct { + Providers []string + CredentialOverrides []string +} + // ListModels will list all the available models. -func (g *GPTScript) ListModels(ctx context.Context) ([]string, error) { - out, err := g.runBasicCommand(ctx, "list-models", nil) +func (g *GPTScript) ListModels(ctx context.Context, opts ...ListModelsOptions) ([]string, error) { + var o ListModelsOptions + for _, opt := range opts { + o.Providers = append(o.Providers, opt.Providers...) + o.CredentialOverrides = append(o.CredentialOverrides, opt.CredentialOverrides...) + } + + if g.globalOpts.DefaultModelProvider != "" { + o.Providers = append(o.Providers, g.globalOpts.DefaultModelProvider) + } + + out, err := g.runBasicCommand(ctx, "list-models", map[string]any{ + "providers": o.Providers, + "env": g.globalOpts.Env, + "credentialOverrides": o.CredentialOverrides, + }) if err != nil { return nil, err } diff --git a/gptscript_test.go b/gptscript_test.go index f298cab..700f0ea 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -19,14 +19,22 @@ func TestMain(m *testing.M) { panic("OPENAI_API_KEY or GPTSCRIPT_URL environment variable must be set") } - var err error + // Start an initial GPTScript instance. + // This one doesn't have any options, but it's there to ensure that using another instance works as expected in all cases. + gFirst, err := NewGPTScript(GlobalOptions{}) + if err != nil { + panic(fmt.Sprintf("error creating gptscript: %s", err)) + } + g, err = NewGPTScript(GlobalOptions{OpenAIAPIKey: os.Getenv("OPENAI_API_KEY")}) if err != nil { + gFirst.Close() panic(fmt.Sprintf("error creating gptscript: %s", err)) } exitCode := m.Run() g.Close() + gFirst.Close() os.Exit(exitCode) } @@ -80,6 +88,59 @@ func TestListModels(t *testing.T) { } } +func TestListModelsWithProvider(t *testing.T) { + if os.Getenv("ANTHROPIC_API_KEY") == "" { + t.Skip("ANTHROPIC_API_KEY not set") + } + models, err := g.ListModels(context.Background(), ListModelsOptions{ + Providers: []string{"github.com/gptscript-ai/claude3-anthropic-provider"}, + CredentialOverrides: []string{"github.com/gptscript-ai/claude3-anthropic-provider/credential:ANTHROPIC_API_KEY"}, + }) + if err != nil { + t.Errorf("Error listing models: %v", err) + } + + if len(models) == 0 { + t.Error("No models found") + } + + for _, model := range models { + if !strings.HasPrefix(model, "claude-3-") || !strings.HasSuffix(model, "from github.com/gptscript-ai/claude3-anthropic-provider") { + t.Errorf("Unexpected model name: %s", model) + } + } +} + +func TestListModelsWithDefaultProvider(t *testing.T) { + if os.Getenv("ANTHROPIC_API_KEY") == "" { + t.Skip("ANTHROPIC_API_KEY not set") + } + g, err := NewGPTScript(GlobalOptions{ + DefaultModelProvider: "github.com/gptscript-ai/claude3-anthropic-provider", + }) + if err != nil { + t.Fatalf("Error creating gptscript: %v", err) + } + defer g.Close() + + models, err := g.ListModels(context.Background(), ListModelsOptions{ + CredentialOverrides: []string{"github.com/gptscript-ai/claude3-anthropic-provider/credential:ANTHROPIC_API_KEY"}, + }) + if err != nil { + t.Errorf("Error listing models: %v", err) + } + + if len(models) == 0 { + t.Error("No models found") + } + + for _, model := range models { + if !strings.HasPrefix(model, "claude-3-") || !strings.HasSuffix(model, "from github.com/gptscript-ai/claude3-anthropic-provider") { + t.Errorf("Unexpected model name: %s", model) + } + } +} + func TestAbortRun(t *testing.T) { tool := ToolDef{Instructions: "What is the capital of the united states?"} From a398de6ca6604c57e0e941a3aaf2627aa4300060 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Fri, 16 Aug 2024 18:07:40 -0400 Subject: [PATCH 18/78] chore: remove extraneous line in tests action file Signed-off-by: Donnie Adams --- .github/workflows/run_tests.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index 8e80a83..c3b3123 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -49,7 +49,6 @@ jobs: - name: Install gptscript run: | curl https://get.gptscript.ai/releases/default_windows_amd64_v1/gptscript.exe -o gptscript.exe - - name: Create config file - name: Run Tests env: GPTSCRIPT_BIN: .\gptscript.exe From 0bba1ded6f3efa634d4b4d2a0b87a2da189b35b7 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Fri, 16 Aug 2024 18:34:50 -0400 Subject: [PATCH 19/78] chore: add anthropic secret to main and pull actions Signed-off-by: Donnie Adams --- .github/workflows/pull_request.yaml | 1 + .github/workflows/push_main.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 8e87f54..474235f 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -38,4 +38,5 @@ jobs: git_ref: ${{ github.event.pull_request.head.sha }} secrets: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} diff --git a/.github/workflows/push_main.yaml b/.github/workflows/push_main.yaml index 2bf364e..edca697 100644 --- a/.github/workflows/push_main.yaml +++ b/.github/workflows/push_main.yaml @@ -13,4 +13,5 @@ jobs: git_ref: '' secrets: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} From d4e7a145b34e1c006e10b8fa9201240bab48de30 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Fri, 16 Aug 2024 18:37:21 -0400 Subject: [PATCH 20/78] chore: add anthropic key as required in test action Signed-off-by: Donnie Adams --- .github/workflows/run_tests.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index c3b3123..c06061c 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -9,6 +9,8 @@ on: secrets: OPENAI_API_KEY: required: true + ANTHROPIC_API_KEY: + required: true jobs: test-linux: From b193865f5cf49a966f07f5e48c408dc69eef689a Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Mon, 19 Aug 2024 17:14:01 -0400 Subject: [PATCH 21/78] feat: add prompt metadata field Signed-off-by: Donnie Adams --- frame.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/frame.go b/frame.go index e3f37f8..d4dfd26 100644 --- a/frame.go +++ b/frame.go @@ -104,16 +104,18 @@ type InputContext struct { } type PromptFrame struct { - ID string `json:"id,omitempty"` - Type EventType `json:"type,omitempty"` - Time time.Time `json:"time,omitempty"` - Message string `json:"message,omitempty"` - Fields []string `json:"fields,omitempty"` - Sensitive bool `json:"sensitive,omitempty"` + ID string `json:"id,omitempty"` + Type EventType `json:"type,omitempty"` + Time time.Time `json:"time,omitempty"` + Message string `json:"message,omitempty"` + Fields []string `json:"fields,omitempty"` + Sensitive bool `json:"sensitive,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` } func (p *PromptFrame) String() string { return fmt.Sprintf(`Message: %s Fields: %v -Sensitive: %v`, p.Message, p.Fields, p.Sensitive) +Sensitive: %v`, p.Message, p.Fields, p.Sensitive, + ) } From 59247128e0a30c5381b5fc1f6cafb0fabdd1f4b2 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Tue, 20 Aug 2024 12:18:08 -0400 Subject: [PATCH 22/78] chore: add test for prompt with metadata Signed-off-by: Donnie Adams --- gptscript_test.go | 62 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/gptscript_test.go b/gptscript_test.go index 700f0ea..8df3dcf 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -1107,6 +1107,68 @@ func TestPrompt(t *testing.T) { } } +func TestPromptWithMetadata(t *testing.T) { + run, err := g.Run(context.Background(), "sys.prompt", Options{IncludeEvents: true, Prompt: true, Input: `{"fields":"first name","metadata":{"key":"value"}}`}) + if err != nil { + t.Errorf("Error executing tool: %v", err) + } + + // Wait for the prompt event + var promptFrame *PromptFrame + for e := range run.Events() { + if e.Prompt != nil { + if e.Prompt.Type == EventTypePrompt { + promptFrame = e.Prompt + break + } + } + } + + if promptFrame == nil { + t.Fatalf("No prompt call event") + } + + if promptFrame.Sensitive { + t.Errorf("Unexpected sensitive prompt event: %v", promptFrame.Sensitive) + } + + if len(promptFrame.Fields) != 1 { + t.Fatalf("Unexpected number of fields: %d", len(promptFrame.Fields)) + } + + if promptFrame.Fields[0] != "first name" { + t.Errorf("Unexpected field: %s", promptFrame.Fields[0]) + } + + if promptFrame.Metadata["key"] != "value" { + t.Errorf("Unexpected metadata: %v", promptFrame.Metadata) + } + + if err = g.PromptResponse(context.Background(), PromptResponse{ + ID: promptFrame.ID, + Responses: map[string]string{promptFrame.Fields[0]: "Clicky"}, + }); err != nil { + t.Errorf("Error responding: %v", err) + } + + // Read the remainder of the events + for range run.Events() { + } + + out, err := run.Text() + if err != nil { + t.Errorf("Error reading output: %v", err) + } + + if !strings.Contains(out, "Clicky") { + t.Errorf("Unexpected output: %s", out) + } + + if len(run.ErrorOutput()) != 0 { + t.Errorf("Should have no stderr output: %v", run.ErrorOutput()) + } +} + func TestPromptWithoutPromptAllowed(t *testing.T) { tools := ToolDef{ Instructions: "Use the sys.prompt user to ask the user for 'first name' which is not sensitive. After you get their first name, say hello.", From b51e30e980ae2065e862929405d927d4ec01be41 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Tue, 20 Aug 2024 14:02:03 -0400 Subject: [PATCH 23/78] chore: fix lint issues in tests Signed-off-by: Donnie Adams --- gptscript_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gptscript_test.go b/gptscript_test.go index 8df3dcf..4e68cee 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -984,6 +984,7 @@ func TestConfirmDeny(t *testing.T) { if confirmCallEvent == nil { t.Fatalf("No confirm call event") + return } if !strings.Contains(confirmCallEvent.Input, "\"ls\"") { @@ -1055,6 +1056,7 @@ func TestPrompt(t *testing.T) { if promptFrame == nil { t.Fatalf("No prompt call event") + return } if promptFrame.Sensitive { @@ -1126,6 +1128,7 @@ func TestPromptWithMetadata(t *testing.T) { if promptFrame == nil { t.Fatalf("No prompt call event") + return } if promptFrame.Sensitive { @@ -1152,6 +1155,7 @@ func TestPromptWithMetadata(t *testing.T) { } // Read the remainder of the events + //nolint:revive for range run.Events() { } From 14735f981d6256d61e431699e72fe3918bed50ec Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Fri, 30 Aug 2024 11:00:33 -0400 Subject: [PATCH 24/78] chore: remove ListTools Signed-off-by: Donnie Adams --- README.md | 26 -------------------------- gptscript.go | 10 ---------- gptscript_test.go | 11 ----------- 3 files changed, 47 deletions(-) diff --git a/README.md b/README.md index a51d067..04fc514 100644 --- a/README.md +++ b/README.md @@ -47,32 +47,6 @@ As noted above, the Global Options are also available to specify here. These opt ## Functions -### listTools - -Lists all the available built-in tools. - -**Usage:** - -```go -package main - -import ( - "context" - - "github.com/gptscript-ai/go-gptscript" -) - -func listTools(ctx context.Context) (string, error) { - g, err := gptscript.NewGPTScript(gptscript.GlobalOptions{}) - if err != nil { - return "", err - } - defer g.Close() - - return g.ListTools(ctx) -} -``` - ### listModels Lists all the available models, returns a list. diff --git a/gptscript.go b/gptscript.go index f3a9e44..1968144 100644 --- a/gptscript.go +++ b/gptscript.go @@ -268,16 +268,6 @@ func (g *GPTScript) Version(ctx context.Context) (string, error) { return out, nil } -// ListTools will list all the available tools. -func (g *GPTScript) ListTools(ctx context.Context) (string, error) { - out, err := g.runBasicCommand(ctx, "list-tools", nil) - if err != nil { - return "", err - } - - return out, nil -} - type ListModelsOptions struct { Providers []string CredentialOverrides []string diff --git a/gptscript_test.go b/gptscript_test.go index 4e68cee..325cb86 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -66,17 +66,6 @@ func TestVersion(t *testing.T) { } } -func TestListTools(t *testing.T) { - tools, err := g.ListTools(context.Background()) - if err != nil { - t.Errorf("Error listing tools: %v", err) - } - - if len(tools) == 0 { - t.Error("No tools found") - } -} - func TestListModels(t *testing.T) { models, err := g.ListModels(context.Background()) if err != nil { From d2e9015e07a85b26b9be4b4922442420ab15f842 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Tue, 3 Sep 2024 09:28:57 -0400 Subject: [PATCH 25/78] chore: move cache dir option to GlobalOptions Signed-off-by: Donnie Adams --- README.md | 1 + gptscript_test.go | 2 +- opts.go | 5 +++-- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 04fc514..725e81c 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ The GPTScript instance allows the caller to run gptscript files, tools, and othe When creating a `GTPScript` instance, you can pass the following global options. These options are also available as run `Options`. Anything specified as a run option will take precedence over the global option. +- `CacheDir`: The directory to use for caching. Default (""), which uses the default cache directory. - `APIKey`: Specify an OpenAI API key for authenticating requests - `BaseURL`: A base URL for an OpenAI compatible API (the default is `https://api.openai.com/v1`) - `DefaultModel`: The default model to use for chat completion requests diff --git a/gptscript_test.go b/gptscript_test.go index 325cb86..3797707 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -414,7 +414,7 @@ func TestRestartFailedRun(t *testing.T) { Instructions: instructions, }, } - run, err := g.Evaluate(context.Background(), Options{DisableCache: true, GlobalOptions: GlobalOptions{Env: []string{"EXIT_CODE=1"}}}, tools...) + run, err := g.Evaluate(context.Background(), Options{GlobalOptions: GlobalOptions{Env: []string{"EXIT_CODE=1"}}, DisableCache: true}, tools...) if err != nil { t.Fatalf("Error executing tool: %v", err) } diff --git a/opts.go b/opts.go index bf91422..191a8d0 100644 --- a/opts.go +++ b/opts.go @@ -7,6 +7,7 @@ type GlobalOptions struct { OpenAIBaseURL string `json:"BaseURL"` DefaultModel string `json:"DefaultModel"` DefaultModelProvider string `json:"DefaultModelProvider"` + CacheDir string `json:"CacheDir"` Env []string `json:"env"` } @@ -31,6 +32,7 @@ func (g GlobalOptions) toEnv() []string { func completeGlobalOptions(opts ...GlobalOptions) GlobalOptions { var result GlobalOptions for _, opt := range opts { + result.CacheDir = firstSet(opt.CacheDir, result.CacheDir) result.OpenAIAPIKey = firstSet(opt.OpenAIAPIKey, result.OpenAIAPIKey) result.OpenAIBaseURL = firstSet(opt.OpenAIBaseURL, result.OpenAIBaseURL) result.DefaultModel = firstSet(opt.DefaultModel, result.DefaultModel) @@ -55,10 +57,9 @@ func firstSet[T comparable](in ...T) T { type Options struct { GlobalOptions `json:",inline"` + DisableCache bool `json:"disableCache"` Confirm bool `json:"confirm"` Input string `json:"input"` - DisableCache bool `json:"disableCache"` - CacheDir string `json:"cacheDir"` SubTool string `json:"subTool"` Workspace string `json:"workspace"` ChatState string `json:"chatState"` From aa14fb67ac83b4f87466517a75110a5db552f7cb Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Tue, 3 Sep 2024 09:34:01 -0400 Subject: [PATCH 26/78] fix: address a flake in the chat tests Signed-off-by: Donnie Adams --- gptscript_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gptscript_test.go b/gptscript_test.go index 3797707..e705c98 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -785,8 +785,8 @@ func TestFileChat(t *testing.T) { } inputs := []string{ "List the 3 largest of the Great Lakes by volume.", - "For the second one in the list: what is the volume cubic miles?", - "For the third one in the list: what is the total area in square miles?", + "What is the second one in the list?", + "What is the third?", } expectedOutputs := []string{ From 66dc3960908b0a09bc2979f1ea02e217fb8c0ef2 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Tue, 27 Aug 2024 15:11:21 -0400 Subject: [PATCH 27/78] feat: add server URL and token options If the server URL has a path, then the SDK will implicitly disable the server since the local SDK server cannot have a path. Signed-off-by: Donnie Adams --- gptscript.go | 57 +++++++++++++++++++++++++++++++++++++++------------- opts.go | 4 ++++ run.go | 34 ++++++++++++++++++++----------- 3 files changed, 69 insertions(+), 26 deletions(-) diff --git a/gptscript.go b/gptscript.go index 1968144..c8b6e64 100644 --- a/gptscript.go +++ b/gptscript.go @@ -10,6 +10,7 @@ import ( "fmt" "io" "log/slog" + "net/url" "os" "os/exec" "path/filepath" @@ -28,7 +29,6 @@ var ( const relativeToBinaryPath = "" type GPTScript struct { - url string globalOpts GlobalOptions } @@ -38,10 +38,11 @@ func NewGPTScript(opts ...GlobalOptions) (*GPTScript, error) { defer lock.Unlock() gptscriptCount++ - disableServer := os.Getenv("GPTSCRIPT_DISABLE_SERVER") == "true" - if serverURL == "" { - serverURL = os.Getenv("GPTSCRIPT_URL") + serverURL = opt.URL + if serverURL == "" { + serverURL = os.Getenv("GPTSCRIPT_URL") + } } if opt.Env == nil { @@ -50,11 +51,31 @@ func NewGPTScript(opts ...GlobalOptions) (*GPTScript, error) { opt.Env = append(opt.Env, opt.toEnv()...) - if serverProcessCancel == nil && !disableServer { + if serverProcessCancel == nil && os.Getenv("GPTSCRIPT_DISABLE_SERVER") != "true" { + if serverURL != "" { + u, err := url.Parse(serverURL) + if err != nil { + return nil, fmt.Errorf("failed to parse server URL: %w", err) + } + + // If the server URL has a path, then this implies that the server is already running. + // In that case, we don't need to start the server. + if u.Path != "" && u.Path != "/" { + opt.URL = serverURL + if !strings.HasPrefix(opt.URL, "http://") && !strings.HasPrefix(opt.URL, "https://") { + opt.URL = "http://" + opt.URL + } + + return &GPTScript{ + globalOpts: opt, + }, nil + } + } + ctx, cancel := context.WithCancel(context.Background()) in, _ := io.Pipe() - serverProcess = exec.CommandContext(ctx, getCommand(), "sys.sdkserver", "--listen-address", serverURL) + serverProcess = exec.CommandContext(ctx, getCommand(), "sys.sdkserver", "--listen-address", strings.TrimPrefix(serverURL, "http://")) serverProcess.Env = opt.Env[:] serverProcess.Stdin = in @@ -95,12 +116,14 @@ func NewGPTScript(opts ...GlobalOptions) (*GPTScript, error) { serverURL = strings.TrimSpace(serverURL) } - g := &GPTScript{ - url: "http://" + serverURL, - globalOpts: opt, - } - return g, nil + opt.URL = serverURL + if !strings.HasPrefix(opt.URL, "http://") && !strings.HasPrefix(opt.URL, "https://") { + opt.URL = "http://" + opt.URL + } + return &GPTScript{ + globalOpts: opt, + }, nil } func readAddress(stdErr io.Reader) (string, error) { @@ -117,6 +140,10 @@ func readAddress(stdErr io.Reader) (string, error) { return addr, nil } +func (g *GPTScript) URL() string { + return g.globalOpts.URL +} + func (g *GPTScript) Close() { lock.Lock() defer lock.Unlock() @@ -131,7 +158,8 @@ func (g *GPTScript) Close() { func (g *GPTScript) Evaluate(ctx context.Context, opts Options, tools ...ToolDef) (*Run, error) { opts.GlobalOptions = completeGlobalOptions(g.globalOpts, opts.GlobalOptions) return (&Run{ - url: g.url, + url: opts.URL, + token: opts.Token, requestPath: "evaluate", state: Creating, opts: opts, @@ -142,7 +170,8 @@ func (g *GPTScript) Evaluate(ctx context.Context, opts Options, tools ...ToolDef func (g *GPTScript) Run(ctx context.Context, toolPath string, opts Options) (*Run, error) { opts.GlobalOptions = completeGlobalOptions(g.globalOpts, opts.GlobalOptions) return (&Run{ - url: g.url, + url: opts.URL, + token: opts.Token, requestPath: "run", state: Creating, opts: opts, @@ -309,7 +338,7 @@ func (g *GPTScript) PromptResponse(ctx context.Context, resp PromptResponse) err func (g *GPTScript) runBasicCommand(ctx context.Context, requestPath string, body any) (string, error) { run := &Run{ - url: g.url, + url: g.globalOpts.URL, requestPath: requestPath, state: Creating, basicCommand: true, diff --git a/opts.go b/opts.go index 191a8d0..125a16f 100644 --- a/opts.go +++ b/opts.go @@ -3,6 +3,8 @@ package gptscript // GlobalOptions allows specification of settings that are used for every call made. // These options can be overridden by the corresponding Options. type GlobalOptions struct { + URL string `json:"url"` + Token string `json:"token"` OpenAIAPIKey string `json:"APIKey"` OpenAIBaseURL string `json:"BaseURL"` DefaultModel string `json:"DefaultModel"` @@ -33,6 +35,8 @@ func completeGlobalOptions(opts ...GlobalOptions) GlobalOptions { var result GlobalOptions for _, opt := range opts { result.CacheDir = firstSet(opt.CacheDir, result.CacheDir) + result.URL = firstSet(opt.URL, result.URL) + result.Token = firstSet(opt.Token, result.Token) result.OpenAIAPIKey = firstSet(opt.OpenAIAPIKey, result.OpenAIAPIKey) result.OpenAIBaseURL = firstSet(opt.OpenAIBaseURL, result.OpenAIBaseURL) result.DefaultModel = firstSet(opt.DefaultModel, result.DefaultModel) diff --git a/run.go b/run.go index 5b716aa..ae5a4a0 100644 --- a/run.go +++ b/run.go @@ -18,15 +18,15 @@ import ( var errAbortRun = errors.New("run aborted") type Run struct { - url, requestPath, toolPath string - tools []ToolDef - opts Options - state RunState - chatState string - cancel context.CancelCauseFunc - err error - wait func() - basicCommand bool + url, token, requestPath, toolPath string + tools []ToolDef + opts Options + state RunState + chatState string + cancel context.CancelCauseFunc + err error + wait func() + basicCommand bool program *Program callsLock sync.RWMutex @@ -175,18 +175,24 @@ func (r *Run) NextChat(ctx context.Context, input string) (*Run, error) { run.opts.ChatState = r.chatState } - var payload any + var ( + payload any + options = run.opts + ) + // Remove the url and token because they shouldn't be sent with the payload. + options.URL = "" + options.Token = "" if len(r.tools) != 0 { payload = requestPayload{ ToolDefs: r.tools, Input: input, - Options: run.opts, + Options: options, } } else if run.toolPath != "" { payload = requestPayload{ File: run.toolPath, Input: input, - Options: run.opts, + Options: options, } } @@ -228,6 +234,10 @@ func (r *Run) request(ctx context.Context, payload any) (err error) { return r.err } + if r.opts.Token != "" { + req.Header.Set("Authorization", "Bearer "+r.opts.Token) + } + resp, err := http.DefaultClient.Do(req) if err != nil { r.state = Error From f243c378c94772bad36ca6c10cb3e06d02a3da03 Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Thu, 12 Sep 2024 10:07:21 -0700 Subject: [PATCH 28/78] chore: add ToolDefsToNodes helper method --- tool.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tool.go b/tool.go index c2360ac..cc2137c 100644 --- a/tool.go +++ b/tool.go @@ -33,6 +33,20 @@ type ToolDef struct { MetaData map[string]string `json:"metadata,omitempty"` } +func ToolDefsToNodes(tools []ToolDef) []Node { + nodes := make([]Node, 0, len(tools)) + for _, tool := range tools { + nodes = append(nodes, Node{ + ToolNode: &ToolNode{ + Tool: Tool{ + ToolDef: tool, + }, + }, + }) + } + return nodes +} + func ObjectSchema(kv ...string) *openapi3.Schema { s := &openapi3.Schema{ Type: &openapi3.Types{"object"}, From 70e878812b42b9f5369d1c09320edc2a43716df0 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Thu, 12 Sep 2024 13:58:39 -0400 Subject: [PATCH 29/78] fix: override the knowledge credential in test The knowledge tool isn't actually used in the test, but it is a good example of a large tool, so it is included. Signed-off-by: Donnie Adams --- gptscript_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gptscript_test.go b/gptscript_test.go index e705c98..6d43e4f 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -828,7 +828,7 @@ func TestToolWithGlobalTools(t *testing.T) { var eventContent string - run, err := g.Run(context.Background(), wd+"/test/global-tools.gpt", Options{DisableCache: true, IncludeEvents: true}) + run, err := g.Run(context.Background(), wd+"/test/global-tools.gpt", Options{DisableCache: true, IncludeEvents: true, CredentialOverrides: []string{"github.com/gptscript-ai/gateway:OPENAI_API_KEY"}}) if err != nil { t.Fatalf("Error executing tool: %v", err) } From 8c8de8b2d3a4bef4e36e77e2cca9b511fa4f5b46 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Thu, 12 Sep 2024 14:39:34 -0400 Subject: [PATCH 30/78] chore: complete the ToolDef fields Signed-off-by: Donnie Adams --- tool.go | 49 +++++++++++++++++++++++++++---------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/tool.go b/tool.go index cc2137c..36eb535 100644 --- a/tool.go +++ b/tool.go @@ -9,28 +9,33 @@ import ( // ToolDef struct represents a tool with various configurations. type ToolDef struct { - Name string `json:"name,omitempty"` - Description string `json:"description,omitempty"` - MaxTokens int `json:"maxTokens,omitempty"` - ModelName string `json:"modelName,omitempty"` - ModelProvider bool `json:"modelProvider,omitempty"` - JSONResponse bool `json:"jsonResponse,omitempty"` - Chat bool `json:"chat,omitempty"` - Temperature *float32 `json:"temperature,omitempty"` - Cache *bool `json:"cache,omitempty"` - InternalPrompt *bool `json:"internalPrompt"` - Arguments *openapi3.Schema `json:"arguments,omitempty"` - Tools []string `json:"tools,omitempty"` - GlobalTools []string `json:"globalTools,omitempty"` - GlobalModelName string `json:"globalModelName,omitempty"` - Context []string `json:"context,omitempty"` - ExportContext []string `json:"exportContext,omitempty"` - Export []string `json:"export,omitempty"` - Agents []string `json:"agents,omitempty"` - Credentials []string `json:"credentials,omitempty"` - Instructions string `json:"instructions,omitempty"` - Type string `json:"type,omitempty"` - MetaData map[string]string `json:"metadata,omitempty"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + MaxTokens int `json:"maxTokens,omitempty"` + ModelName string `json:"modelName,omitempty"` + ModelProvider bool `json:"modelProvider,omitempty"` + JSONResponse bool `json:"jsonResponse,omitempty"` + Chat bool `json:"chat,omitempty"` + Temperature *float32 `json:"temperature,omitempty"` + Cache *bool `json:"cache,omitempty"` + InternalPrompt *bool `json:"internalPrompt"` + Arguments *openapi3.Schema `json:"arguments,omitempty"` + Tools []string `json:"tools,omitempty"` + GlobalTools []string `json:"globalTools,omitempty"` + GlobalModelName string `json:"globalModelName,omitempty"` + Context []string `json:"context,omitempty"` + ExportContext []string `json:"exportContext,omitempty"` + Export []string `json:"export,omitempty"` + Agents []string `json:"agents,omitempty"` + Credentials []string `json:"credentials,omitempty"` + ExportCredentials []string `json:"exportCredentials,omitempty"` + InputFilters []string `json:"inputFilters,omitempty"` + ExportInputFilters []string `json:"exportInputFilters,omitempty"` + OutputFilters []string `json:"outputFilters,omitempty"` + ExportOutputFilters []string `json:"exportOutputFilters,omitempty"` + Instructions string `json:"instructions,omitempty"` + Type string `json:"type,omitempty"` + MetaData map[string]string `json:"metadata,omitempty"` } func ToolDefsToNodes(tools []ToolDef) []Node { From c5466d086c22a3c4becbdcf17eb650bf273f0514 Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Thu, 19 Sep 2024 18:35:24 -0400 Subject: [PATCH 31/78] feat: add credential management (#65) Signed-off-by: Grant Linville --- credentials.go | 27 +++++++++++++++++++ go.mod | 7 ++++- gptscript.go | 61 +++++++++++++++++++++++++++++++++++++++++++ gptscript_test.go | 45 +++++++++++++++++++++++++++++++ opts.go | 1 + run.go | 24 +++++++++++++++++ run_test.go | 46 ++++++++++++++++++++++++++++++++ test/credential.gpt | 13 +++++++++ test/global-tools.gpt | 2 +- 9 files changed, 224 insertions(+), 2 deletions(-) create mode 100644 credentials.go create mode 100644 test/credential.gpt diff --git a/credentials.go b/credentials.go new file mode 100644 index 0000000..4c7c11f --- /dev/null +++ b/credentials.go @@ -0,0 +1,27 @@ +package gptscript + +import "time" + +type CredentialType string + +const ( + CredentialTypeTool CredentialType = "tool" + CredentialTypeModelProvider CredentialType = "modelProvider" +) + +type Credential struct { + Context string `json:"context"` + ToolName string `json:"toolName"` + Type CredentialType `json:"type"` + Env map[string]string `json:"env"` + Ephemeral bool `json:"ephemeral,omitempty"` + ExpiresAt *time.Time `json:"expiresAt"` + RefreshToken string `json:"refreshToken"` +} + +type CredentialRequest struct { + Content string `json:"content"` + AllContexts bool `json:"allContexts"` + Context []string `json:"context"` + Name string `json:"name"` +} diff --git a/go.mod b/go.mod index 7094ab6..283f8d6 100644 --- a/go.mod +++ b/go.mod @@ -2,9 +2,13 @@ module github.com/gptscript-ai/go-gptscript go 1.23.0 -require github.com/getkin/kin-openapi v0.124.0 +require ( + github.com/getkin/kin-openapi v0.124.0 + github.com/stretchr/testify v1.8.4 +) require ( + github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-openapi/jsonpointer v0.20.2 // indirect github.com/go-openapi/swag v0.22.8 // indirect github.com/invopop/yaml v0.2.0 // indirect @@ -12,5 +16,6 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/gptscript.go b/gptscript.go index c8b6e64..6a9723f 100644 --- a/gptscript.go +++ b/gptscript.go @@ -336,6 +336,67 @@ func (g *GPTScript) PromptResponse(ctx context.Context, resp PromptResponse) err return err } +type ListCredentialsOptions struct { + CredentialContexts []string + AllContexts bool +} + +func (g *GPTScript) ListCredentials(ctx context.Context, opts ListCredentialsOptions) ([]Credential, error) { + req := CredentialRequest{} + if opts.AllContexts { + req.AllContexts = true + } else if len(opts.CredentialContexts) > 0 { + req.Context = opts.CredentialContexts + } else { + req.Context = []string{"default"} + } + + out, err := g.runBasicCommand(ctx, "credentials", req) + if err != nil { + return nil, err + } + + var creds []Credential + if err = json.Unmarshal([]byte(out), &creds); err != nil { + return nil, err + } + return creds, nil +} + +func (g *GPTScript) CreateCredential(ctx context.Context, cred Credential) error { + credJSON, err := json.Marshal(cred) + if err != nil { + return fmt.Errorf("failed to marshal credential: %w", err) + } + + _, err = g.runBasicCommand(ctx, "credentials/create", CredentialRequest{Content: string(credJSON)}) + return err +} + +func (g *GPTScript) RevealCredential(ctx context.Context, credCtxs []string, name string) (Credential, error) { + out, err := g.runBasicCommand(ctx, "credentials/reveal", CredentialRequest{ + Context: credCtxs, + Name: name, + }) + if err != nil { + return Credential{}, err + } + + var cred Credential + if err = json.Unmarshal([]byte(out), &cred); err != nil { + return Credential{}, err + } + return cred, nil +} + +func (g *GPTScript) DeleteCredential(ctx context.Context, credCtx, name string) error { + _, err := g.runBasicCommand(ctx, "credentials/delete", CredentialRequest{ + Context: []string{credCtx}, // Only one context can be specified for delete operations + Name: name, + }) + return err +} + func (g *GPTScript) runBasicCommand(ctx context.Context, requestPath string, body any) (string, error) { run := &Run{ url: g.globalOpts.URL, diff --git a/gptscript_test.go b/gptscript_test.go index 6d43e4f..b1a2456 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -2,14 +2,18 @@ package gptscript import ( "context" + "errors" "fmt" + "math/rand" "os" "path/filepath" "runtime" + "strconv" "strings" "testing" "github.com/getkin/kin-openapi/openapi3" + "github.com/stretchr/testify/require" ) var g *GPTScript @@ -1448,3 +1452,44 @@ func TestLoadTools(t *testing.T) { t.Errorf("Unexpected name: %s", prg.Name) } } + +func TestCredentials(t *testing.T) { + // We will test in the following order of create, list, reveal, delete. + name := "test-" + strconv.Itoa(rand.Int()) + if len(name) > 20 { + name = name[:20] + } + + // Create + err := g.CreateCredential(context.Background(), Credential{ + Context: "testing", + ToolName: name, + Type: CredentialTypeTool, + Env: map[string]string{"ENV": "testing"}, + RefreshToken: "my-refresh-token", + }) + require.NoError(t, err) + + // List + creds, err := g.ListCredentials(context.Background(), ListCredentialsOptions{ + CredentialContexts: []string{"testing"}, + }) + require.NoError(t, err) + require.GreaterOrEqual(t, len(creds), 1) + + // Reveal + cred, err := g.RevealCredential(context.Background(), []string{"testing"}, name) + require.NoError(t, err) + require.Contains(t, cred.Env, "ENV") + require.Equal(t, cred.Env["ENV"], "testing") + require.Equal(t, cred.RefreshToken, "my-refresh-token") + + // Delete + err = g.DeleteCredential(context.Background(), "testing", name) + require.NoError(t, err) + + // Delete again and make sure we get a NotFoundError + err = g.DeleteCredential(context.Background(), "testing", name) + require.Error(t, err) + require.True(t, errors.As(err, &ErrNotFound{})) +} diff --git a/opts.go b/opts.go index 125a16f..3184504 100644 --- a/opts.go +++ b/opts.go @@ -70,6 +70,7 @@ type Options struct { IncludeEvents bool `json:"includeEvents"` Prompt bool `json:"prompt"` CredentialOverrides []string `json:"credentialOverrides"` + CredentialContexts []string `json:"credentialContext"` // json tag is left singular to match SDKServer Location string `json:"location"` ForceSequential bool `json:"forceSequential"` } diff --git a/run.go b/run.go index ae5a4a0..f832515 100644 --- a/run.go +++ b/run.go @@ -17,6 +17,14 @@ import ( var errAbortRun = errors.New("run aborted") +type ErrNotFound struct { + Message string +} + +func (e ErrNotFound) Error() string { + return e.Message +} + type Run struct { url, token, requestPath, toolPath string tools []ToolDef @@ -36,6 +44,7 @@ type Run struct { output, errput string events chan Frame lock sync.Mutex + responseCode int } // Text returns the text output of the gptscript. It blocks until the output is ready. @@ -60,6 +69,11 @@ func (r *Run) State() RunState { // Err returns the error that caused the gptscript to fail, if any. func (r *Run) Err() error { if r.err != nil { + if r.responseCode == http.StatusNotFound { + return ErrNotFound{ + Message: fmt.Sprintf("run encountered an error: %s", r.errput), + } + } return fmt.Errorf("run encountered an error: %w with error output: %s", r.err, r.errput) } return nil @@ -245,6 +259,7 @@ func (r *Run) request(ctx context.Context, payload any) (err error) { return r.err } + r.responseCode = resp.StatusCode if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusBadRequest { r.state = Error r.err = fmt.Errorf("run encountered an error") @@ -335,6 +350,15 @@ func (r *Run) request(ctx context.Context, payload any) (err error) { done, _ = out["done"].(bool) r.rawOutput = out + case []any: + b, err := json.Marshal(out) + if err != nil { + r.state = Error + r.err = fmt.Errorf("failed to process stdout: %w", err) + return + } + + r.output = string(b) default: r.state = Error r.err = fmt.Errorf("failed to process stdout, invalid type: %T", out) diff --git a/run_test.go b/run_test.go index f9014a1..ac1e1d8 100644 --- a/run_test.go +++ b/run_test.go @@ -2,8 +2,13 @@ package gptscript import ( "context" + "crypto/rand" + "encoding/hex" + "os" "runtime" "testing" + + "github.com/stretchr/testify/require" ) func TestRestartingErrorRun(t *testing.T) { @@ -42,3 +47,44 @@ func TestRestartingErrorRun(t *testing.T) { t.Errorf("executing run with input of 0 should not fail: %v", err) } } + +func TestStackedContexts(t *testing.T) { + const name = "testcred" + + wd, err := os.Getwd() + require.NoError(t, err) + + bytes := make([]byte, 32) + _, err = rand.Read(bytes) + require.NoError(t, err) + + context1 := hex.EncodeToString(bytes)[:16] + context2 := hex.EncodeToString(bytes)[16:] + + run, err := g.Run(context.Background(), wd+"/test/credential.gpt", Options{ + CredentialContexts: []string{context1, context2}, + }) + require.NoError(t, err) + + _, err = run.Text() + require.NoError(t, err) + + // The credential should exist in context1 now. + cred, err := g.RevealCredential(context.Background(), []string{context1, context2}, name) + require.NoError(t, err) + require.Equal(t, cred.Context, context1) + + // Now change the context order and run the script again. + run, err = g.Run(context.Background(), wd+"/test/credential.gpt", Options{ + CredentialContexts: []string{context2, context1}, + }) + require.NoError(t, err) + + _, err = run.Text() + require.NoError(t, err) + + // Now make sure the credential exists in context1 still. + cred, err = g.RevealCredential(context.Background(), []string{context2, context1}, name) + require.NoError(t, err) + require.Equal(t, cred.Context, context1) +} diff --git a/test/credential.gpt b/test/credential.gpt new file mode 100644 index 0000000..61e656f --- /dev/null +++ b/test/credential.gpt @@ -0,0 +1,13 @@ +name: echocred +credential: mycredentialtool as testcred + +#!/usr/bin/env bash + +echo $VALUE + +--- +name: mycredentialtool + +#!sys.echo + +{"env":{"VALUE":"hello"}} \ No newline at end of file diff --git a/test/global-tools.gpt b/test/global-tools.gpt index 4671fee..7e975be 100644 --- a/test/global-tools.gpt +++ b/test/global-tools.gpt @@ -4,7 +4,7 @@ Runbook 3 --- Name: tool_1 -Global Tools: github.com/gptscript-ai/knowledge, github.com/drpebcak/duckdb, github.com/gptscript-ai/browser, github.com/gptscript-ai/browser-search/google, github.com/gptscript-ai/browser-search/google-question-answerer +Global Tools: github.com/drpebcak/duckdb, github.com/gptscript-ai/browser, github.com/gptscript-ai/browser-search/google, github.com/gptscript-ai/browser-search/google-question-answerer Say "Hello!" From 1aaa0321faffb512794e15989f1b8f681f807eed Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Fri, 20 Sep 2024 09:57:05 -0400 Subject: [PATCH 32/78] fix: update json tag to match sdkserver (#66) Signed-off-by: Grant Linville --- opts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opts.go b/opts.go index 3184504..779dcb9 100644 --- a/opts.go +++ b/opts.go @@ -70,7 +70,7 @@ type Options struct { IncludeEvents bool `json:"includeEvents"` Prompt bool `json:"prompt"` CredentialOverrides []string `json:"credentialOverrides"` - CredentialContexts []string `json:"credentialContext"` // json tag is left singular to match SDKServer + CredentialContexts []string `json:"credentialContexts"` Location string `json:"location"` ForceSequential bool `json:"forceSequential"` } From 64eaa0ac8caf35ab272e52875552d5a941be819a Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Fri, 20 Sep 2024 19:20:51 -0400 Subject: [PATCH 33/78] fix: allow getting program while run is running Signed-off-by: Donnie Adams --- run.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run.go b/run.go index f832515..a111045 100644 --- a/run.go +++ b/run.go @@ -81,8 +81,8 @@ func (r *Run) Err() error { // Program returns the gptscript program for the run. func (r *Run) Program() *Program { - r.lock.Lock() - defer r.lock.Unlock() + r.callsLock.Lock() + defer r.callsLock.Unlock() return r.program } From 73494ba4b39c2bedd2c7851662084d063b3ec8b2 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Wed, 25 Sep 2024 10:49:51 -0400 Subject: [PATCH 34/78] fix: capture Usage, ChatResponseCached, and ToolResults Additionally add tests to ensure these are captured properly. Signed-off-by: Donnie Adams --- frame.go | 18 ++++++------ gptscript_test.go | 73 +++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 11 deletions(-) diff --git a/frame.go b/frame.go index d4dfd26..39d8ba9 100644 --- a/frame.go +++ b/frame.go @@ -52,14 +52,16 @@ type RunFrame struct { type CallFrame struct { CallContext `json:",inline"` - Type EventType `json:"type"` - Start time.Time `json:"start"` - End time.Time `json:"end"` - Input string `json:"input"` - Output []Output `json:"output"` - Usage Usage `json:"usage"` - LLMRequest any `json:"llmRequest"` - LLMResponse any `json:"llmResponse"` + Type EventType `json:"type"` + Start time.Time `json:"start"` + End time.Time `json:"end"` + Input string `json:"input"` + Output []Output `json:"output"` + Usage Usage `json:"usage"` + ChatResponseCached bool `json:"chatResponseCached"` + ToolResults int `json:"toolResults"` + LLMRequest any `json:"llmRequest"` + LLMResponse any `json:"llmResponse"` } type Usage struct { diff --git a/gptscript_test.go b/gptscript_test.go index b1a2456..ec4419c 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -161,9 +161,7 @@ func TestAbortRun(t *testing.T) { func TestSimpleEvaluate(t *testing.T) { tool := ToolDef{Instructions: "What is the capital of the united states?"} - run, err := g.Evaluate(context.Background(), Options{ - GlobalOptions: GlobalOptions{}, - }, tool) + run, err := g.Evaluate(context.Background(), Options{DisableCache: true}, tool) if err != nil { t.Errorf("Error executing tool: %v", err) } @@ -190,6 +188,17 @@ func TestSimpleEvaluate(t *testing.T) { if run.Program() == nil { t.Error("Run program not set") } + + var promptTokens, completionTokens, totalTokens int + for _, c := range run.calls { + promptTokens += c.Usage.PromptTokens + completionTokens += c.Usage.CompletionTokens + totalTokens += c.Usage.TotalTokens + } + + if promptTokens == 0 || completionTokens == 0 || totalTokens == 0 { + t.Errorf("Usage not set: %d, %d, %d", promptTokens, completionTokens, totalTokens) + } } func TestEvaluateWithContext(t *testing.T) { @@ -285,6 +294,16 @@ func TestEvaluateWithToolList(t *testing.T) { if !strings.Contains(out, "hello there") { t.Errorf("Unexpected output: %s", out) } + + // In this case, we expect the total number of tool results to be 1 + var toolResults int + for _, c := range run.calls { + toolResults += c.ToolResults + } + + if toolResults != 1 { + t.Errorf("Unexpected number of tool results: %d", toolResults) + } } func TestEvaluateWithToolListAndSubTool(t *testing.T) { @@ -361,6 +380,54 @@ func TestStreamEvaluate(t *testing.T) { } } +func TestSimpleRun(t *testing.T) { + wd, err := os.Getwd() + if err != nil { + t.Fatalf("Error getting working directory: %v", err) + } + + run, err := g.Run(context.Background(), wd+"/test/catcher.gpt", Options{}) + if err != nil { + t.Fatalf("Error executing file: %v", err) + } + + out, err := run.Text() + if err != nil { + t.Errorf("Error reading output: %v", err) + } + + if !strings.Contains(out, "Salinger") { + t.Errorf("Unexpected output: %s", out) + } + + if len(run.ErrorOutput()) != 0 { + t.Error("Should have no stderr output") + } + + // Run it a second time, ensuring the same output and that a cached response is used + run, err = g.Run(context.Background(), wd+"/test/catcher.gpt", Options{}) + if err != nil { + t.Fatalf("Error executing file: %v", err) + } + + secondOut, err := run.Text() + if err != nil { + t.Errorf("Error reading output: %v", err) + } + + if secondOut != out { + t.Errorf("Unexpected output on second run: %s != %s", out, secondOut) + } + + // In this case, we expect a single call and that the response is cached + for _, c := range run.calls { + if !c.ChatResponseCached { + t.Error("Chat response should be cached") + } + break + } +} + func TestStreamRun(t *testing.T) { wd, err := os.Getwd() if err != nil { From c72e385625365e45980fa2fd390f695c65d2732a Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Thu, 26 Sep 2024 17:01:43 -0400 Subject: [PATCH 35/78] fix: remove the disable server environment variable Now, when the GPTSCRIPT_URL is passed, the SDK will use it and not start its own server. Additionally, the SDK will pass this server URL to child SDK calls. Signed-off-by: Donnie Adams --- gptscript.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gptscript.go b/gptscript.go index 6a9723f..6f543e5 100644 --- a/gptscript.go +++ b/gptscript.go @@ -51,7 +51,7 @@ func NewGPTScript(opts ...GlobalOptions) (*GPTScript, error) { opt.Env = append(opt.Env, opt.toEnv()...) - if serverProcessCancel == nil && os.Getenv("GPTSCRIPT_DISABLE_SERVER") != "true" { + if serverProcessCancel == nil && os.Getenv("GPTSCRIPT_URL") == "" { if serverURL != "" { u, err := url.Parse(serverURL) if err != nil { @@ -66,6 +66,7 @@ func NewGPTScript(opts ...GlobalOptions) (*GPTScript, error) { opt.URL = "http://" + opt.URL } + opt.Env = append(opt.Env, "GPTSCRIPT_URL="+opt.URL) return &GPTScript{ globalOpts: opt, }, nil @@ -121,6 +122,9 @@ func NewGPTScript(opts ...GlobalOptions) (*GPTScript, error) { if !strings.HasPrefix(opt.URL, "http://") && !strings.HasPrefix(opt.URL, "https://") { opt.URL = "http://" + opt.URL } + + opt.Env = append(opt.Env, "GPTSCRIPT_URL="+opt.URL) + return &GPTScript{ globalOpts: opt, }, nil From 0f93d6c6f9b2b54599aac6a99142c0dfc0e2585c Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Fri, 27 Sep 2024 14:06:02 -0400 Subject: [PATCH 36/78] fix: don't start SDK server if URL opt is provided (#69) Signed-off-by: Donnie Adams --- gptscript.go | 55 ++++++++++++++++++++-------------------------------- 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/gptscript.go b/gptscript.go index 6f543e5..1e30d95 100644 --- a/gptscript.go +++ b/gptscript.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "log/slog" - "net/url" "os" "os/exec" "path/filepath" @@ -34,49 +33,27 @@ type GPTScript struct { func NewGPTScript(opts ...GlobalOptions) (*GPTScript, error) { opt := completeGlobalOptions(opts...) - lock.Lock() - defer lock.Unlock() - gptscriptCount++ - - if serverURL == "" { - serverURL = opt.URL - if serverURL == "" { - serverURL = os.Getenv("GPTSCRIPT_URL") - } - } - if opt.Env == nil { opt.Env = os.Environ() } opt.Env = append(opt.Env, opt.toEnv()...) - if serverProcessCancel == nil && os.Getenv("GPTSCRIPT_URL") == "" { - if serverURL != "" { - u, err := url.Parse(serverURL) - if err != nil { - return nil, fmt.Errorf("failed to parse server URL: %w", err) - } + lock.Lock() + defer lock.Unlock() + gptscriptCount++ - // If the server URL has a path, then this implies that the server is already running. - // In that case, we don't need to start the server. - if u.Path != "" && u.Path != "/" { - opt.URL = serverURL - if !strings.HasPrefix(opt.URL, "http://") && !strings.HasPrefix(opt.URL, "https://") { - opt.URL = "http://" + opt.URL - } - - opt.Env = append(opt.Env, "GPTSCRIPT_URL="+opt.URL) - return &GPTScript{ - globalOpts: opt, - }, nil - } - } + startSDK := serverProcess == nil && serverURL == "" && opt.URL == "" + if serverURL == "" { + serverURL = os.Getenv("GPTSCRIPT_URL") + startSDK = startSDK && serverURL == "" + } + if startSDK { ctx, cancel := context.WithCancel(context.Background()) in, _ := io.Pipe() - serverProcess = exec.CommandContext(ctx, getCommand(), "sys.sdkserver", "--listen-address", strings.TrimPrefix(serverURL, "http://")) + serverProcess = exec.CommandContext(ctx, getCommand(), "sys.sdkserver", "--listen-address", "127.0.0.1:0") serverProcess.Env = opt.Env[:] serverProcess.Stdin = in @@ -118,13 +95,23 @@ func NewGPTScript(opts ...GlobalOptions) (*GPTScript, error) { serverURL = strings.TrimSpace(serverURL) } - opt.URL = serverURL + if opt.URL == "" { + opt.URL = serverURL + } + if !strings.HasPrefix(opt.URL, "http://") && !strings.HasPrefix(opt.URL, "https://") { opt.URL = "http://" + opt.URL } opt.Env = append(opt.Env, "GPTSCRIPT_URL="+opt.URL) + if opt.Token == "" { + opt.Token = os.Getenv("GPTSCRIPT_TOKEN") + } + if opt.Token != "" { + opt.Env = append(opt.Env, "GPTSCRIPT_TOKEN="+opt.Token) + } + return &GPTScript{ globalOpts: opt, }, nil From 15782507bdff0cd74c947c48c0365030fdc2491c Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Fri, 27 Sep 2024 15:46:51 -0400 Subject: [PATCH 37/78] chore: stop setting GPTSCRIPT_ env vars for children Signed-off-by: Donnie Adams --- gptscript.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/gptscript.go b/gptscript.go index 1e30d95..8b8cac3 100644 --- a/gptscript.go +++ b/gptscript.go @@ -103,14 +103,9 @@ func NewGPTScript(opts ...GlobalOptions) (*GPTScript, error) { opt.URL = "http://" + opt.URL } - opt.Env = append(opt.Env, "GPTSCRIPT_URL="+opt.URL) - if opt.Token == "" { opt.Token = os.Getenv("GPTSCRIPT_TOKEN") } - if opt.Token != "" { - opt.Env = append(opt.Env, "GPTSCRIPT_TOKEN="+opt.Token) - } return &GPTScript{ globalOpts: opt, From 2af51434b93e34798c95c1ccd1255c7e50b9175f Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Fri, 27 Sep 2024 17:31:53 -0400 Subject: [PATCH 38/78] Revert "chore: stop setting GPTSCRIPT_ env vars for children" This reverts commit 15782507bdff0cd74c947c48c0365030fdc2491c. --- gptscript.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gptscript.go b/gptscript.go index 8b8cac3..1e30d95 100644 --- a/gptscript.go +++ b/gptscript.go @@ -103,9 +103,14 @@ func NewGPTScript(opts ...GlobalOptions) (*GPTScript, error) { opt.URL = "http://" + opt.URL } + opt.Env = append(opt.Env, "GPTSCRIPT_URL="+opt.URL) + if opt.Token == "" { opt.Token = os.Getenv("GPTSCRIPT_TOKEN") } + if opt.Token != "" { + opt.Env = append(opt.Env, "GPTSCRIPT_TOKEN="+opt.Token) + } return &GPTScript{ globalOpts: opt, From 9064f15a037aa31f85f4b788adafc19ff670c5c9 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Fri, 11 Oct 2024 10:16:47 -0400 Subject: [PATCH 39/78] chore: remove windows test workflow Signed-off-by: Donnie Adams --- .github/workflows/run_tests.yaml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index c06061c..28eb3ae 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -36,24 +36,3 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} run: make test - - test-windows: - runs-on: windows-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 1 - ref: ${{ inputs.git_ref }} - - uses: actions/setup-go@v5 - with: - cache: false - go-version: "1.23" - - name: Install gptscript - run: | - curl https://get.gptscript.ai/releases/default_windows_amd64_v1/gptscript.exe -o gptscript.exe - - name: Run Tests - env: - GPTSCRIPT_BIN: .\gptscript.exe - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - run: make test From 3e8b23c009f7cea03bef2fc5387e6126af695e82 Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Fri, 11 Oct 2024 10:27:59 -0400 Subject: [PATCH 40/78] feat: add dataset functions (#70) Signed-off-by: Grant Linville --- datasets.go | 50 ++++++++++++++ gptscript.go | 171 ++++++++++++++++++++++++++++++++++++++++++++++ gptscript_test.go | 47 ++++++++++++- opts.go | 2 + 4 files changed, 268 insertions(+), 2 deletions(-) create mode 100644 datasets.go diff --git a/datasets.go b/datasets.go new file mode 100644 index 0000000..45b665b --- /dev/null +++ b/datasets.go @@ -0,0 +1,50 @@ +package gptscript + +type DatasetElementMeta struct { + Name string `json:"name"` + Description string `json:"description"` +} + +type DatasetElement struct { + DatasetElementMeta `json:",inline"` + Contents string `json:"contents"` +} + +type DatasetMeta struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` +} + +type Dataset struct { + DatasetMeta `json:",inline"` + BaseDir string `json:"baseDir,omitempty"` + Elements map[string]DatasetElementMeta `json:"elements"` +} + +type datasetRequest struct { + Input string `json:"input"` + Workspace string `json:"workspace"` + DatasetToolRepo string `json:"datasetToolRepo"` +} + +type createDatasetArgs struct { + Name string `json:"datasetName"` + Description string `json:"datasetDescription"` +} + +type addDatasetElementArgs struct { + DatasetID string `json:"datasetID"` + ElementName string `json:"elementName"` + ElementDescription string `json:"elementDescription"` + ElementContent string `json:"elementContent"` +} + +type listDatasetElementArgs struct { + DatasetID string `json:"datasetID"` +} + +type getDatasetElementArgs struct { + DatasetID string `json:"datasetID"` + Element string `json:"element"` +} diff --git a/gptscript.go b/gptscript.go index 1e30d95..6178c67 100644 --- a/gptscript.go +++ b/gptscript.go @@ -7,6 +7,7 @@ import ( "context" "encoding/base64" "encoding/json" + "errors" "fmt" "io" "log/slog" @@ -388,6 +389,176 @@ func (g *GPTScript) DeleteCredential(ctx context.Context, credCtx, name string) return err } +// Dataset methods + +func (g *GPTScript) ListDatasets(ctx context.Context, workspace string) ([]DatasetMeta, error) { + if workspace == "" { + workspace = os.Getenv("GPTSCRIPT_WORKSPACE_DIR") + } + + out, err := g.runBasicCommand(ctx, "datasets", datasetRequest{ + Input: "{}", + Workspace: workspace, + DatasetToolRepo: g.globalOpts.DatasetToolRepo, + }) + + if err != nil { + return nil, err + } + + if strings.HasPrefix(out, "ERROR:") { + return nil, errors.New(out) + } + + var datasets []DatasetMeta + if err = json.Unmarshal([]byte(out), &datasets); err != nil { + return nil, err + } + return datasets, nil +} + +func (g *GPTScript) CreateDataset(ctx context.Context, workspace, name, description string) (Dataset, error) { + if workspace == "" { + workspace = os.Getenv("GPTSCRIPT_WORKSPACE_DIR") + } + + args := createDatasetArgs{ + Name: name, + Description: description, + } + argsJSON, err := json.Marshal(args) + if err != nil { + return Dataset{}, fmt.Errorf("failed to marshal dataset args: %w", err) + } + + out, err := g.runBasicCommand(ctx, "datasets/create", datasetRequest{ + Input: string(argsJSON), + Workspace: workspace, + DatasetToolRepo: g.globalOpts.DatasetToolRepo, + }) + + if err != nil { + return Dataset{}, err + } + + if strings.HasPrefix(out, "ERROR:") { + return Dataset{}, errors.New(out) + } + + var dataset Dataset + if err = json.Unmarshal([]byte(out), &dataset); err != nil { + return Dataset{}, err + } + return dataset, nil +} + +func (g *GPTScript) AddDatasetElement(ctx context.Context, workspace, datasetID, elementName, elementDescription, elementContent string) (DatasetElementMeta, error) { + if workspace == "" { + workspace = os.Getenv("GPTSCRIPT_WORKSPACE_DIR") + } + + args := addDatasetElementArgs{ + DatasetID: datasetID, + ElementName: elementName, + ElementDescription: elementDescription, + ElementContent: elementContent, + } + argsJSON, err := json.Marshal(args) + if err != nil { + return DatasetElementMeta{}, fmt.Errorf("failed to marshal element args: %w", err) + } + + out, err := g.runBasicCommand(ctx, "datasets/add-element", datasetRequest{ + Input: string(argsJSON), + Workspace: workspace, + DatasetToolRepo: g.globalOpts.DatasetToolRepo, + }) + + if err != nil { + return DatasetElementMeta{}, err + } + + if strings.HasPrefix(out, "ERROR:") { + return DatasetElementMeta{}, errors.New(out) + } + + var element DatasetElementMeta + if err = json.Unmarshal([]byte(out), &element); err != nil { + return DatasetElementMeta{}, err + } + return element, nil +} + +func (g *GPTScript) ListDatasetElements(ctx context.Context, workspace, datasetID string) ([]DatasetElementMeta, error) { + if workspace == "" { + workspace = os.Getenv("GPTSCRIPT_WORKSPACE_DIR") + } + + args := listDatasetElementArgs{ + DatasetID: datasetID, + } + argsJSON, err := json.Marshal(args) + if err != nil { + return nil, fmt.Errorf("failed to marshal element args: %w", err) + } + + out, err := g.runBasicCommand(ctx, "datasets/list-elements", datasetRequest{ + Input: string(argsJSON), + Workspace: workspace, + DatasetToolRepo: g.globalOpts.DatasetToolRepo, + }) + + if err != nil { + return nil, err + } + + if strings.HasPrefix(out, "ERROR:") { + return nil, errors.New(out) + } + + var elements []DatasetElementMeta + if err = json.Unmarshal([]byte(out), &elements); err != nil { + return nil, err + } + return elements, nil +} + +func (g *GPTScript) GetDatasetElement(ctx context.Context, workspace, datasetID, elementName string) (DatasetElement, error) { + if workspace == "" { + workspace = os.Getenv("GPTSCRIPT_WORKSPACE_DIR") + } + + args := getDatasetElementArgs{ + DatasetID: datasetID, + Element: elementName, + } + argsJSON, err := json.Marshal(args) + if err != nil { + return DatasetElement{}, fmt.Errorf("failed to marshal element args: %w", err) + } + + out, err := g.runBasicCommand(ctx, "datasets/get-element", datasetRequest{ + Input: string(argsJSON), + Workspace: workspace, + DatasetToolRepo: g.globalOpts.DatasetToolRepo, + }) + + if err != nil { + return DatasetElement{}, err + } + + if strings.HasPrefix(out, "ERROR:") { + return DatasetElement{}, errors.New(out) + } + + var element DatasetElement + if err = json.Unmarshal([]byte(out), &element); err != nil { + return DatasetElement{}, err + } + + return element, nil +} + func (g *GPTScript) runBasicCommand(ctx context.Context, requestPath string, body any) (string, error) { run := &Run{ url: g.globalOpts.URL, diff --git a/gptscript_test.go b/gptscript_test.go index ec4419c..eb471c8 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -670,7 +670,7 @@ func TestParseToolWithTextNode(t *testing.T) { t.Fatalf("No text node found") } - if tools[1].TextNode.Text != "hello\n" { + if strings.TrimSpace(tools[1].TextNode.Text) != "hello" { t.Errorf("Unexpected text: %s", tools[1].TextNode.Text) } if tools[1].TextNode.Fmt != "markdown" { @@ -1047,7 +1047,7 @@ func TestConfirmDeny(t *testing.T) { return } - if !strings.Contains(confirmCallEvent.Input, "\"ls\"") { + if !strings.Contains(confirmCallEvent.Input, "ls") { t.Errorf("unexpected confirm input: %s", confirmCallEvent.Input) } @@ -1560,3 +1560,46 @@ func TestCredentials(t *testing.T) { require.Error(t, err) require.True(t, errors.As(err, &ErrNotFound{})) } + +func TestDatasets(t *testing.T) { + workspace, err := os.MkdirTemp("", "go-gptscript-test") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(workspace) + }() + + // Create a dataset + dataset, err := g.CreateDataset(context.Background(), workspace, "test-dataset", "This is a test dataset") + require.NoError(t, err) + require.Equal(t, "test-dataset", dataset.Name) + require.Equal(t, "This is a test dataset", dataset.Description) + require.Equal(t, 0, len(dataset.Elements)) + + // Add an element + elementMeta, err := g.AddDatasetElement(context.Background(), workspace, dataset.ID, "test-element", "This is a test element", "This is the content") + require.NoError(t, err) + require.Equal(t, "test-element", elementMeta.Name) + require.Equal(t, "This is a test element", elementMeta.Description) + + // Get the element + element, err := g.GetDatasetElement(context.Background(), workspace, dataset.ID, "test-element") + require.NoError(t, err) + require.Equal(t, "test-element", element.Name) + require.Equal(t, "This is a test element", element.Description) + require.Equal(t, "This is the content", element.Contents) + + // List elements in the dataset + elements, err := g.ListDatasetElements(context.Background(), workspace, dataset.ID) + require.NoError(t, err) + require.Equal(t, 1, len(elements)) + require.Equal(t, "test-element", elements[0].Name) + require.Equal(t, "This is a test element", elements[0].Description) + + // List datasets + datasets, err := g.ListDatasets(context.Background(), workspace) + require.NoError(t, err) + require.Equal(t, 1, len(datasets)) + require.Equal(t, "test-dataset", datasets[0].Name) + require.Equal(t, "This is a test dataset", datasets[0].Description) + require.Equal(t, dataset.ID, datasets[0].ID) +} diff --git a/opts.go b/opts.go index 779dcb9..283b4ec 100644 --- a/opts.go +++ b/opts.go @@ -11,6 +11,7 @@ type GlobalOptions struct { DefaultModelProvider string `json:"DefaultModelProvider"` CacheDir string `json:"CacheDir"` Env []string `json:"env"` + DatasetToolRepo string `json:"DatasetToolRepo"` } func (g GlobalOptions) toEnv() []string { @@ -41,6 +42,7 @@ func completeGlobalOptions(opts ...GlobalOptions) GlobalOptions { result.OpenAIBaseURL = firstSet(opt.OpenAIBaseURL, result.OpenAIBaseURL) result.DefaultModel = firstSet(opt.DefaultModel, result.DefaultModel) result.DefaultModelProvider = firstSet(opt.DefaultModelProvider, result.DefaultModelProvider) + result.DatasetToolRepo = firstSet(opt.DatasetToolRepo, result.DatasetToolRepo) result.Env = append(result.Env, opt.Env...) } return result From a7d2d945d5e6e8742ffa2c3ebbf12dfe950f754f Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Mon, 14 Oct 2024 07:27:20 -0700 Subject: [PATCH 41/78] bug: remove duplicate field which breaks tooldef rendering --- gptscript_test.go | 32 ++++++++++++++++---------------- tool.go | 1 - 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/gptscript_test.go b/gptscript_test.go index eb471c8..82bf735 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -696,14 +696,14 @@ func TestFmt(t *testing.T) { ToolDef: ToolDef{ Name: "echo", Instructions: "#!/bin/bash\necho hello there", - }, - Arguments: &openapi3.Schema{ - Type: &openapi3.Types{"object"}, - Properties: map[string]*openapi3.SchemaRef{ - "input": { - Value: &openapi3.Schema{ - Description: "The string input to echo", - Type: &openapi3.Types{"string"}, + Arguments: &openapi3.Schema{ + Type: &openapi3.Types{"object"}, + Properties: map[string]*openapi3.SchemaRef{ + "input": { + Value: &openapi3.Schema{ + Description: "The string input to echo", + Type: &openapi3.Types{"string"}, + }, }, }, }, @@ -757,14 +757,14 @@ func TestFmtWithTextNode(t *testing.T) { ToolDef: ToolDef{ Instructions: "#!/bin/bash\necho hello there", Name: "echo", - }, - Arguments: &openapi3.Schema{ - Type: &openapi3.Types{"object"}, - Properties: map[string]*openapi3.SchemaRef{ - "input": { - Value: &openapi3.Schema{ - Description: "The string input to echo", - Type: &openapi3.Types{"string"}, + Arguments: &openapi3.Schema{ + Type: &openapi3.Types{"object"}, + Properties: map[string]*openapi3.SchemaRef{ + "input": { + Value: &openapi3.Schema{ + Description: "The string input to echo", + Type: &openapi3.Types{"string"}, + }, }, }, }, diff --git a/tool.go b/tool.go index 36eb535..306723b 100644 --- a/tool.go +++ b/tool.go @@ -105,7 +105,6 @@ type ToolNode struct { type Tool struct { ToolDef `json:",inline"` ID string `json:"id,omitempty"` - Arguments *openapi3.Schema `json:"arguments,omitempty"` ToolMapping map[string][]ToolReference `json:"toolMapping,omitempty"` LocalTools map[string]string `json:"localTools,omitempty"` Source ToolSource `json:"source,omitempty"` From 91a600f715e76e8573bb73bacd3068b4d1b76b44 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Tue, 15 Oct 2024 16:41:11 -0400 Subject: [PATCH 42/78] feat: add workspace API support (#72) This change also reorganizes some of the code. Signed-off-by: Donnie Adams --- datasets.go | 150 ++++++++++++++++++++++++++++++++++++ datasets_test.go | 52 +++++++++++++ gptscript.go | 171 ----------------------------------------- gptscript_test.go | 43 ----------- opts.go | 1 + workspace.go | 189 ++++++++++++++++++++++++++++++++++++++++++++++ workspace_test.go | 172 +++++++++++++++++++++++++++++++++++++++++ 7 files changed, 564 insertions(+), 214 deletions(-) create mode 100644 datasets_test.go create mode 100644 workspace.go create mode 100644 workspace_test.go diff --git a/datasets.go b/datasets.go index 45b665b..c053a26 100644 --- a/datasets.go +++ b/datasets.go @@ -1,5 +1,12 @@ package gptscript +import ( + "context" + "encoding/json" + "fmt" + "os" +) + type DatasetElementMeta struct { Name string `json:"name"` Description string `json:"description"` @@ -48,3 +55,146 @@ type getDatasetElementArgs struct { DatasetID string `json:"datasetID"` Element string `json:"element"` } + +func (g *GPTScript) ListDatasets(ctx context.Context, workspace string) ([]DatasetMeta, error) { + if workspace == "" { + workspace = os.Getenv("GPTSCRIPT_WORKSPACE_DIR") + } + + out, err := g.runBasicCommand(ctx, "datasets", datasetRequest{ + Input: "{}", + Workspace: workspace, + DatasetToolRepo: g.globalOpts.DatasetToolRepo, + }) + if err != nil { + return nil, err + } + + var datasets []DatasetMeta + if err = json.Unmarshal([]byte(out), &datasets); err != nil { + return nil, err + } + return datasets, nil +} + +func (g *GPTScript) CreateDataset(ctx context.Context, workspace, name, description string) (Dataset, error) { + if workspace == "" { + workspace = os.Getenv("GPTSCRIPT_WORKSPACE_DIR") + } + + args := createDatasetArgs{ + Name: name, + Description: description, + } + argsJSON, err := json.Marshal(args) + if err != nil { + return Dataset{}, fmt.Errorf("failed to marshal dataset args: %w", err) + } + + out, err := g.runBasicCommand(ctx, "datasets/create", datasetRequest{ + Input: string(argsJSON), + Workspace: workspace, + DatasetToolRepo: g.globalOpts.DatasetToolRepo, + }) + if err != nil { + return Dataset{}, err + } + + var dataset Dataset + if err = json.Unmarshal([]byte(out), &dataset); err != nil { + return Dataset{}, err + } + return dataset, nil +} + +func (g *GPTScript) AddDatasetElement(ctx context.Context, workspace, datasetID, elementName, elementDescription, elementContent string) (DatasetElementMeta, error) { + if workspace == "" { + workspace = os.Getenv("GPTSCRIPT_WORKSPACE_DIR") + } + + args := addDatasetElementArgs{ + DatasetID: datasetID, + ElementName: elementName, + ElementDescription: elementDescription, + ElementContent: elementContent, + } + argsJSON, err := json.Marshal(args) + if err != nil { + return DatasetElementMeta{}, fmt.Errorf("failed to marshal element args: %w", err) + } + + out, err := g.runBasicCommand(ctx, "datasets/add-element", datasetRequest{ + Input: string(argsJSON), + Workspace: workspace, + DatasetToolRepo: g.globalOpts.DatasetToolRepo, + }) + if err != nil { + return DatasetElementMeta{}, err + } + + var element DatasetElementMeta + if err = json.Unmarshal([]byte(out), &element); err != nil { + return DatasetElementMeta{}, err + } + return element, nil +} + +func (g *GPTScript) ListDatasetElements(ctx context.Context, workspace, datasetID string) ([]DatasetElementMeta, error) { + if workspace == "" { + workspace = os.Getenv("GPTSCRIPT_WORKSPACE_DIR") + } + + args := listDatasetElementArgs{ + DatasetID: datasetID, + } + argsJSON, err := json.Marshal(args) + if err != nil { + return nil, fmt.Errorf("failed to marshal element args: %w", err) + } + + out, err := g.runBasicCommand(ctx, "datasets/list-elements", datasetRequest{ + Input: string(argsJSON), + Workspace: workspace, + DatasetToolRepo: g.globalOpts.DatasetToolRepo, + }) + if err != nil { + return nil, err + } + + var elements []DatasetElementMeta + if err = json.Unmarshal([]byte(out), &elements); err != nil { + return nil, err + } + return elements, nil +} + +func (g *GPTScript) GetDatasetElement(ctx context.Context, workspace, datasetID, elementName string) (DatasetElement, error) { + if workspace == "" { + workspace = os.Getenv("GPTSCRIPT_WORKSPACE_DIR") + } + + args := getDatasetElementArgs{ + DatasetID: datasetID, + Element: elementName, + } + argsJSON, err := json.Marshal(args) + if err != nil { + return DatasetElement{}, fmt.Errorf("failed to marshal element args: %w", err) + } + + out, err := g.runBasicCommand(ctx, "datasets/get-element", datasetRequest{ + Input: string(argsJSON), + Workspace: workspace, + DatasetToolRepo: g.globalOpts.DatasetToolRepo, + }) + if err != nil { + return DatasetElement{}, err + } + + var element DatasetElement + if err = json.Unmarshal([]byte(out), &element); err != nil { + return DatasetElement{}, err + } + + return element, nil +} diff --git a/datasets_test.go b/datasets_test.go new file mode 100644 index 0000000..3763982 --- /dev/null +++ b/datasets_test.go @@ -0,0 +1,52 @@ +package gptscript + +import ( + "context" + "os" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDatasets(t *testing.T) { + workspace, err := os.MkdirTemp("", "go-gptscript-test") + require.NoError(t, err) + defer func() { + _ = os.RemoveAll(workspace) + }() + + // Create a dataset + dataset, err := g.CreateDataset(context.Background(), workspace, "test-dataset", "This is a test dataset") + require.NoError(t, err) + require.Equal(t, "test-dataset", dataset.Name) + require.Equal(t, "This is a test dataset", dataset.Description) + require.Equal(t, 0, len(dataset.Elements)) + + // Add an element + elementMeta, err := g.AddDatasetElement(context.Background(), workspace, dataset.ID, "test-element", "This is a test element", "This is the content") + require.NoError(t, err) + require.Equal(t, "test-element", elementMeta.Name) + require.Equal(t, "This is a test element", elementMeta.Description) + + // Get the element + element, err := g.GetDatasetElement(context.Background(), workspace, dataset.ID, "test-element") + require.NoError(t, err) + require.Equal(t, "test-element", element.Name) + require.Equal(t, "This is a test element", element.Description) + require.Equal(t, "This is the content", element.Contents) + + // List elements in the dataset + elements, err := g.ListDatasetElements(context.Background(), workspace, dataset.ID) + require.NoError(t, err) + require.Equal(t, 1, len(elements)) + require.Equal(t, "test-element", elements[0].Name) + require.Equal(t, "This is a test element", elements[0].Description) + + // List datasets + datasets, err := g.ListDatasets(context.Background(), workspace) + require.NoError(t, err) + require.Equal(t, 1, len(datasets)) + require.Equal(t, "test-dataset", datasets[0].Name) + require.Equal(t, "This is a test dataset", datasets[0].Description) + require.Equal(t, dataset.ID, datasets[0].ID) +} diff --git a/gptscript.go b/gptscript.go index 6178c67..1e30d95 100644 --- a/gptscript.go +++ b/gptscript.go @@ -7,7 +7,6 @@ import ( "context" "encoding/base64" "encoding/json" - "errors" "fmt" "io" "log/slog" @@ -389,176 +388,6 @@ func (g *GPTScript) DeleteCredential(ctx context.Context, credCtx, name string) return err } -// Dataset methods - -func (g *GPTScript) ListDatasets(ctx context.Context, workspace string) ([]DatasetMeta, error) { - if workspace == "" { - workspace = os.Getenv("GPTSCRIPT_WORKSPACE_DIR") - } - - out, err := g.runBasicCommand(ctx, "datasets", datasetRequest{ - Input: "{}", - Workspace: workspace, - DatasetToolRepo: g.globalOpts.DatasetToolRepo, - }) - - if err != nil { - return nil, err - } - - if strings.HasPrefix(out, "ERROR:") { - return nil, errors.New(out) - } - - var datasets []DatasetMeta - if err = json.Unmarshal([]byte(out), &datasets); err != nil { - return nil, err - } - return datasets, nil -} - -func (g *GPTScript) CreateDataset(ctx context.Context, workspace, name, description string) (Dataset, error) { - if workspace == "" { - workspace = os.Getenv("GPTSCRIPT_WORKSPACE_DIR") - } - - args := createDatasetArgs{ - Name: name, - Description: description, - } - argsJSON, err := json.Marshal(args) - if err != nil { - return Dataset{}, fmt.Errorf("failed to marshal dataset args: %w", err) - } - - out, err := g.runBasicCommand(ctx, "datasets/create", datasetRequest{ - Input: string(argsJSON), - Workspace: workspace, - DatasetToolRepo: g.globalOpts.DatasetToolRepo, - }) - - if err != nil { - return Dataset{}, err - } - - if strings.HasPrefix(out, "ERROR:") { - return Dataset{}, errors.New(out) - } - - var dataset Dataset - if err = json.Unmarshal([]byte(out), &dataset); err != nil { - return Dataset{}, err - } - return dataset, nil -} - -func (g *GPTScript) AddDatasetElement(ctx context.Context, workspace, datasetID, elementName, elementDescription, elementContent string) (DatasetElementMeta, error) { - if workspace == "" { - workspace = os.Getenv("GPTSCRIPT_WORKSPACE_DIR") - } - - args := addDatasetElementArgs{ - DatasetID: datasetID, - ElementName: elementName, - ElementDescription: elementDescription, - ElementContent: elementContent, - } - argsJSON, err := json.Marshal(args) - if err != nil { - return DatasetElementMeta{}, fmt.Errorf("failed to marshal element args: %w", err) - } - - out, err := g.runBasicCommand(ctx, "datasets/add-element", datasetRequest{ - Input: string(argsJSON), - Workspace: workspace, - DatasetToolRepo: g.globalOpts.DatasetToolRepo, - }) - - if err != nil { - return DatasetElementMeta{}, err - } - - if strings.HasPrefix(out, "ERROR:") { - return DatasetElementMeta{}, errors.New(out) - } - - var element DatasetElementMeta - if err = json.Unmarshal([]byte(out), &element); err != nil { - return DatasetElementMeta{}, err - } - return element, nil -} - -func (g *GPTScript) ListDatasetElements(ctx context.Context, workspace, datasetID string) ([]DatasetElementMeta, error) { - if workspace == "" { - workspace = os.Getenv("GPTSCRIPT_WORKSPACE_DIR") - } - - args := listDatasetElementArgs{ - DatasetID: datasetID, - } - argsJSON, err := json.Marshal(args) - if err != nil { - return nil, fmt.Errorf("failed to marshal element args: %w", err) - } - - out, err := g.runBasicCommand(ctx, "datasets/list-elements", datasetRequest{ - Input: string(argsJSON), - Workspace: workspace, - DatasetToolRepo: g.globalOpts.DatasetToolRepo, - }) - - if err != nil { - return nil, err - } - - if strings.HasPrefix(out, "ERROR:") { - return nil, errors.New(out) - } - - var elements []DatasetElementMeta - if err = json.Unmarshal([]byte(out), &elements); err != nil { - return nil, err - } - return elements, nil -} - -func (g *GPTScript) GetDatasetElement(ctx context.Context, workspace, datasetID, elementName string) (DatasetElement, error) { - if workspace == "" { - workspace = os.Getenv("GPTSCRIPT_WORKSPACE_DIR") - } - - args := getDatasetElementArgs{ - DatasetID: datasetID, - Element: elementName, - } - argsJSON, err := json.Marshal(args) - if err != nil { - return DatasetElement{}, fmt.Errorf("failed to marshal element args: %w", err) - } - - out, err := g.runBasicCommand(ctx, "datasets/get-element", datasetRequest{ - Input: string(argsJSON), - Workspace: workspace, - DatasetToolRepo: g.globalOpts.DatasetToolRepo, - }) - - if err != nil { - return DatasetElement{}, err - } - - if strings.HasPrefix(out, "ERROR:") { - return DatasetElement{}, errors.New(out) - } - - var element DatasetElement - if err = json.Unmarshal([]byte(out), &element); err != nil { - return DatasetElement{}, err - } - - return element, nil -} - func (g *GPTScript) runBasicCommand(ctx context.Context, requestPath string, body any) (string, error) { run := &Run{ url: g.globalOpts.URL, diff --git a/gptscript_test.go b/gptscript_test.go index 82bf735..32adff9 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -1560,46 +1560,3 @@ func TestCredentials(t *testing.T) { require.Error(t, err) require.True(t, errors.As(err, &ErrNotFound{})) } - -func TestDatasets(t *testing.T) { - workspace, err := os.MkdirTemp("", "go-gptscript-test") - require.NoError(t, err) - defer func() { - _ = os.RemoveAll(workspace) - }() - - // Create a dataset - dataset, err := g.CreateDataset(context.Background(), workspace, "test-dataset", "This is a test dataset") - require.NoError(t, err) - require.Equal(t, "test-dataset", dataset.Name) - require.Equal(t, "This is a test dataset", dataset.Description) - require.Equal(t, 0, len(dataset.Elements)) - - // Add an element - elementMeta, err := g.AddDatasetElement(context.Background(), workspace, dataset.ID, "test-element", "This is a test element", "This is the content") - require.NoError(t, err) - require.Equal(t, "test-element", elementMeta.Name) - require.Equal(t, "This is a test element", elementMeta.Description) - - // Get the element - element, err := g.GetDatasetElement(context.Background(), workspace, dataset.ID, "test-element") - require.NoError(t, err) - require.Equal(t, "test-element", element.Name) - require.Equal(t, "This is a test element", element.Description) - require.Equal(t, "This is the content", element.Contents) - - // List elements in the dataset - elements, err := g.ListDatasetElements(context.Background(), workspace, dataset.ID) - require.NoError(t, err) - require.Equal(t, 1, len(elements)) - require.Equal(t, "test-element", elements[0].Name) - require.Equal(t, "This is a test element", elements[0].Description) - - // List datasets - datasets, err := g.ListDatasets(context.Background(), workspace) - require.NoError(t, err) - require.Equal(t, 1, len(datasets)) - require.Equal(t, "test-dataset", datasets[0].Name) - require.Equal(t, "This is a test dataset", datasets[0].Description) - require.Equal(t, dataset.ID, datasets[0].ID) -} diff --git a/opts.go b/opts.go index 283b4ec..6c9e1ab 100644 --- a/opts.go +++ b/opts.go @@ -12,6 +12,7 @@ type GlobalOptions struct { CacheDir string `json:"CacheDir"` Env []string `json:"env"` DatasetToolRepo string `json:"DatasetToolRepo"` + WorkspaceTool string `json:"WorkspaceTool"` } func (g GlobalOptions) toEnv() []string { diff --git a/workspace.go b/workspace.go new file mode 100644 index 0000000..86b8c2e --- /dev/null +++ b/workspace.go @@ -0,0 +1,189 @@ +package gptscript + +import ( + "context" + "encoding/base64" + "encoding/json" + "strings" +) + +func (g *GPTScript) CreateWorkspace(ctx context.Context, providerType string) (string, error) { + out, err := g.runBasicCommand(ctx, "workspaces/create", map[string]any{ + "provider": providerType, + "workspaceTool": g.globalOpts.WorkspaceTool, + }) + if err != nil { + return "", err + } + + return strings.TrimSpace(out), nil +} + +type DeleteWorkspaceOptions struct { + IgnoreNotFound bool +} + +func (g *GPTScript) DeleteWorkspace(ctx context.Context, workspaceID string, opts ...DeleteWorkspaceOptions) error { + var opt DeleteWorkspaceOptions + for _, o := range opts { + opt.IgnoreNotFound = opt.IgnoreNotFound || o.IgnoreNotFound + } + _, err := g.runBasicCommand(ctx, "workspaces/delete", map[string]any{ + "id": workspaceID, + "ignoreNotFound": opt.IgnoreNotFound, + "workspaceTool": g.globalOpts.WorkspaceTool, + }) + + return err +} + +type CreateDirectoryInWorkspaceOptions struct { + IgnoreExists bool +} + +func (g *GPTScript) CreateDirectoryInWorkspace(ctx context.Context, workspaceID, dir string, opts ...CreateDirectoryInWorkspaceOptions) error { + var opt CreateDirectoryInWorkspaceOptions + for _, o := range opts { + opt.IgnoreExists = opt.IgnoreExists || o.IgnoreExists + } + + _, err := g.runBasicCommand(ctx, "workspaces/mkdir", map[string]any{ + "id": workspaceID, + "directoryName": dir, + "ignoreExists": opt.IgnoreExists, + "workspaceTool": g.globalOpts.WorkspaceTool, + }) + + return err +} + +type DeleteDirectoryInWorkspaceOptions struct { + IgnoreNotFound bool + MustBeEmpty bool +} + +func (g *GPTScript) DeleteDirectoryInWorkspace(ctx context.Context, workspaceID, dir string, opts ...DeleteDirectoryInWorkspaceOptions) error { + var opt DeleteDirectoryInWorkspaceOptions + for _, o := range opts { + o.IgnoreNotFound = opt.IgnoreNotFound || o.IgnoreNotFound + o.MustBeEmpty = opt.MustBeEmpty || o.MustBeEmpty + } + + _, err := g.runBasicCommand(ctx, "workspaces/rmdir", map[string]any{ + "id": workspaceID, + "directoryName": dir, + "ignoreNotFound": opt.IgnoreNotFound, + "mustBeEmpty": opt.MustBeEmpty, + "workspaceTool": g.globalOpts.WorkspaceTool, + }) + + return err +} + +type ListFilesInWorkspaceOptions struct { + SubDir string + NonRecursive bool + ExcludeHidden bool +} + +type WorkspaceContent struct { + ID, Path, FileName string + Children []WorkspaceContent +} + +func (g *GPTScript) ListFilesInWorkspace(ctx context.Context, workspaceID string, opts ...ListFilesInWorkspaceOptions) (*WorkspaceContent, error) { + var opt ListFilesInWorkspaceOptions + for _, o := range opts { + if o.SubDir != "" { + opt.SubDir = o.SubDir + } + opt.NonRecursive = opt.NonRecursive || o.NonRecursive + opt.ExcludeHidden = opt.ExcludeHidden || o.ExcludeHidden + } + + out, err := g.runBasicCommand(ctx, "workspaces/list", map[string]any{ + "id": workspaceID, + "subDir": opt.SubDir, + "excludeHidden": opt.ExcludeHidden, + "nonRecursive": opt.NonRecursive, + "workspaceTool": g.globalOpts.WorkspaceTool, + "json": true, + }) + if err != nil { + return nil, err + } + + var content []WorkspaceContent + err = json.Unmarshal([]byte(out), &content) + if err != nil { + return nil, err + } + + if len(content) == 0 { + return &WorkspaceContent{ID: workspaceID}, nil + } + + return &content[0], nil +} + +type CreateFileInWorkspaceOptions struct { + MustNotExist bool + WithoutCreate bool + CreateDirs bool +} + +func (g *GPTScript) WriteFileInWorkspace(ctx context.Context, workspaceID, filePath string, contents []byte, opts ...CreateFileInWorkspaceOptions) error { + var opt CreateFileInWorkspaceOptions + for _, o := range opts { + opt.MustNotExist = opt.MustNotExist || o.MustNotExist + opt.WithoutCreate = opt.WithoutCreate || o.WithoutCreate + opt.CreateDirs = opt.CreateDirs || o.CreateDirs + } + + _, err := g.runBasicCommand(ctx, "workspaces/write-file", map[string]any{ + "id": workspaceID, + "contents": base64.StdEncoding.EncodeToString(contents), + "filePath": filePath, + "mustNotExist": opt.MustNotExist, + "withoutCreate": opt.WithoutCreate, + "createDirs": opt.CreateDirs, + "workspaceTool": g.globalOpts.WorkspaceTool, + "base64EncodedInput": true, + }) + + return err +} + +type DeleteFileInWorkspaceOptions struct { + IgnoreNotFound bool +} + +func (g *GPTScript) DeleteFileInWorkspace(ctx context.Context, workspaceID, filePath string, opts ...DeleteFileInWorkspaceOptions) error { + var opt DeleteFileInWorkspaceOptions + for _, o := range opts { + opt.IgnoreNotFound = opt.IgnoreNotFound || o.IgnoreNotFound + } + + _, err := g.runBasicCommand(ctx, "workspaces/delete-file", map[string]any{ + "id": workspaceID, + "filePath": filePath, + "ignoreNotFound": opt.IgnoreNotFound, + "workspaceTool": g.globalOpts.WorkspaceTool, + }) + + return err +} + +func (g *GPTScript) ReadFileInWorkspace(ctx context.Context, workspaceID, filePath string) ([]byte, error) { + out, err := g.runBasicCommand(ctx, "workspaces/read-file", map[string]any{ + "id": workspaceID, + "filePath": filePath, + "workspaceTool": g.globalOpts.WorkspaceTool, + "base64EncodeOutput": true, + }) + if err != nil { + return nil, err + } + + return base64.StdEncoding.DecodeString(out) +} diff --git a/workspace_test.go b/workspace_test.go new file mode 100644 index 0000000..cba683f --- /dev/null +++ b/workspace_test.go @@ -0,0 +1,172 @@ +package gptscript + +import ( + "bytes" + "context" + "testing" +) + +func TestCreateAndDeleteWorkspace(t *testing.T) { + id, err := g.CreateWorkspace(context.Background(), "directory") + if err != nil { + t.Fatalf("Error creating workspace: %v", err) + } + + err = g.DeleteWorkspace(context.Background(), id) + if err != nil { + t.Errorf("Error deleting workspace: %v", err) + } +} + +func TestCreateDirectory(t *testing.T) { + id, err := g.CreateWorkspace(context.Background(), "directory") + if err != nil { + t.Fatalf("Error creating workspace: %v", err) + } + + t.Cleanup(func() { + err := g.DeleteWorkspace(context.Background(), id) + if err != nil { + t.Errorf("Error deleting workspace: %v", err) + } + }) + + err = g.CreateDirectoryInWorkspace(context.Background(), id, "test") + if err != nil { + t.Fatalf("Error creating directory: %v", err) + } + + err = g.DeleteDirectoryInWorkspace(context.Background(), id, "test") + if err != nil { + t.Errorf("Error listing files: %v", err) + } +} + +func TestWriteReadAndDeleteFileFromWorkspace(t *testing.T) { + id, err := g.CreateWorkspace(context.Background(), "directory") + if err != nil { + t.Fatalf("Error creating workspace: %v", err) + } + + t.Cleanup(func() { + err := g.DeleteWorkspace(context.Background(), id) + if err != nil { + t.Errorf("Error deleting workspace: %v", err) + } + }) + + err = g.WriteFileInWorkspace(context.Background(), id, "test.txt", []byte("test")) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + content, err := g.ReadFileInWorkspace(context.Background(), id, "test.txt") + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if !bytes.Equal(content, []byte("test")) { + t.Errorf("Unexpected content: %s", content) + } + + err = g.DeleteFileInWorkspace(context.Background(), id, "test.txt") + if err != nil { + t.Errorf("Error deleting file: %v", err) + } +} + +func TestLsComplexWorkspace(t *testing.T) { + id, err := g.CreateWorkspace(context.Background(), "directory") + if err != nil { + t.Fatalf("Error creating workspace: %v", err) + } + + t.Cleanup(func() { + err := g.DeleteWorkspace(context.Background(), id) + if err != nil { + t.Errorf("Error deleting workspace: %v", err) + } + }) + + err = g.CreateDirectoryInWorkspace(context.Background(), id, "test") + if err != nil { + t.Fatalf("Error creating directory: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), id, "test/test1.txt", []byte("hello1")) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), id, "test1/test2.txt", []byte("hello2"), CreateFileInWorkspaceOptions{CreateDirs: true}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), id, "test1/test2.txt", []byte("hello-2"), CreateFileInWorkspaceOptions{MustNotExist: true}) + if err == nil { + t.Fatalf("Expected error creating file that must not exist") + } + + err = g.WriteFileInWorkspace(context.Background(), id, "test1/test3.txt", []byte("hello3"), CreateFileInWorkspaceOptions{WithoutCreate: true}) + if err == nil { + t.Fatalf("Expected error creating file that doesn't exist") + } + + err = g.WriteFileInWorkspace(context.Background(), id, ".hidden.txt", []byte("hidden")) + if err != nil { + t.Fatalf("Error creating hidden file: %v", err) + } + + // List all files + content, err := g.ListFilesInWorkspace(context.Background(), id) + if err != nil { + t.Fatalf("Error listing files: %v", err) + } + + if content.ID != id { + t.Errorf("Unexpected ID: %s", content.ID) + } + + if content.Path != "" { + t.Errorf("Unexpected path: %s", content.Path) + } + + if content.FileName != "" { + t.Errorf("Unexpected filename: %s", content.FileName) + } + + if len(content.Children) != 3 { + t.Errorf("Unexpected number of files: %d", len(content.Children)) + } + + // List files in subdirectory + content, err = g.ListFilesInWorkspace(context.Background(), id, ListFilesInWorkspaceOptions{SubDir: "test1"}) + if err != nil { + t.Fatalf("Error listing files: %v", err) + } + + if len(content.Children) != 1 { + t.Errorf("Unexpected number of files: %d", len(content.Children)) + } + + // Exclude hidden files + content, err = g.ListFilesInWorkspace(context.Background(), id, ListFilesInWorkspaceOptions{ExcludeHidden: true}) + if err != nil { + t.Fatalf("Error listing files: %v", err) + } + + if len(content.Children) != 2 { + t.Errorf("Unexpected number of files when listing without hidden: %d", len(content.Children)) + } + + // List non-recursive + content, err = g.ListFilesInWorkspace(context.Background(), id, ListFilesInWorkspaceOptions{NonRecursive: true}) + if err != nil { + t.Fatalf("Error listing files: %v", err) + } + + if len(content.Children) != 1 { + t.Errorf("Unexpected number of files when listing non-recursive: %d", len(content.Children)) + } +} From 657dcee11924e7f9ac3436503db491cb3d4ff315 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Thu, 17 Oct 2024 15:32:34 -0400 Subject: [PATCH 43/78] feat: simplify workspace API and add support for s3 (#73) Signed-off-by: Donnie Adams --- opts.go | 24 ++++--- workspace.go | 147 +++++++++----------------------------- workspace_test.go | 178 +++++++++++++++++++++++++++++++++------------- 3 files changed, 175 insertions(+), 174 deletions(-) diff --git a/opts.go b/opts.go index 6c9e1ab..4b71932 100644 --- a/opts.go +++ b/opts.go @@ -3,16 +3,17 @@ package gptscript // GlobalOptions allows specification of settings that are used for every call made. // These options can be overridden by the corresponding Options. type GlobalOptions struct { - URL string `json:"url"` - Token string `json:"token"` - OpenAIAPIKey string `json:"APIKey"` - OpenAIBaseURL string `json:"BaseURL"` - DefaultModel string `json:"DefaultModel"` - DefaultModelProvider string `json:"DefaultModelProvider"` - CacheDir string `json:"CacheDir"` - Env []string `json:"env"` - DatasetToolRepo string `json:"DatasetToolRepo"` - WorkspaceTool string `json:"WorkspaceTool"` + URL string `json:"url"` + Token string `json:"token"` + OpenAIAPIKey string `json:"APIKey"` + OpenAIBaseURL string `json:"BaseURL"` + DefaultModel string `json:"DefaultModel"` + DefaultModelProvider string `json:"DefaultModelProvider"` + CacheDir string `json:"CacheDir"` + Env []string `json:"env"` + DatasetToolRepo string `json:"DatasetToolRepo"` + WorkspaceTool string `json:"WorkspaceTool"` + WorkspaceDirectoryDataHome string `json:"WorkspaceDirectoryDataHome"` } func (g GlobalOptions) toEnv() []string { @@ -29,6 +30,9 @@ func (g GlobalOptions) toEnv() []string { if g.DefaultModelProvider != "" { args = append(args, "GPTSCRIPT_SDKSERVER_DEFAULT_MODEL_PROVIDER="+g.DefaultModelProvider) } + if g.WorkspaceDirectoryDataHome != "" { + args = append(args, "GPTSCRIPT_WORKSPACE_DIR="+g.WorkspaceDirectoryDataHome) + } return args } diff --git a/workspace.go b/workspace.go index 86b8c2e..2b73893 100644 --- a/workspace.go +++ b/workspace.go @@ -3,14 +3,16 @@ package gptscript import ( "context" "encoding/base64" - "encoding/json" "strings" ) -func (g *GPTScript) CreateWorkspace(ctx context.Context, providerType string) (string, error) { +func (g *GPTScript) CreateWorkspace(ctx context.Context, providerType string, fromWorkspaces ...string) (string, error) { out, err := g.runBasicCommand(ctx, "workspaces/create", map[string]any{ - "provider": providerType, - "workspaceTool": g.globalOpts.WorkspaceTool, + "providerType": providerType, + "fromWorkspaceIDs": fromWorkspaces, + "workspaceTool": g.globalOpts.WorkspaceTool, + "directoryDataHome": g.globalOpts.WorkspaceDirectoryDataHome, + "env": g.globalOpts.Env, }) if err != nil { return "", err @@ -19,156 +21,72 @@ func (g *GPTScript) CreateWorkspace(ctx context.Context, providerType string) (s return strings.TrimSpace(out), nil } -type DeleteWorkspaceOptions struct { - IgnoreNotFound bool -} - -func (g *GPTScript) DeleteWorkspace(ctx context.Context, workspaceID string, opts ...DeleteWorkspaceOptions) error { - var opt DeleteWorkspaceOptions - for _, o := range opts { - opt.IgnoreNotFound = opt.IgnoreNotFound || o.IgnoreNotFound - } +func (g *GPTScript) DeleteWorkspace(ctx context.Context, workspaceID string) error { _, err := g.runBasicCommand(ctx, "workspaces/delete", map[string]any{ - "id": workspaceID, - "ignoreNotFound": opt.IgnoreNotFound, - "workspaceTool": g.globalOpts.WorkspaceTool, - }) - - return err -} - -type CreateDirectoryInWorkspaceOptions struct { - IgnoreExists bool -} - -func (g *GPTScript) CreateDirectoryInWorkspace(ctx context.Context, workspaceID, dir string, opts ...CreateDirectoryInWorkspaceOptions) error { - var opt CreateDirectoryInWorkspaceOptions - for _, o := range opts { - opt.IgnoreExists = opt.IgnoreExists || o.IgnoreExists - } - - _, err := g.runBasicCommand(ctx, "workspaces/mkdir", map[string]any{ "id": workspaceID, - "directoryName": dir, - "ignoreExists": opt.IgnoreExists, "workspaceTool": g.globalOpts.WorkspaceTool, - }) - - return err -} - -type DeleteDirectoryInWorkspaceOptions struct { - IgnoreNotFound bool - MustBeEmpty bool -} - -func (g *GPTScript) DeleteDirectoryInWorkspace(ctx context.Context, workspaceID, dir string, opts ...DeleteDirectoryInWorkspaceOptions) error { - var opt DeleteDirectoryInWorkspaceOptions - for _, o := range opts { - o.IgnoreNotFound = opt.IgnoreNotFound || o.IgnoreNotFound - o.MustBeEmpty = opt.MustBeEmpty || o.MustBeEmpty - } - - _, err := g.runBasicCommand(ctx, "workspaces/rmdir", map[string]any{ - "id": workspaceID, - "directoryName": dir, - "ignoreNotFound": opt.IgnoreNotFound, - "mustBeEmpty": opt.MustBeEmpty, - "workspaceTool": g.globalOpts.WorkspaceTool, + "env": g.globalOpts.Env, }) return err } type ListFilesInWorkspaceOptions struct { - SubDir string - NonRecursive bool - ExcludeHidden bool -} - -type WorkspaceContent struct { - ID, Path, FileName string - Children []WorkspaceContent + Prefix string } -func (g *GPTScript) ListFilesInWorkspace(ctx context.Context, workspaceID string, opts ...ListFilesInWorkspaceOptions) (*WorkspaceContent, error) { +func (g *GPTScript) ListFilesInWorkspace(ctx context.Context, workspaceID string, opts ...ListFilesInWorkspaceOptions) ([]string, error) { var opt ListFilesInWorkspaceOptions for _, o := range opts { - if o.SubDir != "" { - opt.SubDir = o.SubDir + if o.Prefix != "" { + opt.Prefix = o.Prefix } - opt.NonRecursive = opt.NonRecursive || o.NonRecursive - opt.ExcludeHidden = opt.ExcludeHidden || o.ExcludeHidden } out, err := g.runBasicCommand(ctx, "workspaces/list", map[string]any{ "id": workspaceID, - "subDir": opt.SubDir, - "excludeHidden": opt.ExcludeHidden, - "nonRecursive": opt.NonRecursive, + "prefix": opt.Prefix, "workspaceTool": g.globalOpts.WorkspaceTool, - "json": true, + "env": g.globalOpts.Env, }) if err != nil { return nil, err } - var content []WorkspaceContent - err = json.Unmarshal([]byte(out), &content) - if err != nil { - return nil, err - } - - if len(content) == 0 { - return &WorkspaceContent{ID: workspaceID}, nil - } - - return &content[0], nil + // The first line of the output is the workspace ID, ignore it. + return strings.Split(strings.TrimSpace(out), "\n")[1:], nil } -type CreateFileInWorkspaceOptions struct { - MustNotExist bool - WithoutCreate bool - CreateDirs bool -} +func (g *GPTScript) RemoveAllWithPrefix(ctx context.Context, workspaceID, prefix string) error { + _, err := g.runBasicCommand(ctx, "workspaces/remove-all-with-prefix", map[string]any{ + "id": workspaceID, + "prefix": prefix, + "workspaceTool": g.globalOpts.WorkspaceTool, + "env": g.globalOpts.Env, + }) -func (g *GPTScript) WriteFileInWorkspace(ctx context.Context, workspaceID, filePath string, contents []byte, opts ...CreateFileInWorkspaceOptions) error { - var opt CreateFileInWorkspaceOptions - for _, o := range opts { - opt.MustNotExist = opt.MustNotExist || o.MustNotExist - opt.WithoutCreate = opt.WithoutCreate || o.WithoutCreate - opt.CreateDirs = opt.CreateDirs || o.CreateDirs - } + return err +} +func (g *GPTScript) WriteFileInWorkspace(ctx context.Context, workspaceID, filePath string, contents []byte) error { _, err := g.runBasicCommand(ctx, "workspaces/write-file", map[string]any{ "id": workspaceID, "contents": base64.StdEncoding.EncodeToString(contents), "filePath": filePath, - "mustNotExist": opt.MustNotExist, - "withoutCreate": opt.WithoutCreate, - "createDirs": opt.CreateDirs, "workspaceTool": g.globalOpts.WorkspaceTool, "base64EncodedInput": true, + "env": g.globalOpts.Env, }) return err } -type DeleteFileInWorkspaceOptions struct { - IgnoreNotFound bool -} - -func (g *GPTScript) DeleteFileInWorkspace(ctx context.Context, workspaceID, filePath string, opts ...DeleteFileInWorkspaceOptions) error { - var opt DeleteFileInWorkspaceOptions - for _, o := range opts { - opt.IgnoreNotFound = opt.IgnoreNotFound || o.IgnoreNotFound - } - +func (g *GPTScript) DeleteFileInWorkspace(ctx context.Context, workspaceID, filePath string) error { _, err := g.runBasicCommand(ctx, "workspaces/delete-file", map[string]any{ - "id": workspaceID, - "filePath": filePath, - "ignoreNotFound": opt.IgnoreNotFound, - "workspaceTool": g.globalOpts.WorkspaceTool, + "id": workspaceID, + "filePath": filePath, + "workspaceTool": g.globalOpts.WorkspaceTool, + "env": g.globalOpts.Env, }) return err @@ -180,6 +98,7 @@ func (g *GPTScript) ReadFileInWorkspace(ctx context.Context, workspaceID, filePa "filePath": filePath, "workspaceTool": g.globalOpts.WorkspaceTool, "base64EncodeOutput": true, + "env": g.globalOpts.Env, }) if err != nil { return nil, err diff --git a/workspace_test.go b/workspace_test.go index cba683f..6df5c3e 100644 --- a/workspace_test.go +++ b/workspace_test.go @@ -3,6 +3,7 @@ package gptscript import ( "bytes" "context" + "os" "testing" ) @@ -18,7 +19,7 @@ func TestCreateAndDeleteWorkspace(t *testing.T) { } } -func TestCreateDirectory(t *testing.T) { +func TestWriteReadAndDeleteFileFromWorkspace(t *testing.T) { id, err := g.CreateWorkspace(context.Background(), "directory") if err != nil { t.Fatalf("Error creating workspace: %v", err) @@ -31,18 +32,27 @@ func TestCreateDirectory(t *testing.T) { } }) - err = g.CreateDirectoryInWorkspace(context.Background(), id, "test") + err = g.WriteFileInWorkspace(context.Background(), id, "test.txt", []byte("test")) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + content, err := g.ReadFileInWorkspace(context.Background(), id, "test.txt") if err != nil { - t.Fatalf("Error creating directory: %v", err) + t.Errorf("Error reading file: %v", err) } - err = g.DeleteDirectoryInWorkspace(context.Background(), id, "test") + if !bytes.Equal(content, []byte("test")) { + t.Errorf("Unexpected content: %s", content) + } + + err = g.DeleteFileInWorkspace(context.Background(), id, "test.txt") if err != nil { - t.Errorf("Error listing files: %v", err) + t.Errorf("Error deleting file: %v", err) } } -func TestWriteReadAndDeleteFileFromWorkspace(t *testing.T) { +func TestLsComplexWorkspace(t *testing.T) { id, err := g.CreateWorkspace(context.Background(), "directory") if err != nil { t.Fatalf("Error creating workspace: %v", err) @@ -55,6 +65,96 @@ func TestWriteReadAndDeleteFileFromWorkspace(t *testing.T) { } }) + err = g.WriteFileInWorkspace(context.Background(), id, "test/test1.txt", []byte("hello1")) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), id, "test1/test2.txt", []byte("hello2")) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), id, "test1/test3.txt", []byte("hello3")) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), id, ".hidden.txt", []byte("hidden")) + if err != nil { + t.Fatalf("Error creating hidden file: %v", err) + } + + // List all files + content, err := g.ListFilesInWorkspace(context.Background(), id) + if err != nil { + t.Fatalf("Error listing files: %v", err) + } + + if len(content) != 4 { + t.Errorf("Unexpected number of files: %d", len(content)) + } + + // List files in subdirectory + content, err = g.ListFilesInWorkspace(context.Background(), id, ListFilesInWorkspaceOptions{Prefix: "test1"}) + if err != nil { + t.Fatalf("Error listing files: %v", err) + } + + if len(content) != 2 { + t.Errorf("Unexpected number of files: %d", len(content)) + } + + // Remove all files with test1 prefix + err = g.RemoveAllWithPrefix(context.Background(), id, "test1") + if err != nil { + t.Fatalf("Error removing files: %v", err) + } + + // List files in subdirectory + content, err = g.ListFilesInWorkspace(context.Background(), id) + if err != nil { + t.Fatalf("Error listing files: %v", err) + } + + if len(content) != 2 { + t.Errorf("Unexpected number of files: %d", len(content)) + } +} + +func TestCreateAndDeleteWorkspaceS3(t *testing.T) { + if os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" || os.Getenv("WORKSPACE_PROVIDER_S3_BUCKET") == "" { + t.Skip("Skipping test because AWS credentials are not set") + } + + id, err := g.CreateWorkspace(context.Background(), "s3") + if err != nil { + t.Fatalf("Error creating workspace: %v", err) + } + + err = g.DeleteWorkspace(context.Background(), id) + if err != nil { + t.Errorf("Error deleting workspace: %v", err) + } +} + +func TestWriteReadAndDeleteFileFromWorkspaceS3(t *testing.T) { + if os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" || os.Getenv("WORKSPACE_PROVIDER_S3_BUCKET") == "" { + t.Skip("Skipping test because AWS credentials are not set") + } + + id, err := g.CreateWorkspace(context.Background(), "s3") + if err != nil { + t.Fatalf("Error creating workspace: %v", err) + } + + t.Cleanup(func() { + err := g.DeleteWorkspace(context.Background(), id) + if err != nil { + t.Errorf("Error deleting workspace: %v", err) + } + }) + err = g.WriteFileInWorkspace(context.Background(), id, "test.txt", []byte("test")) if err != nil { t.Fatalf("Error creating file: %v", err) @@ -75,8 +175,12 @@ func TestWriteReadAndDeleteFileFromWorkspace(t *testing.T) { } } -func TestLsComplexWorkspace(t *testing.T) { - id, err := g.CreateWorkspace(context.Background(), "directory") +func TestLsComplexWorkspaceS3(t *testing.T) { + if os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" || os.Getenv("WORKSPACE_PROVIDER_S3_BUCKET") == "" { + t.Skip("Skipping test because AWS credentials are not set") + } + + id, err := g.CreateWorkspace(context.Background(), "s3") if err != nil { t.Fatalf("Error creating workspace: %v", err) } @@ -88,29 +192,19 @@ func TestLsComplexWorkspace(t *testing.T) { } }) - err = g.CreateDirectoryInWorkspace(context.Background(), id, "test") - if err != nil { - t.Fatalf("Error creating directory: %v", err) - } - err = g.WriteFileInWorkspace(context.Background(), id, "test/test1.txt", []byte("hello1")) if err != nil { t.Fatalf("Error creating file: %v", err) } - err = g.WriteFileInWorkspace(context.Background(), id, "test1/test2.txt", []byte("hello2"), CreateFileInWorkspaceOptions{CreateDirs: true}) + err = g.WriteFileInWorkspace(context.Background(), id, "test1/test2.txt", []byte("hello2")) if err != nil { t.Fatalf("Error creating file: %v", err) } - err = g.WriteFileInWorkspace(context.Background(), id, "test1/test2.txt", []byte("hello-2"), CreateFileInWorkspaceOptions{MustNotExist: true}) - if err == nil { - t.Fatalf("Expected error creating file that must not exist") - } - - err = g.WriteFileInWorkspace(context.Background(), id, "test1/test3.txt", []byte("hello3"), CreateFileInWorkspaceOptions{WithoutCreate: true}) - if err == nil { - t.Fatalf("Expected error creating file that doesn't exist") + err = g.WriteFileInWorkspace(context.Background(), id, "test1/test3.txt", []byte("hello3")) + if err != nil { + t.Fatalf("Error creating file: %v", err) } err = g.WriteFileInWorkspace(context.Background(), id, ".hidden.txt", []byte("hidden")) @@ -124,49 +218,33 @@ func TestLsComplexWorkspace(t *testing.T) { t.Fatalf("Error listing files: %v", err) } - if content.ID != id { - t.Errorf("Unexpected ID: %s", content.ID) - } - - if content.Path != "" { - t.Errorf("Unexpected path: %s", content.Path) - } - - if content.FileName != "" { - t.Errorf("Unexpected filename: %s", content.FileName) - } - - if len(content.Children) != 3 { - t.Errorf("Unexpected number of files: %d", len(content.Children)) + if len(content) != 4 { + t.Errorf("Unexpected number of files: %d", len(content)) } // List files in subdirectory - content, err = g.ListFilesInWorkspace(context.Background(), id, ListFilesInWorkspaceOptions{SubDir: "test1"}) + content, err = g.ListFilesInWorkspace(context.Background(), id, ListFilesInWorkspaceOptions{Prefix: "test1"}) if err != nil { t.Fatalf("Error listing files: %v", err) } - if len(content.Children) != 1 { - t.Errorf("Unexpected number of files: %d", len(content.Children)) + if len(content) != 2 { + t.Errorf("Unexpected number of files: %d", len(content)) } - // Exclude hidden files - content, err = g.ListFilesInWorkspace(context.Background(), id, ListFilesInWorkspaceOptions{ExcludeHidden: true}) + // Remove all files with test1 prefix + err = g.RemoveAllWithPrefix(context.Background(), id, "test1") if err != nil { - t.Fatalf("Error listing files: %v", err) + t.Fatalf("Error removing files: %v", err) } - if len(content.Children) != 2 { - t.Errorf("Unexpected number of files when listing without hidden: %d", len(content.Children)) - } - - // List non-recursive - content, err = g.ListFilesInWorkspace(context.Background(), id, ListFilesInWorkspaceOptions{NonRecursive: true}) + // List files in subdirectory + content, err = g.ListFilesInWorkspace(context.Background(), id) if err != nil { t.Fatalf("Error listing files: %v", err) } - if len(content.Children) != 1 { - t.Errorf("Unexpected number of files when listing non-recursive: %d", len(content.Children)) + if len(content) != 2 { + t.Errorf("Unexpected number of files: %d", len(content)) } } From 82a0de4f919fc605f619be89be78c43fbe8c73e5 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Fri, 18 Oct 2024 09:07:19 -0400 Subject: [PATCH 44/78] fix: complete new workspace global options Signed-off-by: Donnie Adams --- opts.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/opts.go b/opts.go index 4b71932..46b5635 100644 --- a/opts.go +++ b/opts.go @@ -48,6 +48,8 @@ func completeGlobalOptions(opts ...GlobalOptions) GlobalOptions { result.DefaultModel = firstSet(opt.DefaultModel, result.DefaultModel) result.DefaultModelProvider = firstSet(opt.DefaultModelProvider, result.DefaultModelProvider) result.DatasetToolRepo = firstSet(opt.DatasetToolRepo, result.DatasetToolRepo) + result.WorkspaceTool = firstSet(opt.WorkspaceTool, result.WorkspaceTool) + result.WorkspaceDirectoryDataHome = firstSet(opt.WorkspaceDirectoryDataHome, result.WorkspaceDirectoryDataHome) result.Env = append(result.Env, opt.Env...) } return result From d407717daae44f74ec5ac845a329a6b8c5e82712 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Fri, 18 Oct 2024 10:35:36 -0400 Subject: [PATCH 45/78] chore: remove workspace data home option (#74) Signed-off-by: Donnie Adams --- opts.go | 25 ++++++++++--------------- workspace.go | 9 ++++----- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/opts.go b/opts.go index 46b5635..14a9c83 100644 --- a/opts.go +++ b/opts.go @@ -3,17 +3,16 @@ package gptscript // GlobalOptions allows specification of settings that are used for every call made. // These options can be overridden by the corresponding Options. type GlobalOptions struct { - URL string `json:"url"` - Token string `json:"token"` - OpenAIAPIKey string `json:"APIKey"` - OpenAIBaseURL string `json:"BaseURL"` - DefaultModel string `json:"DefaultModel"` - DefaultModelProvider string `json:"DefaultModelProvider"` - CacheDir string `json:"CacheDir"` - Env []string `json:"env"` - DatasetToolRepo string `json:"DatasetToolRepo"` - WorkspaceTool string `json:"WorkspaceTool"` - WorkspaceDirectoryDataHome string `json:"WorkspaceDirectoryDataHome"` + URL string `json:"url"` + Token string `json:"token"` + OpenAIAPIKey string `json:"APIKey"` + OpenAIBaseURL string `json:"BaseURL"` + DefaultModel string `json:"DefaultModel"` + DefaultModelProvider string `json:"DefaultModelProvider"` + CacheDir string `json:"CacheDir"` + Env []string `json:"env"` + DatasetToolRepo string `json:"DatasetToolRepo"` + WorkspaceTool string `json:"WorkspaceTool"` } func (g GlobalOptions) toEnv() []string { @@ -30,9 +29,6 @@ func (g GlobalOptions) toEnv() []string { if g.DefaultModelProvider != "" { args = append(args, "GPTSCRIPT_SDKSERVER_DEFAULT_MODEL_PROVIDER="+g.DefaultModelProvider) } - if g.WorkspaceDirectoryDataHome != "" { - args = append(args, "GPTSCRIPT_WORKSPACE_DIR="+g.WorkspaceDirectoryDataHome) - } return args } @@ -49,7 +45,6 @@ func completeGlobalOptions(opts ...GlobalOptions) GlobalOptions { result.DefaultModelProvider = firstSet(opt.DefaultModelProvider, result.DefaultModelProvider) result.DatasetToolRepo = firstSet(opt.DatasetToolRepo, result.DatasetToolRepo) result.WorkspaceTool = firstSet(opt.WorkspaceTool, result.WorkspaceTool) - result.WorkspaceDirectoryDataHome = firstSet(opt.WorkspaceDirectoryDataHome, result.WorkspaceDirectoryDataHome) result.Env = append(result.Env, opt.Env...) } return result diff --git a/workspace.go b/workspace.go index 2b73893..fc5bfff 100644 --- a/workspace.go +++ b/workspace.go @@ -8,11 +8,10 @@ import ( func (g *GPTScript) CreateWorkspace(ctx context.Context, providerType string, fromWorkspaces ...string) (string, error) { out, err := g.runBasicCommand(ctx, "workspaces/create", map[string]any{ - "providerType": providerType, - "fromWorkspaceIDs": fromWorkspaces, - "workspaceTool": g.globalOpts.WorkspaceTool, - "directoryDataHome": g.globalOpts.WorkspaceDirectoryDataHome, - "env": g.globalOpts.Env, + "providerType": providerType, + "fromWorkspaceIDs": fromWorkspaces, + "workspaceTool": g.globalOpts.WorkspaceTool, + "env": g.globalOpts.Env, }) if err != nil { return "", err From d85190d96d31128f9a030d1c54eb394aa6f5bc14 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Fri, 18 Oct 2024 16:34:27 -0400 Subject: [PATCH 46/78] chore: pass files bytes for workspace API instead of base64 encoding (#75) Signed-off-by: Donnie Adams --- workspace.go | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/workspace.go b/workspace.go index fc5bfff..a8ad102 100644 --- a/workspace.go +++ b/workspace.go @@ -2,7 +2,6 @@ package gptscript import ( "context" - "encoding/base64" "strings" ) @@ -52,8 +51,7 @@ func (g *GPTScript) ListFilesInWorkspace(ctx context.Context, workspaceID string return nil, err } - // The first line of the output is the workspace ID, ignore it. - return strings.Split(strings.TrimSpace(out), "\n")[1:], nil + return strings.Split(strings.TrimSpace(out), "\n"), nil } func (g *GPTScript) RemoveAllWithPrefix(ctx context.Context, workspaceID, prefix string) error { @@ -69,12 +67,11 @@ func (g *GPTScript) RemoveAllWithPrefix(ctx context.Context, workspaceID, prefix func (g *GPTScript) WriteFileInWorkspace(ctx context.Context, workspaceID, filePath string, contents []byte) error { _, err := g.runBasicCommand(ctx, "workspaces/write-file", map[string]any{ - "id": workspaceID, - "contents": base64.StdEncoding.EncodeToString(contents), - "filePath": filePath, - "workspaceTool": g.globalOpts.WorkspaceTool, - "base64EncodedInput": true, - "env": g.globalOpts.Env, + "id": workspaceID, + "contents": contents, + "filePath": filePath, + "workspaceTool": g.globalOpts.WorkspaceTool, + "env": g.globalOpts.Env, }) return err @@ -93,15 +90,14 @@ func (g *GPTScript) DeleteFileInWorkspace(ctx context.Context, workspaceID, file func (g *GPTScript) ReadFileInWorkspace(ctx context.Context, workspaceID, filePath string) ([]byte, error) { out, err := g.runBasicCommand(ctx, "workspaces/read-file", map[string]any{ - "id": workspaceID, - "filePath": filePath, - "workspaceTool": g.globalOpts.WorkspaceTool, - "base64EncodeOutput": true, - "env": g.globalOpts.Env, + "id": workspaceID, + "filePath": filePath, + "workspaceTool": g.globalOpts.WorkspaceTool, + "env": g.globalOpts.Env, }) if err != nil { return nil, err } - return base64.StdEncoding.DecodeString(out) + return []byte(out), nil } From d72fa0553119975b3fca283345e63bc45c39694b Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Sun, 20 Oct 2024 10:52:05 -0400 Subject: [PATCH 47/78] feat: pass the workspace tool as a global option to the SDK server (#76) Signed-off-by: Donnie Adams --- opts.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/opts.go b/opts.go index 14a9c83..e08d217 100644 --- a/opts.go +++ b/opts.go @@ -29,6 +29,9 @@ func (g GlobalOptions) toEnv() []string { if g.DefaultModelProvider != "" { args = append(args, "GPTSCRIPT_SDKSERVER_DEFAULT_MODEL_PROVIDER="+g.DefaultModelProvider) } + if g.WorkspaceTool != "" { + args = append(args, "GPTSCRIPT_SDKSERVER_WORKSPACE_TOOL="+g.WorkspaceTool) + } return args } From 3901872ceda9ff2921a35ae426ca236ac9312c03 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Mon, 21 Oct 2024 14:25:05 -0400 Subject: [PATCH 48/78] feat: make the workspace ID optional in the workspace API If the workspace ID is not specified, then it will be read from the GPTSCRIPT_WORKSPACE_ID environment variable. Signed-off-by: Donnie Adams --- workspace.go | 116 ++++++++++++++++++++++++++++++++++++++++------ workspace_test.go | 56 +++++++++++----------- 2 files changed, 130 insertions(+), 42 deletions(-) diff --git a/workspace.go b/workspace.go index a8ad102..6d79998 100644 --- a/workspace.go +++ b/workspace.go @@ -2,6 +2,7 @@ package gptscript import ( "context" + "os" "strings" ) @@ -19,9 +20,24 @@ func (g *GPTScript) CreateWorkspace(ctx context.Context, providerType string, fr return strings.TrimSpace(out), nil } -func (g *GPTScript) DeleteWorkspace(ctx context.Context, workspaceID string) error { +type DeleteWorkspaceOptions struct { + WorkspaceID string +} + +func (g *GPTScript) DeleteWorkspace(ctx context.Context, opts ...DeleteWorkspaceOptions) error { + var opt DeleteWorkspaceOptions + for _, o := range opts { + if o.WorkspaceID != "" { + opt.WorkspaceID = o.WorkspaceID + } + } + + if opt.WorkspaceID == "" { + opt.WorkspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") + } + _, err := g.runBasicCommand(ctx, "workspaces/delete", map[string]any{ - "id": workspaceID, + "id": opt.WorkspaceID, "workspaceTool": g.globalOpts.WorkspaceTool, "env": g.globalOpts.Env, }) @@ -30,19 +46,27 @@ func (g *GPTScript) DeleteWorkspace(ctx context.Context, workspaceID string) err } type ListFilesInWorkspaceOptions struct { - Prefix string + WorkspaceID string + Prefix string } -func (g *GPTScript) ListFilesInWorkspace(ctx context.Context, workspaceID string, opts ...ListFilesInWorkspaceOptions) ([]string, error) { +func (g *GPTScript) ListFilesInWorkspace(ctx context.Context, opts ...ListFilesInWorkspaceOptions) ([]string, error) { var opt ListFilesInWorkspaceOptions for _, o := range opts { if o.Prefix != "" { opt.Prefix = o.Prefix } + if o.WorkspaceID != "" { + opt.WorkspaceID = o.WorkspaceID + } + } + + if opt.WorkspaceID == "" { + opt.WorkspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") } out, err := g.runBasicCommand(ctx, "workspaces/list", map[string]any{ - "id": workspaceID, + "id": opt.WorkspaceID, "prefix": opt.Prefix, "workspaceTool": g.globalOpts.WorkspaceTool, "env": g.globalOpts.Env, @@ -54,10 +78,29 @@ func (g *GPTScript) ListFilesInWorkspace(ctx context.Context, workspaceID string return strings.Split(strings.TrimSpace(out), "\n"), nil } -func (g *GPTScript) RemoveAllWithPrefix(ctx context.Context, workspaceID, prefix string) error { +type RemoveAllOptions struct { + WorkspaceID string + WithPrefix string +} + +func (g *GPTScript) RemoveAll(ctx context.Context, opts ...RemoveAllOptions) error { + var opt RemoveAllOptions + for _, o := range opts { + if o.WithPrefix != "" { + opt.WithPrefix = o.WithPrefix + } + if o.WorkspaceID != "" { + opt.WorkspaceID = o.WorkspaceID + } + } + + if opt.WorkspaceID == "" { + opt.WorkspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") + } + _, err := g.runBasicCommand(ctx, "workspaces/remove-all-with-prefix", map[string]any{ - "id": workspaceID, - "prefix": prefix, + "id": opt.WorkspaceID, + "prefix": opt.WithPrefix, "workspaceTool": g.globalOpts.WorkspaceTool, "env": g.globalOpts.Env, }) @@ -65,9 +108,24 @@ func (g *GPTScript) RemoveAllWithPrefix(ctx context.Context, workspaceID, prefix return err } -func (g *GPTScript) WriteFileInWorkspace(ctx context.Context, workspaceID, filePath string, contents []byte) error { +type WriteFileInWorkspaceOptions struct { + WorkspaceID string +} + +func (g *GPTScript) WriteFileInWorkspace(ctx context.Context, filePath string, contents []byte, opts ...WriteFileInWorkspaceOptions) error { + var opt WriteFileInWorkspaceOptions + for _, o := range opts { + if o.WorkspaceID != "" { + opt.WorkspaceID = o.WorkspaceID + } + } + + if opt.WorkspaceID == "" { + opt.WorkspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") + } + _, err := g.runBasicCommand(ctx, "workspaces/write-file", map[string]any{ - "id": workspaceID, + "id": opt.WorkspaceID, "contents": contents, "filePath": filePath, "workspaceTool": g.globalOpts.WorkspaceTool, @@ -77,9 +135,24 @@ func (g *GPTScript) WriteFileInWorkspace(ctx context.Context, workspaceID, fileP return err } -func (g *GPTScript) DeleteFileInWorkspace(ctx context.Context, workspaceID, filePath string) error { +type DeleteFileInWorkspaceOptions struct { + WorkspaceID string +} + +func (g *GPTScript) DeleteFileInWorkspace(ctx context.Context, filePath string, opts ...DeleteFileInWorkspaceOptions) error { + var opt DeleteFileInWorkspaceOptions + for _, o := range opts { + if o.WorkspaceID != "" { + opt.WorkspaceID = o.WorkspaceID + } + } + + if opt.WorkspaceID == "" { + opt.WorkspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") + } + _, err := g.runBasicCommand(ctx, "workspaces/delete-file", map[string]any{ - "id": workspaceID, + "id": opt.WorkspaceID, "filePath": filePath, "workspaceTool": g.globalOpts.WorkspaceTool, "env": g.globalOpts.Env, @@ -88,9 +161,24 @@ func (g *GPTScript) DeleteFileInWorkspace(ctx context.Context, workspaceID, file return err } -func (g *GPTScript) ReadFileInWorkspace(ctx context.Context, workspaceID, filePath string) ([]byte, error) { +type ReadFileInWorkspaceOptions struct { + WorkspaceID string +} + +func (g *GPTScript) ReadFileInWorkspace(ctx context.Context, filePath string, opts ...ReadFileInWorkspaceOptions) ([]byte, error) { + var opt ReadFileInWorkspaceOptions + for _, o := range opts { + if o.WorkspaceID != "" { + opt.WorkspaceID = o.WorkspaceID + } + } + + if opt.WorkspaceID == "" { + opt.WorkspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") + } + out, err := g.runBasicCommand(ctx, "workspaces/read-file", map[string]any{ - "id": workspaceID, + "id": opt.WorkspaceID, "filePath": filePath, "workspaceTool": g.globalOpts.WorkspaceTool, "env": g.globalOpts.Env, diff --git a/workspace_test.go b/workspace_test.go index 6df5c3e..a7ff08c 100644 --- a/workspace_test.go +++ b/workspace_test.go @@ -13,7 +13,7 @@ func TestCreateAndDeleteWorkspace(t *testing.T) { t.Fatalf("Error creating workspace: %v", err) } - err = g.DeleteWorkspace(context.Background(), id) + err = g.DeleteWorkspace(context.Background(), DeleteWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Errorf("Error deleting workspace: %v", err) } @@ -26,18 +26,18 @@ func TestWriteReadAndDeleteFileFromWorkspace(t *testing.T) { } t.Cleanup(func() { - err := g.DeleteWorkspace(context.Background(), id) + err := g.DeleteWorkspace(context.Background(), DeleteWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Errorf("Error deleting workspace: %v", err) } }) - err = g.WriteFileInWorkspace(context.Background(), id, "test.txt", []byte("test")) + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test"), WriteFileInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Fatalf("Error creating file: %v", err) } - content, err := g.ReadFileInWorkspace(context.Background(), id, "test.txt") + content, err := g.ReadFileInWorkspace(context.Background(), "test.txt", ReadFileInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Errorf("Error reading file: %v", err) } @@ -46,7 +46,7 @@ func TestWriteReadAndDeleteFileFromWorkspace(t *testing.T) { t.Errorf("Unexpected content: %s", content) } - err = g.DeleteFileInWorkspace(context.Background(), id, "test.txt") + err = g.DeleteFileInWorkspace(context.Background(), "test.txt", DeleteFileInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Errorf("Error deleting file: %v", err) } @@ -59,34 +59,34 @@ func TestLsComplexWorkspace(t *testing.T) { } t.Cleanup(func() { - err := g.DeleteWorkspace(context.Background(), id) + err := g.DeleteWorkspace(context.Background(), DeleteWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Errorf("Error deleting workspace: %v", err) } }) - err = g.WriteFileInWorkspace(context.Background(), id, "test/test1.txt", []byte("hello1")) + err = g.WriteFileInWorkspace(context.Background(), "test/test1.txt", []byte("hello1"), WriteFileInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Fatalf("Error creating file: %v", err) } - err = g.WriteFileInWorkspace(context.Background(), id, "test1/test2.txt", []byte("hello2")) + err = g.WriteFileInWorkspace(context.Background(), "test1/test2.txt", []byte("hello2"), WriteFileInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Fatalf("Error creating file: %v", err) } - err = g.WriteFileInWorkspace(context.Background(), id, "test1/test3.txt", []byte("hello3")) + err = g.WriteFileInWorkspace(context.Background(), "test1/test3.txt", []byte("hello3"), WriteFileInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Fatalf("Error creating file: %v", err) } - err = g.WriteFileInWorkspace(context.Background(), id, ".hidden.txt", []byte("hidden")) + err = g.WriteFileInWorkspace(context.Background(), ".hidden.txt", []byte("hidden"), WriteFileInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Fatalf("Error creating hidden file: %v", err) } // List all files - content, err := g.ListFilesInWorkspace(context.Background(), id) + content, err := g.ListFilesInWorkspace(context.Background(), ListFilesInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Fatalf("Error listing files: %v", err) } @@ -96,7 +96,7 @@ func TestLsComplexWorkspace(t *testing.T) { } // List files in subdirectory - content, err = g.ListFilesInWorkspace(context.Background(), id, ListFilesInWorkspaceOptions{Prefix: "test1"}) + content, err = g.ListFilesInWorkspace(context.Background(), ListFilesInWorkspaceOptions{WorkspaceID: id, Prefix: "test1"}) if err != nil { t.Fatalf("Error listing files: %v", err) } @@ -106,13 +106,13 @@ func TestLsComplexWorkspace(t *testing.T) { } // Remove all files with test1 prefix - err = g.RemoveAllWithPrefix(context.Background(), id, "test1") + err = g.RemoveAll(context.Background(), RemoveAllOptions{WorkspaceID: id, WithPrefix: "test1"}) if err != nil { t.Fatalf("Error removing files: %v", err) } // List files in subdirectory - content, err = g.ListFilesInWorkspace(context.Background(), id) + content, err = g.ListFilesInWorkspace(context.Background(), ListFilesInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Fatalf("Error listing files: %v", err) } @@ -132,7 +132,7 @@ func TestCreateAndDeleteWorkspaceS3(t *testing.T) { t.Fatalf("Error creating workspace: %v", err) } - err = g.DeleteWorkspace(context.Background(), id) + err = g.DeleteWorkspace(context.Background(), DeleteWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Errorf("Error deleting workspace: %v", err) } @@ -149,18 +149,18 @@ func TestWriteReadAndDeleteFileFromWorkspaceS3(t *testing.T) { } t.Cleanup(func() { - err := g.DeleteWorkspace(context.Background(), id) + err := g.DeleteWorkspace(context.Background(), DeleteWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Errorf("Error deleting workspace: %v", err) } }) - err = g.WriteFileInWorkspace(context.Background(), id, "test.txt", []byte("test")) + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test"), WriteFileInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Fatalf("Error creating file: %v", err) } - content, err := g.ReadFileInWorkspace(context.Background(), id, "test.txt") + content, err := g.ReadFileInWorkspace(context.Background(), "test.txt", ReadFileInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Errorf("Error reading file: %v", err) } @@ -169,7 +169,7 @@ func TestWriteReadAndDeleteFileFromWorkspaceS3(t *testing.T) { t.Errorf("Unexpected content: %s", content) } - err = g.DeleteFileInWorkspace(context.Background(), id, "test.txt") + err = g.DeleteFileInWorkspace(context.Background(), "test.txt", DeleteFileInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Errorf("Error deleting file: %v", err) } @@ -186,34 +186,34 @@ func TestLsComplexWorkspaceS3(t *testing.T) { } t.Cleanup(func() { - err := g.DeleteWorkspace(context.Background(), id) + err := g.DeleteWorkspace(context.Background(), DeleteWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Errorf("Error deleting workspace: %v", err) } }) - err = g.WriteFileInWorkspace(context.Background(), id, "test/test1.txt", []byte("hello1")) + err = g.WriteFileInWorkspace(context.Background(), "test/test1.txt", []byte("hello1"), WriteFileInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Fatalf("Error creating file: %v", err) } - err = g.WriteFileInWorkspace(context.Background(), id, "test1/test2.txt", []byte("hello2")) + err = g.WriteFileInWorkspace(context.Background(), "test1/test2.txt", []byte("hello2"), WriteFileInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Fatalf("Error creating file: %v", err) } - err = g.WriteFileInWorkspace(context.Background(), id, "test1/test3.txt", []byte("hello3")) + err = g.WriteFileInWorkspace(context.Background(), "test1/test3.txt", []byte("hello3"), WriteFileInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Fatalf("Error creating file: %v", err) } - err = g.WriteFileInWorkspace(context.Background(), id, ".hidden.txt", []byte("hidden")) + err = g.WriteFileInWorkspace(context.Background(), ".hidden.txt", []byte("hidden"), WriteFileInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Fatalf("Error creating hidden file: %v", err) } // List all files - content, err := g.ListFilesInWorkspace(context.Background(), id) + content, err := g.ListFilesInWorkspace(context.Background(), ListFilesInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Fatalf("Error listing files: %v", err) } @@ -223,7 +223,7 @@ func TestLsComplexWorkspaceS3(t *testing.T) { } // List files in subdirectory - content, err = g.ListFilesInWorkspace(context.Background(), id, ListFilesInWorkspaceOptions{Prefix: "test1"}) + content, err = g.ListFilesInWorkspace(context.Background(), ListFilesInWorkspaceOptions{WorkspaceID: id, Prefix: "test1"}) if err != nil { t.Fatalf("Error listing files: %v", err) } @@ -233,13 +233,13 @@ func TestLsComplexWorkspaceS3(t *testing.T) { } // Remove all files with test1 prefix - err = g.RemoveAllWithPrefix(context.Background(), id, "test1") + err = g.RemoveAll(context.Background(), RemoveAllOptions{WorkspaceID: id, WithPrefix: "test1"}) if err != nil { t.Fatalf("Error removing files: %v", err) } // List files in subdirectory - content, err = g.ListFilesInWorkspace(context.Background(), id) + content, err = g.ListFilesInWorkspace(context.Background(), ListFilesInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Fatalf("Error listing files: %v", err) } From cd749dfce2cefb5ea080e46c52ca3bc5bc4f79f1 Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Wed, 23 Oct 2024 15:42:12 -0400 Subject: [PATCH 49/78] chore: add Add Elements method and update for workspace provider (#78) Signed-off-by: Grant Linville --- datasets.go | 80 ++++++++++++++++++++++++++++++++++-------------- datasets_test.go | 48 +++++++++++++++++++++-------- 2 files changed, 93 insertions(+), 35 deletions(-) diff --git a/datasets.go b/datasets.go index c053a26..7b9f377 100644 --- a/datasets.go +++ b/datasets.go @@ -30,9 +30,10 @@ type Dataset struct { } type datasetRequest struct { - Input string `json:"input"` - Workspace string `json:"workspace"` - DatasetToolRepo string `json:"datasetToolRepo"` + Input string `json:"input"` + WorkspaceID string `json:"workspaceID"` + DatasetToolRepo string `json:"datasetToolRepo"` + Env []string `json:"env"` } type createDatasetArgs struct { @@ -47,6 +48,11 @@ type addDatasetElementArgs struct { ElementContent string `json:"elementContent"` } +type addDatasetElementsArgs struct { + DatasetID string `json:"datasetID"` + Elements []DatasetElement `json:"elements"` +} + type listDatasetElementArgs struct { DatasetID string `json:"datasetID"` } @@ -56,15 +62,16 @@ type getDatasetElementArgs struct { Element string `json:"element"` } -func (g *GPTScript) ListDatasets(ctx context.Context, workspace string) ([]DatasetMeta, error) { - if workspace == "" { - workspace = os.Getenv("GPTSCRIPT_WORKSPACE_DIR") +func (g *GPTScript) ListDatasets(ctx context.Context, workspaceID string) ([]DatasetMeta, error) { + if workspaceID == "" { + workspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") } out, err := g.runBasicCommand(ctx, "datasets", datasetRequest{ Input: "{}", - Workspace: workspace, + WorkspaceID: workspaceID, DatasetToolRepo: g.globalOpts.DatasetToolRepo, + Env: g.globalOpts.Env, }) if err != nil { return nil, err @@ -77,9 +84,9 @@ func (g *GPTScript) ListDatasets(ctx context.Context, workspace string) ([]Datas return datasets, nil } -func (g *GPTScript) CreateDataset(ctx context.Context, workspace, name, description string) (Dataset, error) { - if workspace == "" { - workspace = os.Getenv("GPTSCRIPT_WORKSPACE_DIR") +func (g *GPTScript) CreateDataset(ctx context.Context, workspaceID, name, description string) (Dataset, error) { + if workspaceID == "" { + workspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") } args := createDatasetArgs{ @@ -93,8 +100,9 @@ func (g *GPTScript) CreateDataset(ctx context.Context, workspace, name, descript out, err := g.runBasicCommand(ctx, "datasets/create", datasetRequest{ Input: string(argsJSON), - Workspace: workspace, + WorkspaceID: workspaceID, DatasetToolRepo: g.globalOpts.DatasetToolRepo, + Env: g.globalOpts.Env, }) if err != nil { return Dataset{}, err @@ -107,9 +115,9 @@ func (g *GPTScript) CreateDataset(ctx context.Context, workspace, name, descript return dataset, nil } -func (g *GPTScript) AddDatasetElement(ctx context.Context, workspace, datasetID, elementName, elementDescription, elementContent string) (DatasetElementMeta, error) { - if workspace == "" { - workspace = os.Getenv("GPTSCRIPT_WORKSPACE_DIR") +func (g *GPTScript) AddDatasetElement(ctx context.Context, workspaceID, datasetID, elementName, elementDescription, elementContent string) (DatasetElementMeta, error) { + if workspaceID == "" { + workspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") } args := addDatasetElementArgs{ @@ -125,8 +133,9 @@ func (g *GPTScript) AddDatasetElement(ctx context.Context, workspace, datasetID, out, err := g.runBasicCommand(ctx, "datasets/add-element", datasetRequest{ Input: string(argsJSON), - Workspace: workspace, + WorkspaceID: workspaceID, DatasetToolRepo: g.globalOpts.DatasetToolRepo, + Env: g.globalOpts.Env, }) if err != nil { return DatasetElementMeta{}, err @@ -139,9 +148,32 @@ func (g *GPTScript) AddDatasetElement(ctx context.Context, workspace, datasetID, return element, nil } -func (g *GPTScript) ListDatasetElements(ctx context.Context, workspace, datasetID string) ([]DatasetElementMeta, error) { - if workspace == "" { - workspace = os.Getenv("GPTSCRIPT_WORKSPACE_DIR") +func (g *GPTScript) AddDatasetElements(ctx context.Context, workspaceID, datasetID string, elements []DatasetElement) error { + if workspaceID == "" { + workspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") + } + + args := addDatasetElementsArgs{ + DatasetID: datasetID, + Elements: elements, + } + argsJSON, err := json.Marshal(args) + if err != nil { + return fmt.Errorf("failed to marshal element args: %w", err) + } + + _, err = g.runBasicCommand(ctx, "datasets/add-elements", datasetRequest{ + Input: string(argsJSON), + WorkspaceID: workspaceID, + DatasetToolRepo: g.globalOpts.DatasetToolRepo, + Env: g.globalOpts.Env, + }) + return err +} + +func (g *GPTScript) ListDatasetElements(ctx context.Context, workspaceID, datasetID string) ([]DatasetElementMeta, error) { + if workspaceID == "" { + workspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") } args := listDatasetElementArgs{ @@ -154,8 +186,9 @@ func (g *GPTScript) ListDatasetElements(ctx context.Context, workspace, datasetI out, err := g.runBasicCommand(ctx, "datasets/list-elements", datasetRequest{ Input: string(argsJSON), - Workspace: workspace, + WorkspaceID: workspaceID, DatasetToolRepo: g.globalOpts.DatasetToolRepo, + Env: g.globalOpts.Env, }) if err != nil { return nil, err @@ -168,9 +201,9 @@ func (g *GPTScript) ListDatasetElements(ctx context.Context, workspace, datasetI return elements, nil } -func (g *GPTScript) GetDatasetElement(ctx context.Context, workspace, datasetID, elementName string) (DatasetElement, error) { - if workspace == "" { - workspace = os.Getenv("GPTSCRIPT_WORKSPACE_DIR") +func (g *GPTScript) GetDatasetElement(ctx context.Context, workspaceID, datasetID, elementName string) (DatasetElement, error) { + if workspaceID == "" { + workspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") } args := getDatasetElementArgs{ @@ -184,8 +217,9 @@ func (g *GPTScript) GetDatasetElement(ctx context.Context, workspace, datasetID, out, err := g.runBasicCommand(ctx, "datasets/get-element", datasetRequest{ Input: string(argsJSON), - Workspace: workspace, + WorkspaceID: workspaceID, DatasetToolRepo: g.globalOpts.DatasetToolRepo, + Env: g.globalOpts.Env, }) if err != nil { return DatasetElement{}, err diff --git a/datasets_test.go b/datasets_test.go index 3763982..44d72ab 100644 --- a/datasets_test.go +++ b/datasets_test.go @@ -2,48 +2,72 @@ package gptscript import ( "context" - "os" "testing" "github.com/stretchr/testify/require" ) func TestDatasets(t *testing.T) { - workspace, err := os.MkdirTemp("", "go-gptscript-test") + workspaceID, err := g.CreateWorkspace(context.Background(), "directory") require.NoError(t, err) + defer func() { - _ = os.RemoveAll(workspace) + _ = g.DeleteWorkspace(context.Background(), DeleteWorkspaceOptions{WorkspaceID: workspaceID}) }() // Create a dataset - dataset, err := g.CreateDataset(context.Background(), workspace, "test-dataset", "This is a test dataset") + dataset, err := g.CreateDataset(context.Background(), workspaceID, "test-dataset", "This is a test dataset") require.NoError(t, err) require.Equal(t, "test-dataset", dataset.Name) require.Equal(t, "This is a test dataset", dataset.Description) require.Equal(t, 0, len(dataset.Elements)) // Add an element - elementMeta, err := g.AddDatasetElement(context.Background(), workspace, dataset.ID, "test-element", "This is a test element", "This is the content") + elementMeta, err := g.AddDatasetElement(context.Background(), workspaceID, dataset.ID, "test-element", "This is a test element", "This is the content") require.NoError(t, err) require.Equal(t, "test-element", elementMeta.Name) require.Equal(t, "This is a test element", elementMeta.Description) - // Get the element - element, err := g.GetDatasetElement(context.Background(), workspace, dataset.ID, "test-element") + // Add two more + err = g.AddDatasetElements(context.Background(), workspaceID, dataset.ID, []DatasetElement{ + { + DatasetElementMeta: DatasetElementMeta{ + Name: "test-element-2", + Description: "This is a test element 2", + }, + Contents: "This is the content 2", + }, + { + DatasetElementMeta: DatasetElementMeta{ + Name: "test-element-3", + Description: "This is a test element 3", + }, + Contents: "This is the content 3", + }, + }) + require.NoError(t, err) + + // Get the first element + element, err := g.GetDatasetElement(context.Background(), workspaceID, dataset.ID, "test-element") require.NoError(t, err) require.Equal(t, "test-element", element.Name) require.Equal(t, "This is a test element", element.Description) require.Equal(t, "This is the content", element.Contents) + // Get the third element + element, err = g.GetDatasetElement(context.Background(), workspaceID, dataset.ID, "test-element-3") + require.NoError(t, err) + require.Equal(t, "test-element-3", element.Name) + require.Equal(t, "This is a test element 3", element.Description) + require.Equal(t, "This is the content 3", element.Contents) + // List elements in the dataset - elements, err := g.ListDatasetElements(context.Background(), workspace, dataset.ID) + elements, err := g.ListDatasetElements(context.Background(), workspaceID, dataset.ID) require.NoError(t, err) - require.Equal(t, 1, len(elements)) - require.Equal(t, "test-element", elements[0].Name) - require.Equal(t, "This is a test element", elements[0].Description) + require.Equal(t, 3, len(elements)) // List datasets - datasets, err := g.ListDatasets(context.Background(), workspace) + datasets, err := g.ListDatasets(context.Background(), workspaceID) require.NoError(t, err) require.Equal(t, 1, len(datasets)) require.Equal(t, "test-dataset", datasets[0].Name) From c09e0f56b39b60cbddb7316a67ed43b5a63e0bc7 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Wed, 23 Oct 2024 15:57:50 -0400 Subject: [PATCH 50/78] feat: detect not found errors in workspace API (#79) This will allow for proper error handling in the workspace API. Signed-off-by: Donnie Adams --- workspace.go | 35 ++++++++++++++++++++++++++++++++--- workspace_test.go | 13 +++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/workspace.go b/workspace.go index 6d79998..cc51e58 100644 --- a/workspace.go +++ b/workspace.go @@ -2,10 +2,26 @@ package gptscript import ( "context" + "encoding/base64" + "encoding/json" + "fmt" "os" "strings" ) +type NotFoundInWorkspaceError struct { + id string + name string +} + +func (e *NotFoundInWorkspaceError) Error() string { + return fmt.Sprintf("not found: %s/%s", e.id, e.name) +} + +func newNotFoundInWorkspaceError(id, name string) *NotFoundInWorkspaceError { + return &NotFoundInWorkspaceError{id: id, name: name} +} + func (g *GPTScript) CreateWorkspace(ctx context.Context, providerType string, fromWorkspaces ...string) (string, error) { out, err := g.runBasicCommand(ctx, "workspaces/create", map[string]any{ "providerType": providerType, @@ -75,7 +91,13 @@ func (g *GPTScript) ListFilesInWorkspace(ctx context.Context, opts ...ListFilesI return nil, err } - return strings.Split(strings.TrimSpace(out), "\n"), nil + out = strings.TrimSpace(out) + if len(out) == 0 { + return nil, nil + } + + var files []string + return files, json.Unmarshal([]byte(out), &files) } type RemoveAllOptions struct { @@ -126,7 +148,7 @@ func (g *GPTScript) WriteFileInWorkspace(ctx context.Context, filePath string, c _, err := g.runBasicCommand(ctx, "workspaces/write-file", map[string]any{ "id": opt.WorkspaceID, - "contents": contents, + "contents": base64.StdEncoding.EncodeToString(contents), "filePath": filePath, "workspaceTool": g.globalOpts.WorkspaceTool, "env": g.globalOpts.Env, @@ -158,6 +180,10 @@ func (g *GPTScript) DeleteFileInWorkspace(ctx context.Context, filePath string, "env": g.globalOpts.Env, }) + if err != nil && strings.HasSuffix(err.Error(), fmt.Sprintf("not found: %s/%s", opt.WorkspaceID, filePath)) { + return newNotFoundInWorkspaceError(opt.WorkspaceID, filePath) + } + return err } @@ -184,8 +210,11 @@ func (g *GPTScript) ReadFileInWorkspace(ctx context.Context, filePath string, op "env": g.globalOpts.Env, }) if err != nil { + if strings.HasSuffix(err.Error(), fmt.Sprintf("not found: %s/%s", opt.WorkspaceID, filePath)) { + return nil, newNotFoundInWorkspaceError(opt.WorkspaceID, filePath) + } return nil, err } - return []byte(out), nil + return base64.StdEncoding.DecodeString(out) } diff --git a/workspace_test.go b/workspace_test.go index a7ff08c..8881d00 100644 --- a/workspace_test.go +++ b/workspace_test.go @@ -3,6 +3,7 @@ package gptscript import ( "bytes" "context" + "errors" "os" "testing" ) @@ -46,6 +47,12 @@ func TestWriteReadAndDeleteFileFromWorkspace(t *testing.T) { t.Errorf("Unexpected content: %s", content) } + // Ensure we get the error we expect when trying to read a non-existent file + _, err = g.ReadFileInWorkspace(context.Background(), "test1.txt", ReadFileInWorkspaceOptions{WorkspaceID: id}) + if nf := (*NotFoundInWorkspaceError)(nil); !errors.As(err, &nf) { + t.Errorf("Unexpected error reading non-existent file: %v", err) + } + err = g.DeleteFileInWorkspace(context.Background(), "test.txt", DeleteFileInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Errorf("Error deleting file: %v", err) @@ -169,6 +176,12 @@ func TestWriteReadAndDeleteFileFromWorkspaceS3(t *testing.T) { t.Errorf("Unexpected content: %s", content) } + // Ensure we get the error we expect when trying to read a non-existent file + _, err = g.ReadFileInWorkspace(context.Background(), "test1.txt", ReadFileInWorkspaceOptions{WorkspaceID: id}) + if nf := (*NotFoundInWorkspaceError)(nil); !errors.As(err, &nf) { + t.Errorf("Unexpected error reading non-existent file: %v", err) + } + err = g.DeleteFileInWorkspace(context.Background(), "test.txt", DeleteFileInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Errorf("Error deleting file: %v", err) From d55120b63f4e9d4900a59730f120b377d774c477 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Mon, 28 Oct 2024 06:00:37 -0400 Subject: [PATCH 51/78] chore: make workspace ID required for deleting workspace Signed-off-by: Donnie Adams --- datasets_test.go | 2 +- workspace.go | 19 ++++--------------- workspace_test.go | 18 ++++++++++++------ 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/datasets_test.go b/datasets_test.go index 44d72ab..74c6eb4 100644 --- a/datasets_test.go +++ b/datasets_test.go @@ -12,7 +12,7 @@ func TestDatasets(t *testing.T) { require.NoError(t, err) defer func() { - _ = g.DeleteWorkspace(context.Background(), DeleteWorkspaceOptions{WorkspaceID: workspaceID}) + _ = g.DeleteWorkspace(context.Background(), workspaceID) }() // Create a dataset diff --git a/workspace.go b/workspace.go index cc51e58..58cd72b 100644 --- a/workspace.go +++ b/workspace.go @@ -36,24 +36,13 @@ func (g *GPTScript) CreateWorkspace(ctx context.Context, providerType string, fr return strings.TrimSpace(out), nil } -type DeleteWorkspaceOptions struct { - WorkspaceID string -} - -func (g *GPTScript) DeleteWorkspace(ctx context.Context, opts ...DeleteWorkspaceOptions) error { - var opt DeleteWorkspaceOptions - for _, o := range opts { - if o.WorkspaceID != "" { - opt.WorkspaceID = o.WorkspaceID - } - } - - if opt.WorkspaceID == "" { - opt.WorkspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") +func (g *GPTScript) DeleteWorkspace(ctx context.Context, workspaceID string) error { + if workspaceID == "" { + return fmt.Errorf("workspace ID cannot be empty") } _, err := g.runBasicCommand(ctx, "workspaces/delete", map[string]any{ - "id": opt.WorkspaceID, + "id": workspaceID, "workspaceTool": g.globalOpts.WorkspaceTool, "env": g.globalOpts.Env, }) diff --git a/workspace_test.go b/workspace_test.go index 8881d00..69e99f6 100644 --- a/workspace_test.go +++ b/workspace_test.go @@ -8,13 +8,19 @@ import ( "testing" ) +func TestWorkspaceIDRequiredForDelete(t *testing.T) { + if err := g.DeleteWorkspace(context.Background(), ""); err == nil { + t.Error("Expected error but got nil") + } +} + func TestCreateAndDeleteWorkspace(t *testing.T) { id, err := g.CreateWorkspace(context.Background(), "directory") if err != nil { t.Fatalf("Error creating workspace: %v", err) } - err = g.DeleteWorkspace(context.Background(), DeleteWorkspaceOptions{WorkspaceID: id}) + err = g.DeleteWorkspace(context.Background(), id) if err != nil { t.Errorf("Error deleting workspace: %v", err) } @@ -27,7 +33,7 @@ func TestWriteReadAndDeleteFileFromWorkspace(t *testing.T) { } t.Cleanup(func() { - err := g.DeleteWorkspace(context.Background(), DeleteWorkspaceOptions{WorkspaceID: id}) + err := g.DeleteWorkspace(context.Background(), id) if err != nil { t.Errorf("Error deleting workspace: %v", err) } @@ -66,7 +72,7 @@ func TestLsComplexWorkspace(t *testing.T) { } t.Cleanup(func() { - err := g.DeleteWorkspace(context.Background(), DeleteWorkspaceOptions{WorkspaceID: id}) + err := g.DeleteWorkspace(context.Background(), id) if err != nil { t.Errorf("Error deleting workspace: %v", err) } @@ -139,7 +145,7 @@ func TestCreateAndDeleteWorkspaceS3(t *testing.T) { t.Fatalf("Error creating workspace: %v", err) } - err = g.DeleteWorkspace(context.Background(), DeleteWorkspaceOptions{WorkspaceID: id}) + err = g.DeleteWorkspace(context.Background(), id) if err != nil { t.Errorf("Error deleting workspace: %v", err) } @@ -156,7 +162,7 @@ func TestWriteReadAndDeleteFileFromWorkspaceS3(t *testing.T) { } t.Cleanup(func() { - err := g.DeleteWorkspace(context.Background(), DeleteWorkspaceOptions{WorkspaceID: id}) + err := g.DeleteWorkspace(context.Background(), id) if err != nil { t.Errorf("Error deleting workspace: %v", err) } @@ -199,7 +205,7 @@ func TestLsComplexWorkspaceS3(t *testing.T) { } t.Cleanup(func() { - err := g.DeleteWorkspace(context.Background(), DeleteWorkspaceOptions{WorkspaceID: id}) + err := g.DeleteWorkspace(context.Background(), id) if err != nil { t.Errorf("Error deleting workspace: %v", err) } From 1fce3cceae350e890807b2f751f61d26c46d52ea Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Mon, 28 Oct 2024 07:10:38 -0400 Subject: [PATCH 52/78] feat: add file stat to workspace API Signed-off-by: Donnie Adams --- workspace.go | 46 ++++++++++++++++++++++++++++++++++++++++++++++ workspace_test.go | 44 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/workspace.go b/workspace.go index 58cd72b..aa4f175 100644 --- a/workspace.go +++ b/workspace.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "strings" + "time" ) type NotFoundInWorkspaceError struct { @@ -207,3 +208,48 @@ func (g *GPTScript) ReadFileInWorkspace(ctx context.Context, filePath string, op return base64.StdEncoding.DecodeString(out) } + +type StatFileInWorkspaceOptions struct { + WorkspaceID string +} + +func (g *GPTScript) StatFileInWorkspace(ctx context.Context, filePath string, opts ...StatFileInWorkspaceOptions) (FileInfo, error) { + var opt StatFileInWorkspaceOptions + for _, o := range opts { + if o.WorkspaceID != "" { + opt.WorkspaceID = o.WorkspaceID + } + } + + if opt.WorkspaceID == "" { + opt.WorkspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") + } + + out, err := g.runBasicCommand(ctx, "workspaces/stat-file", map[string]any{ + "id": opt.WorkspaceID, + "filePath": filePath, + "workspaceTool": g.globalOpts.WorkspaceTool, + "env": g.globalOpts.Env, + }) + if err != nil { + if strings.HasSuffix(err.Error(), fmt.Sprintf("not found: %s/%s", opt.WorkspaceID, filePath)) { + return FileInfo{}, newNotFoundInWorkspaceError(opt.WorkspaceID, filePath) + } + return FileInfo{}, err + } + + var info FileInfo + err = json.Unmarshal([]byte(out), &info) + if err != nil { + return FileInfo{}, err + } + + return info, nil +} + +type FileInfo struct { + WorkspaceID string + Name string + Size int64 + ModTime time.Time +} diff --git a/workspace_test.go b/workspace_test.go index 69e99f6..df51a86 100644 --- a/workspace_test.go +++ b/workspace_test.go @@ -53,6 +53,28 @@ func TestWriteReadAndDeleteFileFromWorkspace(t *testing.T) { t.Errorf("Unexpected content: %s", content) } + // Stat the file to ensure it exists + fileInfo, err := g.StatFileInWorkspace(context.Background(), "test.txt", StatFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error statting file: %v", err) + } + + if fileInfo.WorkspaceID != id { + t.Errorf("Unexpected file workspace ID: %v", fileInfo.WorkspaceID) + } + + if fileInfo.Name != "test.txt" { + t.Errorf("Unexpected file name: %s", fileInfo.Name) + } + + if fileInfo.Size != 4 { + t.Errorf("Unexpected file size: %d", fileInfo.Size) + } + + if fileInfo.ModTime.IsZero() { + t.Errorf("Unexpected file mod time: %v", fileInfo.ModTime) + } + // Ensure we get the error we expect when trying to read a non-existent file _, err = g.ReadFileInWorkspace(context.Background(), "test1.txt", ReadFileInWorkspaceOptions{WorkspaceID: id}) if nf := (*NotFoundInWorkspaceError)(nil); !errors.As(err, &nf) { @@ -182,6 +204,28 @@ func TestWriteReadAndDeleteFileFromWorkspaceS3(t *testing.T) { t.Errorf("Unexpected content: %s", content) } + // Stat the file to ensure it exists + fileInfo, err := g.StatFileInWorkspace(context.Background(), "test.txt", StatFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error statting file: %v", err) + } + + if fileInfo.WorkspaceID != id { + t.Errorf("Unexpected file workspace ID: %v", fileInfo.WorkspaceID) + } + + if fileInfo.Name != "test.txt" { + t.Errorf("Unexpected file name: %s", fileInfo.Name) + } + + if fileInfo.Size != 4 { + t.Errorf("Unexpected file size: %d", fileInfo.Size) + } + + if fileInfo.ModTime.IsZero() { + t.Errorf("Unexpected file mod time: %v", fileInfo.ModTime) + } + // Ensure we get the error we expect when trying to read a non-existent file _, err = g.ReadFileInWorkspace(context.Background(), "test1.txt", ReadFileInWorkspaceOptions{WorkspaceID: id}) if nf := (*NotFoundInWorkspaceError)(nil); !errors.As(err, &nf) { From 3a3a70ca44b6e11d3b55f158c85bd4f1bea74732 Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Fri, 1 Nov 2024 10:36:42 -0400 Subject: [PATCH 53/78] enhance: use byte slice for dataset element contents (#81) Signed-off-by: Grant Linville --- datasets.go | 7 ++++--- datasets_test.go | 10 +++++----- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/datasets.go b/datasets.go index 7b9f377..0b39edc 100644 --- a/datasets.go +++ b/datasets.go @@ -2,6 +2,7 @@ package gptscript import ( "context" + "encoding/base64" "encoding/json" "fmt" "os" @@ -14,7 +15,7 @@ type DatasetElementMeta struct { type DatasetElement struct { DatasetElementMeta `json:",inline"` - Contents string `json:"contents"` + Contents []byte `json:"contents"` } type DatasetMeta struct { @@ -115,7 +116,7 @@ func (g *GPTScript) CreateDataset(ctx context.Context, workspaceID, name, descri return dataset, nil } -func (g *GPTScript) AddDatasetElement(ctx context.Context, workspaceID, datasetID, elementName, elementDescription, elementContent string) (DatasetElementMeta, error) { +func (g *GPTScript) AddDatasetElement(ctx context.Context, workspaceID, datasetID, elementName, elementDescription string, elementContent []byte) (DatasetElementMeta, error) { if workspaceID == "" { workspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") } @@ -124,7 +125,7 @@ func (g *GPTScript) AddDatasetElement(ctx context.Context, workspaceID, datasetI DatasetID: datasetID, ElementName: elementName, ElementDescription: elementDescription, - ElementContent: elementContent, + ElementContent: base64.StdEncoding.EncodeToString(elementContent), } argsJSON, err := json.Marshal(args) if err != nil { diff --git a/datasets_test.go b/datasets_test.go index 74c6eb4..6fc7ed9 100644 --- a/datasets_test.go +++ b/datasets_test.go @@ -23,7 +23,7 @@ func TestDatasets(t *testing.T) { require.Equal(t, 0, len(dataset.Elements)) // Add an element - elementMeta, err := g.AddDatasetElement(context.Background(), workspaceID, dataset.ID, "test-element", "This is a test element", "This is the content") + elementMeta, err := g.AddDatasetElement(context.Background(), workspaceID, dataset.ID, "test-element", "This is a test element", []byte("This is the content")) require.NoError(t, err) require.Equal(t, "test-element", elementMeta.Name) require.Equal(t, "This is a test element", elementMeta.Description) @@ -35,14 +35,14 @@ func TestDatasets(t *testing.T) { Name: "test-element-2", Description: "This is a test element 2", }, - Contents: "This is the content 2", + Contents: []byte("This is the content 2"), }, { DatasetElementMeta: DatasetElementMeta{ Name: "test-element-3", Description: "This is a test element 3", }, - Contents: "This is the content 3", + Contents: []byte("This is the content 3"), }, }) require.NoError(t, err) @@ -52,14 +52,14 @@ func TestDatasets(t *testing.T) { require.NoError(t, err) require.Equal(t, "test-element", element.Name) require.Equal(t, "This is a test element", element.Description) - require.Equal(t, "This is the content", element.Contents) + require.Equal(t, []byte("This is the content"), element.Contents) // Get the third element element, err = g.GetDatasetElement(context.Background(), workspaceID, dataset.ID, "test-element-3") require.NoError(t, err) require.Equal(t, "test-element-3", element.Name) require.Equal(t, "This is a test element 3", element.Description) - require.Equal(t, "This is the content 3", element.Contents) + require.Equal(t, []byte("This is the content 3"), element.Contents) // List elements in the dataset elements, err := g.ListDatasetElements(context.Background(), workspaceID, dataset.ID) From df4a8c98fdbc028d172b0b5fd49e262904d7457c Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Fri, 1 Nov 2024 20:44:06 -0400 Subject: [PATCH 54/78] fix: ensure the parent call frame is of category none Signed-off-by: Donnie Adams --- frame.go | 11 +++++++++++ run.go | 28 ++++++++++------------------ 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/frame.go b/frame.go index 39d8ba9..5207e43 100644 --- a/frame.go +++ b/frame.go @@ -49,6 +49,17 @@ type RunFrame struct { Type EventType `json:"type"` } +type CallFrames map[string]CallFrame + +func (c CallFrames) ParentCallFrame() CallFrame { + for _, call := range c { + if call.ParentID == "" && call.ToolCategory == NoCategory { + return call + } + } + return CallFrame{} +} + type CallFrame struct { CallContext `json:",inline"` diff --git a/run.go b/run.go index a111045..4a5865d 100644 --- a/run.go +++ b/run.go @@ -36,15 +36,14 @@ type Run struct { wait func() basicCommand bool - program *Program - callsLock sync.RWMutex - calls map[string]CallFrame - parentCallFrameID string - rawOutput map[string]any - output, errput string - events chan Frame - lock sync.Mutex - responseCode int + program *Program + callsLock sync.RWMutex + calls CallFrames + rawOutput map[string]any + output, errput string + events chan Frame + lock sync.Mutex + responseCode int } // Text returns the text output of the gptscript. It blocks until the output is ready. @@ -104,7 +103,7 @@ func (r *Run) RespondingTool() Tool { } // Calls will return a flattened array of the calls for this run. -func (r *Run) Calls() map[string]CallFrame { +func (r *Run) Calls() CallFrames { r.callsLock.RLock() defer r.callsLock.RUnlock() return maps.Clone(r.calls) @@ -115,11 +114,7 @@ func (r *Run) ParentCallFrame() (CallFrame, bool) { r.callsLock.RLock() defer r.callsLock.RUnlock() - if r.parentCallFrameID == "" { - return CallFrame{}, false - } - - return r.calls[r.parentCallFrameID], true + return r.calls.ParentCallFrame(), true } // ErrorOutput returns the stderr output of the gptscript. @@ -394,9 +389,6 @@ func (r *Run) request(ctx context.Context, payload any) (err error) { if event.Call != nil { r.callsLock.Lock() r.calls[event.Call.ID] = *event.Call - if r.parentCallFrameID == "" && event.Call.ParentID == "" { - r.parentCallFrameID = event.Call.ID - } r.callsLock.Unlock() } else if event.Run != nil { if event.Run.Type == EventTypeRunStart { From ba040ce8f47ba1ba46508aa4e4cc312a5d2caf84 Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Wed, 6 Nov 2024 16:29:14 -0500 Subject: [PATCH 55/78] chore: update for dataset rewrite (#83) Signed-off-by: Grant Linville --- datasets.go | 169 ++++++++++++----------------------------------- datasets_test.go | 74 +++++++++++++-------- opts.go | 4 +- 3 files changed, 92 insertions(+), 155 deletions(-) diff --git a/datasets.go b/datasets.go index 0b39edc..fb408c4 100644 --- a/datasets.go +++ b/datasets.go @@ -2,10 +2,8 @@ package gptscript import ( "context" - "encoding/base64" "encoding/json" "fmt" - "os" ) type DatasetElementMeta struct { @@ -15,7 +13,8 @@ type DatasetElementMeta struct { type DatasetElement struct { DatasetElementMeta `json:",inline"` - Contents []byte `json:"contents"` + Contents string `json:"contents"` + BinaryContents []byte `json:"binaryContents"` } type DatasetMeta struct { @@ -24,34 +23,17 @@ type DatasetMeta struct { Description string `json:"description"` } -type Dataset struct { - DatasetMeta `json:",inline"` - BaseDir string `json:"baseDir,omitempty"` - Elements map[string]DatasetElementMeta `json:"elements"` -} - type datasetRequest struct { - Input string `json:"input"` - WorkspaceID string `json:"workspaceID"` - DatasetToolRepo string `json:"datasetToolRepo"` - Env []string `json:"env"` -} - -type createDatasetArgs struct { - Name string `json:"datasetName"` - Description string `json:"datasetDescription"` -} - -type addDatasetElementArgs struct { - DatasetID string `json:"datasetID"` - ElementName string `json:"elementName"` - ElementDescription string `json:"elementDescription"` - ElementContent string `json:"elementContent"` + Input string `json:"input"` + DatasetTool string `json:"datasetTool"` + Env []string `json:"env"` } type addDatasetElementsArgs struct { - DatasetID string `json:"datasetID"` - Elements []DatasetElement `json:"elements"` + DatasetID string `json:"datasetID"` + Name string `json:"name"` + Description string `json:"description"` + Elements []DatasetElement `json:"elements"` } type listDatasetElementArgs struct { @@ -60,19 +42,14 @@ type listDatasetElementArgs struct { type getDatasetElementArgs struct { DatasetID string `json:"datasetID"` - Element string `json:"element"` + Element string `json:"name"` } -func (g *GPTScript) ListDatasets(ctx context.Context, workspaceID string) ([]DatasetMeta, error) { - if workspaceID == "" { - workspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") - } - +func (g *GPTScript) ListDatasets(ctx context.Context) ([]DatasetMeta, error) { out, err := g.runBasicCommand(ctx, "datasets", datasetRequest{ - Input: "{}", - WorkspaceID: workspaceID, - DatasetToolRepo: g.globalOpts.DatasetToolRepo, - Env: g.globalOpts.Env, + Input: "{}", + DatasetTool: g.globalOpts.DatasetTool, + Env: g.globalOpts.Env, }) if err != nil { return nil, err @@ -85,98 +62,42 @@ func (g *GPTScript) ListDatasets(ctx context.Context, workspaceID string) ([]Dat return datasets, nil } -func (g *GPTScript) CreateDataset(ctx context.Context, workspaceID, name, description string) (Dataset, error) { - if workspaceID == "" { - workspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") - } - - args := createDatasetArgs{ - Name: name, - Description: description, - } - argsJSON, err := json.Marshal(args) - if err != nil { - return Dataset{}, fmt.Errorf("failed to marshal dataset args: %w", err) - } - - out, err := g.runBasicCommand(ctx, "datasets/create", datasetRequest{ - Input: string(argsJSON), - WorkspaceID: workspaceID, - DatasetToolRepo: g.globalOpts.DatasetToolRepo, - Env: g.globalOpts.Env, - }) - if err != nil { - return Dataset{}, err - } - - var dataset Dataset - if err = json.Unmarshal([]byte(out), &dataset); err != nil { - return Dataset{}, err - } - return dataset, nil +type DatasetOptions struct { + Name, Description string } -func (g *GPTScript) AddDatasetElement(ctx context.Context, workspaceID, datasetID, elementName, elementDescription string, elementContent []byte) (DatasetElementMeta, error) { - if workspaceID == "" { - workspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") - } - - args := addDatasetElementArgs{ - DatasetID: datasetID, - ElementName: elementName, - ElementDescription: elementDescription, - ElementContent: base64.StdEncoding.EncodeToString(elementContent), - } - argsJSON, err := json.Marshal(args) - if err != nil { - return DatasetElementMeta{}, fmt.Errorf("failed to marshal element args: %w", err) - } - - out, err := g.runBasicCommand(ctx, "datasets/add-element", datasetRequest{ - Input: string(argsJSON), - WorkspaceID: workspaceID, - DatasetToolRepo: g.globalOpts.DatasetToolRepo, - Env: g.globalOpts.Env, - }) - if err != nil { - return DatasetElementMeta{}, err - } - - var element DatasetElementMeta - if err = json.Unmarshal([]byte(out), &element); err != nil { - return DatasetElementMeta{}, err - } - return element, nil +func (g *GPTScript) CreateDatasetWithElements(ctx context.Context, elements []DatasetElement, options ...DatasetOptions) (string, error) { + return g.AddDatasetElements(ctx, "", elements, options...) } -func (g *GPTScript) AddDatasetElements(ctx context.Context, workspaceID, datasetID string, elements []DatasetElement) error { - if workspaceID == "" { - workspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") - } - +func (g *GPTScript) AddDatasetElements(ctx context.Context, datasetID string, elements []DatasetElement, options ...DatasetOptions) (string, error) { args := addDatasetElementsArgs{ DatasetID: datasetID, Elements: elements, } + + for _, opt := range options { + if opt.Name != "" { + args.Name = opt.Name + } + if opt.Description != "" { + args.Description = opt.Description + } + } + argsJSON, err := json.Marshal(args) if err != nil { - return fmt.Errorf("failed to marshal element args: %w", err) + return "", fmt.Errorf("failed to marshal element args: %w", err) } - _, err = g.runBasicCommand(ctx, "datasets/add-elements", datasetRequest{ - Input: string(argsJSON), - WorkspaceID: workspaceID, - DatasetToolRepo: g.globalOpts.DatasetToolRepo, - Env: g.globalOpts.Env, + return g.runBasicCommand(ctx, "datasets/add-elements", datasetRequest{ + Input: string(argsJSON), + DatasetTool: g.globalOpts.DatasetTool, + Env: g.globalOpts.Env, }) - return err } -func (g *GPTScript) ListDatasetElements(ctx context.Context, workspaceID, datasetID string) ([]DatasetElementMeta, error) { - if workspaceID == "" { - workspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") - } - +func (g *GPTScript) ListDatasetElements(ctx context.Context, datasetID string) ([]DatasetElementMeta, error) { args := listDatasetElementArgs{ DatasetID: datasetID, } @@ -186,10 +107,9 @@ func (g *GPTScript) ListDatasetElements(ctx context.Context, workspaceID, datase } out, err := g.runBasicCommand(ctx, "datasets/list-elements", datasetRequest{ - Input: string(argsJSON), - WorkspaceID: workspaceID, - DatasetToolRepo: g.globalOpts.DatasetToolRepo, - Env: g.globalOpts.Env, + Input: string(argsJSON), + DatasetTool: g.globalOpts.DatasetTool, + Env: g.globalOpts.Env, }) if err != nil { return nil, err @@ -202,11 +122,7 @@ func (g *GPTScript) ListDatasetElements(ctx context.Context, workspaceID, datase return elements, nil } -func (g *GPTScript) GetDatasetElement(ctx context.Context, workspaceID, datasetID, elementName string) (DatasetElement, error) { - if workspaceID == "" { - workspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") - } - +func (g *GPTScript) GetDatasetElement(ctx context.Context, datasetID, elementName string) (DatasetElement, error) { args := getDatasetElementArgs{ DatasetID: datasetID, Element: elementName, @@ -217,10 +133,9 @@ func (g *GPTScript) GetDatasetElement(ctx context.Context, workspaceID, datasetI } out, err := g.runBasicCommand(ctx, "datasets/get-element", datasetRequest{ - Input: string(argsJSON), - WorkspaceID: workspaceID, - DatasetToolRepo: g.globalOpts.DatasetToolRepo, - Env: g.globalOpts.Env, + Input: string(argsJSON), + DatasetTool: g.globalOpts.DatasetTool, + Env: g.globalOpts.Env, }) if err != nil { return DatasetElement{}, err diff --git a/datasets_test.go b/datasets_test.go index 6fc7ed9..c1f1a92 100644 --- a/datasets_test.go +++ b/datasets_test.go @@ -2,6 +2,7 @@ package gptscript import ( "context" + "os" "testing" "github.com/stretchr/testify/require" @@ -11,66 +12,87 @@ func TestDatasets(t *testing.T) { workspaceID, err := g.CreateWorkspace(context.Background(), "directory") require.NoError(t, err) + client, err := NewGPTScript(GlobalOptions{ + OpenAIAPIKey: os.Getenv("OPENAI_API_KEY"), + Env: append(os.Environ(), "GPTSCRIPT_WORKSPACE_ID="+workspaceID), + }) + require.NoError(t, err) + defer func() { _ = g.DeleteWorkspace(context.Background(), workspaceID) }() - // Create a dataset - dataset, err := g.CreateDataset(context.Background(), workspaceID, "test-dataset", "This is a test dataset") - require.NoError(t, err) - require.Equal(t, "test-dataset", dataset.Name) - require.Equal(t, "This is a test dataset", dataset.Description) - require.Equal(t, 0, len(dataset.Elements)) - - // Add an element - elementMeta, err := g.AddDatasetElement(context.Background(), workspaceID, dataset.ID, "test-element", "This is a test element", []byte("This is the content")) + datasetID, err := client.CreateDatasetWithElements(context.Background(), []DatasetElement{ + { + DatasetElementMeta: DatasetElementMeta{ + Name: "test-element-1", + Description: "This is a test element 1", + }, + Contents: "This is the content 1", + }, + }, DatasetOptions{ + Name: "test-dataset", + Description: "this is a test dataset", + }) require.NoError(t, err) - require.Equal(t, "test-element", elementMeta.Name) - require.Equal(t, "This is a test element", elementMeta.Description) - // Add two more - err = g.AddDatasetElements(context.Background(), workspaceID, dataset.ID, []DatasetElement{ + // Add three more elements + _, err = client.AddDatasetElements(context.Background(), datasetID, []DatasetElement{ { DatasetElementMeta: DatasetElementMeta{ Name: "test-element-2", Description: "This is a test element 2", }, - Contents: []byte("This is the content 2"), + Contents: "This is the content 2", }, { DatasetElementMeta: DatasetElementMeta{ Name: "test-element-3", Description: "This is a test element 3", }, - Contents: []byte("This is the content 3"), + Contents: "This is the content 3", + }, + { + DatasetElementMeta: DatasetElementMeta{ + Name: "binary-element", + Description: "this element has binary contents", + }, + BinaryContents: []byte("binary contents"), }, }) require.NoError(t, err) // Get the first element - element, err := g.GetDatasetElement(context.Background(), workspaceID, dataset.ID, "test-element") + element, err := client.GetDatasetElement(context.Background(), datasetID, "test-element-1") require.NoError(t, err) - require.Equal(t, "test-element", element.Name) - require.Equal(t, "This is a test element", element.Description) - require.Equal(t, []byte("This is the content"), element.Contents) + require.Equal(t, "test-element-1", element.Name) + require.Equal(t, "This is a test element 1", element.Description) + require.Equal(t, "This is the content 1", element.Contents) // Get the third element - element, err = g.GetDatasetElement(context.Background(), workspaceID, dataset.ID, "test-element-3") + element, err = client.GetDatasetElement(context.Background(), datasetID, "test-element-3") require.NoError(t, err) require.Equal(t, "test-element-3", element.Name) require.Equal(t, "This is a test element 3", element.Description) - require.Equal(t, []byte("This is the content 3"), element.Contents) + require.Equal(t, "This is the content 3", element.Contents) + + // Get the binary element + element, err = client.GetDatasetElement(context.Background(), datasetID, "binary-element") + require.NoError(t, err) + require.Equal(t, "binary-element", element.Name) + require.Equal(t, "this element has binary contents", element.Description) + require.Equal(t, []byte("binary contents"), element.BinaryContents) // List elements in the dataset - elements, err := g.ListDatasetElements(context.Background(), workspaceID, dataset.ID) + elements, err := client.ListDatasetElements(context.Background(), datasetID) require.NoError(t, err) - require.Equal(t, 3, len(elements)) + require.Equal(t, 4, len(elements)) // List datasets - datasets, err := g.ListDatasets(context.Background(), workspaceID) + datasets, err := client.ListDatasets(context.Background()) require.NoError(t, err) require.Equal(t, 1, len(datasets)) + require.Equal(t, datasetID, datasets[0].ID) require.Equal(t, "test-dataset", datasets[0].Name) - require.Equal(t, "This is a test dataset", datasets[0].Description) - require.Equal(t, dataset.ID, datasets[0].ID) + require.Equal(t, "this is a test dataset", datasets[0].Description) } diff --git a/opts.go b/opts.go index e08d217..07507e2 100644 --- a/opts.go +++ b/opts.go @@ -11,7 +11,7 @@ type GlobalOptions struct { DefaultModelProvider string `json:"DefaultModelProvider"` CacheDir string `json:"CacheDir"` Env []string `json:"env"` - DatasetToolRepo string `json:"DatasetToolRepo"` + DatasetTool string `json:"DatasetTool"` WorkspaceTool string `json:"WorkspaceTool"` } @@ -46,7 +46,7 @@ func completeGlobalOptions(opts ...GlobalOptions) GlobalOptions { result.OpenAIBaseURL = firstSet(opt.OpenAIBaseURL, result.OpenAIBaseURL) result.DefaultModel = firstSet(opt.DefaultModel, result.DefaultModel) result.DefaultModelProvider = firstSet(opt.DefaultModelProvider, result.DefaultModelProvider) - result.DatasetToolRepo = firstSet(opt.DatasetToolRepo, result.DatasetToolRepo) + result.DatasetTool = firstSet(opt.DatasetTool, result.DatasetTool) result.WorkspaceTool = firstSet(opt.WorkspaceTool, result.WorkspaceTool) result.Env = append(result.Env, opt.Env...) } From c6dd53e3a2fdef90b07e524dd3ec3ee474d404e1 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Fri, 15 Nov 2024 11:36:50 -0500 Subject: [PATCH 56/78] fix: only cancel the context if the run is still processing Signed-off-by: Donnie Adams --- run.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/run.go b/run.go index 4a5865d..07f13fb 100644 --- a/run.go +++ b/run.go @@ -135,7 +135,10 @@ func (r *Run) Close() error { return fmt.Errorf("run not started") } - r.cancel(errAbortRun) + if !r.lock.TryLock() { + // If we can't get the lock, then the run is still running. Abort it. + r.cancel(errAbortRun) + } if r.wait == nil { return nil } @@ -285,10 +288,10 @@ func (r *Run) request(ctx context.Context, payload any) (err error) { ) defer func() { resp.Body.Close() - close(r.events) cancel(r.err) r.wait() r.lock.Unlock() + close(r.events) }() r.callsLock.Lock() From aba0474b01f487cc262193729f198702f06d5f71 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Fri, 15 Nov 2024 12:13:01 -0500 Subject: [PATCH 57/78] fix: capture bad response status code Signed-off-by: Donnie Adams --- datasets_test.go | 2 ++ run.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/datasets_test.go b/datasets_test.go index c1f1a92..41553df 100644 --- a/datasets_test.go +++ b/datasets_test.go @@ -9,6 +9,8 @@ import ( ) func TestDatasets(t *testing.T) { + t.Skipf("Changes have been made to the dataset API, this test needs to be updated") + workspaceID, err := g.CreateWorkspace(context.Background(), "directory") require.NoError(t, err) diff --git a/run.go b/run.go index 07f13fb..2ed6c1c 100644 --- a/run.go +++ b/run.go @@ -260,7 +260,7 @@ func (r *Run) request(ctx context.Context, payload any) (err error) { r.responseCode = resp.StatusCode if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusBadRequest { r.state = Error - r.err = fmt.Errorf("run encountered an error") + r.err = fmt.Errorf("run encountered an error: status code %d", resp.StatusCode) } else { r.state = Running } From 7efb3409cfcc1d9c0d24d68f722d9ba846d59470 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Fri, 15 Nov 2024 15:10:52 -0500 Subject: [PATCH 58/78] fix: unlock mutex if trying is successful Signed-off-by: Donnie Adams --- run.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/run.go b/run.go index 2ed6c1c..dac9c15 100644 --- a/run.go +++ b/run.go @@ -135,10 +135,13 @@ func (r *Run) Close() error { return fmt.Errorf("run not started") } - if !r.lock.TryLock() { - // If we can't get the lock, then the run is still running. Abort it. - r.cancel(errAbortRun) + if r.lock.TryLock() { + r.lock.Unlock() + // If we can get the lock, then the run isn't running, so nothing to do. + return nil } + + r.cancel(errAbortRun) if r.wait == nil { return nil } From 9848026fdabcb723b54fb286b7b6e9c2efa9d124 Mon Sep 17 00:00:00 2001 From: Thorsten Klein Date: Wed, 20 Nov 2024 21:17:39 +0100 Subject: [PATCH 59/78] feat: add to workspace fileinfo (#85) --- workspace.go | 1 + workspace_test.go | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/workspace.go b/workspace.go index aa4f175..a1214b8 100644 --- a/workspace.go +++ b/workspace.go @@ -252,4 +252,5 @@ type FileInfo struct { Name string Size int64 ModTime time.Time + MimeType string } diff --git a/workspace_test.go b/workspace_test.go index df51a86..d2d5706 100644 --- a/workspace_test.go +++ b/workspace_test.go @@ -75,6 +75,10 @@ func TestWriteReadAndDeleteFileFromWorkspace(t *testing.T) { t.Errorf("Unexpected file mod time: %v", fileInfo.ModTime) } + if fileInfo.MimeType != "text/plain" { + t.Errorf("Unexpected file mime type: %s", fileInfo.MimeType) + } + // Ensure we get the error we expect when trying to read a non-existent file _, err = g.ReadFileInWorkspace(context.Background(), "test1.txt", ReadFileInWorkspaceOptions{WorkspaceID: id}) if nf := (*NotFoundInWorkspaceError)(nil); !errors.As(err, &nf) { @@ -226,6 +230,10 @@ func TestWriteReadAndDeleteFileFromWorkspaceS3(t *testing.T) { t.Errorf("Unexpected file mod time: %v", fileInfo.ModTime) } + if fileInfo.MimeType != "text/plain" { + t.Errorf("Unexpected file mime type: %s", fileInfo.MimeType) + } + // Ensure we get the error we expect when trying to read a non-existent file _, err = g.ReadFileInWorkspace(context.Background(), "test1.txt", ReadFileInWorkspaceOptions{WorkspaceID: id}) if nf := (*NotFoundInWorkspaceError)(nil); !errors.As(err, &nf) { From 2ddfb8e12f34d133a92b6b29d86a96ec3b0f185f Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Fri, 22 Nov 2024 08:36:27 -0500 Subject: [PATCH 60/78] chore: stop skipping dataset test Signed-off-by: Donnie Adams --- datasets_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/datasets_test.go b/datasets_test.go index 41553df..c1f1a92 100644 --- a/datasets_test.go +++ b/datasets_test.go @@ -9,8 +9,6 @@ import ( ) func TestDatasets(t *testing.T) { - t.Skipf("Changes have been made to the dataset API, this test needs to be updated") - workspaceID, err := g.CreateWorkspace(context.Background(), "directory") require.NoError(t, err) From 79a66826cf829ffe2bbcd086fc2c72ae8062ee32 Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Mon, 16 Dec 2024 16:13:44 -0500 Subject: [PATCH 61/78] enhance: add functions for daemon servers for mTLS (#87) Signed-off-by: Grant Linville --- pkg/daemon/daemon.go | 102 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 pkg/daemon/daemon.go diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go new file mode 100644 index 0000000..b25a49a --- /dev/null +++ b/pkg/daemon/daemon.go @@ -0,0 +1,102 @@ +package daemon + +import ( + "crypto/tls" + "crypto/x509" + "encoding/base64" + "errors" + "fmt" + "net/http" + "os" +) + +type Server struct { + mux *http.ServeMux + tlsConfig *tls.Config +} + +// CreateServer creates a new HTTP server with TLS configured for GPTScript. +// This function should be used when creating a new server for a daemon tool. +// The server should then be started with the StartServer function. +func CreateServer() (*Server, error) { + return CreateServerWithMux(http.DefaultServeMux) +} + +// CreateServerWithMux creates a new HTTP server with TLS configured for GPTScript. +// This function should be used when creating a new server for a daemon tool with a custom ServeMux. +// The server should then be started with the StartServer function. +func CreateServerWithMux(mux *http.ServeMux) (*Server, error) { + tlsConfig, err := getTLSConfig() + if err != nil { + return nil, fmt.Errorf("failed to get TLS config: %v", err) + } + + return &Server{ + mux: mux, + tlsConfig: tlsConfig, + }, nil +} + +// Start starts an HTTP server created by the CreateServer function. +// This is for use with daemon tools. +func (s *Server) Start() error { + server := &http.Server{ + Addr: fmt.Sprintf("127.0.0.1:%s", os.Getenv("PORT")), + TLSConfig: s.tlsConfig, + Handler: s.mux, + } + + if err := server.ListenAndServeTLS("", ""); err != nil && !errors.Is(err, http.ErrServerClosed) { + return fmt.Errorf("stopped serving: %v", err) + } + return nil +} + +func (s *Server) HandleFunc(pattern string, handler http.HandlerFunc) { + s.mux.HandleFunc(pattern, handler) +} + +func getTLSConfig() (*tls.Config, error) { + certB64 := os.Getenv("CERT") + privateKeyB64 := os.Getenv("PRIVATE_KEY") + gptscriptCertB64 := os.Getenv("GPTSCRIPT_CERT") + + if certB64 == "" { + return nil, fmt.Errorf("CERT not set") + } else if privateKeyB64 == "" { + return nil, fmt.Errorf("PRIVATE_KEY not set") + } else if gptscriptCertB64 == "" { + return nil, fmt.Errorf("GPTSCRIPT_CERT not set") + } + + certBytes, err := base64.StdEncoding.DecodeString(certB64) + if err != nil { + return nil, fmt.Errorf("failed to decode cert base64: %v", err) + } + + privateKeyBytes, err := base64.StdEncoding.DecodeString(privateKeyB64) + if err != nil { + return nil, fmt.Errorf("failed to decode private key base64: %v", err) + } + + gptscriptCertBytes, err := base64.StdEncoding.DecodeString(gptscriptCertB64) + if err != nil { + return nil, fmt.Errorf("failed to decode gptscript cert base64: %v", err) + } + + cert, err := tls.X509KeyPair(certBytes, privateKeyBytes) + if err != nil { + return nil, fmt.Errorf("failed to create X509 key pair: %v", err) + } + + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(gptscriptCertBytes) { + return nil, fmt.Errorf("failed to append gptscript cert to pool") + } + + return &tls.Config{ + Certificates: []tls.Certificate{cert}, + ClientCAs: pool, + ClientAuth: tls.RequireAndVerifyClientCert, + }, nil +} From 4c3ef0b9e3e50d1f9ff63bb8db0713e4f85db809 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Tue, 14 Jan 2025 08:09:04 -0500 Subject: [PATCH 62/78] feat: add file revision API for workspaces Signed-off-by: Donnie Adams --- workspace.go | 116 ++++++++++++++++++++++++++-- workspace_test.go | 189 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 300 insertions(+), 5 deletions(-) diff --git a/workspace.go b/workspace.go index a1214b8..321602a 100644 --- a/workspace.go +++ b/workspace.go @@ -209,6 +209,14 @@ func (g *GPTScript) ReadFileInWorkspace(ctx context.Context, filePath string, op return base64.StdEncoding.DecodeString(out) } +type FileInfo struct { + WorkspaceID string + Name string + Size int64 + ModTime time.Time + MimeType string +} + type StatFileInWorkspaceOptions struct { WorkspaceID string } @@ -247,10 +255,108 @@ func (g *GPTScript) StatFileInWorkspace(ctx context.Context, filePath string, op return info, nil } -type FileInfo struct { +type RevisionInfo struct { + FileInfo + RevisionID string +} + +type ListRevisionsForFileInWorkspaceOptions struct { WorkspaceID string - Name string - Size int64 - ModTime time.Time - MimeType string +} + +func (g *GPTScript) ListRevisionsForFileInWorkspace(ctx context.Context, filePath string, opts ...ListRevisionsForFileInWorkspaceOptions) ([]RevisionInfo, error) { + var opt ListRevisionsForFileInWorkspaceOptions + for _, o := range opts { + if o.WorkspaceID != "" { + opt.WorkspaceID = o.WorkspaceID + } + } + + if opt.WorkspaceID == "" { + opt.WorkspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") + } + + out, err := g.runBasicCommand(ctx, "workspaces/list-revisions", map[string]any{ + "id": opt.WorkspaceID, + "filePath": filePath, + "workspaceTool": g.globalOpts.WorkspaceTool, + "env": g.globalOpts.Env, + }) + if err != nil { + if strings.HasSuffix(err.Error(), fmt.Sprintf("not found: %s/%s", opt.WorkspaceID, filePath)) { + return nil, newNotFoundInWorkspaceError(opt.WorkspaceID, filePath) + } + return nil, err + } + + var info []RevisionInfo + err = json.Unmarshal([]byte(out), &info) + if err != nil { + return nil, err + } + + return info, nil +} + +type GetRevisionForFileInWorkspaceOptions struct { + WorkspaceID string +} + +func (g *GPTScript) GetRevisionForFileInWorkspace(ctx context.Context, filePath, revisionID string, opts ...GetRevisionForFileInWorkspaceOptions) ([]byte, error) { + var opt GetRevisionForFileInWorkspaceOptions + for _, o := range opts { + if o.WorkspaceID != "" { + opt.WorkspaceID = o.WorkspaceID + } + } + + if opt.WorkspaceID == "" { + opt.WorkspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") + } + + out, err := g.runBasicCommand(ctx, "workspaces/get-revision", map[string]any{ + "id": opt.WorkspaceID, + "filePath": filePath, + "revisionID": revisionID, + "workspaceTool": g.globalOpts.WorkspaceTool, + "env": g.globalOpts.Env, + }) + if err != nil { + if strings.HasSuffix(err.Error(), fmt.Sprintf("not found: %s/%s", opt.WorkspaceID, filePath)) { + return nil, newNotFoundInWorkspaceError(opt.WorkspaceID, filePath) + } + return nil, err + } + + return base64.StdEncoding.DecodeString(out) +} + +type DeleteRevisionForFileInWorkspaceOptions struct { + WorkspaceID string +} + +func (g *GPTScript) DeleteRevisionForFileInWorkspace(ctx context.Context, filePath, revisionID string, opts ...DeleteRevisionForFileInWorkspaceOptions) error { + var opt DeleteRevisionForFileInWorkspaceOptions + for _, o := range opts { + if o.WorkspaceID != "" { + opt.WorkspaceID = o.WorkspaceID + } + } + + if opt.WorkspaceID == "" { + opt.WorkspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") + } + + _, err := g.runBasicCommand(ctx, "workspaces/delete-revision", map[string]any{ + "id": opt.WorkspaceID, + "filePath": filePath, + "revisionID": revisionID, + "workspaceTool": g.globalOpts.WorkspaceTool, + "env": g.globalOpts.Env, + }) + if err != nil && strings.HasSuffix(err.Error(), fmt.Sprintf("not found: %s/%s", opt.WorkspaceID, filePath)) { + return newNotFoundInWorkspaceError(opt.WorkspaceID, fmt.Sprintf("revision %s for %s", revisionID, filePath)) + } + + return err } diff --git a/workspace_test.go b/workspace_test.go index d2d5706..38bb552 100644 --- a/workspace_test.go +++ b/workspace_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "fmt" "os" "testing" ) @@ -91,6 +92,98 @@ func TestWriteReadAndDeleteFileFromWorkspace(t *testing.T) { } } +func TestRevisionsForFileInWorkspace(t *testing.T) { + id, err := g.CreateWorkspace(context.Background(), "directory") + if err != nil { + t.Fatalf("Error creating workspace: %v", err) + } + + t.Cleanup(func() { + err := g.DeleteWorkspace(context.Background(), id) + if err != nil { + t.Errorf("Error deleting workspace: %v", err) + } + }) + + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test0"), WriteFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test1"), WriteFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test2"), WriteFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + revisions, err := g.ListRevisionsForFileInWorkspace(context.Background(), "test.txt", ListRevisionsForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if len(revisions) != 2 { + t.Errorf("Unexpected number of revisions: %d", len(revisions)) + } + + for i, rev := range revisions { + if rev.WorkspaceID != id { + t.Errorf("Unexpected file workspace ID: %v", rev.WorkspaceID) + } + + if rev.Name != "test.txt" { + t.Errorf("Unexpected file name: %s", rev.Name) + } + + if rev.Size != 5 { + t.Errorf("Unexpected file size: %d", rev.Size) + } + + if rev.ModTime.IsZero() { + t.Errorf("Unexpected file mod time: %v", rev.ModTime) + } + + if rev.MimeType != "text/plain" { + t.Errorf("Unexpected file mime type: %s", rev.MimeType) + } + + if rev.RevisionID != fmt.Sprintf("%d", i+1) { + t.Errorf("Unexpected revision ID: %s", rev.RevisionID) + } + } + + err = g.DeleteRevisionForFileInWorkspace(context.Background(), "test.txt", "1", DeleteRevisionForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error deleting revision for file: %v", err) + } + + revisions, err = g.ListRevisionsForFileInWorkspace(context.Background(), "test.txt", ListRevisionsForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if len(revisions) != 1 { + t.Errorf("Unexpected number of revisions: %d", len(revisions)) + } + + err = g.DeleteFileInWorkspace(context.Background(), "test.txt", DeleteFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error deleting file: %v", err) + } + + revisions, err = g.ListRevisionsForFileInWorkspace(context.Background(), "test.txt", ListRevisionsForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if len(revisions) != 0 { + t.Errorf("Unexpected number of revisions: %d", len(revisions)) + } +} + func TestLsComplexWorkspace(t *testing.T) { id, err := g.CreateWorkspace(context.Background(), "directory") if err != nil { @@ -246,6 +339,102 @@ func TestWriteReadAndDeleteFileFromWorkspaceS3(t *testing.T) { } } +func TestRevisionsForFileInWorkspaceS3(t *testing.T) { + if os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" || os.Getenv("WORKSPACE_PROVIDER_S3_BUCKET") == "" { + t.Skip("Skipping test because AWS credentials are not set") + } + + id, err := g.CreateWorkspace(context.Background(), "s3") + if err != nil { + t.Fatalf("Error creating workspace: %v", err) + } + + t.Cleanup(func() { + err := g.DeleteWorkspace(context.Background(), id) + if err != nil { + t.Errorf("Error deleting workspace: %v", err) + } + }) + + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test0"), WriteFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test1"), WriteFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test2"), WriteFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + revisions, err := g.ListRevisionsForFileInWorkspace(context.Background(), "test.txt", ListRevisionsForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if len(revisions) != 2 { + t.Errorf("Unexpected number of revisions: %d", len(revisions)) + } + + for i, rev := range revisions { + if rev.WorkspaceID != id { + t.Errorf("Unexpected file workspace ID: %v", rev.WorkspaceID) + } + + if rev.Name != "test.txt" { + t.Errorf("Unexpected file name: %s", rev.Name) + } + + if rev.Size != 5 { + t.Errorf("Unexpected file size: %d", rev.Size) + } + + if rev.ModTime.IsZero() { + t.Errorf("Unexpected file mod time: %v", rev.ModTime) + } + + if rev.MimeType != "text/plain" { + t.Errorf("Unexpected file mime type: %s", rev.MimeType) + } + + if rev.RevisionID != fmt.Sprintf("%d", i+1) { + t.Errorf("Unexpected revision ID: %s", rev.RevisionID) + } + } + + err = g.DeleteRevisionForFileInWorkspace(context.Background(), "test.txt", "1", DeleteRevisionForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error deleting revision for file: %v", err) + } + + revisions, err = g.ListRevisionsForFileInWorkspace(context.Background(), "test.txt", ListRevisionsForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if len(revisions) != 1 { + t.Errorf("Unexpected number of revisions: %d", len(revisions)) + } + + err = g.DeleteFileInWorkspace(context.Background(), "test.txt", DeleteFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error deleting file: %v", err) + } + + revisions, err = g.ListRevisionsForFileInWorkspace(context.Background(), "test.txt", ListRevisionsForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if len(revisions) != 0 { + t.Errorf("Unexpected number of revisions: %d", len(revisions)) + } +} + func TestLsComplexWorkspaceS3(t *testing.T) { if os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" || os.Getenv("WORKSPACE_PROVIDER_S3_BUCKET") == "" { t.Skip("Skipping test because AWS credentials are not set") From 07f7c7c78dde31d6103f9a5dde59ab583b8a2261 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Tue, 14 Jan 2025 12:40:58 -0500 Subject: [PATCH 63/78] chore: use model type for listing models Signed-off-by: Donnie Adams --- gptscript.go | 35 +++++++++++++++++++++++++++++++++-- gptscript_test.go | 8 ++++---- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/gptscript.go b/gptscript.go index 1e30d95..28a46a2 100644 --- a/gptscript.go +++ b/gptscript.go @@ -293,8 +293,34 @@ type ListModelsOptions struct { CredentialOverrides []string } +type Model struct { + CreatedAt int64 `json:"created"` + ID string `json:"id"` + Object string `json:"object"` + OwnedBy string `json:"owned_by"` + Permission []Permission `json:"permission"` + Root string `json:"root"` + Parent string `json:"parent"` + Metadata map[string]string `json:"metadata"` +} + +type Permission struct { + CreatedAt int64 `json:"created"` + ID string `json:"id"` + Object string `json:"object"` + AllowCreateEngine bool `json:"allow_create_engine"` + AllowSampling bool `json:"allow_sampling"` + AllowLogprobs bool `json:"allow_logprobs"` + AllowSearchIndices bool `json:"allow_search_indices"` + AllowView bool `json:"allow_view"` + AllowFineTuning bool `json:"allow_fine_tuning"` + Organization string `json:"organization"` + Group interface{} `json:"group"` + IsBlocking bool `json:"is_blocking"` +} + // ListModels will list all the available models. -func (g *GPTScript) ListModels(ctx context.Context, opts ...ListModelsOptions) ([]string, error) { +func (g *GPTScript) ListModels(ctx context.Context, opts ...ListModelsOptions) ([]Model, error) { var o ListModelsOptions for _, opt := range opts { o.Providers = append(o.Providers, opt.Providers...) @@ -314,7 +340,12 @@ func (g *GPTScript) ListModels(ctx context.Context, opts ...ListModelsOptions) ( return nil, err } - return strings.Split(strings.TrimSpace(out), "\n"), nil + var models []Model + if err = json.Unmarshal([]byte(out), &models); err != nil { + return nil, fmt.Errorf("failed to parse models: %w", err) + } + + return models, nil } func (g *GPTScript) Confirm(ctx context.Context, resp AuthResponse) error { diff --git a/gptscript_test.go b/gptscript_test.go index 32adff9..834181e 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -98,8 +98,8 @@ func TestListModelsWithProvider(t *testing.T) { } for _, model := range models { - if !strings.HasPrefix(model, "claude-3-") || !strings.HasSuffix(model, "from github.com/gptscript-ai/claude3-anthropic-provider") { - t.Errorf("Unexpected model name: %s", model) + if !strings.HasPrefix(model.ID, "claude-3-") || !strings.HasSuffix(model.ID, "from github.com/gptscript-ai/claude3-anthropic-provider") { + t.Errorf("Unexpected model name: %s", model.ID) } } } @@ -128,8 +128,8 @@ func TestListModelsWithDefaultProvider(t *testing.T) { } for _, model := range models { - if !strings.HasPrefix(model, "claude-3-") || !strings.HasSuffix(model, "from github.com/gptscript-ai/claude3-anthropic-provider") { - t.Errorf("Unexpected model name: %s", model) + if !strings.HasPrefix(model.ID, "claude-3-") || !strings.HasSuffix(model.ID, "from github.com/gptscript-ai/claude3-anthropic-provider") { + t.Errorf("Unexpected model name: %s", model.ID) } } } From 83490b5890e0f6d30546aeb7eec2227252f0411a Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Mon, 20 Jan 2025 08:46:01 -0500 Subject: [PATCH 64/78] chore: add tests for creating workspaces from workspace IDs Signed-off-by: Donnie Adams --- workspace_test.go | 172 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) diff --git a/workspace_test.go b/workspace_test.go index 38bb552..95829c7 100644 --- a/workspace_test.go +++ b/workspace_test.go @@ -27,6 +27,46 @@ func TestCreateAndDeleteWorkspace(t *testing.T) { } } +func TestCreateAndDeleteWorkspaceFromWorkspace(t *testing.T) { + id, err := g.CreateWorkspace(context.Background(), "directory") + if err != nil { + t.Fatalf("Error creating workspace: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), "file.txt", []byte("hello world"), WriteFileInWorkspaceOptions{ + WorkspaceID: id, + }) + if err != nil { + t.Errorf("Error creating file: %v", err) + } + + newID, err := g.CreateWorkspace(context.Background(), "directory", id) + if err != nil { + t.Errorf("Error creating workspace from workspace: %v", err) + } + + data, err := g.ReadFileInWorkspace(context.Background(), "file.txt", ReadFileInWorkspaceOptions{ + WorkspaceID: newID, + }) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if !bytes.Equal(data, []byte("hello world")) { + t.Errorf("Unexpected content: %s", data) + } + + err = g.DeleteWorkspace(context.Background(), id) + if err != nil { + t.Errorf("Error deleting workspace: %v", err) + } + + err = g.DeleteWorkspace(context.Background(), newID) + if err != nil { + t.Errorf("Error deleting new workspace: %v", err) + } +} + func TestWriteReadAndDeleteFileFromWorkspace(t *testing.T) { id, err := g.CreateWorkspace(context.Background(), "directory") if err != nil { @@ -270,6 +310,138 @@ func TestCreateAndDeleteWorkspaceS3(t *testing.T) { } } +func TestCreateAndDeleteWorkspaceFromWorkspaceS3(t *testing.T) { + if os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" || os.Getenv("WORKSPACE_PROVIDER_S3_BUCKET") == "" { + t.Skip("Skipping test because AWS credentials are not set") + } + + id, err := g.CreateWorkspace(context.Background(), "s3") + if err != nil { + t.Fatalf("Error creating workspace: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), "file.txt", []byte("hello world"), WriteFileInWorkspaceOptions{ + WorkspaceID: id, + }) + if err != nil { + t.Errorf("Error creating file: %v", err) + } + + newID, err := g.CreateWorkspace(context.Background(), "s3", id) + if err != nil { + t.Errorf("Error creating workspace from workspace: %v", err) + } + + data, err := g.ReadFileInWorkspace(context.Background(), "file.txt", ReadFileInWorkspaceOptions{ + WorkspaceID: newID, + }) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if !bytes.Equal(data, []byte("hello world")) { + t.Errorf("Unexpected content: %s", data) + } + + err = g.DeleteWorkspace(context.Background(), id) + if err != nil { + t.Errorf("Error deleting workspace: %v", err) + } + + err = g.DeleteWorkspace(context.Background(), newID) + if err != nil { + t.Errorf("Error deleting new workspace: %v", err) + } +} + +func TestCreateAndDeleteDirectoryWorkspaceFromWorkspaceS3(t *testing.T) { + if os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" || os.Getenv("WORKSPACE_PROVIDER_S3_BUCKET") == "" { + t.Skip("Skipping test because AWS credentials are not set") + } + + id, err := g.CreateWorkspace(context.Background(), "s3") + if err != nil { + t.Fatalf("Error creating workspace: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), "file.txt", []byte("hello world"), WriteFileInWorkspaceOptions{ + WorkspaceID: id, + }) + if err != nil { + t.Errorf("Error creating file: %v", err) + } + + newID, err := g.CreateWorkspace(context.Background(), "directory", id) + if err != nil { + t.Errorf("Error creating workspace from workspace: %v", err) + } + + data, err := g.ReadFileInWorkspace(context.Background(), "file.txt", ReadFileInWorkspaceOptions{ + WorkspaceID: newID, + }) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if !bytes.Equal(data, []byte("hello world")) { + t.Errorf("Unexpected content: %s", data) + } + + err = g.DeleteWorkspace(context.Background(), id) + if err != nil { + t.Errorf("Error deleting workspace: %v", err) + } + + err = g.DeleteWorkspace(context.Background(), newID) + if err != nil { + t.Errorf("Error deleting new workspace: %v", err) + } +} + +func TestCreateAndDeleteS3WorkspaceFromWorkspaceDirectory(t *testing.T) { + if os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" || os.Getenv("WORKSPACE_PROVIDER_S3_BUCKET") == "" { + t.Skip("Skipping test because AWS credentials are not set") + } + + id, err := g.CreateWorkspace(context.Background(), "s3") + if err != nil { + t.Fatalf("Error creating workspace: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), "file.txt", []byte("hello world"), WriteFileInWorkspaceOptions{ + WorkspaceID: id, + }) + if err != nil { + t.Errorf("Error creating file: %v", err) + } + + newID, err := g.CreateWorkspace(context.Background(), "directory", id) + if err != nil { + t.Errorf("Error creating workspace from workspace: %v", err) + } + + data, err := g.ReadFileInWorkspace(context.Background(), "file.txt", ReadFileInWorkspaceOptions{ + WorkspaceID: newID, + }) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if !bytes.Equal(data, []byte("hello world")) { + t.Errorf("Unexpected content: %s", data) + } + + err = g.DeleteWorkspace(context.Background(), id) + if err != nil { + t.Errorf("Error deleting workspace: %v", err) + } + + err = g.DeleteWorkspace(context.Background(), newID) + if err != nil { + t.Errorf("Error deleting new workspace: %v", err) + } +} + func TestWriteReadAndDeleteFileFromWorkspaceS3(t *testing.T) { if os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" || os.Getenv("WORKSPACE_PROVIDER_S3_BUCKET") == "" { t.Skip("Skipping test because AWS credentials are not set") From 926b9da2c8389a54281619822a566533237c8a0c Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Mon, 27 Jan 2025 08:04:00 -0500 Subject: [PATCH 65/78] enhance: add support for disabling the creation of file revisions Signed-off-by: Donnie Adams --- workspace.go | 17 +++-- workspace_test.go | 170 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 181 insertions(+), 6 deletions(-) diff --git a/workspace.go b/workspace.go index 321602a..d04a2e6 100644 --- a/workspace.go +++ b/workspace.go @@ -121,7 +121,8 @@ func (g *GPTScript) RemoveAll(ctx context.Context, opts ...RemoveAllOptions) err } type WriteFileInWorkspaceOptions struct { - WorkspaceID string + WorkspaceID string + CreateRevision *bool } func (g *GPTScript) WriteFileInWorkspace(ctx context.Context, filePath string, contents []byte, opts ...WriteFileInWorkspaceOptions) error { @@ -130,6 +131,9 @@ func (g *GPTScript) WriteFileInWorkspace(ctx context.Context, filePath string, c if o.WorkspaceID != "" { opt.WorkspaceID = o.WorkspaceID } + if o.CreateRevision != nil { + opt.CreateRevision = o.CreateRevision + } } if opt.WorkspaceID == "" { @@ -137,11 +141,12 @@ func (g *GPTScript) WriteFileInWorkspace(ctx context.Context, filePath string, c } _, err := g.runBasicCommand(ctx, "workspaces/write-file", map[string]any{ - "id": opt.WorkspaceID, - "contents": base64.StdEncoding.EncodeToString(contents), - "filePath": filePath, - "workspaceTool": g.globalOpts.WorkspaceTool, - "env": g.globalOpts.Env, + "id": opt.WorkspaceID, + "contents": base64.StdEncoding.EncodeToString(contents), + "filePath": filePath, + "createRevision": opt.CreateRevision, + "workspaceTool": g.globalOpts.WorkspaceTool, + "env": g.globalOpts.Env, }) return err diff --git a/workspace_test.go b/workspace_test.go index 95829c7..601d614 100644 --- a/workspace_test.go +++ b/workspace_test.go @@ -224,6 +224,89 @@ func TestRevisionsForFileInWorkspace(t *testing.T) { } } +func TestDisableCreateRevisionsForFileInWorkspace(t *testing.T) { + id, err := g.CreateWorkspace(context.Background(), "directory") + if err != nil { + t.Fatalf("Error creating workspace: %v", err) + } + + t.Cleanup(func() { + err := g.DeleteWorkspace(context.Background(), id) + if err != nil { + t.Errorf("Error deleting workspace: %v", err) + } + }) + + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test0"), WriteFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test1"), WriteFileInWorkspaceOptions{WorkspaceID: id, CreateRevision: new(bool)}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test2"), WriteFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + revisions, err := g.ListRevisionsForFileInWorkspace(context.Background(), "test.txt", ListRevisionsForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if len(revisions) != 1 { + t.Errorf("Unexpected number of revisions: %d", len(revisions)) + } + + for i, rev := range revisions { + if rev.WorkspaceID != id { + t.Errorf("Unexpected file workspace ID: %v", rev.WorkspaceID) + } + + if rev.Name != "test.txt" { + t.Errorf("Unexpected file name: %s", rev.Name) + } + + if rev.Size != 5 { + t.Errorf("Unexpected file size: %d", rev.Size) + } + + if rev.ModTime.IsZero() { + t.Errorf("Unexpected file mod time: %v", rev.ModTime) + } + + if rev.MimeType != "text/plain" { + t.Errorf("Unexpected file mime type: %s", rev.MimeType) + } + + if rev.RevisionID != fmt.Sprintf("%d", i+1) { + t.Errorf("Unexpected revision ID: %s", rev.RevisionID) + } + } + + err = g.DeleteRevisionForFileInWorkspace(context.Background(), "test.txt", "1", DeleteRevisionForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error deleting revision for file: %v", err) + } + + revisions, err = g.ListRevisionsForFileInWorkspace(context.Background(), "test.txt", ListRevisionsForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if len(revisions) != 0 { + t.Errorf("Unexpected number of revisions: %d", len(revisions)) + } + + err = g.DeleteFileInWorkspace(context.Background(), "test.txt", DeleteFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error deleting file: %v", err) + } +} + func TestLsComplexWorkspace(t *testing.T) { id, err := g.CreateWorkspace(context.Background(), "directory") if err != nil { @@ -607,6 +690,93 @@ func TestRevisionsForFileInWorkspaceS3(t *testing.T) { } } +func TestDisableCreatingRevisionsForFileInWorkspaceS3(t *testing.T) { + if os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" || os.Getenv("WORKSPACE_PROVIDER_S3_BUCKET") == "" { + t.Skip("Skipping test because AWS credentials are not set") + } + + id, err := g.CreateWorkspace(context.Background(), "s3") + if err != nil { + t.Fatalf("Error creating workspace: %v", err) + } + + t.Cleanup(func() { + err := g.DeleteWorkspace(context.Background(), id) + if err != nil { + t.Errorf("Error deleting workspace: %v", err) + } + }) + + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test0"), WriteFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test1"), WriteFileInWorkspaceOptions{WorkspaceID: id, CreateRevision: new(bool)}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test2"), WriteFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + revisions, err := g.ListRevisionsForFileInWorkspace(context.Background(), "test.txt", ListRevisionsForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if len(revisions) != 1 { + t.Errorf("Unexpected number of revisions: %d", len(revisions)) + } + + for i, rev := range revisions { + if rev.WorkspaceID != id { + t.Errorf("Unexpected file workspace ID: %v", rev.WorkspaceID) + } + + if rev.Name != "test.txt" { + t.Errorf("Unexpected file name: %s", rev.Name) + } + + if rev.Size != 5 { + t.Errorf("Unexpected file size: %d", rev.Size) + } + + if rev.ModTime.IsZero() { + t.Errorf("Unexpected file mod time: %v", rev.ModTime) + } + + if rev.MimeType != "text/plain" { + t.Errorf("Unexpected file mime type: %s", rev.MimeType) + } + + if rev.RevisionID != fmt.Sprintf("%d", i+1) { + t.Errorf("Unexpected revision ID: %s", rev.RevisionID) + } + } + + err = g.DeleteRevisionForFileInWorkspace(context.Background(), "test.txt", "1", DeleteRevisionForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error deleting revision for file: %v", err) + } + + revisions, err = g.ListRevisionsForFileInWorkspace(context.Background(), "test.txt", ListRevisionsForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if len(revisions) != 0 { + t.Errorf("Unexpected number of revisions: %d", len(revisions)) + } + + err = g.DeleteFileInWorkspace(context.Background(), "test.txt", DeleteFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error deleting file: %v", err) + } +} + func TestLsComplexWorkspaceS3(t *testing.T) { if os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" || os.Getenv("WORKSPACE_PROVIDER_S3_BUCKET") == "" { t.Skip("Skipping test because AWS credentials are not set") From 744b25b84a617db21fd211846fca3c8f3da3f521 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Tue, 4 Feb 2025 08:34:19 -0500 Subject: [PATCH 66/78] chore: update prompt data structures Signed-off-by: Donnie Adams --- frame.go | 22 +++++++++++++++++----- gptscript_test.go | 12 ++++++------ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/frame.go b/frame.go index 5207e43..953572f 100644 --- a/frame.go +++ b/frame.go @@ -116,16 +116,28 @@ type InputContext struct { Content string `json:"content,omitempty"` } -type PromptFrame struct { - ID string `json:"id,omitempty"` - Type EventType `json:"type,omitempty"` - Time time.Time `json:"time,omitempty"` +type Prompt struct { Message string `json:"message,omitempty"` - Fields []string `json:"fields,omitempty"` + Fields Fields `json:"fields,omitempty"` Sensitive bool `json:"sensitive,omitempty"` Metadata map[string]string `json:"metadata,omitempty"` } +type Field struct { + Name string `json:"name,omitempty"` + Sensitive *bool `json:"sensitive,omitempty"` + Description string `json:"description,omitempty"` +} + +type Fields []Field + +type PromptFrame struct { + Prompt + ID string `json:"id,omitempty"` + Type EventType `json:"type,omitempty"` + Time time.Time `json:"time,omitempty"` +} + func (p *PromptFrame) String() string { return fmt.Sprintf(`Message: %s Fields: %v diff --git a/gptscript_test.go b/gptscript_test.go index 834181e..8f299c8 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -1131,13 +1131,13 @@ func TestPrompt(t *testing.T) { t.Fatalf("Unexpected number of fields: %d", len(promptFrame.Fields)) } - if promptFrame.Fields[0] != "first name" { - t.Errorf("Unexpected field: %s", promptFrame.Fields[0]) + if promptFrame.Fields[0].Name != "first name" { + t.Errorf("Unexpected field: %s", promptFrame.Fields[0].Name) } if err = g.PromptResponse(context.Background(), PromptResponse{ ID: promptFrame.ID, - Responses: map[string]string{promptFrame.Fields[0]: "Clicky"}, + Responses: map[string]string{promptFrame.Fields[0].Name: "Clicky"}, }); err != nil { t.Errorf("Error responding: %v", err) } @@ -1199,8 +1199,8 @@ func TestPromptWithMetadata(t *testing.T) { t.Fatalf("Unexpected number of fields: %d", len(promptFrame.Fields)) } - if promptFrame.Fields[0] != "first name" { - t.Errorf("Unexpected field: %s", promptFrame.Fields[0]) + if promptFrame.Fields[0].Name != "first name" { + t.Errorf("Unexpected field: %s", promptFrame.Fields[0].Name) } if promptFrame.Metadata["key"] != "value" { @@ -1209,7 +1209,7 @@ func TestPromptWithMetadata(t *testing.T) { if err = g.PromptResponse(context.Background(), PromptResponse{ ID: promptFrame.ID, - Responses: map[string]string{promptFrame.Fields[0]: "Clicky"}, + Responses: map[string]string{promptFrame.Fields[0].Name: "Clicky"}, }); err != nil { t.Errorf("Error responding: %v", err) } From f28f1ccf9daf52408120e207bfa2531eb0440e5d Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Tue, 11 Feb 2025 10:20:58 -0500 Subject: [PATCH 67/78] enhance: add latest_revision for writing files to a workspace This new field acts as an optimistic locking for writing to files. Signed-off-by: Donnie Adams --- workspace.go | 33 ++++++++- workspace_test.go | 176 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 208 insertions(+), 1 deletion(-) diff --git a/workspace.go b/workspace.go index d04a2e6..8bd5a6f 100644 --- a/workspace.go +++ b/workspace.go @@ -6,10 +6,13 @@ import ( "encoding/json" "fmt" "os" + "regexp" "strings" "time" ) +var conflictErrParser = regexp.MustCompile(`^.+500 Internal Server Error: conflict: (.+)/([^/]+) \(latest revision: (-?\d+), current revision: (-?\d+)\)$`) + type NotFoundInWorkspaceError struct { id string name string @@ -23,6 +26,29 @@ func newNotFoundInWorkspaceError(id, name string) *NotFoundInWorkspaceError { return &NotFoundInWorkspaceError{id: id, name: name} } +type ConflictInWorkspaceError struct { + ID string + Name string + LatestRevision string + CurrentRevision string +} + +func parsePossibleConflictInWorkspaceError(err error) error { + if err == nil { + return err + } + + matches := conflictErrParser.FindStringSubmatch(err.Error()) + if len(matches) != 5 { + return err + } + return &ConflictInWorkspaceError{ID: matches[1], Name: matches[2], LatestRevision: matches[3], CurrentRevision: matches[4]} +} + +func (e *ConflictInWorkspaceError) Error() string { + return fmt.Sprintf("conflict: %s/%s (latest revision: %s, current revision: %s)", e.ID, e.Name, e.LatestRevision, e.CurrentRevision) +} + func (g *GPTScript) CreateWorkspace(ctx context.Context, providerType string, fromWorkspaces ...string) (string, error) { out, err := g.runBasicCommand(ctx, "workspaces/create", map[string]any{ "providerType": providerType, @@ -123,6 +149,7 @@ func (g *GPTScript) RemoveAll(ctx context.Context, opts ...RemoveAllOptions) err type WriteFileInWorkspaceOptions struct { WorkspaceID string CreateRevision *bool + LatestRevision string } func (g *GPTScript) WriteFileInWorkspace(ctx context.Context, filePath string, contents []byte, opts ...WriteFileInWorkspaceOptions) error { @@ -134,6 +161,9 @@ func (g *GPTScript) WriteFileInWorkspace(ctx context.Context, filePath string, c if o.CreateRevision != nil { opt.CreateRevision = o.CreateRevision } + if o.LatestRevision != "" { + opt.LatestRevision = o.LatestRevision + } } if opt.WorkspaceID == "" { @@ -145,11 +175,12 @@ func (g *GPTScript) WriteFileInWorkspace(ctx context.Context, filePath string, c "contents": base64.StdEncoding.EncodeToString(contents), "filePath": filePath, "createRevision": opt.CreateRevision, + "latestRevision": opt.LatestRevision, "workspaceTool": g.globalOpts.WorkspaceTool, "env": g.globalOpts.Env, }) - return err + return parsePossibleConflictInWorkspaceError(err) } type DeleteFileInWorkspaceOptions struct { diff --git a/workspace_test.go b/workspace_test.go index 601d614..669f2b0 100644 --- a/workspace_test.go +++ b/workspace_test.go @@ -307,6 +307,92 @@ func TestDisableCreateRevisionsForFileInWorkspace(t *testing.T) { } } +func TestConflictsForFileInWorkspace(t *testing.T) { + id, err := g.CreateWorkspace(context.Background(), "directory") + if err != nil { + t.Fatalf("Error creating workspace: %v", err) + } + + t.Cleanup(func() { + err := g.DeleteWorkspace(context.Background(), id) + if err != nil { + t.Errorf("Error deleting workspace: %v", err) + } + }) + + ce := (*ConflictInWorkspaceError)(nil) + // Writing a new file with a non-zero latest revision should fail + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test0"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevision: "1"}) + if err == nil || !errors.As(err, &ce) { + t.Errorf("Expected error writing file with non-zero latest revision: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test0"), WriteFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test1"), WriteFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + revisions, err := g.ListRevisionsForFileInWorkspace(context.Background(), "test.txt", ListRevisionsForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if len(revisions) != 1 { + t.Errorf("Unexpected number of revisions: %d", len(revisions)) + } + + // Writing to the file with the latest revision should succeed + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test2"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevision: revisions[0].RevisionID}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + revisions, err = g.ListRevisionsForFileInWorkspace(context.Background(), "test.txt", ListRevisionsForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if len(revisions) != 2 { + t.Errorf("Unexpected number of revisions: %d", len(revisions)) + } + + // Writing to the file with the same revision should fail + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test3"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevision: revisions[0].RevisionID}) + if err == nil || !errors.As(err, &ce) { + t.Errorf("Expected error writing file with same revision: %v", err) + } + + err = g.DeleteRevisionForFileInWorkspace(context.Background(), "test.txt", revisions[1].RevisionID, DeleteRevisionForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error deleting revision for file: %v", err) + } + + revisions, err = g.ListRevisionsForFileInWorkspace(context.Background(), "test.txt", ListRevisionsForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if len(revisions) != 1 { + t.Errorf("Unexpected number of revisions: %d", len(revisions)) + } + + // Ensure we can write a new file after deleting the latest revision + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test4"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevision: revisions[0].RevisionID}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + err = g.DeleteFileInWorkspace(context.Background(), "test.txt", DeleteFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error deleting file: %v", err) + } +} + func TestLsComplexWorkspace(t *testing.T) { id, err := g.CreateWorkspace(context.Background(), "directory") if err != nil { @@ -690,6 +776,96 @@ func TestRevisionsForFileInWorkspaceS3(t *testing.T) { } } +func TestConflictsForFileInWorkspaceS3(t *testing.T) { + if os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" || os.Getenv("WORKSPACE_PROVIDER_S3_BUCKET") == "" { + t.Skip("Skipping test because AWS credentials are not set") + } + + id, err := g.CreateWorkspace(context.Background(), "s3") + if err != nil { + t.Fatalf("Error creating workspace: %v", err) + } + + t.Cleanup(func() { + err := g.DeleteWorkspace(context.Background(), id) + if err != nil { + t.Errorf("Error deleting workspace: %v", err) + } + }) + + ce := (*ConflictInWorkspaceError)(nil) + // Writing a new file with a non-zero latest revision should fail + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test0"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevision: "1"}) + if err == nil || !errors.As(err, &ce) { + t.Errorf("Expected error writing file with non-zero latest revision: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test0"), WriteFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test1"), WriteFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + revisions, err := g.ListRevisionsForFileInWorkspace(context.Background(), "test.txt", ListRevisionsForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if len(revisions) != 1 { + t.Errorf("Unexpected number of revisions: %d", len(revisions)) + } + + // Writing to the file with the latest revision should succeed + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test2"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevision: revisions[0].RevisionID}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + revisions, err = g.ListRevisionsForFileInWorkspace(context.Background(), "test.txt", ListRevisionsForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if len(revisions) != 2 { + t.Errorf("Unexpected number of revisions: %d", len(revisions)) + } + + // Writing to the file with the same revision should fail + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test3"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevision: revisions[0].RevisionID}) + if err == nil || !errors.As(err, &ce) { + t.Errorf("Expected error writing file with same revision: %v", err) + } + + err = g.DeleteRevisionForFileInWorkspace(context.Background(), "test.txt", revisions[1].RevisionID, DeleteRevisionForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error deleting revision for file: %v", err) + } + + revisions, err = g.ListRevisionsForFileInWorkspace(context.Background(), "test.txt", ListRevisionsForFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if len(revisions) != 1 { + t.Errorf("Unexpected number of revisions: %d", len(revisions)) + } + + // Ensure we can write a new file after deleting the latest revision + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test4"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevision: revisions[0].RevisionID}) + if err != nil { + t.Fatalf("Error creating file: %v", err) + } + + err = g.DeleteFileInWorkspace(context.Background(), "test.txt", DeleteFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error deleting file: %v", err) + } +} + func TestDisableCreatingRevisionsForFileInWorkspaceS3(t *testing.T) { if os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" || os.Getenv("WORKSPACE_PROVIDER_S3_BUCKET") == "" { t.Skip("Skipping test because AWS credentials are not set") From 834896a4bb9f94b5a2ff1c7c41c0006b99c818f7 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Tue, 11 Feb 2025 13:42:09 -0500 Subject: [PATCH 68/78] chore: remove unneeded err-nil check Signed-off-by: Donnie Adams --- run.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.go b/run.go index dac9c15..52ad6e4 100644 --- a/run.go +++ b/run.go @@ -414,7 +414,7 @@ func (r *Run) request(ctx context.Context, payload any) (err error) { } } - if err != nil && !errors.Is(err, io.EOF) { + if !errors.Is(err, io.EOF) { slog.Debug("failed to read events from response", "error", err) r.err = fmt.Errorf("failed to read events: %w", err) } From eee4337500a631fb78a7eebb9d6f49ca7095060b Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Sat, 22 Feb 2025 12:08:45 -0500 Subject: [PATCH 69/78] enhance: add ability to get revision ID when opening file in a workspace (#93) enhance: add ability to get revision ID when opening file in a workspace Signed-off-by: Donnie Adams --- go.mod | 13 ++-- go.sum | 27 ++++---- workspace.go | 86 ++++++++++++++++++-------- workspace_test.go | 152 +++++++++++++++++++++++++++++++++++----------- 4 files changed, 200 insertions(+), 78 deletions(-) diff --git a/go.mod b/go.mod index 283f8d6..0ca27e7 100644 --- a/go.mod +++ b/go.mod @@ -3,18 +3,19 @@ module github.com/gptscript-ai/go-gptscript go 1.23.0 require ( - github.com/getkin/kin-openapi v0.124.0 - github.com/stretchr/testify v1.8.4 + github.com/getkin/kin-openapi v0.129.0 + github.com/stretchr/testify v1.10.0 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/go-openapi/jsonpointer v0.20.2 // indirect - github.com/go-openapi/swag v0.22.8 // indirect - github.com/invopop/yaml v0.2.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/mailru/easyjson v0.7.7 // indirect + github.com/mailru/easyjson v0.9.0 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/oasdiff/yaml v0.0.0-20241210131133-6b86fb107d80 // indirect + github.com/oasdiff/yaml3 v0.0.0-20241210130736-a94c01f36349 // indirect github.com/perimeterx/marshmallow v1.1.5 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 9d94c4d..1c7e6b3 100644 --- a/go.sum +++ b/go.sum @@ -1,38 +1,39 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/getkin/kin-openapi v0.124.0 h1:VSFNMB9C9rTKBnQ/fpyDU8ytMTr4dWI9QovSKj9kz/M= -github.com/getkin/kin-openapi v0.124.0/go.mod h1:wb1aSZA/iWmorQP9KTAS/phLj/t17B5jT7+fS8ed9NM= -github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= -github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= -github.com/go-openapi/swag v0.22.8 h1:/9RjDSQ0vbFR+NyjGMkFTsA1IA0fmhKSThmfGZjicbw= -github.com/go-openapi/swag v0.22.8/go.mod h1:6QT22icPLEqAM/z/TChgb4WAveCHF92+2gF0CNjHpPI= +github.com/getkin/kin-openapi v0.129.0 h1:QGYTNcmyP5X0AtFQ2Dkou9DGBJsUETeLH9rFrJXZh30= +github.com/getkin/kin-openapi v0.129.0/go.mod h1:gmWI+b/J45xqpyK5wJmRRZse5wefA5H0RDMK46kLUtI= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/invopop/yaml v0.2.0 h1:7zky/qH+O0DwAyoobXUqvVBwgBFRxKoQ/3FjcVpjTMY= -github.com/invopop/yaml v0.2.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/oasdiff/yaml v0.0.0-20241210131133-6b86fb107d80 h1:nZspmSkneBbtxU9TopEAE0CY+SBJLxO8LPUlw2vG4pU= +github.com/oasdiff/yaml v0.0.0-20241210131133-6b86fb107d80/go.mod h1:7tFDb+Y51LcDpn26GccuUgQXUk6t0CXZsivKjyimYX8= +github.com/oasdiff/yaml3 v0.0.0-20241210130736-a94c01f36349 h1:t05Ww3DxZutOqbMN+7OIuqDwXbhl32HiZGpLy26BAPc= +github.com/oasdiff/yaml3 v0.0.0-20241210130736-a94c01f36349/go.mod h1:y5+oSEHCPT/DGrS++Wc/479ERge0zTFxaF8PbGKcg2o= github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/workspace.go b/workspace.go index 8bd5a6f..f384a85 100644 --- a/workspace.go +++ b/workspace.go @@ -147,9 +147,9 @@ func (g *GPTScript) RemoveAll(ctx context.Context, opts ...RemoveAllOptions) err } type WriteFileInWorkspaceOptions struct { - WorkspaceID string - CreateRevision *bool - LatestRevision string + WorkspaceID string + CreateRevision *bool + LatestRevisionID string } func (g *GPTScript) WriteFileInWorkspace(ctx context.Context, filePath string, contents []byte, opts ...WriteFileInWorkspaceOptions) error { @@ -161,8 +161,8 @@ func (g *GPTScript) WriteFileInWorkspace(ctx context.Context, filePath string, c if o.CreateRevision != nil { opt.CreateRevision = o.CreateRevision } - if o.LatestRevision != "" { - opt.LatestRevision = o.LatestRevision + if o.LatestRevisionID != "" { + opt.LatestRevisionID = o.LatestRevisionID } } @@ -171,13 +171,13 @@ func (g *GPTScript) WriteFileInWorkspace(ctx context.Context, filePath string, c } _, err := g.runBasicCommand(ctx, "workspaces/write-file", map[string]any{ - "id": opt.WorkspaceID, - "contents": base64.StdEncoding.EncodeToString(contents), - "filePath": filePath, - "createRevision": opt.CreateRevision, - "latestRevision": opt.LatestRevision, - "workspaceTool": g.globalOpts.WorkspaceTool, - "env": g.globalOpts.Env, + "id": opt.WorkspaceID, + "contents": base64.StdEncoding.EncodeToString(contents), + "filePath": filePath, + "createRevision": opt.CreateRevision, + "latestRevisionID": opt.LatestRevisionID, + "workspaceTool": g.globalOpts.WorkspaceTool, + "env": g.globalOpts.Env, }) return parsePossibleConflictInWorkspaceError(err) @@ -245,16 +245,57 @@ func (g *GPTScript) ReadFileInWorkspace(ctx context.Context, filePath string, op return base64.StdEncoding.DecodeString(out) } +type ReadFileWithRevisionInWorkspaceResponse struct { + Content []byte `json:"content"` + RevisionID string `json:"revisionID"` +} + +func (g *GPTScript) ReadFileWithRevisionInWorkspace(ctx context.Context, filePath string, opts ...ReadFileInWorkspaceOptions) (*ReadFileWithRevisionInWorkspaceResponse, error) { + var opt ReadFileInWorkspaceOptions + for _, o := range opts { + if o.WorkspaceID != "" { + opt.WorkspaceID = o.WorkspaceID + } + } + + if opt.WorkspaceID == "" { + opt.WorkspaceID = os.Getenv("GPTSCRIPT_WORKSPACE_ID") + } + + out, err := g.runBasicCommand(ctx, "workspaces/read-file-with-revision", map[string]any{ + "id": opt.WorkspaceID, + "filePath": filePath, + "workspaceTool": g.globalOpts.WorkspaceTool, + "env": g.globalOpts.Env, + }) + if err != nil { + if strings.HasSuffix(err.Error(), fmt.Sprintf("not found: %s/%s", opt.WorkspaceID, filePath)) { + return nil, newNotFoundInWorkspaceError(opt.WorkspaceID, filePath) + } + return nil, err + } + + var resp ReadFileWithRevisionInWorkspaceResponse + err = json.Unmarshal([]byte(out), &resp) + if err != nil { + return nil, err + } + + return &resp, nil +} + type FileInfo struct { WorkspaceID string Name string Size int64 ModTime time.Time MimeType string + RevisionID string } type StatFileInWorkspaceOptions struct { - WorkspaceID string + WorkspaceID string + WithLatestRevisionID bool } func (g *GPTScript) StatFileInWorkspace(ctx context.Context, filePath string, opts ...StatFileInWorkspaceOptions) (FileInfo, error) { @@ -263,6 +304,7 @@ func (g *GPTScript) StatFileInWorkspace(ctx context.Context, filePath string, op if o.WorkspaceID != "" { opt.WorkspaceID = o.WorkspaceID } + opt.WithLatestRevisionID = opt.WithLatestRevisionID || o.WithLatestRevisionID } if opt.WorkspaceID == "" { @@ -270,10 +312,11 @@ func (g *GPTScript) StatFileInWorkspace(ctx context.Context, filePath string, op } out, err := g.runBasicCommand(ctx, "workspaces/stat-file", map[string]any{ - "id": opt.WorkspaceID, - "filePath": filePath, - "workspaceTool": g.globalOpts.WorkspaceTool, - "env": g.globalOpts.Env, + "id": opt.WorkspaceID, + "filePath": filePath, + "withLatestRevisionID": opt.WithLatestRevisionID, + "workspaceTool": g.globalOpts.WorkspaceTool, + "env": g.globalOpts.Env, }) if err != nil { if strings.HasSuffix(err.Error(), fmt.Sprintf("not found: %s/%s", opt.WorkspaceID, filePath)) { @@ -291,16 +334,11 @@ func (g *GPTScript) StatFileInWorkspace(ctx context.Context, filePath string, op return info, nil } -type RevisionInfo struct { - FileInfo - RevisionID string -} - type ListRevisionsForFileInWorkspaceOptions struct { WorkspaceID string } -func (g *GPTScript) ListRevisionsForFileInWorkspace(ctx context.Context, filePath string, opts ...ListRevisionsForFileInWorkspaceOptions) ([]RevisionInfo, error) { +func (g *GPTScript) ListRevisionsForFileInWorkspace(ctx context.Context, filePath string, opts ...ListRevisionsForFileInWorkspaceOptions) ([]FileInfo, error) { var opt ListRevisionsForFileInWorkspaceOptions for _, o := range opts { if o.WorkspaceID != "" { @@ -325,7 +363,7 @@ func (g *GPTScript) ListRevisionsForFileInWorkspace(ctx context.Context, filePat return nil, err } - var info []RevisionInfo + var info []FileInfo err = json.Unmarshal([]byte(out), &info) if err != nil { return nil, err diff --git a/workspace_test.go b/workspace_test.go index 669f2b0..eb895d3 100644 --- a/workspace_test.go +++ b/workspace_test.go @@ -33,6 +33,13 @@ func TestCreateAndDeleteWorkspaceFromWorkspace(t *testing.T) { t.Fatalf("Error creating workspace: %v", err) } + t.Cleanup(func() { + err = g.DeleteWorkspace(context.Background(), id) + if err != nil { + t.Errorf("Error deleting workspace: %v", err) + } + }) + err = g.WriteFileInWorkspace(context.Background(), "file.txt", []byte("hello world"), WriteFileInWorkspaceOptions{ WorkspaceID: id, }) @@ -45,26 +52,21 @@ func TestCreateAndDeleteWorkspaceFromWorkspace(t *testing.T) { t.Errorf("Error creating workspace from workspace: %v", err) } - data, err := g.ReadFileInWorkspace(context.Background(), "file.txt", ReadFileInWorkspaceOptions{ + content, err := g.ReadFileInWorkspace(context.Background(), "file.txt", ReadFileInWorkspaceOptions{ WorkspaceID: newID, }) if err != nil { - t.Errorf("Error reading file: %v", err) + t.Fatalf("Error reading file: %v", err) } - if !bytes.Equal(data, []byte("hello world")) { - t.Errorf("Unexpected content: %s", data) + if !bytes.Equal(content, []byte("hello world")) { + t.Errorf("Unexpected content: %s", content) } err = g.DeleteWorkspace(context.Background(), id) if err != nil { t.Errorf("Error deleting workspace: %v", err) } - - err = g.DeleteWorkspace(context.Background(), newID) - if err != nil { - t.Errorf("Error deleting new workspace: %v", err) - } } func TestWriteReadAndDeleteFileFromWorkspace(t *testing.T) { @@ -94,6 +96,20 @@ func TestWriteReadAndDeleteFileFromWorkspace(t *testing.T) { t.Errorf("Unexpected content: %s", content) } + // Read the file and request the revision ID + contentWithRevision, err := g.ReadFileWithRevisionInWorkspace(context.Background(), "test.txt", ReadFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if !bytes.Equal(contentWithRevision.Content, []byte("test")) { + t.Errorf("Unexpected content: %s", contentWithRevision.Content) + } + + if contentWithRevision.RevisionID == "" { + t.Errorf("Expected file revision ID when requesting it: %s", contentWithRevision.RevisionID) + } + // Stat the file to ensure it exists fileInfo, err := g.StatFileInWorkspace(context.Background(), "test.txt", StatFileInWorkspaceOptions{WorkspaceID: id}) if err != nil { @@ -120,6 +136,24 @@ func TestWriteReadAndDeleteFileFromWorkspace(t *testing.T) { t.Errorf("Unexpected file mime type: %s", fileInfo.MimeType) } + if fileInfo.RevisionID != "" { + t.Errorf("Unexpected file revision ID when not requesting it: %s", fileInfo.RevisionID) + } + + // Stat file and request the revision ID + fileInfo, err = g.StatFileInWorkspace(context.Background(), "test.txt", StatFileInWorkspaceOptions{WorkspaceID: id, WithLatestRevisionID: true}) + if err != nil { + t.Errorf("Error statting file: %v", err) + } + + if fileInfo.WorkspaceID != id { + t.Errorf("Unexpected file workspace ID: %v", fileInfo.WorkspaceID) + } + + if fileInfo.RevisionID == "" { + t.Errorf("Expected file revision ID when requesting it: %s", fileInfo.RevisionID) + } + // Ensure we get the error we expect when trying to read a non-existent file _, err = g.ReadFileInWorkspace(context.Background(), "test1.txt", ReadFileInWorkspaceOptions{WorkspaceID: id}) if nf := (*NotFoundInWorkspaceError)(nil); !errors.As(err, &nf) { @@ -322,7 +356,7 @@ func TestConflictsForFileInWorkspace(t *testing.T) { ce := (*ConflictInWorkspaceError)(nil) // Writing a new file with a non-zero latest revision should fail - err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test0"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevision: "1"}) + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test0"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevisionID: "1"}) if err == nil || !errors.As(err, &ce) { t.Errorf("Expected error writing file with non-zero latest revision: %v", err) } @@ -347,7 +381,7 @@ func TestConflictsForFileInWorkspace(t *testing.T) { } // Writing to the file with the latest revision should succeed - err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test2"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevision: revisions[0].RevisionID}) + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test2"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevisionID: revisions[0].RevisionID}) if err != nil { t.Fatalf("Error creating file: %v", err) } @@ -362,12 +396,13 @@ func TestConflictsForFileInWorkspace(t *testing.T) { } // Writing to the file with the same revision should fail - err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test3"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevision: revisions[0].RevisionID}) + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test3"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevisionID: revisions[0].RevisionID}) if err == nil || !errors.As(err, &ce) { t.Errorf("Expected error writing file with same revision: %v", err) } - err = g.DeleteRevisionForFileInWorkspace(context.Background(), "test.txt", revisions[1].RevisionID, DeleteRevisionForFileInWorkspaceOptions{WorkspaceID: id}) + latestRevisionID := revisions[1].RevisionID + err = g.DeleteRevisionForFileInWorkspace(context.Background(), "test.txt", latestRevisionID, DeleteRevisionForFileInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Errorf("Error deleting revision for file: %v", err) } @@ -381,10 +416,16 @@ func TestConflictsForFileInWorkspace(t *testing.T) { t.Errorf("Unexpected number of revisions: %d", len(revisions)) } + // Ensure we cannot write a new file with the zero-th revision ID + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test4"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevisionID: revisions[0].RevisionID}) + if err == nil || !errors.As(err, &ce) { + t.Errorf("Unexpected error writing to file: %v", err) + } + // Ensure we can write a new file after deleting the latest revision - err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test4"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevision: revisions[0].RevisionID}) + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test4"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevisionID: latestRevisionID}) if err != nil { - t.Fatalf("Error creating file: %v", err) + t.Errorf("Error writing file: %v", err) } err = g.DeleteFileInWorkspace(context.Background(), "test.txt", DeleteFileInWorkspaceOptions{WorkspaceID: id}) @@ -501,15 +542,15 @@ func TestCreateAndDeleteWorkspaceFromWorkspaceS3(t *testing.T) { t.Errorf("Error creating workspace from workspace: %v", err) } - data, err := g.ReadFileInWorkspace(context.Background(), "file.txt", ReadFileInWorkspaceOptions{ + content, err := g.ReadFileInWorkspace(context.Background(), "file.txt", ReadFileInWorkspaceOptions{ WorkspaceID: newID, }) if err != nil { t.Errorf("Error reading file: %v", err) } - if !bytes.Equal(data, []byte("hello world")) { - t.Errorf("Unexpected content: %s", data) + if !bytes.Equal(content, []byte("hello world")) { + t.Errorf("Unexpected content: %s", content) } err = g.DeleteWorkspace(context.Background(), id) @@ -545,15 +586,15 @@ func TestCreateAndDeleteDirectoryWorkspaceFromWorkspaceS3(t *testing.T) { t.Errorf("Error creating workspace from workspace: %v", err) } - data, err := g.ReadFileInWorkspace(context.Background(), "file.txt", ReadFileInWorkspaceOptions{ + content, err := g.ReadFileInWorkspace(context.Background(), "file.txt", ReadFileInWorkspaceOptions{ WorkspaceID: newID, }) if err != nil { t.Errorf("Error reading file: %v", err) } - if !bytes.Equal(data, []byte("hello world")) { - t.Errorf("Unexpected content: %s", data) + if !bytes.Equal(content, []byte("hello world")) { + t.Errorf("Unexpected content: %s", content) } err = g.DeleteWorkspace(context.Background(), id) @@ -577,6 +618,13 @@ func TestCreateAndDeleteS3WorkspaceFromWorkspaceDirectory(t *testing.T) { t.Fatalf("Error creating workspace: %v", err) } + t.Cleanup(func() { + err = g.DeleteWorkspace(context.Background(), id) + if err != nil { + t.Errorf("Error deleting workspace: %v", err) + } + }) + err = g.WriteFileInWorkspace(context.Background(), "file.txt", []byte("hello world"), WriteFileInWorkspaceOptions{ WorkspaceID: id, }) @@ -589,26 +637,21 @@ func TestCreateAndDeleteS3WorkspaceFromWorkspaceDirectory(t *testing.T) { t.Errorf("Error creating workspace from workspace: %v", err) } - data, err := g.ReadFileInWorkspace(context.Background(), "file.txt", ReadFileInWorkspaceOptions{ + content, err := g.ReadFileInWorkspace(context.Background(), "file.txt", ReadFileInWorkspaceOptions{ WorkspaceID: newID, }) if err != nil { - t.Errorf("Error reading file: %v", err) + t.Fatalf("Error reading file: %v", err) } - if !bytes.Equal(data, []byte("hello world")) { - t.Errorf("Unexpected content: %s", data) + if !bytes.Equal(content, []byte("hello world")) { + t.Errorf("Unexpected content: %s", content) } err = g.DeleteWorkspace(context.Background(), id) if err != nil { t.Errorf("Error deleting workspace: %v", err) } - - err = g.DeleteWorkspace(context.Background(), newID) - if err != nil { - t.Errorf("Error deleting new workspace: %v", err) - } } func TestWriteReadAndDeleteFileFromWorkspaceS3(t *testing.T) { @@ -642,6 +685,20 @@ func TestWriteReadAndDeleteFileFromWorkspaceS3(t *testing.T) { t.Errorf("Unexpected content: %s", content) } + // Read the file and request the revision ID + contentWithRevision, err := g.ReadFileWithRevisionInWorkspace(context.Background(), "test.txt", ReadFileInWorkspaceOptions{WorkspaceID: id}) + if err != nil { + t.Errorf("Error reading file: %v", err) + } + + if !bytes.Equal(contentWithRevision.Content, []byte("test")) { + t.Errorf("Unexpected content: %s", contentWithRevision.Content) + } + + if contentWithRevision.RevisionID == "" { + t.Errorf("Expected file revision ID when requesting it: %s", contentWithRevision.RevisionID) + } + // Stat the file to ensure it exists fileInfo, err := g.StatFileInWorkspace(context.Background(), "test.txt", StatFileInWorkspaceOptions{WorkspaceID: id}) if err != nil { @@ -668,6 +725,24 @@ func TestWriteReadAndDeleteFileFromWorkspaceS3(t *testing.T) { t.Errorf("Unexpected file mime type: %s", fileInfo.MimeType) } + if fileInfo.RevisionID != "" { + t.Errorf("Unexpected file revision ID when not requesting it: %s", fileInfo.RevisionID) + } + + // Stat file and request the revision ID + fileInfo, err = g.StatFileInWorkspace(context.Background(), "test.txt", StatFileInWorkspaceOptions{WorkspaceID: id, WithLatestRevisionID: true}) + if err != nil { + t.Errorf("Error statting file: %v", err) + } + + if fileInfo.WorkspaceID != id { + t.Errorf("Unexpected file workspace ID: %v", fileInfo.WorkspaceID) + } + + if fileInfo.RevisionID == "" { + t.Errorf("Expected file revision ID when requesting it: %s", fileInfo.RevisionID) + } + // Ensure we get the error we expect when trying to read a non-existent file _, err = g.ReadFileInWorkspace(context.Background(), "test1.txt", ReadFileInWorkspaceOptions{WorkspaceID: id}) if nf := (*NotFoundInWorkspaceError)(nil); !errors.As(err, &nf) { @@ -795,7 +870,7 @@ func TestConflictsForFileInWorkspaceS3(t *testing.T) { ce := (*ConflictInWorkspaceError)(nil) // Writing a new file with a non-zero latest revision should fail - err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test0"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevision: "1"}) + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test0"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevisionID: "1"}) if err == nil || !errors.As(err, &ce) { t.Errorf("Expected error writing file with non-zero latest revision: %v", err) } @@ -820,7 +895,7 @@ func TestConflictsForFileInWorkspaceS3(t *testing.T) { } // Writing to the file with the latest revision should succeed - err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test2"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevision: revisions[0].RevisionID}) + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test2"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevisionID: revisions[0].RevisionID}) if err != nil { t.Fatalf("Error creating file: %v", err) } @@ -835,12 +910,13 @@ func TestConflictsForFileInWorkspaceS3(t *testing.T) { } // Writing to the file with the same revision should fail - err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test3"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevision: revisions[0].RevisionID}) + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test3"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevisionID: revisions[0].RevisionID}) if err == nil || !errors.As(err, &ce) { t.Errorf("Expected error writing file with same revision: %v", err) } - err = g.DeleteRevisionForFileInWorkspace(context.Background(), "test.txt", revisions[1].RevisionID, DeleteRevisionForFileInWorkspaceOptions{WorkspaceID: id}) + latestRevisionID := revisions[1].RevisionID + err = g.DeleteRevisionForFileInWorkspace(context.Background(), "test.txt", latestRevisionID, DeleteRevisionForFileInWorkspaceOptions{WorkspaceID: id}) if err != nil { t.Errorf("Error deleting revision for file: %v", err) } @@ -854,8 +930,14 @@ func TestConflictsForFileInWorkspaceS3(t *testing.T) { t.Errorf("Unexpected number of revisions: %d", len(revisions)) } + // Ensure we cannot write a new file with the zero-th revision ID + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test4"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevisionID: revisions[0].RevisionID}) + if err == nil || !errors.As(err, &ce) { + t.Fatalf("Error creating file: %v", err) + } + // Ensure we can write a new file after deleting the latest revision - err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test4"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevision: revisions[0].RevisionID}) + err = g.WriteFileInWorkspace(context.Background(), "test.txt", []byte("test4"), WriteFileInWorkspaceOptions{WorkspaceID: id, LatestRevisionID: latestRevisionID}) if err != nil { t.Fatalf("Error creating file: %v", err) } From ff48fd21ead9ae281d35ecf3d724ee366b3bc32e Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Tue, 25 Feb 2025 06:16:23 -0500 Subject: [PATCH 70/78] enhance: Add proper aborting of runs (#94) Aborting a run is different from "closing" it. Closing a run will result in an error. Aborting a run will cause it to stop at the next available event and not return any error. Instead, the run will have its text appended with "ABORTED BY USER" and all the chat state will be preserved. Signed-off-by: Donnie Adams --- gptscript.go | 5 ++ gptscript_test.go | 139 +++++++++++++++++++++++++++++++++++++++++++++- run.go | 2 + 3 files changed, 144 insertions(+), 2 deletions(-) diff --git a/gptscript.go b/gptscript.go index 28a46a2..8ca9747 100644 --- a/gptscript.go +++ b/gptscript.go @@ -170,6 +170,11 @@ func (g *GPTScript) Run(ctx context.Context, toolPath string, opts Options) (*Ru }).NextChat(ctx, opts.Input) } +func (g *GPTScript) AbortRun(ctx context.Context, run *Run) error { + _, err := g.runBasicCommand(ctx, "abort/"+run.id, (map[string]any)(nil)) + return err +} + type ParseOptions struct { DisableCache bool } diff --git a/gptscript_test.go b/gptscript_test.go index 8f299c8..476492d 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" "testing" + "time" "github.com/getkin/kin-openapi/openapi3" "github.com/stretchr/testify/require" @@ -134,7 +135,7 @@ func TestListModelsWithDefaultProvider(t *testing.T) { } } -func TestAbortRun(t *testing.T) { +func TestCancelRun(t *testing.T) { tool := ToolDef{Instructions: "What is the capital of the united states?"} run, err := g.Evaluate(context.Background(), Options{DisableCache: true, IncludeEvents: true}, tool) @@ -146,7 +147,7 @@ func TestAbortRun(t *testing.T) { <-run.Events() if err := run.Close(); err != nil { - t.Errorf("Error aborting run: %v", err) + t.Errorf("Error canceling run: %v", err) } if run.State() != Error { @@ -158,6 +159,77 @@ func TestAbortRun(t *testing.T) { } } +func TestAbortChatCompletionRun(t *testing.T) { + tool := ToolDef{Instructions: "What is the capital of the united states?"} + + run, err := g.Evaluate(context.Background(), Options{DisableCache: true, IncludeEvents: true}, tool) + if err != nil { + t.Errorf("Error executing tool: %v", err) + } + + // Abort the run after the first event from the LLM + for e := range run.Events() { + if e.Call != nil && e.Call.Type == EventTypeCallProgress && len(e.Call.Output) > 0 && e.Call.Output[0].Content != "Waiting for model response..." { + break + } + } + + if err := g.AbortRun(context.Background(), run); err != nil { + t.Errorf("Error aborting run: %v", err) + } + + // Wait for run to stop + for range run.Events() { + continue + } + + if run.State() != Finished { + t.Errorf("Unexpected run state: %s", run.State()) + } + + if out, err := run.Text(); err != nil { + t.Errorf("Error reading output: %v", err) + } else if strings.TrimSpace(out) != "ABORTED BY USER" && !strings.HasSuffix(out, "\nABORTED BY USER") { + t.Errorf("Unexpected output: %s", out) + } +} + +func TestAbortCommandRun(t *testing.T) { + tool := ToolDef{Instructions: "#!/usr/bin/env bash\necho Hello, world!\nsleep 5\necho Hello, again!\nsleep 5"} + + run, err := g.Evaluate(context.Background(), Options{DisableCache: true, IncludeEvents: true}, tool) + if err != nil { + t.Errorf("Error executing tool: %v", err) + } + + // Abort the run after the first event. + for e := range run.Events() { + if e.Call != nil && e.Call.Type == EventTypeChat { + time.Sleep(2 * time.Second) + break + } + } + + if err := g.AbortRun(context.Background(), run); err != nil { + t.Errorf("Error aborting run: %v", err) + } + + // Wait for run to stop + for range run.Events() { + continue + } + + if run.State() != Finished { + t.Errorf("Unexpected run state: %s", run.State()) + } + + if out, err := run.Text(); err != nil { + t.Errorf("Error reading output: %v", err) + } else if !strings.Contains(out, "Hello, world!") || strings.Contains(out, "Hello, again!") || !strings.HasSuffix(out, "\nABORTED BY USER") { + t.Errorf("Unexpected output: %s", out) + } +} + func TestSimpleEvaluate(t *testing.T) { tool := ToolDef{Instructions: "What is the capital of the united states?"} @@ -844,6 +916,69 @@ func TestToolChat(t *testing.T) { } } +func TestAbortChat(t *testing.T) { + tool := ToolDef{ + Chat: true, + Instructions: "You are a chat bot. Don't finish the conversation until I say 'bye'.", + Tools: []string{"sys.chat.finish"}, + } + + run, err := g.Evaluate(context.Background(), Options{DisableCache: true, IncludeEvents: true}, tool) + if err != nil { + t.Fatalf("Error executing tool: %v", err) + } + inputs := []string{ + "Tell me a joke.", + "What was my first message?", + } + + // Just wait for the chat to start up. + for range run.Events() { + continue + } + + for i, input := range inputs { + run, err = run.NextChat(context.Background(), input) + if err != nil { + t.Fatalf("Error sending next input %q: %v", input, err) + } + + // Abort the run after the first event from the LLM + for e := range run.Events() { + if e.Call != nil && e.Call.Type == EventTypeCallProgress && len(e.Call.Output) > 0 && e.Call.Output[0].Content != "Waiting for model response..." { + break + } + } + + if i == 0 { + if err := g.AbortRun(context.Background(), run); err != nil { + t.Fatalf("Error aborting run: %v", err) + } + } + + // Wait for the run to complete + for range run.Events() { + continue + } + + out, err := run.Text() + if err != nil { + t.Errorf("Error reading output: %s", run.ErrorOutput()) + t.Fatalf("Error reading output: %v", err) + } + + if i == 0 { + if strings.TrimSpace(out) != "ABORTED BY USER" && !strings.HasSuffix(out, "\nABORTED BY USER") { + t.Fatalf("Unexpected output: %s", out) + } + } else { + if !strings.Contains(out, "Tell me a joke") { + t.Errorf("Unexpected output: %s", out) + } + } + } +} + func TestFileChat(t *testing.T) { wd, err := os.Getwd() if err != nil { diff --git a/run.go b/run.go index 52ad6e4..558b388 100644 --- a/run.go +++ b/run.go @@ -37,6 +37,7 @@ type Run struct { basicCommand bool program *Program + id string callsLock sync.RWMutex calls CallFrames rawOutput map[string]any @@ -400,6 +401,7 @@ func (r *Run) request(ctx context.Context, payload any) (err error) { if event.Run.Type == EventTypeRunStart { r.callsLock.Lock() r.program = &event.Run.Program + r.id = event.Run.ID r.callsLock.Unlock() } else if event.Run.Type == EventTypeRunFinish && event.Run.Error != "" { r.state = Error From 8d1f06fa87a40e33465ca12a416d9eb4ba0e42b3 Mon Sep 17 00:00:00 2001 From: Grant Linville Date: Fri, 14 Mar 2025 11:01:04 -0400 Subject: [PATCH 71/78] chore: add function for recreate credentials (#95) Signed-off-by: Grant Linville --- gptscript.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gptscript.go b/gptscript.go index 8ca9747..3356982 100644 --- a/gptscript.go +++ b/gptscript.go @@ -400,6 +400,11 @@ func (g *GPTScript) CreateCredential(ctx context.Context, cred Credential) error return err } +func (g *GPTScript) RecreateAllCredentials(ctx context.Context) error { + _, err := g.runBasicCommand(ctx, "credentials/recreate-all", struct{}{}) + return err +} + func (g *GPTScript) RevealCredential(ctx context.Context, credCtxs []string, name string) (Credential, error) { out, err := g.runBasicCommand(ctx, "credentials/reveal", CredentialRequest{ Context: credCtxs, From 5df35b4a3df5127be41df17cfffc330079bf496d Mon Sep 17 00:00:00 2001 From: Daishan Peng Date: Wed, 26 Mar 2025 16:20:14 -0700 Subject: [PATCH 72/78] Enhance: Add ability to do inline prompting to choose oauth/pat Signed-off-by: Daishan Peng --- frame.go | 7 ++++--- gptscript_test.go | 49 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/frame.go b/frame.go index 953572f..4b77860 100644 --- a/frame.go +++ b/frame.go @@ -124,9 +124,10 @@ type Prompt struct { } type Field struct { - Name string `json:"name,omitempty"` - Sensitive *bool `json:"sensitive,omitempty"` - Description string `json:"description,omitempty"` + Name string `json:"name,omitempty"` + Sensitive *bool `json:"sensitive,omitempty"` + Description string `json:"description,omitempty"` + Options []string `json:"options,omitempty"` } type Fields []Field diff --git a/gptscript_test.go b/gptscript_test.go index 476492d..213a7a6 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -160,7 +160,7 @@ func TestCancelRun(t *testing.T) { } func TestAbortChatCompletionRun(t *testing.T) { - tool := ToolDef{Instructions: "What is the capital of the united states?"} + tool := ToolDef{Instructions: "Generate a real long essay about the meaning of life."} run, err := g.Evaluate(context.Background(), Options{DisableCache: true, IncludeEvents: true}, tool) if err != nil { @@ -1404,6 +1404,53 @@ func TestPromptWithoutPromptAllowed(t *testing.T) { } } +func TestPromptWithOptions(t *testing.T) { + run, err := g.Run(context.Background(), "sys.prompt", Options{IncludeEvents: true, Prompt: true, Input: `{"fields":[{"name":"Authentication Method","description":"The authentication token for the user","options":["API Key","OAuth"]}]}`}) + if err != nil { + t.Errorf("Error executing tool: %v", err) + } + + // Wait for the prompt event + var promptFrame *PromptFrame + for e := range run.Events() { + if e.Prompt != nil { + if e.Prompt.Type == EventTypePrompt { + promptFrame = e.Prompt + break + } + } + } + + if promptFrame == nil { + t.Fatalf("No prompt call event") + return + } + + if len(promptFrame.Fields) != 1 { + t.Fatalf("Unexpected number of fields: %d", len(promptFrame.Fields)) + } + + if promptFrame.Fields[0].Name != "Authentication Method" { + t.Errorf("Unexpected field: %s", promptFrame.Fields[0].Name) + } + + if promptFrame.Fields[0].Description != "The authentication token for the user" { + t.Errorf("Unexpected description: %s", promptFrame.Fields[0].Description) + } + + if len(promptFrame.Fields[0].Options) != 2 { + t.Fatalf("Unexpected number of options: %d", len(promptFrame.Fields[0].Options)) + } + + if promptFrame.Fields[0].Options[0] != "API Key" { + t.Errorf("Unexpected option: %s", promptFrame.Fields[0].Options[0]) + } + + if promptFrame.Fields[0].Options[1] != "OAuth" { + t.Errorf("Unexpected option: %s", promptFrame.Fields[0].Options[1]) + } +} + func TestGetCommand(t *testing.T) { currentEnvVar := os.Getenv("GPTSCRIPT_BIN") t.Cleanup(func() { From c8eaf67ab0d651d71b00a7aaa368edc13e2e467b Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Mon, 21 Apr 2025 17:00:17 -0400 Subject: [PATCH 73/78] enhance: add run usage method (#97) Signed-off-by: Donnie Adams --- run.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/run.go b/run.go index 558b388..983dccc 100644 --- a/run.go +++ b/run.go @@ -118,6 +118,21 @@ func (r *Run) ParentCallFrame() (CallFrame, bool) { return r.calls.ParentCallFrame(), true } +// Usage returns all the usage for this run. +func (r *Run) Usage() Usage { + var u Usage + r.callsLock.RLock() + defer r.callsLock.RUnlock() + + for _, c := range r.calls { + u.CompletionTokens += c.Usage.CompletionTokens + u.PromptTokens += c.Usage.PromptTokens + u.TotalTokens += c.Usage.TotalTokens + } + + return u +} + // ErrorOutput returns the stderr output of the gptscript. // Should only be called after Bytes or Text has returned an error. func (r *Run) ErrorOutput() string { From af453989e88f0f6a28e2d4ade8d33e4a84f6c29f Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Thu, 24 Apr 2025 16:49:37 -0400 Subject: [PATCH 74/78] chore: add checkParam credential field (#98) Signed-off-by: Donnie Adams --- credentials.go | 1 + gptscript_test.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/credentials.go b/credentials.go index 4c7c11f..ca04c6c 100644 --- a/credentials.go +++ b/credentials.go @@ -15,6 +15,7 @@ type Credential struct { Type CredentialType `json:"type"` Env map[string]string `json:"env"` Ephemeral bool `json:"ephemeral,omitempty"` + CheckParam string `json:"checkParam"` ExpiresAt *time.Time `json:"expiresAt"` RefreshToken string `json:"refreshToken"` } diff --git a/gptscript_test.go b/gptscript_test.go index 213a7a6..af14f98 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -1716,6 +1716,7 @@ func TestCredentials(t *testing.T) { Type: CredentialTypeTool, Env: map[string]string{"ENV": "testing"}, RefreshToken: "my-refresh-token", + CheckParam: "my-check-param", }) require.NoError(t, err) @@ -1732,6 +1733,7 @@ func TestCredentials(t *testing.T) { require.Contains(t, cred.Env, "ENV") require.Equal(t, cred.Env["ENV"], "testing") require.Equal(t, cred.RefreshToken, "my-refresh-token") + require.Equal(t, cred.CheckParam, "my-check-param") // Delete err = g.DeleteCredential(context.Background(), "testing", name) From f1616a06f1b0513cf49f9b00f958631699e375b2 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Tue, 20 May 2025 11:46:49 -0400 Subject: [PATCH 75/78] chore: switch to huma v2 for OpenAPI schema (#99) Signed-off-by: Donnie Adams --- go.mod | 13 ++++--------- go.sum | 30 ++++++++---------------------- gptscript_test.go | 26 +++++++++++--------------- tool.go | 20 +++++++++----------- 4 files changed, 32 insertions(+), 57 deletions(-) diff --git a/go.mod b/go.mod index 0ca27e7..b2062db 100644 --- a/go.mod +++ b/go.mod @@ -3,20 +3,15 @@ module github.com/gptscript-ai/go-gptscript go 1.23.0 require ( - github.com/getkin/kin-openapi v0.129.0 + github.com/danielgtaylor/huma/v2 v2.32.0 github.com/stretchr/testify v1.10.0 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/swag v0.23.0 // indirect - github.com/josharian/intern v1.0.0 // indirect - github.com/mailru/easyjson v0.9.0 // indirect - github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect - github.com/oasdiff/yaml v0.0.0-20241210131133-6b86fb107d80 // indirect - github.com/oasdiff/yaml3 v0.0.0-20241210130736-a94c01f36349 // indirect - github.com/perimeterx/marshmallow v1.1.5 // indirect + github.com/kr/text v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 1c7e6b3..b774272 100644 --- a/go.sum +++ b/go.sum @@ -1,37 +1,23 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/danielgtaylor/huma/v2 v2.32.0 h1:ytU9ExG/axC434+soXxwNzv0uaxOb3cyCgjj8y3PmBE= +github.com/danielgtaylor/huma/v2 v2.32.0/go.mod h1:9BxJwkeoPPDEJ2Bg4yPwL1mM1rYpAwCAWFKoo723spk= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/getkin/kin-openapi v0.129.0 h1:QGYTNcmyP5X0AtFQ2Dkou9DGBJsUETeLH9rFrJXZh30= -github.com/getkin/kin-openapi v0.129.0/go.mod h1:gmWI+b/J45xqpyK5wJmRRZse5wefA5H0RDMK46kLUtI= -github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= -github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= -github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= -github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= -github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= -github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= -github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= -github.com/oasdiff/yaml v0.0.0-20241210131133-6b86fb107d80 h1:nZspmSkneBbtxU9TopEAE0CY+SBJLxO8LPUlw2vG4pU= -github.com/oasdiff/yaml v0.0.0-20241210131133-6b86fb107d80/go.mod h1:7tFDb+Y51LcDpn26GccuUgQXUk6t0CXZsivKjyimYX8= -github.com/oasdiff/yaml3 v0.0.0-20241210130736-a94c01f36349 h1:t05Ww3DxZutOqbMN+7OIuqDwXbhl32HiZGpLy26BAPc= -github.com/oasdiff/yaml3 v0.0.0-20241210130736-a94c01f36349/go.mod h1:y5+oSEHCPT/DGrS++Wc/479ERge0zTFxaF8PbGKcg2o= -github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= -github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= -github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/gptscript_test.go b/gptscript_test.go index af14f98..9349ccd 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -13,7 +13,7 @@ import ( "testing" "time" - "github.com/getkin/kin-openapi/openapi3" + humav2 "github.com/danielgtaylor/huma/v2" "github.com/stretchr/testify/require" ) @@ -768,14 +768,12 @@ func TestFmt(t *testing.T) { ToolDef: ToolDef{ Name: "echo", Instructions: "#!/bin/bash\necho hello there", - Arguments: &openapi3.Schema{ - Type: &openapi3.Types{"object"}, - Properties: map[string]*openapi3.SchemaRef{ + Arguments: &humav2.Schema{ + Type: humav2.TypeObject, + Properties: map[string]*humav2.Schema{ "input": { - Value: &openapi3.Schema{ - Description: "The string input to echo", - Type: &openapi3.Types{"string"}, - }, + Description: "The string input to echo", + Type: humav2.TypeString, }, }, }, @@ -829,14 +827,12 @@ func TestFmtWithTextNode(t *testing.T) { ToolDef: ToolDef{ Instructions: "#!/bin/bash\necho hello there", Name: "echo", - Arguments: &openapi3.Schema{ - Type: &openapi3.Types{"object"}, - Properties: map[string]*openapi3.SchemaRef{ + Arguments: &humav2.Schema{ + Type: humav2.TypeObject, + Properties: map[string]*humav2.Schema{ "input": { - Value: &openapi3.Schema{ - Description: "The string input to echo", - Type: &openapi3.Types{"string"}, - }, + Description: "The string input to echo", + Type: humav2.TypeString, }, }, }, diff --git a/tool.go b/tool.go index 306723b..5603ec3 100644 --- a/tool.go +++ b/tool.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - "github.com/getkin/kin-openapi/openapi3" + humav2 "github.com/danielgtaylor/huma/v2" ) // ToolDef struct represents a tool with various configurations. @@ -19,7 +19,7 @@ type ToolDef struct { Temperature *float32 `json:"temperature,omitempty"` Cache *bool `json:"cache,omitempty"` InternalPrompt *bool `json:"internalPrompt"` - Arguments *openapi3.Schema `json:"arguments,omitempty"` + Arguments *humav2.Schema `json:"arguments,omitempty"` Tools []string `json:"tools,omitempty"` GlobalTools []string `json:"globalTools,omitempty"` GlobalModelName string `json:"globalModelName,omitempty"` @@ -52,18 +52,16 @@ func ToolDefsToNodes(tools []ToolDef) []Node { return nodes } -func ObjectSchema(kv ...string) *openapi3.Schema { - s := &openapi3.Schema{ - Type: &openapi3.Types{"object"}, - Properties: openapi3.Schemas{}, +func ObjectSchema(kv ...string) *humav2.Schema { + s := &humav2.Schema{ + Type: humav2.TypeObject, + Properties: make(map[string]*humav2.Schema, len(kv)/2), } for i, v := range kv { if i%2 == 1 { - s.Properties[kv[i-1]] = &openapi3.SchemaRef{ - Value: &openapi3.Schema{ - Description: v, - Type: &openapi3.Types{"string"}, - }, + s.Properties[kv[i-1]] = &humav2.Schema{ + Description: v, + Type: humav2.TypeString, } } } From 9129819aea512c475cb334c25dc88cdf621c1c9e Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Tue, 17 Jun 2025 09:17:50 -0400 Subject: [PATCH 76/78] chore: use forked version of huma for schema unmarshal fixes (#100) Signed-off-by: Donnie Adams --- go.mod | 6 +++--- go.sum | 13 ++++--------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index b2062db..b642e52 100644 --- a/go.mod +++ b/go.mod @@ -2,16 +2,16 @@ module github.com/gptscript-ai/go-gptscript go 1.23.0 +replace github.com/danielgtaylor/huma/v2 => github.com/gptscript-ai/huma v0.0.0-20250617131016-b2081da6c65b + require ( - github.com/danielgtaylor/huma/v2 v2.32.0 + github.com/danielgtaylor/huma/v2 v2.32.1-0.20250509235652-c7ead6f3c67f github.com/stretchr/testify v1.10.0 ) require ( github.com/davecgh/go-spew v1.1.1 // indirect - github.com/kr/text v0.2.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/rogpeppe/go-internal v1.12.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index b774272..c9b2029 100644 --- a/go.sum +++ b/go.sum @@ -1,21 +1,16 @@ -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/danielgtaylor/huma/v2 v2.32.0 h1:ytU9ExG/axC434+soXxwNzv0uaxOb3cyCgjj8y3PmBE= -github.com/danielgtaylor/huma/v2 v2.32.0/go.mod h1:9BxJwkeoPPDEJ2Bg4yPwL1mM1rYpAwCAWFKoo723spk= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gptscript-ai/huma v0.0.0-20250617131016-b2081da6c65b h1:QReUetqY+ep2sj6g83oqldPHzwH2T2TG1sv0IWE2hL0= +github.com/gptscript-ai/huma v0.0.0-20250617131016-b2081da6c65b/go.mod h1:y2Eq35Y5Xy6+MZRPgn81/bjNBiEHqEQba+vY+fLigjU= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From fb8d698f610c3092c905d2f63baa624f1a36a28d Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Mon, 14 Jul 2025 12:43:38 -0400 Subject: [PATCH 77/78] chore: switch to MCP go-sdk for JSON Schema Signed-off-by: Donnie Adams --- go.mod | 6 ++--- go.sum | 8 +++--- gptscript_test.go | 18 ++++++------- tool.go | 68 +++++++++++++++++++++++------------------------ 4 files changed, 49 insertions(+), 51 deletions(-) diff --git a/go.mod b/go.mod index b642e52..a625857 100644 --- a/go.mod +++ b/go.mod @@ -1,11 +1,9 @@ module github.com/gptscript-ai/go-gptscript -go 1.23.0 - -replace github.com/danielgtaylor/huma/v2 => github.com/gptscript-ai/huma v0.0.0-20250617131016-b2081da6c65b +go 1.24.2 require ( - github.com/danielgtaylor/huma/v2 v2.32.1-0.20250509235652-c7ead6f3c67f + github.com/modelcontextprotocol/go-sdk v0.2.0 github.com/stretchr/testify v1.10.0 ) diff --git a/go.sum b/go.sum index c9b2029..10b0c7a 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,14 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gptscript-ai/huma v0.0.0-20250617131016-b2081da6c65b h1:QReUetqY+ep2sj6g83oqldPHzwH2T2TG1sv0IWE2hL0= -github.com/gptscript-ai/huma v0.0.0-20250617131016-b2081da6c65b/go.mod h1:y2Eq35Y5Xy6+MZRPgn81/bjNBiEHqEQba+vY+fLigjU= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/modelcontextprotocol/go-sdk v0.2.0 h1:PESNYOmyM1c369tRkzXLY5hHrazj8x9CY1Xu0fLCryM= +github.com/modelcontextprotocol/go-sdk v0.2.0/go.mod h1:0sL9zUKKs2FTTkeCCVnKqbLJTw5TScefPAzojjU459E= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= diff --git a/gptscript_test.go b/gptscript_test.go index 9349ccd..e74fe98 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -13,7 +13,7 @@ import ( "testing" "time" - humav2 "github.com/danielgtaylor/huma/v2" + "github.com/modelcontextprotocol/go-sdk/jsonschema" "github.com/stretchr/testify/require" ) @@ -768,12 +768,12 @@ func TestFmt(t *testing.T) { ToolDef: ToolDef{ Name: "echo", Instructions: "#!/bin/bash\necho hello there", - Arguments: &humav2.Schema{ - Type: humav2.TypeObject, - Properties: map[string]*humav2.Schema{ + Arguments: &jsonschema.Schema{ + Type: "object", + Properties: map[string]*jsonschema.Schema{ "input": { Description: "The string input to echo", - Type: humav2.TypeString, + Type: "string", }, }, }, @@ -827,12 +827,12 @@ func TestFmtWithTextNode(t *testing.T) { ToolDef: ToolDef{ Instructions: "#!/bin/bash\necho hello there", Name: "echo", - Arguments: &humav2.Schema{ - Type: humav2.TypeObject, - Properties: map[string]*humav2.Schema{ + Arguments: &jsonschema.Schema{ + Type: "object", + Properties: map[string]*jsonschema.Schema{ "input": { Description: "The string input to echo", - Type: humav2.TypeString, + Type: "string", }, }, }, diff --git a/tool.go b/tool.go index 5603ec3..18e8486 100644 --- a/tool.go +++ b/tool.go @@ -4,38 +4,38 @@ import ( "fmt" "strings" - humav2 "github.com/danielgtaylor/huma/v2" + "github.com/modelcontextprotocol/go-sdk/jsonschema" ) // ToolDef struct represents a tool with various configurations. type ToolDef struct { - Name string `json:"name,omitempty"` - Description string `json:"description,omitempty"` - MaxTokens int `json:"maxTokens,omitempty"` - ModelName string `json:"modelName,omitempty"` - ModelProvider bool `json:"modelProvider,omitempty"` - JSONResponse bool `json:"jsonResponse,omitempty"` - Chat bool `json:"chat,omitempty"` - Temperature *float32 `json:"temperature,omitempty"` - Cache *bool `json:"cache,omitempty"` - InternalPrompt *bool `json:"internalPrompt"` - Arguments *humav2.Schema `json:"arguments,omitempty"` - Tools []string `json:"tools,omitempty"` - GlobalTools []string `json:"globalTools,omitempty"` - GlobalModelName string `json:"globalModelName,omitempty"` - Context []string `json:"context,omitempty"` - ExportContext []string `json:"exportContext,omitempty"` - Export []string `json:"export,omitempty"` - Agents []string `json:"agents,omitempty"` - Credentials []string `json:"credentials,omitempty"` - ExportCredentials []string `json:"exportCredentials,omitempty"` - InputFilters []string `json:"inputFilters,omitempty"` - ExportInputFilters []string `json:"exportInputFilters,omitempty"` - OutputFilters []string `json:"outputFilters,omitempty"` - ExportOutputFilters []string `json:"exportOutputFilters,omitempty"` - Instructions string `json:"instructions,omitempty"` - Type string `json:"type,omitempty"` - MetaData map[string]string `json:"metadata,omitempty"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + MaxTokens int `json:"maxTokens,omitempty"` + ModelName string `json:"modelName,omitempty"` + ModelProvider bool `json:"modelProvider,omitempty"` + JSONResponse bool `json:"jsonResponse,omitempty"` + Chat bool `json:"chat,omitempty"` + Temperature *float32 `json:"temperature,omitempty"` + Cache *bool `json:"cache,omitempty"` + InternalPrompt *bool `json:"internalPrompt"` + Arguments *jsonschema.Schema `json:"arguments,omitempty"` + Tools []string `json:"tools,omitempty"` + GlobalTools []string `json:"globalTools,omitempty"` + GlobalModelName string `json:"globalModelName,omitempty"` + Context []string `json:"context,omitempty"` + ExportContext []string `json:"exportContext,omitempty"` + Export []string `json:"export,omitempty"` + Agents []string `json:"agents,omitempty"` + Credentials []string `json:"credentials,omitempty"` + ExportCredentials []string `json:"exportCredentials,omitempty"` + InputFilters []string `json:"inputFilters,omitempty"` + ExportInputFilters []string `json:"exportInputFilters,omitempty"` + OutputFilters []string `json:"outputFilters,omitempty"` + ExportOutputFilters []string `json:"exportOutputFilters,omitempty"` + Instructions string `json:"instructions,omitempty"` + Type string `json:"type,omitempty"` + MetaData map[string]string `json:"metadata,omitempty"` } func ToolDefsToNodes(tools []ToolDef) []Node { @@ -52,16 +52,16 @@ func ToolDefsToNodes(tools []ToolDef) []Node { return nodes } -func ObjectSchema(kv ...string) *humav2.Schema { - s := &humav2.Schema{ - Type: humav2.TypeObject, - Properties: make(map[string]*humav2.Schema, len(kv)/2), +func ObjectSchema(kv ...string) *jsonschema.Schema { + s := &jsonschema.Schema{ + Type: "object", + Properties: make(map[string]*jsonschema.Schema, len(kv)/2), } for i, v := range kv { if i%2 == 1 { - s.Properties[kv[i-1]] = &humav2.Schema{ + s.Properties[kv[i-1]] = &jsonschema.Schema{ Description: v, - Type: humav2.TypeString, + Type: "string", } } } From 17ad44ae8c54e02d0e678fc1a5c6fe1c29a48eb2 Mon Sep 17 00:00:00 2001 From: Donnie Adams Date: Mon, 14 Jul 2025 12:48:58 -0400 Subject: [PATCH 78/78] chore: update golangci-lint version Signed-off-by: Donnie Adams --- .golangci.yaml | 37 ++++++++++++++++++++++++------------- Makefile | 2 +- gptscript_test.go | 14 ++++++++------ 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index 03be8a7..c2c3ee3 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -1,24 +1,35 @@ +version: "2" run: timeout: 5m - -output: - formats: - - format: colored-line-number - linters: - disable-all: true + default: none enable: - errcheck - - gofmt - - gosimple - govet - ineffassign + - revive - staticcheck - - typecheck - thelper - unused - - goimports - whitespace - - revive - fast: false - max-same-issues: 50 + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + enable: + - gofmt + - goimports + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/Makefile b/Makefile index ecb61c5..0227d0a 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ tidy: test: go test -v ./... -GOLANGCI_LINT_VERSION ?= v1.60.1 +GOLANGCI_LINT_VERSION ?= v2.1.2 lint: if ! command -v golangci-lint &> /dev/null; then \ echo "Could not find golangci-lint, installing version $(GOLANGCI_LINT_VERSION)."; \ diff --git a/gptscript_test.go b/gptscript_test.go index e74fe98..4cf7a6b 100644 --- a/gptscript_test.go +++ b/gptscript_test.go @@ -567,7 +567,7 @@ func TestRestartFailedRun(t *testing.T) { t.Errorf("Expected error but got nil") } - run.opts.GlobalOptions.Env = nil + run.opts.Env = nil run, err = run.NextChat(context.Background(), "") if err != nil { t.Fatalf("Error executing next run: %v", err) @@ -1037,21 +1037,23 @@ func TestToolWithGlobalTools(t *testing.T) { for e := range run.Events() { if e.Run != nil { - if e.Run.Type == EventTypeRunStart { + switch e.Run.Type { + case EventTypeRunStart: runStartSeen = true - } else if e.Run.Type == EventTypeRunFinish { + case EventTypeRunFinish: runFinishSeen = true } } else if e.Call != nil { - if e.Call.Type == EventTypeCallStart { + switch e.Call.Type { + case EventTypeCallStart: callStartSeen = true - } else if e.Call.Type == EventTypeCallFinish { + case EventTypeCallFinish: callFinishSeen = true for _, o := range e.Call.Output { eventContent += o.Content } - } else if e.Call.Type == EventTypeCallProgress { + case EventTypeCallProgress: callProgressSeen = true } }