Merge branch 'xqdoo00o-master' into Add-Gemini

auth.go

@@ -58,9 +58,13 @@ func AppendIfNone(slice []string, i string) []string {
 }
 
 func getSecret() (string, string) {
-	account := validAccounts[0]
-	validAccounts = append(validAccounts[1:], account)
-	return ACCESS_TOKENS.GetSecret(account)
+	if len(validAccounts) != 0 {
+		account := validAccounts[0]
+		validAccounts = append(validAccounts[1:], account)
+		return ACCESS_TOKENS.GetSecret(account)
+	} else {
+		return "", ""
+	}
 }
 
 // Read accounts.txt and create a list of accounts

conversion/requests/chatgpt/convert.go

@@ -9,9 +9,12 @@ import (
 	arkose "github.com/xqdoo00o/funcaptcha"
 )
 
-func ConvertAPIRequest(api_request official_types.APIRequest, puid string, requireArk bool, proxy string) chatgpt_types.ChatGPTRequest {
+func ConvertAPIRequest(api_request official_types.APIRequest, puid string, requireArk bool, dx string, proxy string) chatgpt_types.ChatGPTRequest {
 	chatgpt_request := chatgpt_types.NewChatGPTRequest()
 	var api_version int
+	if puid == "" {
+		api_request.Model = "gpt-3.5"
+	}
 	if strings.HasPrefix(api_request.Model, "gpt-3.5") {
 		api_version = 3
 		chatgpt_request.Model = "text-davinci-002-render-sha"
@@ -24,7 +27,7 @@ func ConvertAPIRequest(api_request official_types.APIRequest, puid string, requi
 		}
 	}
 	if requireArk {
-		token, err := arkose.GetOpenAIToken(api_version, puid, proxy)
+		token, err := arkose.GetOpenAIToken(api_version, puid, dx, proxy)
 		if err == nil {
 			chatgpt_request.ArkoseToken = token
 		} else {
@@ -44,14 +47,14 @@ func ConvertAPIRequest(api_request official_types.APIRequest, puid string, requi
 	return chatgpt_request
 }
 
-func RenewTokenForRequest(request *chatgpt_types.ChatGPTRequest, puid string, proxy string) {
+func RenewTokenForRequest(request *chatgpt_types.ChatGPTRequest, puid string, dx string, proxy string) {
 	var api_version int
 	if strings.HasPrefix(request.Model, "gpt-4") {
 		api_version = 4
 	} else {
 		api_version = 3
 	}
-	token, err := arkose.GetOpenAIToken(api_version, puid, proxy)
+	token, err := arkose.GetOpenAIToken(api_version, puid, dx, proxy)
 	if err == nil {
 		request.ArkoseToken = token
 	} else {

go.mod

@@ -17,8 +17,8 @@ require (
 	github.com/joho/godotenv v1.5.1
 	github.com/sashabaranov/go-openai v1.20.3
 	github.com/tidwall/gjson v1.14.4
-	github.com/xqdoo00o/OpenAIAuth v0.0.0-20240320160511-93065690aa0e
-	github.com/xqdoo00o/funcaptcha v0.0.0-20240313153914-4ab805804232
+	github.com/xqdoo00o/OpenAIAuth v0.0.0-20240403091529-7ef147706fc4
+	github.com/xqdoo00o/funcaptcha v0.0.0-20240403090732-1b604d808f6c
 	github.com/zhu327/gemini-openai-proxy v0.0.0-20240314045503-f6afa6badbd2
 	google.golang.org/api v0.169.0
 	k8s.io/apimachinery v0.27.2

go.sum

@@ -164,10 +164,10 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
 github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
-github.com/xqdoo00o/OpenAIAuth v0.0.0-20240320160511-93065690aa0e h1:bV8VUCkeHGmgvx4HrOcEldIs+SD3WOwb6/ak3bGewy4=
-github.com/xqdoo00o/OpenAIAuth v0.0.0-20240320160511-93065690aa0e/go.mod h1:lsPJ6JX/GLhM0fEoPrJRdKxTmN5n0j4TqjDmnY6rFu0=
-github.com/xqdoo00o/funcaptcha v0.0.0-20240313153914-4ab805804232 h1:K8+8YdWdDzmmhVqNflEXV4RLpZPf56wwxo0k0nAtQO4=
-github.com/xqdoo00o/funcaptcha v0.0.0-20240313153914-4ab805804232/go.mod h1:7aCyoW5MHDUsoooMVLqKe0F7W9HMPUvDG3bXqw++8XA=
+github.com/xqdoo00o/OpenAIAuth v0.0.0-20240403091529-7ef147706fc4 h1:Goj6yoMgADshscKkfhRpDeMxyGm2wdcHCUPj0YQ+B3c=
+github.com/xqdoo00o/OpenAIAuth v0.0.0-20240403091529-7ef147706fc4/go.mod h1:XHxG7wI3Erk4nS0rwjXVF3FllYW5QxsRynnJ0GqAMCQ=
+github.com/xqdoo00o/funcaptcha v0.0.0-20240403090732-1b604d808f6c h1:nj17XsSTwprsZUDXLldOUZmqz7VlHsLCeXXFOE6Q+Mk=
+github.com/xqdoo00o/funcaptcha v0.0.0-20240403090732-1b604d808f6c/go.mod h1:7aCyoW5MHDUsoooMVLqKe0F7W9HMPUvDG3bXqw++8XA=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/zhu327/gemini-openai-proxy v0.0.0-20240314045503-f6afa6badbd2 h1:jt7dpvsz47vZU1FbHUBQQjTnXUj99NidNL8nYZ+ySNg=
 github.com/zhu327/gemini-openai-proxy v0.0.0-20240314045503-f6afa6badbd2/go.mod h1:tL/SICLAKUSC5+ew0OhiW+ifcC8sEzQD7nnqL0I+Rm0=

handlers.go

@@ -141,14 +141,21 @@ func nightmare(c *gin.Context) {
 		proxies = append(proxies[1:], proxies[0])
 	}
 	uid := uuid.NewString()
+	if token == "" {
+		chatgpt.SetOAICookie(uid)
+	}
 	var err error
 	var chat_require *chatgpt.ChatRequire
 	var wg sync.WaitGroup
-	wg.Add(2)
-	go func() {
-		defer wg.Done()
-		err = chatgpt.InitWSConn(token, uid, proxy_url)
-	}()
+	if token == "" {
+		wg.Add(1)
+	} else {
+		wg.Add(2)
+		go func() {
+			defer wg.Done()
+			err = chatgpt.InitWSConn(token, uid, proxy_url)
+		}()
+	}
 	go func() {
 		defer wg.Done()
 		chat_require = chatgpt.CheckRequire(token, puid, proxy_url)
@@ -163,7 +170,7 @@ func nightmare(c *gin.Context) {
 		return
 	}
 	// Convert the chat request to a ChatGPT request
-	translated_request := chatgpt_request_converter.ConvertAPIRequest(original_request, puid, chat_require.Arkose.Required, proxy_url)
+	translated_request := chatgpt_request_converter.ConvertAPIRequest(original_request, puid, chat_require.Arkose.Required, chat_require.Arkose.DX, proxy_url)
 
 	response, err := chatgpt.POSTconversation(translated_request, token, puid, chat_require.Token, proxy_url)
 	if err != nil {
@@ -191,7 +198,7 @@ func nightmare(c *gin.Context) {
 		translated_request.ConversationID = continue_info.ConversationID
 		translated_request.ParentMessageID = continue_info.ParentID
 		if chat_require.Arkose.Required {
-			chatgpt_request_converter.RenewTokenForRequest(&translated_request, puid, proxy_url)
+			chatgpt_request_converter.RenewTokenForRequest(&translated_request, puid, chat_require.Arkose.DX, proxy_url)
 		}
 		response, err = chatgpt.POSTconversation(translated_request, token, puid, chat_require.Token, proxy_url)
 		if err != nil {

internal/chatgpt/request.go

@@ -10,6 +10,7 @@ import (
 	chatgpt_types "freechatgpt/typings/chatgpt"
 	"io"
 	"net"
+	"net/url"
 	"os"
 	"strconv"
 	"strings"
@@ -189,6 +190,14 @@ func InitWSConn(token string, uuid string, proxy string) error {
 	}
 }
 
+func SetOAICookie(uuid string) {
+	u, _ := url.Parse("https://openai.com")
+	client.GetCookieJar().SetCookies(u, []*http.Cookie{{
+		Name:  "oai-did",
+		Value: uuid,
+	}})
+}
+
 type ChatRequire struct {
 	Token  string `json:"token"`
 	Arkose struct {
@@ -201,10 +210,16 @@ func CheckRequire(access_token string, puid string, proxy string) *ChatRequire {
 	if proxy != "" {
 		client.SetProxy(proxy)
 	}
-
-	apiUrl := "https://chat.openai.com/backend-api/sentinel/chat-requirements"
-
-	request, err := http.NewRequest(http.MethodPost, apiUrl, bytes.NewBuffer([]byte(`{"conversation_mode_kind":"primary_assistant"}`)))
+	var body *bytes.Buffer
+	var apiUrl string
+	if access_token == "" {
+		body = bytes.NewBuffer([]byte(`{}`))
+		apiUrl = "https://chat.openai.com/backend-anon/sentinel/chat-requirements"
+	} else {
+		body = bytes.NewBuffer([]byte(`{"conversation_mode_kind":"primary_assistant"}`))
+		apiUrl = "https://chat.openai.com/backend-api/sentinel/chat-requirements"
+	}
+	request, err := http.NewRequest(http.MethodPost, apiUrl, body)
 	if err != nil {
 		return nil
 	}
@@ -217,9 +232,6 @@ func CheckRequire(access_token string, puid string, proxy string) *ChatRequire {
 	if access_token != "" {
 		request.Header.Set("Authorization", "Bearer "+access_token)
 	}
-	if err != nil {
-		return nil
-	}
 	response, err := client.Do(request)
 	if err != nil {
 		return nil
@@ -233,6 +245,43 @@ func CheckRequire(access_token string, puid string, proxy string) *ChatRequire {
 	return &require
 }
 
+var urlAttrMap = make(map[string]string)
+
+type urlAttr struct {
+	Url         string `json:"url"`
+	Attribution string `json:"attribution"`
+}
+
+func getURLAttribution(access_token string, puid string, url string) string {
+	request, err := http.NewRequest(http.MethodPost, "https://chat.openai.com/backend-api/attributions", bytes.NewBuffer([]byte(`{"urls":["`+url+`"]}`)))
+	if err != nil {
+		return ""
+	}
+	if puid != "" {
+		request.Header.Set("Cookie", "_puid="+puid+";")
+	}
+	request.Header.Set("Content-Type", "application/json")
+	request.Header.Set("User-Agent", userAgent)
+	request.Header.Set("Oai-Language", "en-US")
+	if access_token != "" {
+		request.Header.Set("Authorization", "Bearer "+access_token)
+	}
+	if err != nil {
+		return ""
+	}
+	response, err := client.Do(request)
+	if err != nil {
+		return ""
+	}
+	defer response.Body.Close()
+	var attr urlAttr
+	err = json.NewDecoder(response.Body).Decode(&attr)
+	if err != nil {
+		return ""
+	}
+	return attr.Attribution
+}
+
 func POSTconversation(message chatgpt_types.ChatGPTRequest, access_token string, puid string, chat_token string, proxy string) (*http.Response, error) {
 	if proxy != "" {
 		client.SetProxy(proxy)
@@ -498,6 +547,9 @@ func Handler(c *gin.Context, response *http.Response, token string, puid string,
 			if !(original_response.Message.Author.Role == "assistant" || (original_response.Message.Author.Role == "tool" && original_response.Message.Content.ContentType != "text")) || original_response.Message.Content.Parts == nil {
 				continue
 			}
+			if original_response.Message.Metadata.MessageType == "" {
+				continue
+			}
 			if original_response.Message.Metadata.MessageType != "next" && original_response.Message.Metadata.MessageType != "continue" || !strings.HasSuffix(original_response.Message.Content.ContentType, "text") {
 				continue
 			}
@@ -517,9 +569,18 @@ func Handler(c *gin.Context, response *http.Response, token string, puid string,
 					}
 				}
 				offset := 0
-				for i, citation := range original_response.Message.Metadata.Citations {
+				for _, citation := range original_response.Message.Metadata.Citations {
 					rl := len(r)
-					original_response.Message.Content.Parts[0] = string(r[:citation.StartIx+offset]) + "[^" + strconv.Itoa(i+1) + "^](" + citation.Metadata.URL + " \"" + citation.Metadata.Title + "\")" + string(r[citation.EndIx+offset:])
+					u, _ := url.Parse(citation.Metadata.URL)
+					baseURL := u.Scheme + "://" + u.Host + "/"
+					attr := urlAttrMap[baseURL]
+					if attr == "" {
+						attr = getURLAttribution(token, puid, baseURL)
+						if attr != "" {
+							urlAttrMap[baseURL] = attr
+						}
+					}
+					original_response.Message.Content.Parts[0] = string(r[:citation.StartIx+offset]) + " ([" + attr + "](" + citation.Metadata.URL + " \"" + citation.Metadata.Title + "\"))" + string(r[citation.EndIx+offset:])
 					r = []rune(original_response.Message.Content.Parts[0].(string))
 					offset += len(r) - rl
 				}