whisper models

ehalsey · Sep 27, 2023 · f24066d · f24066d
1 parent 6dce6a9
commit f24066d
Show file tree

Hide file tree

Showing 6 changed files with 37 additions and 64 deletions.
diff --git a/src/backend/engine/serviceCatalog.ts b/src/backend/engine/serviceCatalog.ts
@@ -1126,6 +1126,20 @@ const convertToChatCopilotService: BpaService = {
     }
 }
 
+const whisperBatchService: BpaService = {
+    bpaServiceId: "abc123",
+    inputTypes: ["wav", "mp3", "mp4"],
+    outputTypes: ["whisper"],
+    name: "whisper",
+    process: speech.processWhisperBatch,
+    serviceSpecificConfig: {
+
+    },
+    serviceSpecificConfigDefaults: {
+
+    }
+}
+
 export const serviceCatalog = {
     // "copy" : copyService,
     //"redactPdf" : redactPdfService,
@@ -1201,6 +1215,7 @@ export const serviceCatalog = {
     "formatKMAccelerator": formatKMAcceleratorService,
     "jsonToText": jsonToTextService,
     "piiStt": piiSttService,
-    "piiToOpenaiRest": piiToOpenaiRestService
+    "piiToOpenaiRest": piiToOpenaiRestService,
+    "whisper" : whisperBatchService
 }
 
diff --git a/src/backend/local.settings.example.json b/src/backend/local.settings.example.json
diff --git a/src/backend/services/speech.ts b/src/backend/services/speech.ts
@@ -1,7 +1,6 @@
 import * as sdk from "microsoft-cognitiveservices-speech-sdk";
 import { BpaServiceObject } from '../engine/types'
 import { BlobServiceClient, ContainerClient, BlockBlobClient, ContainerGenerateSasUrlOptions, ContainerSASPermissions } from "@azure/storage-blob"
-
 import axios, { AxiosRequestConfig, AxiosResponse } from 'axios'
 import { DB } from "./db";
 import MessageQueue from "./messageQueue";
@@ -144,12 +143,12 @@ export class Speech {
                 sasUrl
             ],
             "properties": {
-                "diarizationEnabled": true,
-                "wordLevelTimestampsEnabled": true,
-                "punctuationMode": "DictatedAndAutomatic",
-                "profanityFilterMode": "Masked"
+                "wordLevelTimestampsEnabled": false
             },
             "locale": "en-US",
+            "model": {
+                "self": process.env.WHISPER_MODEL//"https://eastus.api.cognitive.microsoft.com/speechtotext/v3.2-preview.1/models/base/71cbd7af-3212-43ab-8695-666fb28ffef7"
+              },
             "displayName": "Transcription of file using default model for en-US"
         }
         if (input?.serviceSpecificConfig?.to) {
@@ -158,11 +157,11 @@ export class Speech {
                     sasUrl
                 ],
                 "properties": {
-                    "diarizationEnabled": true,
-                    "wordLevelTimestampsEnabled": true,
-                    "punctuationMode": "DictatedAndAutomatic",
-                    "profanityFilterMode": "Masked"
+                    "wordLevelTimestampsEnabled": false
                 },
+                "model": {
+                    "self": process.env.WHISPER_MODEL
+                  },
                 "locale": input.serviceSpecificConfig.to,
                 "displayName": "Transcription of file using default model for en-US"
             }
@@ -173,7 +172,7 @@ export class Speech {
                 "Ocp-Apim-Subscription-Key": process.env.SPEECH_SUB_KEY
             }
         }
-        axiosResp = await axios.post(process.env.SPEECH_SUB_ENDPOINT + 'speechtotext/v3.0/transcriptions', payload, axiosParams)
+        axiosResp = await axios.post(process.env.SPEECH_SUB_ENDPOINT + 'speechtotext/v3.1/transcriptions', payload, axiosParams)
         //httpResult = axiosResp.status
 
         input.aggregatedResults["speechToText"] = {

diff --git a/templates/oneclick.json b/templates/oneclick.json
@@ -772,6 +772,10 @@
             {
               "name": "HUGGINGFACE_ENDPOINT",
               "value": "[format('https://{0}.azurewebsites.net', parameters('functionAppNamePython'))]"
+            },
+            {
+              "name": "WHISPER_MODEL",
+              "value": "https://eastus.api.cognitive.microsoft.com/speechtotext/v3.2-preview.1/models/base/71cbd7af-3212-43ab-8695-666fb28ffef7"
             }
           ]
         }

diff --git a/templates/oneclickoai.json b/templates/oneclickoai.json
@@ -851,6 +851,10 @@
             {
               "name": "HUGGINGFACE_ENDPOINT",
               "value": "[format('https://{0}.azurewebsites.net', parameters('functionAppNamePython'))]"
+            },
+            {
+              "name": "WHISPER_MODEL",
+              "value": "https://eastus.api.cognitive.microsoft.com/speechtotext/v3.2-preview.1/models/base/71cbd7af-3212-43ab-8695-666fb28ffef7"
             }
           ]
         }

diff --git a/templates/privatedeploy.json b/templates/privatedeploy.json
@@ -1563,8 +1563,11 @@
                         {
                             "name": "OPENAI_DEPLOYMENT_SEARCH_QUERY",
                             "value": "text-embedding-ada-002"
+                        },
+                        {
+                            "name": "WHISPER_MODEL",
+                            "value": "https://eastus.api.cognitive.microsoft.com/speechtotext/v3.2-preview.1/models/base/71cbd7af-3212-43ab-8695-666fb28ffef7"
                         }
-
                     ]
                 }
             },