Twilio to Azure Continuous Speech To Text

45 viewsazure speechnode.jstwilio
0

I would like to use the Twilio verb to send audio to Azure’s Continuous Speech to Text API to do real time transcription on a call. I’ve successfully used the code below to send an audio file to Azure Speech to Text but when I push the Twilio Stream data I get no transcription results. What am I doing wrong?

#!/usr/bin/env node
const WebSocket = require("ws");
const fs = require("fs");
const mulaw = require("mulaw-js");
const base64 = require("js-base64");
const express = require("express");
const app = express();
const server = require("http").createServer(app);
const wss = new WebSocket.Server({ server });
const subscriptionKey = "6038f4a6669540bd89547b19a9135657";
const serviceRegion = "eastus"; // e.g., "westus"
const language = "en-US";

const sdk = require("microsoft-cognitiveservices-speech-sdk");
const stream = require("stream");
const azurePusher = sdk.AudioInputStream.createPushStream(sdk.AudioStreamFormat.getWaveFormatPCM(8000, 16, 1));
const audioConfig = sdk.AudioConfig.fromStreamInput(azurePusher);
//const audioConfig = sdk.AudioConfig.fromDefaultSpeakerOutput();
//const audioConfig = sdk.AudioConfig.fromWavFileInput(fs.readFileSync("C:\Users\kenar\Downloads\ACCDownload_20210904094910\Audio\MFA IDMission Demo Audio - 1.wav"))
const speechConfig = sdk.SpeechConfig.fromSubscription(subscriptionKey,serviceRegion);

speechConfig.speechRecognitionLanguage = language;
const recognizer = new sdk.SpeechRecognizer(speechConfig, audioConfig);

recognizer.recognizing = (s, e) => {
    console.log(`RECOGNIZING: Text=${e.result.text}`);
};

recognizer.recognized = (s, e) => {
    if (e.result.reason == sdk.ResultReason.RecognizedSpeech) {
        console.log(`RECOGNIZED: Text=${e.result.text}`);
    }
    else if (e.result.reason == sdk.ResultReason.NoMatch) {
        console.log("NOMATCH: Speech could not be recognized.");
    }
};

recognizer.canceled = (s, e) => {
    console.log(`CANCELED: Reason=${e.reason}`);

    if (e.reason == sdk.CancellationReason.Error) {
        console.log(`"CANCELED: ErrorCode=${e.errorCode}`);
        console.log(`"CANCELED: ErrorDetails=${e.errorDetails}`);
        console.log("CANCELED: Did you update the key and location/region info?");
    }

    recognizer.stopContinuousRecognitionAsync();
};

recognizer.sessionStopped = (s, e) => {
    console.log("n    Session stopped event.");
    recognizer.stopContinuousRecognitionAsync();
};

recognizer.startContinuousRecognitionAsync(() => {
    console.log("Continuous Reco Started");
},
    err => {
        console.trace("err - " + err);
        recognizer.close();
        recognizer = undefined;
    });

// Handle Web Socket Connection
wss.on("connection", function connection(ws) {
    console.log("New Connection Initiated");
    let recostream = null;

    ws.on("message", function incoming(message) {
        const msg = JSON.parse(message);
        switch (msg.event) {
            case "connected":
                console.log(`A new call has connected.`);

                break;
            case "start":
                console.log(`Starting Media Stream ${msg.streamSid}`);
                break;
            case "media":
                process.stdout.write(msg.media.payload + " " + " bytes33[0G");
                streampayload = base64.decode(msg.media.payload);
                let data = Buffer.from(streampayload);
                azurePusher.write(mulaw.decode(data));
                break;
            case "stop":
                console.log(`Call Has Ended`);
                azurePusher.close();
                recognizer.stopContinuousRecognitionAsync();
            break;
        }
    });

});


 Here are the results after running with attached audio:
"C:Program Filesnodejsnode.exe"                 
C:UserskenarWebstormProjectstwiliostreams1twiliostream.js
Listening at Port 8080
Continuous Reco Started
New Connection Initiated
A new call has connected.
Starting Media Stream MZ8dc3ec47f7b9bd3b37e1b4896beb354e
RECOGNIZED: Text=
Call Has Ended
RECOGNIZED: Text=
NOMATCH: Speech could not be recognized.

Session stopped event.