I am trying to set up a server to receive audio from a client browser using SocketIO, then process it through Google Speech-to-Text, and finally reply back to the client with the text.
Originally and ideally, I wanted to set up to function somewhat like the tool on this page: https://cloud.google.com/speech-to-text/
I tried using getUserMedia and streaming it through SocketIO-Stream, but I couldn't figure out how to 'pipe' MediaStream.
Instead, now I've decided to use MediaRecorder on the client side, and then send the data altogether as a blob(seen in this example).
I then apply toString('base64') to the blob and call google-cloud/speech's client.recognize() on the blob.
Client Side(i'm using VueJS):
new Vue({
el: '#app',
data: function () {
return ({
msgs: [],
socket: null,
recorder: null,
: []
})
},
mounted: function () {
this.socket = io.connect('localhost:3000/user');
console.log('Connected!')
this.socket.on('text', function (text) {
this.msgs.push(text)
})
},
methods: {
startRecording: function () {
if (this.recorder && this.recorder.state == 'recording') {
console.log("Stopping!")
this.recorder.stop()
} else {
console.log("Starting!")
navigator.mediaDevices.getUserMedia({ audio: true, video: false })
.then(this.handleSuccess);
}
},
handleSuccess: function (stream) {
this.recorder = new MediaRecorder(stream)
this.recorder.start(10000)
this.recorder.ondataavailable = (e) => {
this.chunks.push(e.data)
console.log(e.data)
}
this.recorder.onstop = (e) => {
const blob = new Blob(this.chunks, { 'type': 'audio/webm; codecs=opus' })
this.socket.emit('audio', blob)
}
}
}
})
Server Side:
const speech = require('@google-cloud/speech');
const client = new speech.SpeechClient();
const io = require('socket.io').listen(3000)
const ss = require('socket.io-stream')
const encoding = 'LINEAR16';
const sampleRateHertz = 16000;
const languageCode = 'en-US';
const audio = {
content: null
}
const config = {
encoding: encoding,
sampleRateHertz: sampleRateHertz,
languageCode: languageCode,
}
async function main() {
const [response] = await client.recognize({
audio: audio,
config: config
})
const transcription = response.results
.map(result => result.alternatives[0].transcript)
.join('\n');
console.log(`Transcription: ${transcription}`);
}
io.of('/user').on('connection', function (socket) {
console.log('Connection made!')
socket.on('audio', function (data) {
audio.content = data.toString('base64')
main().catch(console.error)
});
});
The log from the main() function in the Server side is always:
"Transcription: "
-- which is empty!
It should contain the text from the audio sent. Thank you in advance!
