Send audio over WebSocket

Question

I'm implementig getUserMedia() to record audio messages. I want to record the audio and then pass it using websocket to another peer. How I can accomplish this?I've searched on SO and I've found only some interesting question, but nothing that can guide me to a working solution. I'm using Pusher API for the websocket part of my app. Here is the code I'm testing.

$(document).on("click", ".audio-chat",function(){
  console.log('clicked');
  var channel = $('input[name="channelName"]').val();
  navigator.mediaDevices.getUserMedia({
    audio: true
  })
  .then(function(stream){
    var mediaRecorder = new MediaRecorder(stream);
    mediaRecorder.start();
    console.log(mediaRecorder.state);
    console.log("recorder started");

    mediaRecorder.ondataavailable = function(e) {
      chunks.push(e.data);
      console.log(chunks);
    }
    setTimeout(function(){
      mediaRecorder.stop();
      console.log(mediaRecorder.state);
      console.log("recorder stopped");
      var blob = new Blob(chunks, {'type':'audio/webm; codecs=opus'});
      //console.log(blob);
      chunks = [];
      const audioUrl = window.URL.createObjectURL(blob);
      var data = {channel:channel,message:audioUrl,socketId:socketId}
      $.post('api/message.php', data);
    }, 10000);
  });
});

Try this backend is on node but frontend will remain same https://stackoverflow.com/a/50976085/4518930 — Adil Liaqat, Sep 06 '19 at 09:51
@MosèRaguzzini I just want to send the recorded audio (stream) without saving it on the server. I know that there is webRTC but it will consume a lot of resources of my pusher api quota for signaling so I prefere to use websockets — jihuuNI, Sep 06 '19 at 09:53
@AdilLiaqat I will give it a look! I'm on php for the backend. — jihuuNI, Sep 06 '19 at 09:54

score 2 · Accepted Answer · answered Sep 06 '19 at 10:14

instead of relying on Blob/CreateObjectURL, try to rely on binary transfer by arrayBuffer as done in https://github.com/Ivan-Feofanov/ws-audio-api.

You'll need also an encoder and a _resampler.

Relevant code to look at

https://github.com/Ivan-Feofanov/ws-audio-api/blob/master/src/ws-audio-api.js

Streamer: function(config, socket) {
            navigator.getUserMedia = (navigator.getUserMedia ||
                navigator.webkitGetUserMedia ||
                navigator.mozGetUserMedia ||
                navigator.msGetUserMedia);

            this.config = {};
            this.config.codec = this.config.codec || defaultConfig.codec;
            this.config.server = this.config.server || defaultConfig.server;
            this.sampler = new Resampler(44100, this.config.codec.sampleRate, 1, this.config.codec.bufferSize);
            this.parentSocket = socket;
            this.encoder = new OpusEncoder(this.config.codec.sampleRate, this.config.codec.channels, this.config.codec.app, this.config.codec.frameDuration);
            var _this = this;
            this._makeStream = function(onError) {
                navigator.getUserMedia({ audio: true }, function(stream) {
                    _this.stream = stream;
                    _this.audioInput = audioContext.createMediaStreamSource(stream);
                    _this.gainNode = audioContext.createGain();
                    _this.recorder = audioContext.createScriptProcessor(_this.config.codec.bufferSize, 1, 1);
                    _this.recorder.onaudioprocess = function(e) {
                        var resampled = _this.sampler.resampler(e.inputBuffer.getChannelData(0));
                        var packets = _this.encoder.encode_float(resampled);
                        for (var i = 0; i < packets.length; i++) {
                            if (_this.socket.readyState == 1) _this.socket.send(packets[i]);
                        }
                    };
                    _this.audioInput.connect(_this.gainNode);
                    _this.gainNode.connect(_this.recorder);
                    _this.recorder.connect(audioContext.destination);
                }, onError || _this.onError);
            }
        }

Streaming start

WSAudioAPI.Streamer.prototype.start = function(onError) {
        var _this = this;

        if (!this.parentSocket) {
            this.socket = new WebSocket('wss://' + this.config.server.host + ':' + this.config.server.port);
        } else {
            this.socket = this.parentSocket;
        }

        this.socket.binaryType = 'arraybuffer';

        if (this.socket.readyState == WebSocket.OPEN) {
            this._makeStream(onError);
        } else if (this.socket.readyState == WebSocket.CONNECTING) {
            var _onopen = this.socket.onopen;
            this.socket.onopen = function() {
                if (_onopen) {
                    _onopen();
                }
                _this._makeStream(onError);
            }
        } else {
            console.error('Socket is in CLOSED state');
        }

        var _onclose = this.socket.onclose;
        this.socket.onclose = function() {
            if (_onclose) {
                _onclose();
            }
            if (_this.audioInput) {
                _this.audioInput.disconnect();
                _this.audioInput = null;
            }
            if (_this.gainNode) {
                _this.gainNode.disconnect();
                _this.gainNode = null;
            }
            if (_this.recorder) {
                _this.recorder.disconnect();
                _this.recorder = null;
            }
            _this.stream.getTracks()[0].stop();
            console.log('Disconnected from server');
        };
    };

Player

WSAudioAPI.Player.prototype.start = function() {
        var _this = this;

        this.audioQueue = {
            buffer: new Float32Array(0),

            write: function(newAudio) {
                var currentQLength = this.buffer.length;
                newAudio = _this.sampler.resampler(newAudio);
                var newBuffer = new Float32Array(currentQLength + newAudio.length);
                newBuffer.set(this.buffer, 0);
                newBuffer.set(newAudio, currentQLength);
                this.buffer = newBuffer;
            },

            read: function(nSamples) {
                var samplesToPlay = this.buffer.subarray(0, nSamples);
                this.buffer = this.buffer.subarray(nSamples, this.buffer.length);
                return samplesToPlay;
            },

            length: function() {
                return this.buffer.length;
            }
        };

        this.scriptNode = audioContext.createScriptProcessor(this.config.codec.bufferSize, 1, 1);
        this.scriptNode.onaudioprocess = function(e) {
            if (_this.audioQueue.length()) {
                e.outputBuffer.getChannelData(0).set(_this.audioQueue.read(_this.config.codec.bufferSize));
            } else {
                e.outputBuffer.getChannelData(0).set(_this.silence);
            }
        };
        this.gainNode = audioContext.createGain();
        this.scriptNode.connect(this.gainNode);
        this.gainNode.connect(audioContext.destination);

        if (!this.parentSocket) {
            this.socket = new WebSocket('wss://' + this.config.server.host + ':' + this.config.server.port);
        } else {
            this.socket = this.parentSocket;
        }
        //this.socket.onopen = function () {
        //    console.log('Connected to server ' + _this.config.server.host + ' as listener');
        //};
        var _onmessage = this.parentOnmessage = this.socket.onmessage;
        this.socket.onmessage = function(message) {
            if (_onmessage) {
                _onmessage(message);
            }
            if (message.data instanceof Blob) {
                var reader = new FileReader();
                reader.onload = function() {
                    _this.audioQueue.write(_this.decoder.decode_float(reader.result));
                };
                reader.readAsArrayBuffer(message.data);
            }
        };
        //this.socket.onclose = function () {
        //    console.log('Connection to server closed');
        //};
        //this.socket.onerror = function (err) {
        //    console.log('Getting audio data error:', err);
        //};
      };

this will work with the pusher API? I see from the code that it rely on the websocket API to create a socket. I need also some explainations on how the code works. — jihuuNI, Sep 06 '19 at 10:23
it seems that "binary WebSocket frames are not supported", https://pusher.com/docs/channels/library_auth_reference/pusher-websockets-protocol#websocket-data-messages...if you want to send a stream maybe pusher is not the right spot. — Mosè Raguzzini, Sep 06 '19 at 10:57
Ok, thanks. This is why I was trying without success to pass the data between the users on the same channel. I will implement my code on a shared hosting service, can you suggest me another service for websocket? Or if possible, a reliable webRTC solution where the socket are used only once for signaling? I don't know if you tested the pusher webRTC demo, but it will not work correctly — jihuuNI, Sep 06 '19 at 11:05
Hi mate, you can use socket.io/ws/sockjs or just use the implementation of library mentioned above — Mosè Raguzzini, Sep 06 '19 at 11:10
On a shared hosting it will not work because they don't allow for sockets. This is why initially I choose to use pusher. As a workaroudn, if i compress the data into json? Is it possible ? — jihuuNI, Sep 06 '19 at 11:12
A stream is usually pretty intensive, a shared hosting for the node server is not suitable IMHO, you can search for a specific audio message api service, instead — Mosè Raguzzini, Sep 06 '19 at 11:18
I will try with socket.io with the pusher credentials. An experiment I want to do is to compress the data into JSON format and then pass it to the socket connection. Unfortunately I'm using php so I can't apply some of the solutions I've found searching on duckduckgo because are written for node. Thanks for the support, if you have suggestions on how to compress the stream data into JSON please share it.! — jihuuNI, Sep 06 '19 at 11:22

score 1 · Answer 2 · answered Sep 06 '19 at 11:25

1

Binary data are by far more effective than JSON, I do not recomend to pick that way

answered Sep 06 '19 at 11:25

Mosè Raguzzini

12,776
26
36

ok, I'm reading about sockjs. For the server part is there any php server library? It's a nice library – jihuuNI Sep 06 '19 at 11:30

score 0 · Answer 3 · answered Mar 13 '21 at 14:56

0

You can use ratchet php websocket

Or in case you would like to use laravel framework , you can use laravel websockets with Echo

answered Mar 13 '21 at 14:56

Cleyverson Costa

1
1

Send audio over WebSocket

3 Answers3