From 09c4a4b90ecc9b3c045b07285b352b11bc5d5d12 Mon Sep 17 00:00:00 2001 From: Sergiu Toma Date: Tue, 29 Nov 2022 14:19:02 +0200 Subject: [PATCH] LH-265: Added audio on client and server --- app.js | 261 +++++++++++++++++++++++++++++----------- build.sh | 4 +- public/config.js | 3 +- public/index.html | 2 +- public/index.js | 296 ++++++++++++++++++++++++++-------------------- 5 files changed, 363 insertions(+), 203 deletions(-) diff --git a/app.js b/app.js index 702eb14..8b33249 100644 --- a/app.js +++ b/app.js @@ -90,28 +90,81 @@ worker = createWorker(); // https://mediasoup.org/documentation/v3/mediasoup/rtp-parameters-and-capabilities/#RtpCodecCapability // list of media codecs supported by mediasoup ... // https://github.com/versatica/mediasoup/blob/v3/src/supportedRtpCapabilities.ts -const mediaCodecs = [ +const mediaCodecs = [ { - kind: 'audio', - mimeType: 'audio/opus', - clockRate: 48000, - channels: 2, + kind : 'audio', + mimeType : 'audio/opus', + clockRate : 48000, + channels : 2 }, { - kind: 'video', - mimeType: 'video/VP8', - clockRate: 90000, - parameters: { - 'x-google-start-bitrate': 1000, + kind : 'video', + mimeType : 'video/VP8', + clockRate : 90000, + parameters : + { + 'x-google-start-bitrate' : 1000 }, + channels : 2 }, + { + kind : 'video', + mimeType : 'video/VP9', + clockRate : 90000, + parameters : + { + 'profile-id' : 2, + 'x-google-start-bitrate' : 1000 + } + }, + { + kind : 'video', + mimeType : 'video/h264', + clockRate : 90000, + parameters : + { + 'packetization-mode' : 1, + 'profile-level-id' : '4d0032', + 'level-asymmetry-allowed' : 1, + 'x-google-start-bitrate' : 1000 + } + }, + { + kind : 'video', + mimeType : 'video/h264', + clockRate : 90000, + parameters : + { + 'packetization-mode' : 1, + 'profile-level-id' : '42e01f', + 'level-asymmetry-allowed' : 1, + 'x-google-start-bitrate' : 1000 + } + } +// { +// kind: 'audio', +// mimeType: 'audio/opus', +// clockRate: 48000, +// channels: 2, +// }, +// { +// kind: 'video', +// mimeType: 'video/VP8', +// clockRate: 90000, +// parameters: { +// 'x-google-start-bitrate': 1000, +// }, +// }, ]; const closeCall = (callId) => { try { if (callId && videoCalls[callId]) { - videoCalls[callId].producer?.close(); - videoCalls[callId].consumer?.close(); + videoCalls[callId].producerVideo?.close(); + videoCalls[callId].producerAudio?.close(); + videoCalls[callId].consumerVideo?.close(); + videoCalls[callId].consumerAudio?.close(); + videoCalls[callId]?.consumerTransport?.close(); videoCalls[callId]?.producerTransport?.close(); videoCalls[callId]?.router?.close(); @@ -230,29 +283,54 @@ peers.on('connection', async socket => { - The event sent by the client (PRODUCER) after successfully connecting to producerTransport - For the router with the id callId, we make produce on producerTransport - Create the handler on producer at the 'transportclose' event - */ - socket.on('transport-produce', async ({ kind, rtpParameters, appData }, callback) => { - try { - const callId = socketDetails[socket.id]; - if (typeof rtpParameters === 'string') rtpParameters = JSON.parse(rtpParameters); + */ + socket.on('transport-produce', async ({ kind, rtpParameters, appData }, callback) => { + try { + const callId = socketDetails[socket.id]; + if (typeof rtpParameters === 'string') rtpParameters = JSON.parse(rtpParameters); + + console.log(`[transport-produce] kind: ${kind} | socket.id: ${socket.id} | callId: ${callId}`); + console.log('kind', kind); + console.log('rtpParameters', rtpParameters); - console.log('[transport-produce] | socket.id', socket.id, '| callId', callId); - videoCalls[callId].producer = await videoCalls[callId].producerTransport.produce({ - kind, - rtpParameters, - }); - console.log(`[transport-produce] Producer ID: ${videoCalls[callId].producer.id} | kind: ${videoCalls[callId].producer.kind}`); - - videoCalls[callId].producer.on('transportclose', () => { - const callId = socketDetails[socket.id]; - console.log('transport for this producer closed', callId) - closeCall(callId); - }); + if (kind === 'video') { + videoCalls[callId].producerVideo = await videoCalls[callId].producerTransport.produce({ + kind, + rtpParameters, + }); + - // Send back to the client the Producer's id - callback && callback({ - id: videoCalls[callId].producer.id - }); + console.log(`[transport-produce] Producer ID: ${videoCalls[callId].producerVideo.id} | kind: ${videoCalls[callId].producerVideo.kind}`); + + videoCalls[callId].producerVideo.on('transportclose', () => { + const callId = socketDetails[socket.id]; + console.log('transport for this producer closed', callId) + closeCall(callId); + }); + + // Send back to the client the Producer's id + callback && callback({ + id: videoCalls[callId].producerVideo.id + }); + } else if (kind === 'audio') { + videoCalls[callId].producerAudio = await videoCalls[callId].producerTransport.produce({ + kind, + rtpParameters, + }); + + console.log(`[transport-produce] Producer ID: ${videoCalls[callId].producerAudio.id} | kind: ${videoCalls[callId].producerAudio.kind}`); + + videoCalls[callId].producerAudio.on('transportclose', () => { + const callId = socketDetails[socket.id]; + console.log('transport for this producer closed', callId) + closeCall(callId); + }); + + // Send back to the client the Producer's id + callback && callback({ + id: videoCalls[callId].producerAudio.id + }); + } } catch (error) { console.log(`ERROR | transport-produce | callId ${socketDetails[socket.id]} | ${error.message}`); } @@ -281,48 +359,36 @@ peers.on('connection', async socket => { */ socket.on('consume', async ({ rtpCapabilities }, callback) => { try { + console.log(`[consume] rtpCapabilities: ${JSON.stringify(rtpCapabilities)}`); + const callId = socketDetails[socket.id]; console.log('[consume] callId', callId); - // Check if the router can consume the specified producer - if (videoCalls[callId].router.canConsume({ - producerId: videoCalls[callId].producer.id, + const canConsumeVideo = !!videoCalls[callId].producerVideo && !!videoCalls[callId].router.canConsume({ + producerId: videoCalls[callId].producerVideo.id, rtpCapabilities - })) { - console.log('[consume] Can consume', callId); - // Transport can now consume and return a consumer - videoCalls[callId].consumer = await videoCalls[callId].consumerTransport.consume({ - producerId: videoCalls[callId].producer.id, - rtpCapabilities, - paused: true, - }); + }) - // https://mediasoup.org/documentation/v3/mediasoup/api/#consumer-on-transportclose - videoCalls[callId].consumer.on('transportclose', () => { - const callId = socketDetails[socket.id]; - console.log('transport close from consumer', callId); - closeCall(callId); - }); + const canConsumeAudio = !!videoCalls[callId].producerAudio && !!videoCalls[callId].router.canConsume({ + producerId: videoCalls[callId].producerAudio.id, + rtpCapabilities + }) - // https://mediasoup.org/documentation/v3/mediasoup/api/#consumer-on-producerclose - videoCalls[callId].consumer.on('producerclose', () => { - const callId = socketDetails[socket.id]; - console.log('producer of consumer closed', callId); - closeCall(callId); - }); + console.log('[consume] canConsumeVideo', canConsumeVideo); + console.log('[consume] canConsumeAudio', canConsumeAudio); - // From the consumer extract the following params to send back to the Client - const params = { - id: videoCalls[callId].consumer.id, - producerId: videoCalls[callId].producer.id, - kind: videoCalls[callId].consumer.kind, - rtpParameters: videoCalls[callId].consumer.rtpParameters, - }; - - // Send the parameters to the client - callback({ params }); + if (canConsumeVideo && !canConsumeAudio) { + console.log('1'); + const videoParams = await consumeVideo(callId, rtpCapabilities) + console.log('videoParams', videoParams); + callback({ videoParams, audioParams: null }); + } else if (canConsumeVideo && canConsumeAudio) { + console.log('2'); + const videoParams = await consumeVideo(callId, rtpCapabilities) + const audioParams = await consumeAudio(callId, rtpCapabilities) + callback({ videoParams, audioParams }); } else { - console.log(`[canConsume] Can't consume | callId ${callId}`); + console.log(`[consume] Can't consume | callId ${callId}`); callback(null); } } catch (error) { @@ -339,13 +405,71 @@ peers.on('connection', async socket => { try { const callId = socketDetails[socket.id]; console.log(`[consumer-resume] callId ${callId}`) - await videoCalls[callId].consumer.resume(); + await videoCalls[callId].consumerVideo.resume(); + await videoCalls[callId].consumerAudio.resume(); } catch (error) { console.log(`ERROR | consumer-resume | callId ${socketDetails[socket.id]} | ${error.message}`); } }); }); +const consumeVideo = async (callId, rtpCapabilities) => { + videoCalls[callId].consumerVideo = await videoCalls[callId].consumerTransport.consume({ + producerId: videoCalls[callId].producerVideo.id, + rtpCapabilities, + paused: true, + }); + + // https://mediasoup.org/documentation/v3/mediasoup/api/#consumer-on-transportclose + videoCalls[callId].consumerVideo.on('transportclose', () => { + const callId = socketDetails[socket.id]; + console.log('transport close from consumer', callId); + closeCall(callId); + }); + + // https://mediasoup.org/documentation/v3/mediasoup/api/#consumer-on-producerclose + videoCalls[callId].consumerVideo.on('producerclose', () => { + const callId = socketDetails[socket.id]; + console.log('producer of consumer closed', callId); + closeCall(callId); + }); + + return { + id: videoCalls[callId].consumerVideo.id, + producerId: videoCalls[callId].producerVideo.id, + kind: 'video', + rtpParameters: videoCalls[callId].consumerVideo.rtpParameters, + } +} + +const consumeAudio = async (callId, rtpCapabilities) => { + videoCalls[callId].consumerAudio = await videoCalls[callId].consumerTransport.consume({ + producerId: videoCalls[callId].producerAudio.id, + rtpCapabilities, + paused: true, + }); + + // https://mediasoup.org/documentation/v3/mediasoup/api/#consumer-on-transportclose + videoCalls[callId].consumerAudio.on('transportclose', () => { + const callId = socketDetails[socket.id]; + console.log('transport close from consumer', callId); + closeCall(callId); + }); + + // https://mediasoup.org/documentation/v3/mediasoup/api/#consumer-on-producerclose + videoCalls[callId].consumerAudio.on('producerclose', () => { + const callId = socketDetails[socket.id]; + console.log('producer of consumer closed', callId); + closeCall(callId); + }); + return { + id: videoCalls[callId].consumerAudio.id, + producerId: videoCalls[callId].producerAudio.id, + kind: 'audio', + rtpParameters: videoCalls[callId].consumerAudio.rtpParameters, + } +} + /* - Called from at event 'createWebRtcTransport' and assigned to the consumer or producer transport - It will return parameters, these are required for the client to create the RecvTransport @@ -393,6 +517,7 @@ const createWebRtcTransportLayer = async (callId, callback) => { dtlsParameters: transport.dtlsParameters, }; + console.log('[createWebRtcTransportLayer] callback params', params); // Send back to the client the params callback({ params }); diff --git a/build.sh b/build.sh index 041c169..5ee5695 100755 --- a/build.sh +++ b/build.sh @@ -43,6 +43,4 @@ fi ## POST BUILD -cd - - - +cd - \ No newline at end of file diff --git a/public/config.js b/public/config.js index d12c148..fa3612f 100644 --- a/public/config.js +++ b/public/config.js @@ -1,5 +1,4 @@ module.exports = { hubAddress: 'https://hub.dev.linx.safemobile.com/', - mediasoupAddress: 'https://video.safemobile.org/mediasoup', - // mediasoupAddress: 'http://localhost:3000/mediasoup', + mediasoupAddress: 'https://video.safemobile.org', } \ No newline at end of file diff --git a/public/index.html b/public/index.html index fb0f078..71e0467 100644 --- a/public/index.html +++ b/public/index.html @@ -43,7 +43,7 @@
- +
diff --git a/public/index.js b/public/index.js index affb71b..86b9d90 100644 --- a/public/index.js +++ b/public/index.js @@ -12,145 +12,157 @@ let callId = parseInt(urlParams.get('callId')) || null; const IS_PRODUCER = urlParams.get('producer') === 'true' ? true : false console.log('[URL] ASSET_ID', ASSET_ID, '| ACCOUNT_ID', ACCOUNT_ID, '| callId', callId, ' | IS_PRODUCER', IS_PRODUCER) -let socket -hub = io(config.hubAddress) - -const connectToMediasoup = () => { - - socket = io(config.mediasoupAddress, { - reconnection: true, - reconnectionDelay: 1000, - reconnectionDelayMax : 5000, - reconnectionAttempts: Infinity - }) - - socket.on('connection-success', ({ _socketId, existsProducer }) => { - console.log(`[MEDIA] ${config.mediasoupAddress} | connected: ${socket.connected} | existsProducer: ${existsProducer}`) - if (!IS_PRODUCER && existsProducer && consumer === undefined) { - goConnect() - // document.getElementById('btnRecvSendTransport').click(); - } - if (IS_PRODUCER && urlParams.get('testing') === 'true') { getLocalStream() } - }) -} - -if (IS_PRODUCER === true) { - hub.on('connect', async () => { - console.log(`[HUB] ${config.hubAddress} | connected: ${hub.connected}`) - connectToMediasoup() - - hub.emit( - 'ars', - JSON.stringify({ - ars: true, - asset_id: ASSET_ID, - account_id: ACCOUNT_ID, - }) - ) - - hub.on('video', (data) => { - const parsedData = JSON.parse(data); - - if (parsedData.type === 'notify-request') { - console.log('video', parsedData) - originAssetId = parsedData.origin_asset_id; - // originAssetName = parsedData.origin_asset_name; - // originAssetTypeName = parsedData.origin_asset_type_name; - callId = parsedData.video_call_id; - - console.log('[VIDEO] notify-request | IS_PRODUCER', IS_PRODUCER, 'callId', callId); - getLocalStream() - } - - if (parsedData.type === 'notify-end') { - console.log('[VIDEO] notify-end | IS_PRODUCER', IS_PRODUCER, 'callId', callId); - resetCallSettings() - } - }) - }) - - hub.on('connect_error', (error) => { - console.log('connect_error', error); - }); - - hub.on('connection', () => { - console.log('connection') - }) - - hub.on('disconnect', () => { - console.log('disconnect') - }) -} else { - connectToMediasoup() -} +console.log('🟩 config', config) +let socket, hub let device let rtpCapabilities let producerTransport let consumerTransport -let producer +let producerVideo +let producerAudio let consumer let originAssetId -// let originAssetName = 'Adi' -// let originAssetTypeName = 'linx' // https://mediasoup.org/documentation/v3/mediasoup-client/api/#ProducerOptions // https://mediasoup.org/documentation/v3/mediasoup-client/api/#transport-produce -let params = { - // mediasoup params +let videoParams = { encodings: [ - { - rid: 'r0', - maxBitrate: 100000, - scalabilityMode: 'S1T3', - }, - { - rid: 'r1', - maxBitrate: 300000, - scalabilityMode: 'S1T3', - }, - { - rid: 'r2', - maxBitrate: 900000, - scalabilityMode: 'S1T3', - }, + { scaleResolutionDownBy: 4, maxBitrate: 500000 }, + { scaleResolutionDownBy: 2, maxBitrate: 1000000 }, + { scaleResolutionDownBy: 1, maxBitrate: 5000000 }, + { scalabilityMode: 'S3T3_KEY' } ], - // https://mediasoup.org/documentation/v3/mediasoup-client/api/#ProducerCodecOptions codecOptions: { videoGoogleStartBitrate: 1000 } } -const streamSuccess = (stream) => { - console.log('[streamSuccess]'); - localVideo.srcObject = stream - const track = stream.getVideoTracks()[0] - params = { - track, - ...params +let audioParams = { + codecOptions : + { + opusStereo : true, + opusDtx : true } +} + +setTimeout(() => { + hub = io(config.hubAddress) + + const connectToMediasoup = () => { + + socket = io(config.mediasoupAddress, { + reconnection: true, + reconnectionDelay: 1000, + reconnectionDelayMax : 5000, + reconnectionAttempts: Infinity + }) + + socket.on('connection-success', ({ _socketId, existsProducer }) => { + console.log(`[MEDIA] ${config.mediasoupAddress} | connected: ${socket.connected} | existsProducer: ${existsProducer}`) + if (!IS_PRODUCER && existsProducer && consumer === undefined) { + goConnect() + // document.getElementById('btnRecvSendTransport').click(); + } + if (IS_PRODUCER && urlParams.get('testing') === 'true') { getLocalStream() } + }) + } + + if (IS_PRODUCER === true) { + hub.on('connect', async () => { + console.log(`[HUB]! ${config.hubAddress} | connected: ${hub.connected}`) + connectToMediasoup() + + hub.emit( + 'ars', + JSON.stringify({ + ars: true, + asset_id: ASSET_ID, + account_id: ACCOUNT_ID, + }) + ) + + hub.on('video', (data) => { + const parsedData = JSON.parse(data); + + if (parsedData.type === 'notify-request') { + console.log('video', parsedData) + originAssetId = parsedData.origin_asset_id; + // originAssetName = parsedData.origin_asset_name; + // originAssetTypeName = parsedData.origin_asset_type_name; + callId = parsedData.video_call_id; + + console.log('[VIDEO] notify-request | IS_PRODUCER', IS_PRODUCER, 'callId', callId); + getLocalStream() + } + + if (parsedData.type === 'notify-end') { + console.log('[VIDEO] notify-end | IS_PRODUCER', IS_PRODUCER, 'callId', callId); + resetCallSettings() + } + }) + }) + + hub.on('connect_error', (error) => { + console.log('connect_error', error); + }); + + hub.on('connection', () => { + console.log('connection') + }) + + hub.on('disconnect', () => { + console.log('disconnect') + }) + } else { + connectToMediasoup() + } + +}, 1600); + +const streamSuccess = (stream) => { + console.log('[streamSuccess] device', device); + localVideo.srcObject = stream + console.log('stream', stream); + const videoTrack = stream.getVideoTracks()[0] + const audioTrack = stream.getAudioTracks()[0] + + videoParams = { + track: videoTrack, + ...videoParams + } + + audioParams = { + track: audioTrack, + ...audioParams + } + + console.log('[streamSuccess] videoParams', videoParams, ' | audioParams', audioParams); goConnect() } const getLocalStream = () => { console.log('[getLocalStream]'); navigator.mediaDevices.getUserMedia({ - audio: false, + audio: true, video: { - width: { - min: 640, - max: 1920, - }, - height: { - min: 400, - max: 1080, - } + qvga : { width: { ideal: 320 }, height: { ideal: 240 } }, + vga : { width: { ideal: 640 }, height: { ideal: 480 } }, + hd : { width: { ideal: 1280 }, height: { ideal: 720 } } } }) .then(streamSuccess) .catch(error => { console.log(error.message) }) + + navigator.permissions.query( + { name: 'microphone' } + ).then((permissionStatus) =>{ + console.log('🟨 [PERMISSION] permissionStatus', permissionStatus); // granted, denied, prompt + // It will block the code from execution and display "Permission denied" if we don't have microphone permissions + }) + } const goConnect = () => { @@ -167,7 +179,6 @@ const goCreateTransport = () => { // server side to send/recive media const createDevice = async () => { try { - console.log('[createDevice]'); device = new mediasoupClient.Device() // https://mediasoup.org/documentation/v3/mediasoup-client/api/#device-load @@ -178,7 +189,8 @@ const createDevice = async () => { }) console.log('Device RTP Capabilities', device.rtpCapabilities) - + console.log('[createDevice] device', device); + // once the device loads, create transport goCreateTransport() @@ -207,18 +219,20 @@ const getRtpCapabilities = () => { } const createSendTransport = () => { + console.log('[createSendTransport'); // see server's socket.on('createWebRtcTransport', sender?, ...) // this is a call from Producer, so sender = true - socket.emit('createWebRtcTransport', { sender: true, callId }, ({ params }) => { + socket.emit('createWebRtcTransport', { sender: true, callId }, (value) => { + + console.log(`[createWebRtcTransport] value: ${JSON.stringify(value)}`); + + const params = value.params; // The server sends back params needed // to create Send Transport on the client side if (params.error) { console.log(params.error) return } - - console.log(params) - // creates a new WebRTC Transport to send media // based on the server's producer transport params // https://mediasoup.org/documentation/v3/mediasoup-client/api/#TransportOptions @@ -244,10 +258,10 @@ const createSendTransport = () => { }) producerTransport.on('produce', async (parameters, callback, errback) => { - console.log(parameters) + console.log('[produce] parameters', parameters) try { - // tell the server to create a Producer + // Tell the server to create a Producer // with the following parameters and produce // and expect back a server side producer id // see server's socket.on('transport-produce', ...) @@ -270,20 +284,42 @@ const createSendTransport = () => { } const connectSendTransport = async () => { - // we now call produce() to instruct the producer transport + + console.log('[connectSendTransport] producerTransport'); + + // We now call produce() to instruct the producer transport // to send media to the Router // https://mediasoup.org/documentation/v3/mediasoup-client/api/#transport-produce // this action will trigger the 'connect' and 'produce' events above - producer = await producerTransport.produce(params) + + // Produce video + producerVideo = await producerTransport.produce(videoParams) + console.log('videoParams', videoParams); + console.log('producerVideo', producerVideo); - producer.on('trackended', () => { + producerVideo.on('trackended', () => { console.log('track ended') // close video track - }) + }) - producer.on('transportclose', () => { + producerVideo.on('transportclose', () => { console.log('transport ended') // close video track + }) + + // Produce audio + producerAudio = await producerTransport.produce(audioParams) + console.log('audioParams', audioParams); + console.log('producerAudio', producerAudio); + + producerAudio.on('trackended', () => { + console.log('track ended') + // close audio track + }) + + producerAudio.on('transportclose', () => { + console.log('transport ended') + // close audio track }) const answer = { @@ -294,7 +330,7 @@ const connectSendTransport = async () => { origin_asset_type_name: ASSET_TYPE, origin_asset_name: ASSET_NAME, video_call_id: callId, - answer: 'accepted', // answer: 'rejected' + answer: 'accepted', // answer: accepted/rejected }; console.log('SEND answer', answer); @@ -310,7 +346,7 @@ const connectSendTransport = async () => { const createRecvTransport = async () => { console.log('createRecvTransport'); - // see server's socket.on('consume', sender?, ...) + // See server's socket.on('consume', sender?, ...) // this is a call from Consumer, so sender = false await socket.emit('createWebRtcTransport', { sender: false, callId }, ({ params }) => { // The server sends back params needed @@ -320,15 +356,15 @@ const createRecvTransport = async () => { return } - console.log(params) + console.log('[createRecvTransport] params', params) - // creates a new WebRTC Transport to receive media + // Creates a new WebRTC Transport to receive media // based on server's consumer transport params // https://mediasoup.org/documentation/v3/mediasoup-client/api/#device-createRecvTransport consumerTransport = device.createRecvTransport(params) // https://mediasoup.org/documentation/v3/communication-between-client-and-server/#producing-media - // this event is raised when a first call to transport.produce() is made + // This event is raised when a first call to transport.produce() is made // see connectRecvTransport() below consumerTransport.on('connect', async ({ dtlsParameters }, callback, errback) => { try { @@ -353,7 +389,8 @@ const resetCallSettings = () => { localVideo.srcObject = null remoteVideo.srcObject = null consumer = null - producer = null + producerVideo = null + producerAudio = null producerTransport = null consumerTransport = null device = undefined @@ -361,7 +398,7 @@ const resetCallSettings = () => { const connectRecvTransport = async () => { console.log('connectRecvTransport'); - // for consumer, we need to tell the server first + // For consumer, we need to tell the server first // to create a consumer based on the rtpCapabilities and consume // if the router can consume, it will send back a set of params as below await socket.emit('consume', { @@ -373,7 +410,7 @@ const connectRecvTransport = async () => { return } - // then consume with the local consumer transport + // Then consume with the local consumer transport // which creates a consumer consumer = await consumerTransport.consume({ id: params.id, @@ -416,6 +453,7 @@ const closeCall = () => { resetCallSettings() } + btnLocalVideo.addEventListener('click', getLocalStream) btnRecvSendTransport.addEventListener('click', goConnect) btnCloseCall.addEventListener('click', closeCall) \ No newline at end of file