From bde02fe250a4dd22e83cd6b7fa0e178846573040 Mon Sep 17 00:00:00 2001 From: Sergiu Toma Date: Tue, 21 Feb 2023 02:41:48 +0200 Subject: [PATCH 1/9] LH-276: Add close-producer event handler; Update client --- app.js | 206 +++++++++++++++++------------------------------ public/bundle.js | 17 ++-- public/config.js | 2 +- public/index.js | 15 ++-- 4 files changed, 95 insertions(+), 145 deletions(-) diff --git a/app.js b/app.js index 7e97cec..e99d904 100644 --- a/app.js +++ b/app.js @@ -76,7 +76,7 @@ const createWorker = async () => { }); return worker; } catch (error) { - console.log(`ERROR | createWorker | ${error.message}`); + console.error(`[createWorker] | ERROR | error: ${error.message}`); } }; @@ -148,11 +148,10 @@ const closeCall = (callId) => { videoCalls[callId]?.receiverProducerTransport?.close(); videoCalls[callId]?.router?.close(); delete videoCalls[callId]; - } else { - console.log(`The call with id ${callId} has already been deleted`); + console.log(`[closeCall] | callId: ${callId}`); } } catch (error) { - console.log(`ERROR | closeCall | callid ${callId} | ${error.message}`); + console.error(`[closeCall] | ERROR | callId: ${callId} | error: ${error.message}`); } }; @@ -189,22 +188,21 @@ peers.on('connection', async (socket) => { console.log(`[createRoom] socket.id ${socket.id} callId ${callId}`); if (!videoCalls[callId]) { videoCalls[callId] = { router: await worker.createRouter({ mediaCodecs }) }; - console.log(`[createRoom] Router ID: ${videoCalls[callId].router.id}`); + console.log(`[createRoom] Generate Router ID: ${videoCalls[callId].router.id}`); videoCalls[callId].receiverSocket = socket; } else { videoCalls[callId].initiatorSocket = socket; } socketDetails[socket.id] = callId; // rtpCapabilities is set for callback - console.log('[getRtpCapabilities] callId', callId); callbackResponse = { rtpCapabilities: videoCalls[callId].router.rtpCapabilities, }; } else { - console.log(`[createRoom] missing callId ${callId}`); + console.log(`[createRoom] missing callId: ${callId}`); } } catch (error) { - console.log(`ERROR | createRoom | callId ${callId} | ${error.message}`); + console.error(`[createRoom] | ERROR | callId: ${callId} | error: ${error.message}`); } finally { callback(callbackResponse); } @@ -239,9 +237,7 @@ peers.on('connection', async (socket) => { } } } catch (error) { - console.log( - `ERROR | createWebRtcTransport | callId ${socketDetails[socket.id]} | sender ${sender} | ${error.message}` - ); + console.error(`[createWebRtcTransport] | ERROR | callId: ${socketDetails[socket.id]} | sender: ${sender} | error: ${error.message}`); callback(error); } }); @@ -261,7 +257,7 @@ peers.on('connection', async (socket) => { ? await videoCalls[callId].initiatorProducerTransport.connect({ dtlsParameters }) : await videoCalls[callId].receiverProducerTransport.connect({ dtlsParameters }); } catch (error) { - console.log(`ERROR | transport-connect | callId ${socketDetails[socket.id]} | ${error.message}`); + console.error(`[transport-connect] | ERROR | callId: ${socketDetails[socket.id]} | error: ${error.message}`); } }); @@ -275,7 +271,7 @@ peers.on('connection', async (socket) => { const callId = socketDetails[socket.id]; if (typeof rtpParameters === 'string') rtpParameters = JSON.parse(rtpParameters); - console.log(`[transport-produce] kind: ${kind} | socket: ${socket.id} | callId: ${callId}`); + console.log(`[transport-produce] callId: ${callId} | kind: ${kind} | socket: ${socket.id}`); if (kind === 'video') { if (!isInitiator(callId, socket.id)) { @@ -284,12 +280,7 @@ peers.on('connection', async (socket) => { rtpParameters, }); - console.log( - `[transport-produce] receiverVideoProducer Producer ID: ${videoCalls[callId].receiverVideoProducer.id} | kind: ${videoCalls[callId].receiverVideoProducer.kind}` - ); - videoCalls[callId].receiverVideoProducer.on('transportclose', () => { - const callId = socketDetails[socket.id]; console.log('transport for this producer closed', callId); closeCall(callId); }); @@ -305,12 +296,7 @@ peers.on('connection', async (socket) => { rtpParameters, }); - console.log( - `[transport-produce] initiatorVideoProducer Producer ID: ${videoCalls[callId].initiatorVideoProducer.id} | kind: ${videoCalls[callId].initiatorVideoProducer.kind}` - ); - videoCalls[callId].initiatorVideoProducer.on('transportclose', () => { - const callId = socketDetails[socket.id]; console.log('transport for this producer closed', callId); closeCall(callId); }); @@ -327,12 +313,7 @@ peers.on('connection', async (socket) => { rtpParameters, }); - console.log( - `[transport-produce] receiverAudioProducer Producer ID: ${videoCalls[callId].receiverAudioProducer.id} | kind: ${videoCalls[callId].receiverAudioProducer.kind}` - ); - videoCalls[callId].receiverAudioProducer.on('transportclose', () => { - const callId = socketDetails[socket.id]; console.log('transport for this producer closed', callId); closeCall(callId); }); @@ -348,12 +329,7 @@ peers.on('connection', async (socket) => { rtpParameters, }); - console.log( - `[transport-produce] initiatorAudioProducer Producer ID: ${videoCalls[callId].initiatorAudioProducer.id} | kind: ${videoCalls[callId].initiatorAudioProducer.kind}` - ); - videoCalls[callId].initiatorAudioProducer.on('transportclose', () => { - const callId = socketDetails[socket.id]; console.log('transport for this producer closed', callId); closeCall(callId); }); @@ -372,9 +348,9 @@ peers.on('connection', async (socket) => { // callId - Id of the call // kind - producer type: audio/video - socketToEmit.emit('new-producer', { callId, kind }); + socketToEmit?.emit('new-producer', { callId, kind }); } catch (error) { - console.log(`ERROR | transport-produce | callId ${socketDetails[socket.id]} | ${error.message}`); + console.error(`[transport-produce] | ERROR | callId: ${socketDetails[socket.id]} | error: ${error.message}`); } }); @@ -394,7 +370,7 @@ peers.on('connection', async (socket) => { await videoCalls[callId].initiatorConsumerTransport.connect({ dtlsParameters }); } } catch (error) { - console.log(`ERROR | transport-recv-connect | callId ${socketDetails[socket.id]} | ${error.message}`); + console.error(`[transport-recv-connect] | ERROR | callId: ${socketDetails[socket.id]} | error: ${error.message}`); } }); @@ -408,44 +384,44 @@ peers.on('connection', async (socket) => { socket.on('consume', async ({ rtpCapabilities }, callback) => { try { const callId = socketDetails[socket.id]; - console.log( - `[consume] socket ${socket.id} | callId ${callId} | rtpCapabilities: ${JSON.stringify(rtpCapabilities)}` - ); if (typeof rtpCapabilities === 'string') rtpCapabilities = JSON.parse(rtpCapabilities); - console.log('[consume] callId', callId); let canConsumeVideo, canConsumeAudio; - if (isInitiator(callId, socket.id)) { - canConsumeVideo = - !!videoCalls[callId].receiverVideoProducer && - !!videoCalls[callId].router.canConsume({ - producerId: videoCalls[callId].receiverVideoProducer.id, - rtpCapabilities, - }); - canConsumeAudio = - !!videoCalls[callId].receiverAudioProducer && - !!videoCalls[callId].router.canConsume({ - producerId: videoCalls[callId].receiverAudioProducer.id, - rtpCapabilities, - }); - } else { - canConsumeVideo = - !!videoCalls[callId].initiatorVideoProducer && - !!videoCalls[callId].router.canConsume({ - producerId: videoCalls[callId].initiatorVideoProducer.id, - rtpCapabilities, - }); - - canConsumeAudio = - !!videoCalls[callId].initiatorAudioProducer && - !!videoCalls[callId].router.canConsume({ - producerId: videoCalls[callId].initiatorAudioProducer.id, - rtpCapabilities, - }); + try { + if (isInitiator(callId, socket.id)) { + canConsumeVideo = + !!videoCalls[callId].receiverVideoProducer && + !!videoCalls[callId].router.canConsume({ + producerId: videoCalls[callId].receiverVideoProducer.id, + rtpCapabilities, + }); + canConsumeAudio = + !!videoCalls[callId].receiverAudioProducer && + !!videoCalls[callId].router.canConsume({ + producerId: videoCalls[callId].receiverAudioProducer.id, + rtpCapabilities, + }); + } else { + canConsumeVideo = + !!videoCalls[callId].initiatorVideoProducer && + !!videoCalls[callId].router.canConsume({ + producerId: videoCalls[callId].initiatorVideoProducer.id, + rtpCapabilities, + }); + + canConsumeAudio = + !!videoCalls[callId].initiatorAudioProducer && + !!videoCalls[callId].router.canConsume({ + producerId: videoCalls[callId].initiatorAudioProducer.id, + rtpCapabilities, + }); + } + } catch (error) { + console.error(`[consume] | ERROR | callId: ${callId} | error: ${error.message}`); } - console.log('[consume] canConsumeVideo', canConsumeVideo); - console.log('[consume] canConsumeAudio', canConsumeAudio); + + console.log(`[consume] socket ${socket.id} | callId: ${callId} | canConsumeVideo: ${canConsumeVideo} | canConsumeAudio: ${canConsumeAudio}`); if (canConsumeVideo && !canConsumeAudio) { const videoParams = await consumeVideo(callId, socket.id, rtpCapabilities); @@ -463,7 +439,7 @@ peers.on('connection', async (socket) => { callback(null); } } catch (error) { - console.log(`ERROR | consume | callId ${socketDetails[socket.id]} | ${error.message}`); + console.error(`[consume] | ERROR | callId: ${socketDetails[socket.id]} | error: ${error.message}`); callback({ params: { error } }); } }); @@ -473,24 +449,41 @@ peers.on('connection', async (socket) => { - When consuming on consumerTransport, it is initially done with paused: true, here we will resume - For the initiator we resume the initiatorConsumerAUDIO/VIDEO and for receiver the receiverConsumerAUDIO/VIDEO */ - socket.on('consumer-resume', () => { + socket.on('consumer-resume', () => { + try { + const callId = socketDetails[socket.id]; + const isInitiatorValue = isInitiator(callId, socket.id); + console.log(`[consumer-resume] callId: ${callId} | isInitiator: ${isInitiatorValue}`); + if (isInitiatorValue) { + videoCalls[callId].initiatorConsumerVideo && videoCalls[callId].initiatorConsumerVideo.resume(); + videoCalls[callId].initiatorConsumerAudio && videoCalls[callId].initiatorConsumerAudio.resume(); + } else { + videoCalls[callId].receiverConsumerVideo && videoCalls[callId].receiverConsumerVideo.resume(); + videoCalls[callId].receiverConsumerAudio && videoCalls[callId].receiverConsumerAudio.resume(); + } + } catch (error) { + console.error(`[consumer-resume] | ERROR | callId: ${socketDetails[socket.id]} | isInitiator: ${isInitiator} | error: ${error.message}`); + } + }); + + socket.on('close-producer', ({ callId, kind}) => { try { - const callId = socketDetails[socket.id]; - console.log(`[consumer-resume] callId ${callId}`); if (isInitiator(callId, socket.id)) { - videoCalls[callId]?.initiatorConsumerVideo?.resume(); - videoCalls[callId]?.initiatorConsumerAudio?.resume(); + console.log(`[close-producer] initiator --EMIT--> receiver | callId: ${callId} | kind: ${kind}`); + videoCalls[callId].receiverSocket.emit('close-producer', { callId, kind }); } else { - videoCalls[callId]?.receiverConsumerVideo?.resume(); - videoCalls[callId]?.receiverConsumerAudio?.resume(); + console.log(`[close-producer] receiver --EMIT--> initiator | callId: ${callId} | kind: ${kind}`); + videoCalls[callId].initiatorSocket.emit('close-producer', { callId, kind }); } } catch (error) { - console.log(`ERROR | consumer-resume | callId ${socketDetails[socket.id]} | ${error.message}`); + console.error(`[close-producer] | ERROR | callId: ${socketDetails[socket.id]} | error: ${error.message}`); } }); + }); const consumeVideo = async (callId, socketId, rtpCapabilities) => { + // Handlers for transports https://mediasoup.org/documentation/v3/mediasoup/api/#consumer-on-transportclose if (isInitiator(callId, socketId)) { videoCalls[callId].initiatorConsumerVideo = await videoCalls[callId].initiatorConsumerTransport.consume({ producerId: videoCalls[callId].receiverVideoProducer.id, @@ -498,20 +491,6 @@ const consumeVideo = async (callId, socketId, rtpCapabilities) => { paused: true, }); - // https://mediasoup.org/documentation/v3/mediasoup/api/#consumer-on-transportclose - videoCalls[callId].initiatorConsumerVideo.on('transportclose', () => { - const callId = socketDetails[socket.id]; - console.log('transport close from consumer', callId); - closeCall(callId); - }); - - // https://mediasoup.org/documentation/v3/mediasoup/api/#consumer-on-producerclose - videoCalls[callId].initiatorConsumerVideo.on('producerclose', () => { - const callId = socketDetails[socket.id]; - console.log('producer of consumer closed', callId); - closeCall(callId); - }); - return { id: videoCalls[callId].initiatorConsumerVideo.id, producerId: videoCalls[callId].receiverVideoProducer.id, @@ -525,18 +504,6 @@ const consumeVideo = async (callId, socketId, rtpCapabilities) => { paused: true, }); - videoCalls[callId].receiverConsumerVideo.on('transportclose', () => { - const callId = socketDetails[socket.id]; - console.log('transport close from consumer', callId); - closeCall(callId); - }); - - videoCalls[callId].receiverConsumerVideo.on('producerclose', () => { - const callId = socketDetails[socket.id]; - console.log('producer of consumer closed', callId); - closeCall(callId); - }); - return { id: videoCalls[callId].receiverConsumerVideo.id, producerId: videoCalls[callId].initiatorVideoProducer.id, @@ -554,20 +521,6 @@ const consumeAudio = async (callId, socketId, rtpCapabilities) => { paused: true, }); - // https://mediasoup.org/documentation/v3/mediasoup/api/#consumer-on-transportclose - videoCalls[callId].initiatorConsumerAudio.on('transportclose', () => { - const callId = socketDetails[socket.id]; - console.log('transport close from consumer', callId); - closeCall(callId); - }); - - // https://mediasoup.org/documentation/v3/mediasoup/api/#consumer-on-producerclose - videoCalls[callId].initiatorConsumerAudio.on('producerclose', () => { - const callId = socketDetails[socket.id]; - console.log('producer of consumer closed', callId); - closeCall(callId); - }); - return { id: videoCalls[callId].initiatorConsumerAudio.id, producerId: videoCalls[callId].receiverAudioProducer.id, @@ -581,18 +534,6 @@ const consumeAudio = async (callId, socketId, rtpCapabilities) => { paused: true, }); - videoCalls[callId].receiverConsumerAudio.on('transportclose', () => { - const callId = socketDetails[socket.id]; - console.log('transport close from consumer', callId); - closeCall(callId); - }); - - videoCalls[callId].receiverConsumerAudio.on('producerclose', () => { - const callId = socketDetails[socket.id]; - console.log('producer of consumer closed', callId); - closeCall(callId); - }); - return { id: videoCalls[callId].receiverConsumerAudio.id, producerId: videoCalls[callId].initiatorAudioProducer.id, @@ -615,7 +556,7 @@ const isInitiator = (callId, socketId) => { */ const createWebRtcTransportLayer = async (callId, callback) => { try { - console.log('[createWebRtcTransportLayer] callId', callId); + console.log(`[createWebRtcTransportLayer] callId: ${callId}`); // https://mediasoup.org/documentation/v3/mediasoup/api/#WebRtcTransportOptions const webRtcTransport_options = { listenIps: [ @@ -652,14 +593,13 @@ const createWebRtcTransportLayer = async (callId, callback) => { dtlsParameters: transport.dtlsParameters, }; - console.log('[createWebRtcTransportLayer] callback params', params); // Send back to the client the params callback({ params }); // Set transport to producerTransport or consumerTransport return transport; } catch (error) { - console.log(`ERROR | createWebRtcTransportLayer | callId ${socketDetails[socket.id]} | ${error.message}`); + console.error(`[createWebRtcTransportLayer] | ERROR | callId: ${socketDetails[socket.id]} | error: ${error.message}`); callback({ params: { error } }); } }; diff --git a/public/bundle.js b/public/bundle.js index ee7b33e..a2b43cb 100644 --- a/public/bundle.js +++ b/public/bundle.js @@ -20353,7 +20353,7 @@ module.exports = yeast; },{}],94:[function(require,module,exports){ module.exports = { hubAddress: 'https://hub.dev.linx.safemobile.com/', - mediasoupAddress: 'https://video.safemobile.org/', + mediasoupAddress: 'https://testing.video.safemobile.org/', } },{}],95:[function(require,module,exports){ const io = require('socket.io-client') @@ -20457,6 +20457,15 @@ setTimeout(() => { console.log(`🟢 new-producer | callId: ${callId} | kind: ${kind} | Ready to consume`); connectRecvTransport(); }) + + socket.on('close-producer', ({ callId, kind }) => { + console.log(`🔴 close-producer | callId: ${callId} | kind: ${kind}`); + if (kind === 'video') { + consumerVideo.close() + remoteVideo.srcObject = null + } + else if (kind === 'audio') consumerAudio.close() + }) } if (IS_PRODUCER === true) { @@ -20833,7 +20842,7 @@ const connectRecvTransport = async () => { console.log('remoteVideo PLAY') }) .catch((error) => { - displayError(`remoteVideo PLAY ERROR | ${error.message}`) + console.error(`remoteVideo PLAY ERROR | ${error.message}`) }) }) } @@ -20846,10 +20855,6 @@ const getVideoTrask = async (videoParams) => { rtpParameters: videoParams.rtpParameters }) - consumerVideo.on('transportclose', () => { - console.log('transport closed so consumer closed') - }) - return consumerVideo.track } diff --git a/public/config.js b/public/config.js index 4d1b6a4..233f24c 100644 --- a/public/config.js +++ b/public/config.js @@ -1,4 +1,4 @@ module.exports = { hubAddress: 'https://hub.dev.linx.safemobile.com/', - mediasoupAddress: 'https://video.safemobile.org/', + mediasoupAddress: 'https://testing.video.safemobile.org/', } \ No newline at end of file diff --git a/public/index.js b/public/index.js index 0fdb4f0..06625c0 100644 --- a/public/index.js +++ b/public/index.js @@ -99,6 +99,15 @@ setTimeout(() => { console.log(`🟢 new-producer | callId: ${callId} | kind: ${kind} | Ready to consume`); connectRecvTransport(); }) + + socket.on('close-producer', ({ callId, kind }) => { + console.log(`🔴 close-producer | callId: ${callId} | kind: ${kind}`); + if (kind === 'video') { + consumerVideo.close() + remoteVideo.srcObject = null + } + else if (kind === 'audio') consumerAudio.close() + }) } if (IS_PRODUCER === true) { @@ -475,7 +484,7 @@ const connectRecvTransport = async () => { console.log('remoteVideo PLAY') }) .catch((error) => { - displayError(`remoteVideo PLAY ERROR | ${error.message}`) + console.error(`remoteVideo PLAY ERROR | ${error.message}`) }) }) } @@ -488,10 +497,6 @@ const getVideoTrask = async (videoParams) => { rtpParameters: videoParams.rtpParameters }) - consumerVideo.on('transportclose', () => { - console.log('transport closed so consumer closed') - }) - return consumerVideo.track } From 4d8adf9eac7cc0086433e4538718e268db14fe82 Mon Sep 17 00:00:00 2001 From: Sergiu Toma Date: Wed, 22 Feb 2023 18:21:53 +0200 Subject: [PATCH 2/9] LH-276: Refactor consume(consumeAudio/consumeVideo); Format code --- app.js | 149 ++++++++++++++++++++++++++------------------------------- 1 file changed, 67 insertions(+), 82 deletions(-) diff --git a/app.js b/app.js index e99d904..e3a2600 100644 --- a/app.js +++ b/app.js @@ -237,7 +237,11 @@ peers.on('connection', async (socket) => { } } } catch (error) { - console.error(`[createWebRtcTransport] | ERROR | callId: ${socketDetails[socket.id]} | sender: ${sender} | error: ${error.message}`); + console.error( + `[createWebRtcTransport] | ERROR | callId: ${socketDetails[socket.id]} | sender: ${sender} | error: ${ + error.message + }` + ); callback(error); } }); @@ -384,60 +388,16 @@ peers.on('connection', async (socket) => { socket.on('consume', async ({ rtpCapabilities }, callback) => { try { const callId = socketDetails[socket.id]; + const socketId = socket.id; + + console.log(`[consume] socket ${socket.id} | callId: ${callId}`); if (typeof rtpCapabilities === 'string') rtpCapabilities = JSON.parse(rtpCapabilities); - let canConsumeVideo, canConsumeAudio; - try { - if (isInitiator(callId, socket.id)) { - canConsumeVideo = - !!videoCalls[callId].receiverVideoProducer && - !!videoCalls[callId].router.canConsume({ - producerId: videoCalls[callId].receiverVideoProducer.id, - rtpCapabilities, - }); - canConsumeAudio = - !!videoCalls[callId].receiverAudioProducer && - !!videoCalls[callId].router.canConsume({ - producerId: videoCalls[callId].receiverAudioProducer.id, - rtpCapabilities, - }); - } else { - canConsumeVideo = - !!videoCalls[callId].initiatorVideoProducer && - !!videoCalls[callId].router.canConsume({ - producerId: videoCalls[callId].initiatorVideoProducer.id, - rtpCapabilities, - }); - - canConsumeAudio = - !!videoCalls[callId].initiatorAudioProducer && - !!videoCalls[callId].router.canConsume({ - producerId: videoCalls[callId].initiatorAudioProducer.id, - rtpCapabilities, - }); - } - } catch (error) { - console.error(`[consume] | ERROR | callId: ${callId} | error: ${error.message}`); - } - - console.log(`[consume] socket ${socket.id} | callId: ${callId} | canConsumeVideo: ${canConsumeVideo} | canConsumeAudio: ${canConsumeAudio}`); - - if (canConsumeVideo && !canConsumeAudio) { - const videoParams = await consumeVideo(callId, socket.id, rtpCapabilities); - callback({ videoParams, audioParams: null }); - } else if (canConsumeVideo && canConsumeAudio) { - const videoParams = await consumeVideo(callId, socket.id, rtpCapabilities); - const audioParams = await consumeAudio(callId, socket.id, rtpCapabilities); - callback({ videoParams, audioParams }); - } else if (!canConsumeVideo && canConsumeAudio) { - const audioParams = await consumeAudio(callId, socket.id, rtpCapabilities); - const data = { videoParams: null, audioParams }; - callback(data); - } else { - console.log(`[consume] Can't consume | callId ${callId}`); - callback(null); - } + callback({ + videoParams: await consumeVideo({ callId, socketId, rtpCapabilities }), + audioParams: await consumeAudio({ callId, socketId, rtpCapabilities }), + }); } catch (error) { console.error(`[consume] | ERROR | callId: ${socketDetails[socket.id]} | error: ${error.message}`); callback({ params: { error } }); @@ -449,24 +409,28 @@ peers.on('connection', async (socket) => { - When consuming on consumerTransport, it is initially done with paused: true, here we will resume - For the initiator we resume the initiatorConsumerAUDIO/VIDEO and for receiver the receiverConsumerAUDIO/VIDEO */ - socket.on('consumer-resume', () => { - try { - const callId = socketDetails[socket.id]; - const isInitiatorValue = isInitiator(callId, socket.id); - console.log(`[consumer-resume] callId: ${callId} | isInitiator: ${isInitiatorValue}`); - if (isInitiatorValue) { - videoCalls[callId].initiatorConsumerVideo && videoCalls[callId].initiatorConsumerVideo.resume(); - videoCalls[callId].initiatorConsumerAudio && videoCalls[callId].initiatorConsumerAudio.resume(); - } else { - videoCalls[callId].receiverConsumerVideo && videoCalls[callId].receiverConsumerVideo.resume(); - videoCalls[callId].receiverConsumerAudio && videoCalls[callId].receiverConsumerAudio.resume(); - } - } catch (error) { - console.error(`[consumer-resume] | ERROR | callId: ${socketDetails[socket.id]} | isInitiator: ${isInitiator} | error: ${error.message}`); - } - }); + socket.on('consumer-resume', () => { + try { + const callId = socketDetails[socket.id]; + const isInitiatorValue = isInitiator(callId, socket.id); + console.log(`[consumer-resume] callId: ${callId} | isInitiator: ${isInitiatorValue}`); + if (isInitiatorValue) { + videoCalls[callId].initiatorConsumerVideo && videoCalls[callId].initiatorConsumerVideo.resume(); + videoCalls[callId].initiatorConsumerAudio && videoCalls[callId].initiatorConsumerAudio.resume(); + } else { + videoCalls[callId].receiverConsumerVideo && videoCalls[callId].receiverConsumerVideo.resume(); + videoCalls[callId].receiverConsumerAudio && videoCalls[callId].receiverConsumerAudio.resume(); + } + } catch (error) { + console.error( + `[consumer-resume] | ERROR | callId: ${socketDetails[socket.id]} | isInitiator: ${isInitiator} | error: ${ + error.message + }` + ); + } + }); - socket.on('close-producer', ({ callId, kind}) => { + socket.on('close-producer', ({ callId, kind }) => { try { if (isInitiator(callId, socket.id)) { console.log(`[close-producer] initiator --EMIT--> receiver | callId: ${callId} | kind: ${kind}`); @@ -479,64 +443,83 @@ peers.on('connection', async (socket) => { console.error(`[close-producer] | ERROR | callId: ${socketDetails[socket.id]} | error: ${error.message}`); } }); - }); -const consumeVideo = async (callId, socketId, rtpCapabilities) => { - // Handlers for transports https://mediasoup.org/documentation/v3/mediasoup/api/#consumer-on-transportclose +const canConsume = ({ callId, producerId, rtpCapabilities }) => { + return !!videoCalls[callId].router.canConsume({ + producerId, + rtpCapabilities, + }); +}; + +const consumeVideo = async ({ callId, socketId, rtpCapabilities }) => { + // Handlers for consumer transport https://mediasoup.org/documentation/v3/mediasoup/api/#consumer-on-transportclose if (isInitiator(callId, socketId)) { + const producerId = videoCalls[callId].receiverVideoProducer.id; + if (!canConsume({ callId, producerId, rtpCapabilities })) return null; + videoCalls[callId].initiatorConsumerVideo = await videoCalls[callId].initiatorConsumerTransport.consume({ - producerId: videoCalls[callId].receiverVideoProducer.id, + producerId, rtpCapabilities, paused: true, }); return { id: videoCalls[callId].initiatorConsumerVideo.id, - producerId: videoCalls[callId].receiverVideoProducer.id, + producerId, kind: 'video', rtpParameters: videoCalls[callId].initiatorConsumerVideo.rtpParameters, }; } else { + const producerId = videoCalls[callId].initiatorVideoProducer.id; + if (!canConsume({ callId, producerId, rtpCapabilities })) return null; + videoCalls[callId].receiverConsumerVideo = await videoCalls[callId].receiverConsumerTransport.consume({ - producerId: videoCalls[callId].initiatorVideoProducer.id, + producerId, rtpCapabilities, paused: true, }); return { id: videoCalls[callId].receiverConsumerVideo.id, - producerId: videoCalls[callId].initiatorVideoProducer.id, + producerId, kind: 'video', rtpParameters: videoCalls[callId].receiverConsumerVideo.rtpParameters, }; } }; -const consumeAudio = async (callId, socketId, rtpCapabilities) => { +const consumeAudio = async ({ callId, socketId, rtpCapabilities }) => { + // Handlers for consumer transport https://mediasoup.org/documentation/v3/mediasoup/api/#consumer-on-transportclose if (isInitiator(callId, socketId)) { + const producerId = videoCalls[callId].receiverAudioProducer.id; + if (!canConsume({ callId, producerId, rtpCapabilities })) return null; + videoCalls[callId].initiatorConsumerAudio = await videoCalls[callId].initiatorConsumerTransport.consume({ - producerId: videoCalls[callId].receiverAudioProducer.id, + producerId, rtpCapabilities, paused: true, }); return { id: videoCalls[callId].initiatorConsumerAudio.id, - producerId: videoCalls[callId].receiverAudioProducer.id, + producerId, kind: 'audio', rtpParameters: videoCalls[callId].initiatorConsumerAudio.rtpParameters, }; } else { + const producerId = videoCalls[callId].initiatorAudioProducer.id; + if (!canConsume({ callId, producerId, rtpCapabilities })) return null; + videoCalls[callId].receiverConsumerAudio = await videoCalls[callId].receiverConsumerTransport.consume({ - producerId: videoCalls[callId].initiatorAudioProducer.id, + producerId, rtpCapabilities, paused: true, }); return { id: videoCalls[callId].receiverConsumerAudio.id, - producerId: videoCalls[callId].initiatorAudioProducer.id, + producerId, kind: 'audio', rtpParameters: videoCalls[callId].receiverConsumerAudio.rtpParameters, }; @@ -599,7 +582,9 @@ const createWebRtcTransportLayer = async (callId, callback) => { // Set transport to producerTransport or consumerTransport return transport; } catch (error) { - console.error(`[createWebRtcTransportLayer] | ERROR | callId: ${socketDetails[socket.id]} | error: ${error.message}`); + console.error( + `[createWebRtcTransportLayer] | ERROR | callId: ${socketDetails[socket.id]} | error: ${error.message}` + ); callback({ params: { error } }); } }; From 6e3ce9fbb34076a6f42b66d23e8d3e1d302f09a3 Mon Sep 17 00:00:00 2001 From: Sergiu Toma Date: Wed, 22 Feb 2023 18:30:28 +0200 Subject: [PATCH 3/9] LH-276: Format code --- app.js | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/app.js b/app.js index e3a2600..640c56a 100644 --- a/app.js +++ b/app.js @@ -389,19 +389,32 @@ peers.on('connection', async (socket) => { try { const callId = socketDetails[socket.id]; const socketId = socket.id; + let audioParams, videoParams; - console.log(`[consume] socket ${socket.id} | callId: ${callId}`); + console.log(`[consume] socket ${socketId} | callId: ${callId}`); if (typeof rtpCapabilities === 'string') rtpCapabilities = JSON.parse(rtpCapabilities); - - callback({ - videoParams: await consumeVideo({ callId, socketId, rtpCapabilities }), - audioParams: await consumeAudio({ callId, socketId, rtpCapabilities }), - }); } catch (error) { - console.error(`[consume] | ERROR | callId: ${socketDetails[socket.id]} | error: ${error.message}`); + console.error(`[consume] | ERROR | callId: ${callId} | error: ${error.message}`); callback({ params: { error } }); } + + try { + videoParams = await consumeVideo({ callId, socketId, rtpCapabilities }); + } catch (error) { + console.error(`[consume] | ERROR | videoParams | callId: ${callId} | error: ${error.message}`); + } + + try { + audioParams = await consumeAudio({ callId, socketId, rtpCapabilities }); + } catch (error) { + console.error(`[consume] | ERROR | audioParams | callId: ${callId} | error: ${error.message}`); + } + + callback({ + videoParams, + audioParams, + }); }); /* From abb1533c9bd0aa32bb32d5c5fcd301e86a658dfc Mon Sep 17 00:00:00 2001 From: Sergiu Toma Date: Wed, 22 Feb 2023 18:32:41 +0200 Subject: [PATCH 4/9] LH-276: Format code --- app.js | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/app.js b/app.js index 640c56a..3482956 100644 --- a/app.js +++ b/app.js @@ -386,18 +386,13 @@ peers.on('connection', async (socket) => { - The consumer does consumerTransport.consume(params) */ socket.on('consume', async ({ rtpCapabilities }, callback) => { - try { - const callId = socketDetails[socket.id]; - const socketId = socket.id; - let audioParams, videoParams; + const callId = socketDetails[socket.id]; + const socketId = socket.id; + let audioParams, videoParams; - console.log(`[consume] socket ${socketId} | callId: ${callId}`); + console.log(`[consume] socket ${socketId} | callId: ${callId}`); - if (typeof rtpCapabilities === 'string') rtpCapabilities = JSON.parse(rtpCapabilities); - } catch (error) { - console.error(`[consume] | ERROR | callId: ${callId} | error: ${error.message}`); - callback({ params: { error } }); - } + if (typeof rtpCapabilities === 'string') rtpCapabilities = JSON.parse(rtpCapabilities); try { videoParams = await consumeVideo({ callId, socketId, rtpCapabilities }); From c80265fe25d28138926f0715ca3341d62f8e5a5b Mon Sep 17 00:00:00 2001 From: Sergiu Toma Date: Wed, 22 Feb 2023 18:36:50 +0200 Subject: [PATCH 5/9] LH-276: Format code --- app.js | 72 +++++++++++++++++++++++++++++++--------------------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/app.js b/app.js index 3482956..583c79f 100644 --- a/app.js +++ b/app.js @@ -498,39 +498,45 @@ const consumeVideo = async ({ callId, socketId, rtpCapabilities }) => { }; const consumeAudio = async ({ callId, socketId, rtpCapabilities }) => { - // Handlers for consumer transport https://mediasoup.org/documentation/v3/mediasoup/api/#consumer-on-transportclose - if (isInitiator(callId, socketId)) { - const producerId = videoCalls[callId].receiverAudioProducer.id; - if (!canConsume({ callId, producerId, rtpCapabilities })) return null; - - videoCalls[callId].initiatorConsumerAudio = await videoCalls[callId].initiatorConsumerTransport.consume({ - producerId, - rtpCapabilities, - paused: true, - }); - - return { - id: videoCalls[callId].initiatorConsumerAudio.id, - producerId, - kind: 'audio', - rtpParameters: videoCalls[callId].initiatorConsumerAudio.rtpParameters, - }; - } else { - const producerId = videoCalls[callId].initiatorAudioProducer.id; - if (!canConsume({ callId, producerId, rtpCapabilities })) return null; - - videoCalls[callId].receiverConsumerAudio = await videoCalls[callId].receiverConsumerTransport.consume({ - producerId, - rtpCapabilities, - paused: true, - }); - - return { - id: videoCalls[callId].receiverConsumerAudio.id, - producerId, - kind: 'audio', - rtpParameters: videoCalls[callId].receiverConsumerAudio.rtpParameters, - }; + try { + // Handlers for consumer transport https://mediasoup.org/documentation/v3/mediasoup/api/#consumer-on-transportclose + if (isInitiator(callId, socketId)) { + const producerId = videoCalls[callId].receiverAudioProducer.id; + if (!canConsume({ callId, producerId, rtpCapabilities })) return null; + + videoCalls[callId].initiatorConsumerAudio = await videoCalls[callId].initiatorConsumerTransport.consume({ + producerId, + rtpCapabilities, + paused: true, + }); + const returnData = { + id: videoCalls[callId].initiatorConsumerAudio.id, + producerId, + kind: 'audio', + rtpParameters: videoCalls[callId].initiatorConsumerAudio.rtpParameters, + }; + console.log('audio returnData', returnData); + return returnData; + } else { + const producerId = videoCalls[callId].initiatorAudioProducer.id; + if (!canConsume({ callId, producerId, rtpCapabilities })) return null; + + videoCalls[callId].receiverConsumerAudio = await videoCalls[callId].receiverConsumerTransport.consume({ + producerId, + rtpCapabilities, + paused: true, + }); + const returnData = { + id: videoCalls[callId].receiverConsumerAudio.id, + producerId, + kind: 'audio', + rtpParameters: videoCalls[callId].receiverConsumerAudio.rtpParameters, + }; + console.log('audio returnData', returnData); + return returnData; + } + } catch (error) { + console.error(`[consumeAudio] | ERROR | error: ${error.message}`) } }; From 29a4cd7227766823dfc06762add1236b7a032bb7 Mon Sep 17 00:00:00 2001 From: Sergiu Toma Date: Wed, 22 Feb 2023 18:40:39 +0200 Subject: [PATCH 6/9] LH-276: Format code --- app.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app.js b/app.js index 583c79f..5f3a652 100644 --- a/app.js +++ b/app.js @@ -501,6 +501,7 @@ const consumeAudio = async ({ callId, socketId, rtpCapabilities }) => { try { // Handlers for consumer transport https://mediasoup.org/documentation/v3/mediasoup/api/#consumer-on-transportclose if (isInitiator(callId, socketId)) { + console.log('1videoCalls[callId].receiverAudioProducer', videoCalls[callId].receiverAudioProducer); const producerId = videoCalls[callId].receiverAudioProducer.id; if (!canConsume({ callId, producerId, rtpCapabilities })) return null; @@ -518,6 +519,7 @@ const consumeAudio = async ({ callId, socketId, rtpCapabilities }) => { console.log('audio returnData', returnData); return returnData; } else { + console.log('2videoCalls[callId].initiatorAudioProducer', videoCalls[callId].initiatorAudioProducer); const producerId = videoCalls[callId].initiatorAudioProducer.id; if (!canConsume({ callId, producerId, rtpCapabilities })) return null; @@ -536,7 +538,7 @@ const consumeAudio = async ({ callId, socketId, rtpCapabilities }) => { return returnData; } } catch (error) { - console.error(`[consumeAudio] | ERROR | error: ${error.message}`) + console.error(`[consumeAudio] | ERROR | error: ${error}`) } }; From 9b3f2f94c88eaf08838f45f3d4c5f418149115be Mon Sep 17 00:00:00 2001 From: Sergiu Toma Date: Wed, 22 Feb 2023 18:46:01 +0200 Subject: [PATCH 7/9] LH-276: Fix audio when initiator is not set yet(at start) --- app.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/app.js b/app.js index 5f3a652..f25b330 100644 --- a/app.js +++ b/app.js @@ -500,11 +500,11 @@ const consumeVideo = async ({ callId, socketId, rtpCapabilities }) => { const consumeAudio = async ({ callId, socketId, rtpCapabilities }) => { try { // Handlers for consumer transport https://mediasoup.org/documentation/v3/mediasoup/api/#consumer-on-transportclose - if (isInitiator(callId, socketId)) { + if (isInitiator(callId, socketId) && videoCalls[callId].receiverAudioProducer) { console.log('1videoCalls[callId].receiverAudioProducer', videoCalls[callId].receiverAudioProducer); const producerId = videoCalls[callId].receiverAudioProducer.id; if (!canConsume({ callId, producerId, rtpCapabilities })) return null; - + videoCalls[callId].initiatorConsumerAudio = await videoCalls[callId].initiatorConsumerTransport.consume({ producerId, rtpCapabilities, @@ -518,11 +518,11 @@ const consumeAudio = async ({ callId, socketId, rtpCapabilities }) => { }; console.log('audio returnData', returnData); return returnData; - } else { + } else if (videoCalls[callId].initiatorAudioProducer) { console.log('2videoCalls[callId].initiatorAudioProducer', videoCalls[callId].initiatorAudioProducer); const producerId = videoCalls[callId].initiatorAudioProducer.id; if (!canConsume({ callId, producerId, rtpCapabilities })) return null; - + videoCalls[callId].receiverConsumerAudio = await videoCalls[callId].receiverConsumerTransport.consume({ producerId, rtpCapabilities, @@ -536,9 +536,11 @@ const consumeAudio = async ({ callId, socketId, rtpCapabilities }) => { }; console.log('audio returnData', returnData); return returnData; + } else { + return null; } } catch (error) { - console.error(`[consumeAudio] | ERROR | error: ${error}`) + console.error(`[consumeAudio] | ERROR | error: ${error}`); } }; From 4591617b1e787018e2a54664ad3910483fc4711a Mon Sep 17 00:00:00 2001 From: Sergiu Toma Date: Wed, 22 Feb 2023 18:50:37 +0200 Subject: [PATCH 8/9] LH-276: Fix video consume when initiator is not set yet(at start) --- app.js | 35 ++++++++++------------------------- 1 file changed, 10 insertions(+), 25 deletions(-) diff --git a/app.js b/app.js index f25b330..dfdf39c 100644 --- a/app.js +++ b/app.js @@ -388,27 +388,14 @@ peers.on('connection', async (socket) => { socket.on('consume', async ({ rtpCapabilities }, callback) => { const callId = socketDetails[socket.id]; const socketId = socket.id; - let audioParams, videoParams; console.log(`[consume] socket ${socketId} | callId: ${callId}`); if (typeof rtpCapabilities === 'string') rtpCapabilities = JSON.parse(rtpCapabilities); - try { - videoParams = await consumeVideo({ callId, socketId, rtpCapabilities }); - } catch (error) { - console.error(`[consume] | ERROR | videoParams | callId: ${callId} | error: ${error.message}`); - } - - try { - audioParams = await consumeAudio({ callId, socketId, rtpCapabilities }); - } catch (error) { - console.error(`[consume] | ERROR | audioParams | callId: ${callId} | error: ${error.message}`); - } - callback({ - videoParams, - audioParams, + videoParams: await consumeVideo({ callId, socketId, rtpCapabilities }), + audioParams: await consumeAudio({ callId, socketId, rtpCapabilities }), }); }); @@ -462,7 +449,7 @@ const canConsume = ({ callId, producerId, rtpCapabilities }) => { const consumeVideo = async ({ callId, socketId, rtpCapabilities }) => { // Handlers for consumer transport https://mediasoup.org/documentation/v3/mediasoup/api/#consumer-on-transportclose - if (isInitiator(callId, socketId)) { + if (isInitiator(callId, socketId) && videoCalls[callId].receiverVideoProducer) { const producerId = videoCalls[callId].receiverVideoProducer.id; if (!canConsume({ callId, producerId, rtpCapabilities })) return null; @@ -478,7 +465,7 @@ const consumeVideo = async ({ callId, socketId, rtpCapabilities }) => { kind: 'video', rtpParameters: videoCalls[callId].initiatorConsumerVideo.rtpParameters, }; - } else { + } else if (videoCalls[callId].initiatorVideoProducer) { const producerId = videoCalls[callId].initiatorVideoProducer.id; if (!canConsume({ callId, producerId, rtpCapabilities })) return null; @@ -494,6 +481,8 @@ const consumeVideo = async ({ callId, socketId, rtpCapabilities }) => { kind: 'video', rtpParameters: videoCalls[callId].receiverConsumerVideo.rtpParameters, }; + } else { + return null; } }; @@ -501,7 +490,6 @@ const consumeAudio = async ({ callId, socketId, rtpCapabilities }) => { try { // Handlers for consumer transport https://mediasoup.org/documentation/v3/mediasoup/api/#consumer-on-transportclose if (isInitiator(callId, socketId) && videoCalls[callId].receiverAudioProducer) { - console.log('1videoCalls[callId].receiverAudioProducer', videoCalls[callId].receiverAudioProducer); const producerId = videoCalls[callId].receiverAudioProducer.id; if (!canConsume({ callId, producerId, rtpCapabilities })) return null; @@ -510,16 +498,14 @@ const consumeAudio = async ({ callId, socketId, rtpCapabilities }) => { rtpCapabilities, paused: true, }); - const returnData = { + + return { id: videoCalls[callId].initiatorConsumerAudio.id, producerId, kind: 'audio', rtpParameters: videoCalls[callId].initiatorConsumerAudio.rtpParameters, }; - console.log('audio returnData', returnData); - return returnData; } else if (videoCalls[callId].initiatorAudioProducer) { - console.log('2videoCalls[callId].initiatorAudioProducer', videoCalls[callId].initiatorAudioProducer); const producerId = videoCalls[callId].initiatorAudioProducer.id; if (!canConsume({ callId, producerId, rtpCapabilities })) return null; @@ -528,14 +514,13 @@ const consumeAudio = async ({ callId, socketId, rtpCapabilities }) => { rtpCapabilities, paused: true, }); - const returnData = { + + return { id: videoCalls[callId].receiverConsumerAudio.id, producerId, kind: 'audio', rtpParameters: videoCalls[callId].receiverConsumerAudio.rtpParameters, }; - console.log('audio returnData', returnData); - return returnData; } else { return null; } From 7842953faf5368a47518bdb5aa5fd9d58ab0c893 Mon Sep 17 00:00:00 2001 From: Sergiu Toma Date: Wed, 22 Feb 2023 18:57:57 +0200 Subject: [PATCH 9/9] LH-276: Refactor consumer-resume --- app.js | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/app.js b/app.js index dfdf39c..cf9ddee 100644 --- a/app.js +++ b/app.js @@ -409,13 +409,17 @@ peers.on('connection', async (socket) => { const callId = socketDetails[socket.id]; const isInitiatorValue = isInitiator(callId, socket.id); console.log(`[consumer-resume] callId: ${callId} | isInitiator: ${isInitiatorValue}`); - if (isInitiatorValue) { - videoCalls[callId].initiatorConsumerVideo && videoCalls[callId].initiatorConsumerVideo.resume(); - videoCalls[callId].initiatorConsumerAudio && videoCalls[callId].initiatorConsumerAudio.resume(); - } else { - videoCalls[callId].receiverConsumerVideo && videoCalls[callId].receiverConsumerVideo.resume(); - videoCalls[callId].receiverConsumerAudio && videoCalls[callId].receiverConsumerAudio.resume(); - } + + const consumerVideo = isInitiatorValue + ? videoCalls[callId].initiatorConsumerVideo + : videoCalls[callId].receiverConsumerVideo; + + const consumerAudio = isInitiatorValue + ? videoCalls[callId].initiatorConsumerAudio + : videoCalls[callId].receiverConsumerAudio; + + consumerVideo?.resume(); + consumerAudio?.resume(); } catch (error) { console.error( `[consumer-resume] | ERROR | callId: ${socketDetails[socket.id]} | isInitiator: ${isInitiator} | error: ${