idillon | 989d654 | 2023-01-30 15:49:58 -0500 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2022 Savoir-faire Linux Inc. |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or modify |
| 5 | * it under the terms of the GNU Affero General Public License as |
| 6 | * published by the Free Software Foundation; either version 3 of the |
| 7 | * License, or (at your option) any later version. |
| 8 | * |
| 9 | * This program is distributed in the hope that it will be useful, |
| 10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 12 | * GNU Affero General Public License for more details. |
| 13 | * |
| 14 | * You should have received a copy of the GNU Affero General Public |
| 15 | * License along with this program. If not, see |
| 16 | * <https://www.gnu.org/licenses/>. |
| 17 | */ |
| 18 | |
idillon | 255682e | 2023-02-06 13:25:26 -0500 | [diff] [blame] | 19 | import { WebRtcIceCandidate, WebRtcSdp, WebSocketMessageTable, WebSocketMessageType } from 'jami-web-common'; |
idillon | 989d654 | 2023-01-30 15:49:58 -0500 | [diff] [blame] | 20 | |
idillon | 989d654 | 2023-01-30 15:49:58 -0500 | [diff] [blame] | 21 | import { IWebSocketContext } from '../contexts/WebSocketProvider'; |
| 22 | import { Account } from '../models/account'; |
idillon | c45a43d | 2023-02-10 18:12:10 -0500 | [diff] [blame^] | 23 | import { CallData } from '../services/CallManager'; |
idillon | 989d654 | 2023-01-30 15:49:58 -0500 | [diff] [blame] | 24 | import { Listener } from '../utils/utils'; |
| 25 | |
| 26 | export type RTCPeerConnectionInfos = { |
| 27 | remoteStreams: readonly MediaStream[]; |
| 28 | iceConnectionState: RTCIceConnectionState; |
| 29 | }; |
| 30 | |
| 31 | export class RTCPeerConnectionHandler { |
| 32 | private readonly connection: RTCPeerConnection; |
| 33 | private remoteStreams: readonly MediaStream[] = []; |
| 34 | private iceConnectionState: RTCIceConnectionState = 'new'; |
| 35 | private isReadyForIceCandidates = false; |
| 36 | |
| 37 | private audioRtcRtpSenders: RTCRtpSender[] | null = null; |
| 38 | private videoRtcRtpSenders: RTCRtpSender[] | null = null; |
| 39 | |
| 40 | // TODO: The ICE candidate queue is used to cache candidates that were received before `setRemoteDescription` was |
| 41 | // called. This is currently necessary, because the jami-daemon is unreliable as a WebRTC signaling channel, |
| 42 | // because messages can be received with a delay or out of order. This queue is a temporary workaround that |
| 43 | // should be replaced if there is a better way to send messages with the daemon. |
| 44 | // Relevant links: |
| 45 | // - https://github.com/w3c/webrtc-pc/issues/2519#issuecomment-622055440 |
| 46 | // - https://stackoverflow.com/questions/57256828/how-to-fix-invalidstateerror-cannot-add-ice-candidate-when-there-is-no-remote-s |
| 47 | private iceCandidateQueue: RTCIceCandidate[] = []; |
| 48 | |
| 49 | private cleaningFunctions: (() => void)[] = []; |
| 50 | private listener: Listener; |
| 51 | |
| 52 | getInfos(): RTCPeerConnectionInfos { |
| 53 | return { |
| 54 | remoteStreams: this.remoteStreams, |
| 55 | iceConnectionState: this.iceConnectionState, |
| 56 | }; |
| 57 | } |
| 58 | |
| 59 | constructor( |
| 60 | webSocket: IWebSocketContext, |
| 61 | account: Account, |
| 62 | contactUri: string, |
| 63 | callData: CallData, |
| 64 | localStream: MediaStream | undefined, |
| 65 | screenShareLocalStream: MediaStream | undefined, |
| 66 | listener: Listener |
| 67 | ) { |
idillon | 255682e | 2023-02-06 13:25:26 -0500 | [diff] [blame] | 68 | console.log('constructor', callData); |
idillon | 989d654 | 2023-01-30 15:49:58 -0500 | [diff] [blame] | 69 | this.listener = listener; |
| 70 | const iceServers = this.getIceServers(account); |
| 71 | this.connection = new RTCPeerConnection({ iceServers }); |
| 72 | this.setConnectionListeners(webSocket, callData.conversationId, contactUri); |
| 73 | this.setWebSocketListeners(webSocket, callData.conversationId, contactUri); |
| 74 | this.updateLocalStreams(localStream, screenShareLocalStream); |
| 75 | |
| 76 | if (callData.role === 'caller') { |
| 77 | this.startNegociation(webSocket, contactUri, callData.conversationId); |
| 78 | } |
| 79 | } |
| 80 | |
| 81 | updateLocalStreams(localStream: MediaStream | undefined, screenShareLocalStream: MediaStream | undefined) { |
| 82 | if (this.connection.iceConnectionState === 'closed') { |
| 83 | return; |
| 84 | } |
| 85 | |
| 86 | const updateTracks = async (stream: MediaStream, kind: 'audio' | 'video') => { |
| 87 | const senders = kind === 'audio' ? this.audioRtcRtpSenders : this.videoRtcRtpSenders; |
| 88 | const tracks = kind === 'audio' ? stream.getAudioTracks() : stream.getVideoTracks(); |
| 89 | if (senders) { |
| 90 | const promises: Promise<void>[] = []; |
| 91 | for (let i = 0; i < senders.length; i++) { |
| 92 | // TODO: There is a bug where calling multiple times `addTrack` when changing an input device doesn't work. |
| 93 | // Calling `addTrack` doesn't trigger the `track` event listener for the other user. |
| 94 | // This workaround makes it possible to replace a track, but it could be improved by figuring out the |
| 95 | // proper way of changing a track. |
| 96 | promises.push( |
| 97 | senders[i].replaceTrack(tracks[i]).catch((e) => { |
| 98 | console.error('Error replacing track:', e); |
| 99 | }) |
| 100 | ); |
| 101 | } |
| 102 | return Promise.all(promises); |
| 103 | } |
| 104 | |
| 105 | // TODO: Currently, we do not support adding new devices. To enable this feature, we would need to implement |
| 106 | // the "Perfect negotiation" pattern to renegotiate after `addTrack`. |
| 107 | // https://blog.mozilla.org/webrtc/perfect-negotiation-in-webrtc/ |
| 108 | const newSenders = tracks.map((track) => this.connection.addTrack(track, stream)); |
| 109 | if (kind === 'audio') { |
| 110 | this.audioRtcRtpSenders = newSenders; |
| 111 | } else { |
| 112 | this.videoRtcRtpSenders = newSenders; |
| 113 | } |
| 114 | }; |
| 115 | |
| 116 | if (localStream) { |
| 117 | updateTracks(localStream, 'audio'); |
| 118 | updateTracks(localStream, 'video'); |
| 119 | } |
| 120 | |
| 121 | if (screenShareLocalStream) { |
| 122 | updateTracks(screenShareLocalStream, 'video'); |
| 123 | } |
| 124 | } |
| 125 | |
| 126 | disconnect() { |
| 127 | this.connection?.close(); |
| 128 | this.cleaningFunctions.forEach((func) => func()); |
| 129 | this.cleaningFunctions = []; |
| 130 | this.emitChange(); |
| 131 | } |
| 132 | |
| 133 | private getIceServers(account: Account) { |
| 134 | const iceServers: RTCIceServer[] = []; |
| 135 | |
| 136 | if (account.details['TURN.enable'] === 'true') { |
| 137 | iceServers.push({ |
| 138 | urls: 'turn:' + account.details['TURN.server'], |
| 139 | username: account.details['TURN.username'], |
| 140 | credential: account.details['TURN.password'], |
| 141 | }); |
| 142 | } |
| 143 | |
| 144 | if (account.details['STUN.enable'] === 'true') { |
| 145 | iceServers.push({ |
| 146 | urls: 'stun:' + account.details['STUN.server'], |
| 147 | }); |
| 148 | } |
| 149 | |
| 150 | return iceServers; |
| 151 | } |
| 152 | |
| 153 | private startNegociation(webSocket: IWebSocketContext, contactUri: string, conversationId: string) { |
| 154 | this.sendWebRtcOffer(webSocket, contactUri, conversationId); |
| 155 | } |
| 156 | |
| 157 | private async sendWebRtcOffer(webSocket: IWebSocketContext, contactUri: string, conversationId: string) { |
| 158 | const sdp = await this.connection.createOffer({ |
| 159 | offerToReceiveAudio: true, |
| 160 | offerToReceiveVideo: true, |
| 161 | }); |
| 162 | |
idillon | 255682e | 2023-02-06 13:25:26 -0500 | [diff] [blame] | 163 | const webRtcOffer: WebSocketMessageTable['sendWebRtcOffer'] = { |
| 164 | receiverId: contactUri, |
idillon | 989d654 | 2023-01-30 15:49:58 -0500 | [diff] [blame] | 165 | conversationId: conversationId, |
| 166 | sdp, |
| 167 | }; |
| 168 | |
| 169 | await this.connection.setLocalDescription(new RTCSessionDescription(sdp)); |
| 170 | console.info('Sending WebRtcOffer', webRtcOffer); |
idillon | 255682e | 2023-02-06 13:25:26 -0500 | [diff] [blame] | 171 | webSocket.send(WebSocketMessageType.sendWebRtcOffer, webRtcOffer); |
idillon | 989d654 | 2023-01-30 15:49:58 -0500 | [diff] [blame] | 172 | } |
| 173 | |
| 174 | private setWebSocketListeners(webSocket: IWebSocketContext, conversationId: string, contactUri: string) { |
| 175 | const sendWebRtcAnswer = async () => { |
| 176 | const sdp = await this.connection.createAnswer({ |
| 177 | offerToReceiveAudio: true, |
| 178 | offerToReceiveVideo: true, |
| 179 | }); |
| 180 | |
idillon | 255682e | 2023-02-06 13:25:26 -0500 | [diff] [blame] | 181 | const webRtcAnswer: WebSocketMessageTable['sendWebRtcAnswer'] = { |
| 182 | receiverId: contactUri, |
idillon | 989d654 | 2023-01-30 15:49:58 -0500 | [diff] [blame] | 183 | conversationId: conversationId, |
| 184 | sdp, |
| 185 | }; |
| 186 | |
| 187 | await this.connection.setLocalDescription(new RTCSessionDescription(sdp)); |
| 188 | console.info('Sending WebRtcAnswer', webRtcAnswer); |
idillon | 255682e | 2023-02-06 13:25:26 -0500 | [diff] [blame] | 189 | webSocket.send(WebSocketMessageType.sendWebRtcAnswer, webRtcAnswer); |
idillon | 989d654 | 2023-01-30 15:49:58 -0500 | [diff] [blame] | 190 | }; |
| 191 | |
| 192 | const addQueuedIceCandidates = async () => { |
| 193 | console.info('WebRTC remote description has been set. Ready to receive ICE candidates'); |
| 194 | this.isReadyForIceCandidates = true; |
| 195 | if (this.iceCandidateQueue.length !== 0) { |
| 196 | console.warn( |
| 197 | 'Found queued ICE candidates that were added before `setRemoteDescription` was called. ' + |
| 198 | 'Adding queued ICE candidates...', |
| 199 | this.iceCandidateQueue |
| 200 | ); |
| 201 | |
| 202 | await Promise.all(this.iceCandidateQueue.map((iceCandidate) => this.connection.addIceCandidate(iceCandidate))); |
| 203 | } |
| 204 | }; |
| 205 | |
| 206 | const webRtcOfferListener = async (data: WebRtcSdp) => { |
| 207 | console.debug('receive webrtcoffer'); |
| 208 | console.info('Received event on WebRtcOffer', data); |
| 209 | if (data.conversationId !== conversationId) { |
| 210 | console.warn('Wrong incoming conversationId, ignoring action'); |
| 211 | return; |
| 212 | } |
| 213 | |
| 214 | await this.connection.setRemoteDescription(new RTCSessionDescription(data.sdp)); |
| 215 | await sendWebRtcAnswer(); |
| 216 | await addQueuedIceCandidates(); |
| 217 | }; |
| 218 | |
| 219 | const webRtcAnswerListener = async (data: WebRtcSdp) => { |
| 220 | console.info('Received event on WebRtcAnswer', data); |
| 221 | if (data.conversationId !== conversationId) { |
| 222 | console.warn('Wrong incoming conversationId, ignoring action'); |
| 223 | return; |
| 224 | } |
| 225 | |
| 226 | await this.connection.setRemoteDescription(new RTCSessionDescription(data.sdp)); |
| 227 | await addQueuedIceCandidates(); |
| 228 | }; |
| 229 | |
| 230 | const webRtcIceCandidateListener = async (data: WebRtcIceCandidate) => { |
| 231 | if (data.conversationId !== conversationId) { |
| 232 | console.warn('Wrong incoming conversationId, ignoring action'); |
| 233 | return; |
| 234 | } |
| 235 | |
| 236 | if (!data.candidate) { |
| 237 | return; |
| 238 | } |
| 239 | |
| 240 | if (this.isReadyForIceCandidates) { |
| 241 | await this.connection.addIceCandidate(data.candidate); |
| 242 | } else { |
| 243 | this.iceCandidateQueue.push(data.candidate); |
| 244 | } |
| 245 | }; |
| 246 | |
idillon | 255682e | 2023-02-06 13:25:26 -0500 | [diff] [blame] | 247 | webSocket.bind(WebSocketMessageType.onWebRtcOffer, webRtcOfferListener); |
| 248 | webSocket.bind(WebSocketMessageType.onWebRtcAnswer, webRtcAnswerListener); |
| 249 | webSocket.bind(WebSocketMessageType.onWebRtcIceCandidate, webRtcIceCandidateListener); |
idillon | 989d654 | 2023-01-30 15:49:58 -0500 | [diff] [blame] | 250 | |
| 251 | this.cleaningFunctions.push(() => { |
idillon | 255682e | 2023-02-06 13:25:26 -0500 | [diff] [blame] | 252 | webSocket.unbind(WebSocketMessageType.onWebRtcOffer, webRtcOfferListener); |
| 253 | webSocket.unbind(WebSocketMessageType.onWebRtcAnswer, webRtcAnswerListener); |
| 254 | webSocket.unbind(WebSocketMessageType.onWebRtcIceCandidate, webRtcIceCandidateListener); |
idillon | 989d654 | 2023-01-30 15:49:58 -0500 | [diff] [blame] | 255 | }); |
| 256 | } |
| 257 | |
| 258 | private setConnectionListeners(webSocket: IWebSocketContext, conversationId: string, contactUri: string) { |
| 259 | this.connection.onicecandidate = (event: RTCPeerConnectionIceEvent) => { |
| 260 | if (event.candidate) { |
idillon | 255682e | 2023-02-06 13:25:26 -0500 | [diff] [blame] | 261 | const webRtcIceCandidate: WebSocketMessageTable['sendWebRtcIceCandidate'] = { |
| 262 | receiverId: contactUri, |
idillon | 989d654 | 2023-01-30 15:49:58 -0500 | [diff] [blame] | 263 | conversationId: conversationId, |
| 264 | candidate: event.candidate, |
| 265 | }; |
| 266 | |
| 267 | // Send ice candidates as soon as they're found. This is called "trickle ice" |
idillon | 255682e | 2023-02-06 13:25:26 -0500 | [diff] [blame] | 268 | webSocket.send(WebSocketMessageType.sendWebRtcIceCandidate, webRtcIceCandidate); |
idillon | 989d654 | 2023-01-30 15:49:58 -0500 | [diff] [blame] | 269 | } |
| 270 | }; |
| 271 | |
| 272 | this.connection.ontrack = (event: RTCTrackEvent) => { |
| 273 | console.info('Received WebRTC event on track', event); |
| 274 | this.remoteStreams = event.streams; |
| 275 | this.emitChange(); |
| 276 | }; |
| 277 | |
| 278 | this.connection.oniceconnectionstatechange = (event: Event) => { |
| 279 | console.info(`Received WebRTC event on iceconnectionstatechange: ${this.connection.iceConnectionState}`, event); |
| 280 | this.iceConnectionState = this.connection.iceConnectionState; |
| 281 | this.emitChange(); |
| 282 | }; |
| 283 | } |
| 284 | |
| 285 | private emitChange() { |
| 286 | this.listener(); |
| 287 | } |
| 288 | } |