Implement proper WebRTC call management
This is a big CR that implements the whole proper call logic.
-- Other Contributors --
This CR was created in pair programming with:
- Charlie Duquette <charlie_duquette@hotmail.fr>
-- New files --
- `ConversationProvider`: Provides the conversation object to its children. Also contains the function to begin a call (first step in the flow) which sends the `BeginCall` event.
- `CallProvider`: Contains the call logic that was previously in WebRTCProvider. From now on, WebRTCProvider contains only the WebRTC logic, while CallProvider
contains everything related to the call (get the media devices, start/accept calls...)
- `NotificationManager`: Wrapper component to bind the WebSocket CallBegin event listener. That listener will fire when a `BeginCall` event is received and will then redirect the user to the call receiving page.
-- New routes --
When a `conversationId` is included in the URL, all pages are inside a `<ConversationProvider />`.
When starting a call, the caller is redirected to:
> http://localhost:3000/conversation/:conversationId/call?role=caller
When receiving a call, the receiver is redirected to:
> http://localhost:3000/conversation/:conversationId/call?role=receiver&hostId={HOST_ID}
When the user is in a `.../call` route, the `WebRTCContext` and
`CallContext` are provided
The `hostId` is the account URI of the caller. It's used when the receiver wants to send their answer back to the caller.
```
/
|-- login: <Welcome />
|-- settings: <AccountSettings />
|-- ...
`-- conversation: <Messenger />
|-- add-contact/:contactId
`-- :conversationId: <ConversationProvider />
|-- /: <ConversationView />
`-- /call: <WebRTCProvider>
<CallProvider>
<CallInterface />
</CallProvider>
</WebRTCProvider>
```
-- Call flow --
1. Caller:
- Clicks "Call" button
- Sends `BeginCall` event
- Redirects to call page `/call?role=caller`
- Sets `callStatus` to "Ringing"
2. Receiver:
- Receieves `BeginCall` event
- The callback in `NotificationManager` is called
- Redirects to the call receiving page `/conversation/{CONVERSATION_ID}/call?role=receiver`
3. Receiver:
- Clicks the "Answer call" button
- Sends a `CallAccept` event
- Sets `callStatus` to "Connecting"
4. Caller:
- Receives `CallAccept` event
- The callback in `CallProvider` is called
- Sets `callStatus` to "Connecting"
- Creates WebRTC Offer
- Sends `WebRTCOffer` event containing the offer SDP
5. Receiver:
- Receives `WebRTCOffer` event
- The callback in `WebRTCProvider` is called
- Sets WebRTC remote description.
- WebRTC `icecandidate` event fired. Sends `IceCandidate` WebSocket event
- Creates WebRTC answer
- Sends `WebRTCAnswer` event
- Sets WebRTC local description
- Sets connected status to true. Call page now shows the call interface
6. Caller:
- Receives `WebRTCAnswer` event
- Sets WebRTC local description
- Sets WebRTC remote description
- WebRTC `icecandidate` event fired. Sends `IceCandidate` WebSocket event
- Sets connected status to true. Call page now shows the call interface
-- Misc Changes --
- Improve CallPending and CallInterface UI
- Move `useUrlParams` hook from the (now deleted) `client/src/utils/hooks.ts` file to `client/src/hooks/useUrlParams.ts`
- Disable StrictMode. This was causing issues, because some event would be sent twice. There is a TODO comment to fix the problem and reenable it.
- Improvements in server `webrtc-handler.ts`. This is still a WIP
- Rename dash-case client files to camelCase
GitLab: #70
Change-Id: I6c75f6b867e8acb9ccaaa118b0123bba30431f78
diff --git a/client/src/contexts/CallProvider.tsx b/client/src/contexts/CallProvider.tsx
new file mode 100644
index 0000000..0cc667a
--- /dev/null
+++ b/client/src/contexts/CallProvider.tsx
@@ -0,0 +1,249 @@
+/*
+ * Copyright (C) 2022 Savoir-faire Linux Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation; either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public
+ * License along with this program. If not, see
+ * <https://www.gnu.org/licenses/>.
+ */
+import { AccountTextMessage, WebSocketMessageType } from 'jami-web-common';
+import React, { createContext, useCallback, useContext, useEffect, useMemo, useState } from 'react';
+import { Navigate } from 'react-router-dom';
+
+import { useUrlParams } from '../hooks/useUrlParams';
+import { CallRouteParams } from '../router';
+import { WithChildren } from '../utils/utils';
+import { useAuthContext } from './AuthProvider';
+import { ConversationContext } from './ConversationProvider';
+import { WebRTCContext } from './WebRTCProvider';
+import { WebSocketContext } from './WebSocketProvider';
+
+export type CallRole = 'caller' | 'receiver';
+
+export enum CallStatus {
+ Ringing,
+ Connecting,
+ InCall,
+}
+
+export interface ICallContext {
+ mediaDevices: Record<MediaDeviceKind, MediaDeviceInfo[]>;
+
+ localStream: MediaStream | undefined;
+ remoteStream: MediaStream | undefined; // TODO: should be an array of participants. find a way to map MediaStream id to contactid https://stackoverflow.com/a/68663155/6592293
+
+ isAudioOn: boolean;
+ setAudioStatus: (isOn: boolean) => void;
+ isVideoOn: boolean;
+ setVideoStatus: (isOn: boolean) => void;
+ callRole: CallRole;
+ callStatus: CallStatus;
+
+ acceptCall: () => void;
+}
+
+const defaultCallContext: ICallContext = {
+ mediaDevices: {
+ audioinput: [],
+ audiooutput: [],
+ videoinput: [],
+ },
+
+ localStream: undefined,
+ remoteStream: undefined,
+
+ isAudioOn: false,
+ setAudioStatus: () => {},
+ isVideoOn: false,
+ setVideoStatus: () => {},
+ callRole: 'caller',
+ callStatus: CallStatus.Ringing,
+
+ acceptCall: () => {},
+};
+
+export const CallContext = createContext<ICallContext>(defaultCallContext);
+
+export default ({ children }: WithChildren) => {
+ const {
+ queryParams: { role: callRole },
+ } = useUrlParams<CallRouteParams>();
+ const { account } = useAuthContext();
+ const webSocket = useContext(WebSocketContext);
+ const { webRTCConnection, remoteStreams, sendWebRTCOffer, isConnected } = useContext(WebRTCContext);
+ const { conversation } = useContext(ConversationContext);
+
+ const [mediaDevices, setMediaDevices] = useState<Record<MediaDeviceKind, MediaDeviceInfo[]>>(
+ defaultCallContext.mediaDevices
+ );
+ const [localStream, setLocalStream] = useState<MediaStream>();
+
+ const [isAudioOn, setIsAudioOn] = useState(false);
+ const [isVideoOn, setIsVideoOn] = useState(false);
+ const [callStatus, setCallStatus] = useState(CallStatus.Ringing);
+
+ // TODO: This logic will have to change to support multiple people in a call
+ const contactUri = useMemo(() => conversation.getFirstMember().contact.getUri(), [conversation]);
+
+ useEffect(() => {
+ // TODO: Wait until status is `InCall` before getting devices
+ navigator.mediaDevices.enumerateDevices().then((devices) => {
+ const newMediaDevices: Record<MediaDeviceKind, MediaDeviceInfo[]> = {
+ audioinput: [],
+ audiooutput: [],
+ videoinput: [],
+ };
+
+ for (const device of devices) {
+ newMediaDevices[device.kind].push(device);
+ }
+
+ setMediaDevices(newMediaDevices);
+ });
+ }, []);
+
+ useEffect(() => {
+ // TODO: Only ask media permission once the call has been accepted
+ try {
+ // TODO: When toggling mute on/off, the camera flickers
+ // https://git.jami.net/savoirfairelinux/jami-web/-/issues/90
+ navigator.mediaDevices
+ .getUserMedia({
+ audio: true, // TODO: Set both to false by default
+ video: true,
+ })
+ .then((stream) => {
+ for (const track of stream.getTracks()) {
+ // TODO: Set default from isVideoOn and isMicOn values
+ track.enabled = false;
+ }
+ setLocalStream(stream);
+ });
+ } catch (e) {
+ // TODO: Better handle user denial
+ console.error('Could not get media devices:', e);
+ }
+ }, [setLocalStream]);
+
+ useEffect(() => {
+ if (localStream && webRTCConnection) {
+ for (const track of localStream.getTracks()) {
+ webRTCConnection.addTrack(track, localStream);
+ }
+ }
+ }, [localStream, webRTCConnection]);
+
+ const setAudioStatus = useCallback(
+ (isOn: boolean) => {
+ if (!localStream) {
+ return;
+ }
+
+ for (const track of localStream.getAudioTracks()) {
+ track.enabled = isOn;
+ }
+
+ setIsAudioOn(isOn);
+ },
+ [localStream]
+ );
+
+ const setVideoStatus = useCallback(
+ (isOn: boolean) => {
+ if (!localStream) {
+ return;
+ }
+
+ for (const track of localStream.getVideoTracks()) {
+ track.enabled = isOn;
+ }
+
+ setIsVideoOn(isOn);
+ },
+ [localStream]
+ );
+
+ useEffect(() => {
+ if (!webSocket || !webRTCConnection) {
+ return;
+ }
+
+ if (callRole === 'caller' && callStatus === CallStatus.Ringing) {
+ const callAcceptListener = (data: AccountTextMessage<undefined>) => {
+ console.info('Received event on CallAccept', data);
+ setCallStatus(CallStatus.Connecting);
+
+ webRTCConnection
+ .createOffer({
+ offerToReceiveAudio: true,
+ offerToReceiveVideo: true,
+ })
+ .then((offerSDP) => {
+ sendWebRTCOffer(offerSDP);
+ });
+ };
+
+ webSocket.bind(WebSocketMessageType.CallAccept, callAcceptListener);
+
+ return () => {
+ webSocket.unbind(WebSocketMessageType.CallAccept, callAcceptListener);
+ };
+ }
+ }, [callRole, webSocket, webRTCConnection, sendWebRTCOffer, callStatus]);
+
+ useEffect(() => {
+ if (callStatus === CallStatus.Connecting && isConnected) {
+ console.info('Changing call status to InCall');
+ setCallStatus(CallStatus.InCall);
+ }
+ }, [isConnected, callStatus]);
+
+ const acceptCall = useCallback(() => {
+ if (!webSocket) {
+ throw new Error('Could not accept call');
+ }
+
+ const callAccept = {
+ from: account.getId(),
+ to: contactUri,
+ message: undefined,
+ };
+
+ console.info('Sending CallAccept', callAccept);
+ webSocket.send(WebSocketMessageType.CallAccept, callAccept);
+ setCallStatus(CallStatus.Connecting);
+ }, [webSocket, account, contactUri]);
+
+ if (!callRole) {
+ console.error('Call role not defined. Redirecting...');
+ return <Navigate to={'/'} />;
+ }
+
+ return (
+ <CallContext.Provider
+ value={{
+ mediaDevices,
+ localStream,
+ remoteStream: remoteStreams?.at(-1),
+ isAudioOn,
+ setAudioStatus,
+ isVideoOn,
+ setVideoStatus,
+ callRole,
+ callStatus,
+ acceptCall,
+ }}
+ >
+ {children}
+ </CallContext.Provider>
+ );
+};