Improve permission handling in call flow
Improve permission handling by asking the user to give mic and camera permissions before sending `CallBegin` or `CallAccept` for the caller and receiver respectively.
Followed the flow described here: https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Connectivity#session_descriptions
CallProvider:
- Change functions order to place listeners under the function that sends the corresponding WebSocket message.
- Replace `Default` CallStatus with `Loading` for when asking user permissions before sending the `CallBegin`/`CallAccept` message.
- Remove `localStream` and `remoteStream` from `CallContext`. They are now available only in `WebRtcContext`.
- Replace `setAudioStatus` and `setVideoStatus` with `setIsAudioOn` and `setIsVideoOn`. A `useEffect` is now used to disable the tracks when the audio/video status changes.
WebRtcProvider:
- Move WebRTC connection close logic to WebRtcProvider
- Remove `webRtcConnection` from `WebRtcContext`. `WebRtcProvider` is now in charge of setting everything related to the WebRTC Connection.
UI:
- Add `CallPermissionDenied` page for when permissions are denied.
- Rework `CallPending` to display `Loading...` when waiting for user permissions
Change-Id: I48153577cca4c73cdb9b81d2fa78cfdfe2e06d69
diff --git a/client/src/contexts/CallProvider.tsx b/client/src/contexts/CallProvider.tsx
index 8db38e0..31b6673 100644
--- a/client/src/contexts/CallProvider.tsx
+++ b/client/src/contexts/CallProvider.tsx
@@ -21,6 +21,7 @@
import LoadingPage from '../components/Loading';
import { useUrlParams } from '../hooks/useUrlParams';
+import CallPermissionDenied from '../pages/CallPermissionDenied';
import { CallRouteParams } from '../router';
import { callTimeoutMs } from '../utils/constants';
import { SetState, WithChildren } from '../utils/utils';
@@ -32,21 +33,18 @@
export enum CallStatus {
Default,
+ Loading,
Ringing,
Connecting,
InCall,
+ PermissionsDenied,
}
export interface ICallContext {
- mediaDevices: Record<MediaDeviceKind, MediaDeviceInfo[]>;
-
- localStream: MediaStream | undefined;
- remoteStream: MediaStream | undefined; // TODO: should be an array of participants. find a way to map MediaStream id to contactid https://stackoverflow.com/a/68663155/6592293
-
isAudioOn: boolean;
- setAudioStatus: (isOn: boolean) => void;
+ setIsAudioOn: SetState<boolean>;
isVideoOn: boolean;
- setVideoStatus: (isOn: boolean) => void;
+ setIsVideoOn: SetState<boolean>;
isChatShown: boolean;
setIsChatShown: SetState<boolean>;
isFullscreen: boolean;
@@ -60,19 +58,10 @@
}
const defaultCallContext: ICallContext = {
- mediaDevices: {
- audioinput: [],
- audiooutput: [],
- videoinput: [],
- },
-
- localStream: undefined,
- remoteStream: undefined,
-
isAudioOn: false,
- setAudioStatus: () => {},
+ setIsAudioOn: () => {},
isVideoOn: false,
- setVideoStatus: () => {},
+ setIsVideoOn: () => {},
isChatShown: false,
setIsChatShown: () => {},
isFullscreen: false,
@@ -89,37 +78,25 @@
export default ({ children }: WithChildren) => {
const webSocket = useContext(WebSocketContext);
- const { webRtcConnection } = useContext(WebRtcContext);
- if (!webSocket || !webRtcConnection) {
+ if (!webSocket) {
return <LoadingPage />;
}
- return (
- <CallProvider webSocket={webSocket} webRtcConnection={webRtcConnection}>
- {children}
- </CallProvider>
- );
+ return <CallProvider webSocket={webSocket}>{children}</CallProvider>;
};
const CallProvider = ({
children,
webSocket,
- webRtcConnection,
}: WithChildren & {
webSocket: IWebSocketContext;
- webRtcConnection: RTCPeerConnection;
}) => {
const { state: routeState } = useUrlParams<CallRouteParams>();
- const { remoteStreams, sendWebRtcOffer, iceConnectionState } = useContext(WebRtcContext);
+ const { localStream, sendWebRtcOffer, iceConnectionState, closeConnection, getUserMedia } = useContext(WebRtcContext);
const { conversationId, conversation } = useContext(ConversationContext);
const navigate = useNavigate();
- const [mediaDevices, setMediaDevices] = useState<Record<MediaDeviceKind, MediaDeviceInfo[]>>(
- defaultCallContext.mediaDevices
- );
- const [localStream, setLocalStream] = useState<MediaStream>();
-
const [isAudioOn, setIsAudioOn] = useState(false);
const [isVideoOn, setIsVideoOn] = useState(false);
const [isChatShown, setIsChatShown] = useState(false);
@@ -134,96 +111,20 @@
const contactUri = useMemo(() => conversation.getFirstMember().contact.getUri(), [conversation]);
useEffect(() => {
- try {
- // TODO: Wait until status is `InCall` before getting devices
- navigator.mediaDevices.enumerateDevices().then((devices) => {
- const newMediaDevices: Record<MediaDeviceKind, MediaDeviceInfo[]> = {
- audioinput: [],
- audiooutput: [],
- videoinput: [],
- };
-
- for (const device of devices) {
- newMediaDevices[device.kind].push(device);
- }
-
- setMediaDevices(newMediaDevices);
- });
- } catch (e) {
- console.error('Could not get media devices:', e);
+ if (localStream) {
+ for (const track of localStream.getAudioTracks()) {
+ track.enabled = isAudioOn;
+ }
}
-
- try {
- navigator.mediaDevices
- .getUserMedia({
- audio: true, // TODO: Set both to false by default
- video: true,
- })
- .then((stream) => {
- for (const track of stream.getTracks()) {
- // TODO: Set default from isVideoOn and isMicOn values
- track.enabled = false;
- }
- setLocalStream(stream);
- });
- } catch (e) {
- // TODO: Better handle user denial
- console.error('Could not get media devices:', e);
- }
- }, []);
+ }, [isAudioOn, localStream]);
useEffect(() => {
if (localStream) {
- for (const track of localStream.getTracks()) {
- webRtcConnection.addTrack(track, localStream);
- }
- }
- }, [localStream, webRtcConnection]);
-
- const setAudioStatus = useCallback(
- (isOn: boolean) => {
- if (!localStream) {
- return;
- }
-
- for (const track of localStream.getAudioTracks()) {
- track.enabled = isOn;
- }
-
- setIsAudioOn(isOn);
- },
- [localStream]
- );
-
- const setVideoStatus = useCallback(
- (isOn: boolean) => {
- if (!localStream) {
- return;
- }
-
for (const track of localStream.getVideoTracks()) {
- track.enabled = isOn;
+ track.enabled = isVideoOn;
}
-
- setIsVideoOn(isOn);
- },
- [localStream]
- );
-
- useEffect(() => {
- if (callRole === 'caller' && callStatus === CallStatus.Default) {
- const callBegin: CallBegin = {
- contactId: contactUri,
- conversationId,
- withVideoOn: routeState?.isVideoOn ?? false,
- };
-
- console.info('Sending CallBegin', callBegin);
- webSocket.send(WebSocketMessageType.CallBegin, callBegin);
- setCallStatus(CallStatus.Ringing);
- setIsVideoOn(routeState?.isVideoOn ?? false);
}
- }, [webSocket, callRole, callStatus, contactUri, conversationId, routeState]);
+ }, [isVideoOn, localStream]);
useEffect(() => {
const onFullscreenChange = () => {
@@ -237,6 +138,53 @@
}, []);
useEffect(() => {
+ if (callRole === 'caller' && callStatus === CallStatus.Default) {
+ setCallStatus(CallStatus.Loading);
+ getUserMedia()
+ .then(() => {
+ const callBegin: CallBegin = {
+ contactId: contactUri,
+ conversationId,
+ withVideoOn: routeState?.isVideoOn ?? false,
+ };
+
+ setCallStatus(CallStatus.Ringing);
+ setIsVideoOn(routeState?.isVideoOn ?? false);
+ console.info('Sending CallBegin', callBegin);
+ webSocket.send(WebSocketMessageType.CallBegin, callBegin);
+ })
+ .catch((e) => {
+ console.error(e);
+ setCallStatus(CallStatus.PermissionsDenied);
+ });
+ }
+ }, [webSocket, getUserMedia, callRole, callStatus, contactUri, conversationId, routeState]);
+
+ const acceptCall = useCallback(
+ (withVideoOn: boolean) => {
+ setCallStatus(CallStatus.Loading);
+
+ getUserMedia()
+ .then(() => {
+ const callAccept: CallAction = {
+ contactId: contactUri,
+ conversationId,
+ };
+
+ setIsVideoOn(withVideoOn);
+ setCallStatus(CallStatus.Connecting);
+ console.info('Sending CallAccept', callAccept);
+ webSocket.send(WebSocketMessageType.CallAccept, callAccept);
+ })
+ .catch((e) => {
+ console.error(e);
+ setCallStatus(CallStatus.PermissionsDenied);
+ });
+ },
+ [webSocket, getUserMedia, contactUri, conversationId]
+ );
+
+ useEffect(() => {
if (callRole === 'caller' && callStatus === CallStatus.Ringing) {
const callAcceptListener = (data: CallAction) => {
console.info('Received event on CallAccept', data);
@@ -247,14 +195,7 @@
setCallStatus(CallStatus.Connecting);
- webRtcConnection
- .createOffer({
- offerToReceiveAudio: true,
- offerToReceiveVideo: true,
- })
- .then((sdp) => {
- sendWebRtcOffer(sdp);
- });
+ sendWebRtcOffer();
};
webSocket.bind(WebSocketMessageType.CallAccept, callAcceptListener);
@@ -263,19 +204,20 @@
webSocket.unbind(WebSocketMessageType.CallAccept, callAcceptListener);
};
}
- }, [callRole, webSocket, webRtcConnection, sendWebRtcOffer, callStatus, conversationId]);
+ }, [callRole, webSocket, sendWebRtcOffer, callStatus, conversationId]);
- const quitCall = useCallback(() => {
- const localTracks = localStream?.getTracks();
- if (localTracks) {
- for (const track of localTracks) {
- track.stop();
- }
- }
+ const endCall = useCallback(() => {
+ const callEnd: CallAction = {
+ contactId: contactUri,
+ conversationId,
+ };
- webRtcConnection.close();
+ console.info('Sending CallEnd', callEnd);
+ closeConnection();
+ webSocket.send(WebSocketMessageType.CallEnd, callEnd);
navigate(`/conversation/${conversationId}`);
- }, [webRtcConnection, localStream, navigate, conversationId]);
+ // TODO: write in chat that the call ended
+ }, [webSocket, contactUri, conversationId, closeConnection, navigate]);
useEffect(() => {
const callEndListener = (data: CallAction) => {
@@ -285,7 +227,8 @@
return;
}
- quitCall();
+ closeConnection();
+ navigate(`/conversation/${conversationId}`);
// TODO: write in chat that the call ended
};
@@ -293,7 +236,7 @@
return () => {
webSocket.unbind(WebSocketMessageType.CallEnd, callEndListener);
};
- }, [webSocket, navigate, conversationId, quitCall]);
+ }, [webSocket, navigate, conversationId, closeConnection]);
useEffect(() => {
if (
@@ -302,44 +245,16 @@
) {
console.info('Changing call status to InCall');
setCallStatus(CallStatus.InCall);
- setVideoStatus(isVideoOn);
setCallStartTime(new Date());
}
- }, [iceConnectionState, callStatus, setVideoStatus, isVideoOn]);
-
- const acceptCall = useCallback(
- (withVideoOn: boolean) => {
- const callAccept: CallAction = {
- contactId: contactUri,
- conversationId,
- };
-
- console.info('Sending CallAccept', callAccept);
- webSocket.send(WebSocketMessageType.CallAccept, callAccept);
- setIsVideoOn(withVideoOn);
- setCallStatus(CallStatus.Connecting);
- },
- [webSocket, contactUri, conversationId]
- );
-
- const endCall = useCallback(() => {
- const callEnd: CallAction = {
- contactId: contactUri,
- conversationId,
- };
-
- console.info('Sending CallEnd', callEnd);
- webSocket.send(WebSocketMessageType.CallEnd, callEnd);
- quitCall();
- // TODO: write in chat that the call ended
- }, [webSocket, contactUri, conversationId, quitCall]);
+ }, [iceConnectionState, callStatus]);
useEffect(() => {
if (iceConnectionState === 'disconnected' || iceConnectionState === 'failed') {
console.info('ICE connection disconnected or failed, ending call');
endCall();
}
- }, [iceConnectionState, callStatus, setVideoStatus, isVideoOn, endCall]);
+ }, [iceConnectionState, callStatus, isVideoOn, endCall]);
useEffect(() => {
const checkStatusTimeout = () => {
@@ -369,13 +284,10 @@
return (
<CallContext.Provider
value={{
- mediaDevices,
- localStream,
- remoteStream: remoteStreams?.at(-1),
isAudioOn,
- setAudioStatus,
+ setIsAudioOn,
isVideoOn,
- setVideoStatus,
+ setIsVideoOn,
isChatShown,
setIsChatShown,
isFullscreen,
@@ -387,7 +299,7 @@
endCall,
}}
>
- {children}
+ {callStatus === CallStatus.PermissionsDenied ? <CallPermissionDenied /> : children}
</CallContext.Provider>
);
};
diff --git a/client/src/contexts/WebRtcProvider.tsx b/client/src/contexts/WebRtcProvider.tsx
index 63e2ca2..42ac308 100644
--- a/client/src/contexts/WebRtcProvider.tsx
+++ b/client/src/contexts/WebRtcProvider.tsx
@@ -28,17 +28,27 @@
interface IWebRtcContext {
iceConnectionState: RTCIceConnectionState | undefined;
+ mediaDevices: Record<MediaDeviceKind, MediaDeviceInfo[]>;
+ localStream: MediaStream | undefined;
remoteStreams: readonly MediaStream[] | undefined;
- webRtcConnection: RTCPeerConnection | undefined;
+ getUserMedia: () => Promise<void>;
- sendWebRtcOffer: (sdp: RTCSessionDescriptionInit) => Promise<void>;
+ sendWebRtcOffer: () => Promise<void>;
+ closeConnection: () => void;
}
const defaultWebRtcContext: IWebRtcContext = {
iceConnectionState: undefined,
+ mediaDevices: {
+ audioinput: [],
+ audiooutput: [],
+ videoinput: [],
+ },
+ localStream: undefined,
remoteStreams: undefined,
- webRtcConnection: undefined,
+ getUserMedia: async () => {},
sendWebRtcOffer: async () => {},
+ closeConnection: () => {},
};
export const WebRtcContext = createContext<IWebRtcContext>(defaultWebRtcContext);
@@ -66,7 +76,7 @@
});
}
- setWebRtcConnection(new RTCPeerConnection({ iceServers: iceServers }));
+ setWebRtcConnection(new RTCPeerConnection({ iceServers }));
}
}, [account, webRtcConnection]);
@@ -90,40 +100,117 @@
webSocket: IWebSocketContext;
}) => {
const { conversation, conversationId } = useContext(ConversationContext);
+ const [localStream, setLocalStream] = useState<MediaStream>();
const [remoteStreams, setRemoteStreams] = useState<readonly MediaStream[]>();
const [iceConnectionState, setIceConnectionState] = useState<RTCIceConnectionState | undefined>();
+ const [mediaDevices, setMediaDevices] = useState<Record<MediaDeviceKind, MediaDeviceInfo[]>>(
+ defaultWebRtcContext.mediaDevices
+ );
// TODO: This logic will have to change to support multiple people in a call
const contactUri = useMemo(() => conversation.getFirstMember().contact.getUri(), [conversation]);
- const sendWebRtcOffer = useCallback(
- async (sdp: RTCSessionDescriptionInit) => {
- const webRtcOffer: WebRtcSdp = {
- contactId: contactUri,
- conversationId: conversationId,
- sdp,
+ const getMediaDevices = useCallback(async () => {
+ try {
+ const devices = await navigator.mediaDevices.enumerateDevices();
+ const newMediaDevices: Record<MediaDeviceKind, MediaDeviceInfo[]> = {
+ audioinput: [],
+ audiooutput: [],
+ videoinput: [],
};
- await webRtcConnection.setLocalDescription(new RTCSessionDescription(sdp));
- console.info('Sending WebRtcOffer', webRtcOffer);
- webSocket.send(WebSocketMessageType.WebRtcOffer, webRtcOffer);
- },
- [webRtcConnection, webSocket, conversationId, contactUri]
- );
+ for (const device of devices) {
+ newMediaDevices[device.kind].push(device);
+ }
- const sendWebRtcAnswer = useCallback(
- (sdp: RTCSessionDescriptionInit) => {
- const webRtcAnswer: WebRtcSdp = {
- contactId: contactUri,
- conversationId: conversationId,
- sdp,
- };
+ return newMediaDevices;
+ } catch (e) {
+ throw new Error('Could not get media devices', { cause: e });
+ }
+ }, []);
- console.info('Sending WebRtcAnswer', webRtcAnswer);
- webSocket.send(WebSocketMessageType.WebRtcAnswer, webRtcAnswer);
- },
- [contactUri, conversationId, webSocket]
- );
+ useEffect(() => {
+ if (iceConnectionState !== 'connected' && iceConnectionState !== 'completed') {
+ return;
+ }
+
+ const updateMediaDevices = async () => {
+ try {
+ const newMediaDevices = await getMediaDevices();
+ setMediaDevices(newMediaDevices);
+ } catch (e) {
+ console.error('Could not update media devices:', e);
+ }
+ };
+
+ navigator.mediaDevices.addEventListener('devicechange', updateMediaDevices);
+ updateMediaDevices();
+
+ return () => {
+ navigator.mediaDevices.removeEventListener('devicechange', updateMediaDevices);
+ };
+ }, [getMediaDevices, iceConnectionState]);
+
+ const getUserMedia = useCallback(async () => {
+ const devices = await getMediaDevices();
+
+ const shouldGetAudio = devices.audioinput.length !== 0;
+ const shouldGetVideo = devices.videoinput.length !== 0;
+
+ if (!shouldGetAudio && !shouldGetVideo) {
+ return;
+ }
+
+ try {
+ const stream = await navigator.mediaDevices.getUserMedia({
+ audio: shouldGetAudio,
+ video: shouldGetVideo,
+ });
+
+ for (const track of stream.getTracks()) {
+ track.enabled = false;
+ webRtcConnection.addTrack(track, stream);
+ }
+
+ setLocalStream(stream);
+ } catch (e) {
+ throw new Error('Could not get media devices', { cause: e });
+ }
+ }, [webRtcConnection, getMediaDevices]);
+
+ const sendWebRtcOffer = useCallback(async () => {
+ const sdp = await webRtcConnection.createOffer({
+ offerToReceiveAudio: true,
+ offerToReceiveVideo: true,
+ });
+
+ const webRtcOffer: WebRtcSdp = {
+ contactId: contactUri,
+ conversationId: conversationId,
+ sdp,
+ };
+
+ await webRtcConnection.setLocalDescription(new RTCSessionDescription(sdp));
+ console.info('Sending WebRtcOffer', webRtcOffer);
+ webSocket.send(WebSocketMessageType.WebRtcOffer, webRtcOffer);
+ }, [webRtcConnection, webSocket, conversationId, contactUri]);
+
+ const sendWebRtcAnswer = useCallback(async () => {
+ const sdp = await webRtcConnection.createAnswer({
+ offerToReceiveAudio: true,
+ offerToReceiveVideo: true,
+ });
+
+ const webRtcAnswer: WebRtcSdp = {
+ contactId: contactUri,
+ conversationId: conversationId,
+ sdp,
+ };
+
+ await webRtcConnection.setLocalDescription(new RTCSessionDescription(sdp));
+ console.info('Sending WebRtcAnswer', webRtcAnswer);
+ webSocket.send(WebSocketMessageType.WebRtcAnswer, webRtcAnswer);
+ }, [contactUri, conversationId, webRtcConnection, webSocket]);
/* WebSocket Listeners */
@@ -136,13 +223,7 @@
}
await webRtcConnection.setRemoteDescription(new RTCSessionDescription(data.sdp));
-
- const sdp = await webRtcConnection.createAnswer({
- offerToReceiveAudio: true,
- offerToReceiveVideo: true,
- });
- await webRtcConnection.setLocalDescription(new RTCSessionDescription(sdp));
- sendWebRtcAnswer(sdp);
+ await sendWebRtcAnswer();
};
const webRtcAnswerListener = async (data: WebRtcSdp) => {
@@ -154,6 +235,7 @@
await webRtcConnection.setRemoteDescription(new RTCSessionDescription(data.sdp));
};
+
webSocket.bind(WebSocketMessageType.WebRtcOffer, webRtcOfferListener);
webSocket.bind(WebSocketMessageType.WebRtcAnswer, webRtcAnswerListener);
@@ -194,7 +276,6 @@
webSocket.send(WebSocketMessageType.WebRtcIceCandidate, webRtcIceCandidate);
}
};
-
webRtcConnection.addEventListener('icecandidate', iceCandidateEventListener);
return () => {
@@ -208,8 +289,8 @@
setRemoteStreams(event.streams);
};
- const iceConnectionStateChangeEventListener = () => {
- console.info('ICE connection state changed:', webRtcConnection.iceConnectionState);
+ const iceConnectionStateChangeEventListener = (event: Event) => {
+ console.info(`Received WebRTC event on iceconnectionstatechange: ${webRtcConnection.iceConnectionState}`, event);
setIceConnectionState(webRtcConnection.iceConnectionState);
};
@@ -222,13 +303,27 @@
};
}, [webRtcConnection]);
+ const closeConnection = useCallback(() => {
+ const localTracks = localStream?.getTracks();
+ if (localTracks) {
+ for (const track of localTracks) {
+ track.stop();
+ }
+ }
+
+ webRtcConnection.close();
+ }, [webRtcConnection, localStream]);
+
return (
<WebRtcContext.Provider
value={{
iceConnectionState,
+ mediaDevices,
+ localStream,
remoteStreams,
- webRtcConnection,
+ getUserMedia,
sendWebRtcOffer,
+ closeConnection,
}}
>
{children}