Add remote video overlay
Fix remote audio not working when viewing another conversation.
Add a remote video overlay when in a call, but viewing a different page.
New components:
- VideoOverlay
- RemoteVideoOverlay: Video overlay for the remote stream when in a
different page. Clicking it will redirect to the current call page.
- VideoStream: renders a video from a MediaStream and can
output audio to the device with id `audioOutDeviceId`
Misc changes:
- Can click local video to make bigger
Change-Id: If1bb0b10c137944c405d540f041ebfe8005f9251
diff --git a/client/src/components/Button.tsx b/client/src/components/Button.tsx
index 4e0d5ac..ddd99e9 100644
--- a/client/src/components/Button.tsx
+++ b/client/src/components/Button.tsx
@@ -34,7 +34,7 @@
} from '@mui/material';
import { styled } from '@mui/material/styles';
import EmojiPicker, { IEmojiData } from 'emoji-picker-react';
-import React, { ComponentType, MouseEvent, ReactNode, useCallback, useState } from 'react';
+import { ComponentType, MouseEvent, ReactNode, useCallback, useState } from 'react';
import {
Arrow2Icon,
diff --git a/client/src/components/VideoOverlay.tsx b/client/src/components/VideoOverlay.tsx
new file mode 100644
index 0000000..93106ab
--- /dev/null
+++ b/client/src/components/VideoOverlay.tsx
@@ -0,0 +1,106 @@
+/*
+ * Copyright (C) 2022 Savoir-faire Linux Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation; either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public
+ * License along with this program. If not, see
+ * <https://www.gnu.org/licenses/>.
+ */
+import { Box } from '@mui/material';
+import { useContext, useRef, useState } from 'react';
+import Draggable, { DraggableEventHandler } from 'react-draggable';
+import { useNavigate } from 'react-router-dom';
+
+import { CallContext } from '../contexts/CallProvider';
+import { WebRtcContext } from '../contexts/WebRtcProvider';
+import { VideoElementWithSinkId } from '../utils/utils';
+import VideoStream, { VideoStreamProps } from './VideoStream';
+
+type Size = 'small' | 'medium' | 'large';
+export type VideoOverlayProps = VideoStreamProps & {
+ onClick?: DraggableEventHandler;
+ size?: Size;
+};
+
+const sizeToDimentions: Record<Size, string> = {
+ small: '25%',
+ medium: '50%',
+ large: '75%',
+};
+
+const VideoOverlay = ({ onClick, size = 'medium', ...props }: VideoOverlayProps) => {
+ const videoRef = useRef<VideoElementWithSinkId | null>(null);
+ const [dragging, setDragging] = useState(false);
+
+ return (
+ <Box position="relative" width="100%" height="100%">
+ <Draggable
+ onDrag={() => setDragging(true)}
+ onStop={(...args) => {
+ if (!dragging && onClick) {
+ onClick(...args);
+ }
+
+ setDragging(false);
+ }}
+ bounds="parent"
+ nodeRef={videoRef}
+ >
+ <VideoStream
+ ref={videoRef}
+ {...props}
+ style={{
+ position: 'absolute',
+ right: 0,
+ borderRadius: '12px',
+ maxHeight: sizeToDimentions[size],
+ maxWidth: sizeToDimentions[size],
+ zIndex: 2,
+ transition: 'max-width, max-height .2s ease-in-out',
+ ...props.style,
+ }}
+ />
+ </Draggable>
+ </Box>
+ );
+};
+
+export const RemoteVideoOverlay = ({ callConversationId }: { callConversationId: string }) => {
+ const { remoteStreams } = useContext(WebRtcContext);
+ const {
+ currentMediaDeviceIds: {
+ audiooutput: { id: audioOutDeviceId },
+ },
+ } = useContext(CallContext);
+ const navigate = useNavigate();
+
+ // TODO: For now, `remoteStream` is the first remote stream in the array.
+ // There should only be one in the array, but we should make sure this is right.
+ const stream = remoteStreams?.at(0);
+
+ return (
+ <Box position="absolute" width="100%" height="100%" display="flex">
+ <Box margin={2} flexGrow={1}>
+ <VideoOverlay
+ stream={stream}
+ audioOutDeviceId={audioOutDeviceId}
+ onClick={() => {
+ navigate(`/conversation/${callConversationId}`);
+ }}
+ size={'small'}
+ />
+ </Box>
+ </Box>
+ );
+};
+
+export default VideoOverlay;
diff --git a/client/src/components/VideoStream.tsx b/client/src/components/VideoStream.tsx
new file mode 100644
index 0000000..4740e81
--- /dev/null
+++ b/client/src/components/VideoStream.tsx
@@ -0,0 +1,64 @@
+/*
+ * Copyright (C) 2022 Savoir-faire Linux Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation; either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public
+ * License along with this program. If not, see
+ * <https://www.gnu.org/licenses/>.
+ */
+import { forwardRef, useEffect, useRef, VideoHTMLAttributes } from 'react';
+
+import { VideoElementWithSinkId } from '../utils/utils';
+
+export type VideoStreamProps = Partial<VideoHTMLAttributes<VideoElementWithSinkId>> & {
+ stream: MediaStream | undefined;
+ audioOutDeviceId?: string;
+};
+
+const VideoStream = forwardRef<VideoElementWithSinkId, VideoStreamProps>(
+ ({ stream, audioOutDeviceId, ...props }, ref) => {
+ const videoRef = useRef<VideoElementWithSinkId | null>(null);
+
+ useEffect(() => {
+ if (!ref) {
+ return;
+ }
+
+ if (typeof ref === 'function') {
+ ref(videoRef.current);
+ } else {
+ ref.current = videoRef.current;
+ }
+ }, [ref]);
+
+ useEffect(() => {
+ if (stream && videoRef.current) {
+ videoRef.current.srcObject = stream;
+ }
+ }, [stream, videoRef]);
+
+ useEffect(() => {
+ if (!audioOutDeviceId) {
+ return;
+ }
+
+ if (videoRef.current?.setSinkId) {
+ // This only work on chrome and other browsers that support `setSinkId`
+ // https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/setSinkId#browser_compatibility
+ videoRef.current.setSinkId(audioOutDeviceId);
+ }
+ }, [audioOutDeviceId, videoRef]);
+
+ return <video ref={videoRef} autoPlay {...props} />;
+ }
+);
+export default VideoStream;
diff --git a/client/src/contexts/CallManagerProvider.tsx b/client/src/contexts/CallManagerProvider.tsx
index 4261f07..9bde186 100644
--- a/client/src/contexts/CallManagerProvider.tsx
+++ b/client/src/contexts/CallManagerProvider.tsx
@@ -19,7 +19,10 @@
import { createContext, useCallback, useContext, useEffect, useState } from 'react';
import { useNavigate } from 'react-router-dom';
+import { RemoteVideoOverlay } from '../components/VideoOverlay';
+import { useUrlParams } from '../hooks/useUrlParams';
import { Conversation } from '../models/conversation';
+import { ConversationRouteParams } from '../router';
import { useConversationQuery } from '../services/conversationQueries';
import { SetState, WithChildren } from '../utils/utils';
import CallProvider, { CallRole } from './CallProvider';
@@ -103,6 +106,8 @@
const CallManagerProvider = ({ children }: WithChildren) => {
const { callData } = useContext(CallManagerContext);
+ const { urlParams } = useUrlParams<ConversationRouteParams>();
+ const conversationId = urlParams.conversationId;
if (!callData) {
return <>{children}</>;
@@ -110,7 +115,12 @@
return (
<WebRtcProvider>
- <CallProvider>{children}</CallProvider>
+ <CallProvider>
+ {callData.conversationId !== conversationId && (
+ <RemoteVideoOverlay callConversationId={callData.conversationId} />
+ )}
+ {children}
+ </CallProvider>
</WebRtcProvider>
);
};
diff --git a/client/src/pages/CallInterface.tsx b/client/src/pages/CallInterface.tsx
index 4feecc2..3c98784 100644
--- a/client/src/pages/CallInterface.tsx
+++ b/client/src/pages/CallInterface.tsx
@@ -29,7 +29,6 @@
useRef,
useState,
} from 'react';
-import Draggable from 'react-draggable';
import { ExpandableButtonProps } from '../components/Button';
import {
@@ -46,6 +45,8 @@
CallingVolumeButton,
} from '../components/CallButtons';
import CallChatDrawer from '../components/CallChatDrawer';
+import VideoOverlay from '../components/VideoOverlay';
+import VideoStream from '../components/VideoStream';
import { CallContext, CallStatus, VideoStatus } from '../contexts/CallProvider';
import { useConversationContext } from '../contexts/ConversationProvider';
import { WebRtcContext } from '../contexts/WebRtcProvider';
@@ -89,58 +90,51 @@
}
const CallInterface = () => {
- const { remoteStreams } = useContext(WebRtcContext);
+ const { localStream, screenShareLocalStream, remoteStreams } = useContext(WebRtcContext);
const {
currentMediaDeviceIds: {
audiooutput: { id: audioOutDeviceId },
},
+ videoStatus,
} = useContext(CallContext);
const remoteVideoRef = useRef<VideoElementWithSinkId | null>(null);
const gridItemRef = useRef<HTMLDivElement | null>(null);
+ const [isLocalVideoZoomed, setIsLocalVideoZoomed] = useState(false);
- useEffect(() => {
- // TODO: For now, `remoteStream` is the first remote stream in the array.
- // There should only be one in the array, but we should make sure this is right.
- const remoteStream = remoteStreams?.at(0);
- if (remoteStream && remoteVideoRef.current) {
- remoteVideoRef.current.srcObject = remoteStream;
+ const stream = useMemo(() => {
+ switch (videoStatus) {
+ case VideoStatus.Camera:
+ return localStream;
+ case VideoStatus.ScreenShare:
+ return screenShareLocalStream;
}
- }, [remoteStreams, remoteVideoRef]);
-
- useEffect(() => {
- if (!audioOutDeviceId) {
- return;
- }
-
- if (remoteVideoRef.current?.setSinkId) {
- // This only work on chrome and other browsers that support `setSinkId`
- // https://developer.mozilla.org/en-US/docs/Web/API/HTMLMediaElement/setSinkId#browser_compatibility
- remoteVideoRef.current.setSinkId(audioOutDeviceId);
- }
- }, [audioOutDeviceId, remoteVideoRef]);
+ }, [videoStatus, localStream, screenShareLocalStream]);
const hasSetSinkId = remoteVideoRef.current?.setSinkId != null;
+ // TODO: For now, `remoteStream` is the first remote stream in the array.
+ // There should only be one in the array, but we should make sure this is right.
+ const remoteStream = remoteStreams?.at(0);
+
return (
<Box display="flex" flexGrow={1}>
- <video
+ <VideoStream
ref={remoteVideoRef}
- autoPlay
+ stream={remoteStream}
+ audioOutDeviceId={audioOutDeviceId}
style={{ zIndex: -1, backgroundColor: 'black', position: 'absolute', height: '100%', width: '100%' }}
/>
<Box flexGrow={1} margin={2} display="flex" flexDirection="column">
{/* Guest video, takes the whole screen */}
<CallInterfaceInformation />
<Box flexGrow={1} marginY={2} position="relative">
- <Box
- sx={{
- position: 'absolute',
- width: '100%',
- height: '100%',
- }}
- >
- <LocalVideo />
- </Box>
+ <VideoOverlay
+ stream={stream}
+ hidden={!stream}
+ muted
+ size={isLocalVideoZoomed ? 'large' : 'medium'}
+ onClick={() => setIsLocalVideoZoomed((v) => !v)}
+ />
</Box>
<Grid container>
<Grid item xs />
@@ -158,45 +152,6 @@
);
};
-const LocalVideo = () => {
- const { localStream, screenShareLocalStream } = useContext(WebRtcContext);
- const { videoStatus } = useContext(CallContext);
- const videoRef = useRef<VideoElementWithSinkId | null>(null);
-
- const stream = useMemo(() => {
- switch (videoStatus) {
- case VideoStatus.Camera:
- return localStream;
- case VideoStatus.ScreenShare:
- return screenShareLocalStream;
- }
- }, [videoStatus, localStream, screenShareLocalStream]);
-
- useEffect(() => {
- if (stream && videoRef.current) {
- videoRef.current.srcObject = stream;
- }
- }, [stream, videoRef]);
-
- return (
- <Draggable bounds="parent" nodeRef={videoRef ?? undefined}>
- <video
- ref={videoRef}
- autoPlay
- muted
- style={{
- position: 'absolute',
- borderRadius: '12px',
- maxHeight: '50%',
- maxWidth: '50%',
- right: 0,
- visibility: stream ? 'visible' : 'hidden',
- }}
- />
- </Draggable>
- );
-};
-
const formatElapsedSeconds = (elapsedSeconds: number): string => {
const seconds = Math.floor(elapsedSeconds % 60);
elapsedSeconds = Math.floor(elapsedSeconds / 60);