chore: POC media calling Video/screen sharing

apps/meteor/client/views/room/Room.tsx

@@ -11,6 +11,7 @@ import RoomE2EESetup from './E2EESetup/RoomE2EESetup';
 import Header from './Header';
 import MessageHighlightProvider from './MessageList/providers/MessageHighlightProvider';
 import RoomInvite from './RoomInvite';
+import MediaCallRoom from './body/MediaCallRoom';
 import RoomBody from './body/RoomBody';
 import { useRoom, useRoomSubscription } from './contexts/RoomContext';
 import { useAppsContextualBar } from './hooks/useAppsContextualBar';
@@ -50,7 +51,7 @@ const Room = (): ReactElement => {
 							data-qa-rc-room={room._id}
 							aria-label={roomLabel}
 							header={<Header room={room} />}
-							body={shouldDisplayE2EESetup ? <RoomE2EESetup /> : <RoomBody />}
+							body={shouldDisplayE2EESetup ? <RoomE2EESetup /> : <MediaCallRoom body={<RoomBody />} />}
 							aside={
 								(toolbox.tab?.tabComponent && (
 									<ErrorBoundary fallback={null}>

apps/meteor/client/views/room/body/MediaCallRoom.tsx

@@ -0,0 +1,36 @@
+// import { Box } from '@rocket.chat/fuselage';
+import type { IRoom } from '@rocket.chat/core-typings';
+import { isDirectMessageRoom } from '@rocket.chat/core-typings';
+import type { PeerInfo } from '@rocket.chat/ui-voip';
+import { MediaCallRoom as MediaCallRoomComponent, useMediaCallContext } from '@rocket.chat/ui-voip';
+import type { ReactNode } from 'react';
+import { memo } from 'react';
+
+import { useRoom } from '../contexts/RoomContext';
+
+const isMediaCallRoom = (room: IRoom, peerInfo?: PeerInfo) => {
+	if (!peerInfo || 'number' in peerInfo) {
+		return false;
+	}
+	if (!isDirectMessageRoom(room)) {
+		return false;
+	}
+	if (!room.uids || room.uids.length !== 2) {
+		return false;
+	}
+
+	return room.uids.includes(peerInfo.userId);
+};
+
+const MediaCallRoom = ({ body }: { body: ReactNode }) => {
+	const { peerInfo, state } = useMediaCallContext();
+	const room = useRoom();
+	console.log({ room, peerInfo });
+	if (state !== 'ongoing' || !isMediaCallRoom(room, peerInfo)) {
+		return body;
+	}
+
+	return <MediaCallRoomComponent body={body} />;
+};
+
+export default memo(MediaCallRoom);

apps/meteor/client/views/room/layout/RoomLayout.tsx

@@ -60,7 +60,7 @@ const RoomLayout = ({ header, body, footer, aside, ...props }: RoomLayoutProps):
 				<Suspense fallback={<HeaderSkeleton />}>{header}</Suspense>
 				<Box display='flex' flexGrow={1} overflow='hidden' height='full' position='relative'>
 					<Box display='flex' flexDirection='column' flexGrow={1} minWidth={0}>
-						<Box is='div' display='flex' flexDirection='column' flexGrow={1}>
+						<Box is='div' display='flex' flexDirection='column' flexGrow={1} maxHeight='100%'>
 							<Suspense fallback={null}>{body}</Suspense>
 						</Box>
 						{footer && <Suspense fallback={null}>{footer}</Suspense>}

ee/packages/media-calls/src/internal/agents/CallSignalProcessor.ts

@@ -96,7 +96,7 @@ export class UserActorSignalProcessor {
 		// 4. It's a hangup request with reason = 'another-client' and the request came from any valid client of either user
 		switch (signal.type) {
 			case 'local-sdp':
-				return this.saveLocalDescription(signal.sdp, signal.negotiationId);
+				return this.saveLocalDescription(signal.sdp, signal.negotiationId, signal.streams);
 			case 'answer':
 				return this.processAnswer(signal.answer);
 			case 'hangup':
@@ -118,12 +118,16 @@ export class UserActorSignalProcessor {
 		return mediaCallDirector.hangup(this.call, this.agent, reason);
 	}
 
-	protected async saveLocalDescription(sdp: RTCSessionDescriptionInit, negotiationId: string): Promise<void> {
+	protected async saveLocalDescription(
+		sdp: RTCSessionDescriptionInit,
+		negotiationId: string,
+		streams?: { tag: string; id: string }[],
+	): Promise<void> {
 		if (!this.signed) {
 			return;
 		}
 
-		await mediaCallDirector.saveWebrtcSession(this.call, this.agent, { sdp, negotiationId }, this.contractId);
+		await mediaCallDirector.saveWebrtcSession(this.call, this.agent, { sdp, negotiationId, streams }, this.contractId);
 	}
 
 	private async processAnswer(answer: CallAnswer): Promise<void> {

ee/packages/media-calls/src/internal/agents/UserActorAgent.ts

@@ -44,6 +44,7 @@ export class UserActorAgent extends BaseMediaCallAgent {
 			type: 'remote-sdp',
 			sdp: negotiation.offer,
 			negotiationId: negotiation._id,
+			streams: negotiation.offerStreams,
 		});
 	}
 
@@ -112,6 +113,7 @@ export class UserActorAgent extends BaseMediaCallAgent {
 				type: 'remote-sdp',
 				sdp: negotiation.answer,
 				negotiationId,
+				streams: negotiation.answerStreams,
 			});
 			return;
 		}
@@ -126,6 +128,7 @@ export class UserActorAgent extends BaseMediaCallAgent {
 			type: 'remote-sdp',
 			sdp: negotiation.offer,
 			negotiationId,
+			streams: negotiation.offerStreams,
 		});
 	}
 

ee/packages/media-calls/src/server/CallDirector.ts

@@ -1,4 +1,11 @@
-import type { IMediaCall, IMediaCallNegotiation, MediaCallContact, MediaCallSignedContact, ServerActor } from '@rocket.chat/core-typings';
+import type {
+	IMediaCall,
+	IMediaCallNegotiation,
+	MediaCallContact,
+	MediaCallSignedContact,
+	ServerActor,
+	MediaCallNegotiationStream,
+} from '@rocket.chat/core-typings';
 import type { CallHangupReason, CallRole } from '@rocket.chat/media-signaling';
 import type { InsertionModel } from '@rocket.chat/model-typings';
 import { MediaCallNegotiations, MediaCalls } from '@rocket.chat/models';
@@ -131,7 +138,7 @@ class MediaCallDirector {
 	public async saveWebrtcSession(
 		call: IMediaCall,
 		fromAgent: IMediaCallAgent,
-		session: { sdp: RTCSessionDescriptionInit; negotiationId: string },
+		session: { sdp: RTCSessionDescriptionInit; negotiationId: string; streams?: MediaCallNegotiationStream[] },
 		contractId: string,
 	): Promise<void> {
 		logger.debug({ msg: 'MediaCallDirector.saveWebrtcSession', callId: call?._id });
@@ -153,8 +160,8 @@ class MediaCallDirector {
 		}
 
 		const updater = isOffer
-			? MediaCallNegotiations.setOfferById(negotiation._id, session.sdp)
-			: MediaCallNegotiations.setAnswerById(negotiation._id, session.sdp);
+			? MediaCallNegotiations.setOfferById(negotiation._id, session.sdp, session.streams)
+			: MediaCallNegotiations.setAnswerById(negotiation._id, session.sdp, session.streams);
 		const updateResult = await updater;
 
 		if (!updateResult.modifiedCount) {

packages/core-typings/src/mediaCalls/IMediaCallNegotiation.ts

@@ -1,5 +1,10 @@
 import type { IRocketChatRecord } from '../IRocketChatRecord';
 
+export type MediaCallNegotiationStream = {
+	tag: string;
+	id: string;
+};
+
 export interface IMediaCallNegotiation extends IRocketChatRecord {
 	callId: string;
 
@@ -12,4 +17,7 @@ export interface IMediaCallNegotiation extends IRocketChatRecord {
 
 	offer?: RTCSessionDescriptionInit;
 	answer?: RTCSessionDescriptionInit;
+
+	offerStreams?: MediaCallNegotiationStream[];
+	answerStreams?: MediaCallNegotiationStream[];
 }

packages/media-signaling/src/definition/call/CallEvents.ts

@@ -28,4 +28,10 @@ export type CallEvents = {
 
 	/* Triggered when the call's state on the server changes to 'hangup' */
 	ended: void;
+
+	/* Triggered when screen share is toggled */
+	screenShareRequestChange: void;
+
+	/* Triggered when any of the streams or tracks have changed */
+	streamChange: void;
 };

packages/media-signaling/src/definition/call/IClientMediaCall.ts

@@ -1,6 +1,7 @@
 import type { Emitter } from '@rocket.chat/emitter';
 
 import type { CallEvents } from './CallEvents';
+import type { IMediaStreamWrapper } from '../media/IMediaStreamWrapper';
 
 export type CallActorType = 'user' | 'sip';
 
@@ -95,15 +96,19 @@ export interface IClientMediaCall {
 	/** confirmed indicates if the call exists on the server */
 	readonly confirmed: boolean;
 
+	readonly screenShareRequested: boolean;
+
 	emitter: Emitter<CallEvents>;
 
-	getRemoteMediaStream(): MediaStream | null;
+	getLocalMediaStream(tag?: string): IMediaStreamWrapper | null;
+	getRemoteMediaStream(tag?: string): IMediaStreamWrapper | null;
 
 	accept(): void;
 	reject(): void;
 	hangup(): void;
 	setMuted(muted: boolean): void;
 	setHeld(onHold: boolean): void;
+	setScreenShareRequested(requested: boolean): void;
 	transfer(callee: { type: CallActorType; id: string }): void;
 
 	sendDTMF(dtmf: string, duration?: number): void;

packages/media-signaling/src/definition/index.ts

@@ -1,5 +1,6 @@
 export * from './call';
 export * from './services';
+export * from './media';
 export * from './signals';
 export * from './client';
 export * from './logger';

packages/media-signaling/src/definition/media/IMediaStreamManager.ts

@@ -0,0 +1,26 @@
+import type { Emitter } from '@rocket.chat/emitter';
+
+import type { IMediaStreamWrapper } from './IMediaStreamWrapper';
+
+export type MediaStreamManagerEvents = {
+	streamChanged: void;
+};
+
+export interface IMediaStreamManager {
+	readonly emitter: Emitter<MediaStreamManagerEvents>;
+
+	readonly mainLocal: IMediaStreamWrapper;
+
+	readonly screenShareLocal: IMediaStreamWrapper;
+
+	readonly mainRemote: IMediaStreamWrapper;
+
+	readonly screenShareRemote: IMediaStreamWrapper;
+
+	getStreams(): IMediaStreamWrapper[];
+	getLocalStreams(): IMediaStreamWrapper[];
+	getRemoteStreams(): IMediaStreamWrapper[];
+
+	getLocalStreamByTag(tag: string): IMediaStreamWrapper | null;
+	getRemoteStreamByTag(tag: string): IMediaStreamWrapper | null;
+}

packages/media-signaling/src/definition/media/IMediaStreamWrapper.ts

@@ -0,0 +1,59 @@
+import type { Emitter } from '@rocket.chat/emitter';
+
+export type MediaStreamEvents = {
+	trackChanged: {
+		track: MediaStreamTrack | null;
+		kind: MediaStreamTrack['kind'];
+	};
+	stateChanged: void;
+};
+
+/**
+ * An object that holds a reference to a single MediaStream, plus additional data related to that specific stream
+ *
+ * We make a few assumptions about the stream based on our intendend use cases; Use multiple streams if you need, but make sure each individual stream follow the rules:
+ *
+ * 1. A stream MAY be empty (have no tracks)
+ *
+ * 2. A stream MAY have both an audio track and a video track at the same time, but only if they are related
+ *
+ * 3. A stream MAY NOT have multiple tracks of the same kind (audio/video).
+ *
+ * Audio and video are related if they come from the same source. For example: if the user is sharing their screen, the stream with the video may only include audio if that audio is from the user's screen - it may not include the user's mic.
+ * The audio from the user's mic and the video from the user's camera also count as being related.
+ *
+ * We have no control over what remote peers may send us if they are not also a rocket.chat client. If they send us multiple tracks in the same stream, we'll use the first and ignore the rest. It's likely that all tracks would have the same data anyway - just with different encodings.
+ * */
+export interface IMediaStreamWrapper {
+	readonly emitter: Emitter<MediaStreamEvents>;
+
+	readonly remote: boolean;
+
+	readonly stream: MediaStream;
+
+	readonly localId: string;
+
+	readonly active: boolean;
+
+	/**
+	 * Indicates if there's any track on this stream that is receiving or may receive audio
+	 **/
+	hasAudio(): boolean;
+
+	/**
+	 * Indicates if there's any track on this stream that is receiving or may receive video
+	 **/
+	hasVideo(): boolean;
+
+	/**
+	 * indicates if the audio track is not receiving audio from the system or network; beyond what the rocket.chat client can control
+	 * */
+	isAudioMuted(): boolean;
+
+	/**
+	 * indicates if the audio track is enabled by the rocket.chat client (aka not muted by the user nor placed on hold)
+	 * */
+	isAudioEnabled(): boolean;
+
+	isStopped(): boolean;
+}

packages/media-signaling/src/definition/media/MediaStreamIdentification.ts

@@ -0,0 +1,4 @@
+export type MediaStreamIdentification = {
+	tag: string;
+	id: string;
+};

packages/media-signaling/src/definition/media/index.ts

@@ -0,0 +1,3 @@
+export * from './IMediaStreamManager';
+export * from './IMediaStreamWrapper';
+export * from './MediaStreamIdentification';

packages/media-signaling/src/definition/services/negotiation.ts

@@ -27,4 +27,5 @@ export type NegotiationData = {
 
 export interface INegotiationCompatibleMediaCall extends IClientMediaCall {
 	hasInputTrack(): boolean;
+	hasVideoTrack(): boolean;
 }

packages/media-signaling/src/definition/services/webrtc/IWebRTCProcessor.ts

@@ -2,6 +2,8 @@ import type { Emitter } from '@rocket.chat/emitter';
 
 import type { IClientMediaCall } from '../../call';
 import type { IMediaSignalLogger } from '../../logger';
+import type { IMediaStreamManager } from '../../media/IMediaStreamManager';
+import type { MediaStreamIdentification } from '../../media/MediaStreamIdentification';
 import type { IServiceProcessor, ServiceProcessorEvents } from '../IServiceProcessor';
 
 export type WebRTCInternalStateMap = {
@@ -15,6 +17,7 @@ export type WebRTCInternalStateMap = {
 
 export type WebRTCUniqueEvents = {
 	negotiationNeeded: void;
+	streamChanged: void;
 };
 
 export type WebRTCProcessorEvents = ServiceProcessorEvents<WebRTCInternalStateMap> & WebRTCUniqueEvents;
@@ -28,7 +31,10 @@ export interface IWebRTCProcessor extends IServiceProcessor<WebRTCInternalStateM
 	setHeld(held: boolean): void;
 	stop(): void;
 
+	readonly streams: IMediaStreamManager;
+
 	setInputTrack(newInputTrack: MediaStreamTrack | null): Promise<void>;
+	setVideoTrack(newVideoTrack: MediaStreamTrack | null): Promise<void>;
 	createOffer(params: { iceRestart?: boolean }): Promise<RTCSessionDescriptionInit>;
 	createAnswer(): Promise<RTCSessionDescriptionInit>;
 
@@ -37,19 +43,21 @@ export interface IWebRTCProcessor extends IServiceProcessor<WebRTCInternalStateM
 	waitForIceGathering(): Promise<void>;
 	getLocalDescription(): RTCSessionDescriptionInit | null;
 
-	getRemoteMediaStream(): MediaStream;
-
 	audioLevel: number;
 	localAudioLevel: number;
 
 	getStats(selector?: MediaStreamTrack | null): Promise<RTCStatsReport | null>;
 	isRemoteHeld(): boolean;
 	isRemoteMute(): boolean;
+
+	setRemoteIds(streams: MediaStreamIdentification[]): void;
+	getLocalStreamIds(): MediaStreamIdentification[];
 }
 
 export type WebRTCProcessorConfig = {
 	call: IClientMediaCall;
 	inputTrack: MediaStreamTrack | null;
+	videoTrack?: MediaStreamTrack | null;
 	iceGatheringTimeout: number;
 	logger?: IMediaSignalLogger;
 	rtc?: RTCConfiguration;