feat(multi-stream-support) Add screenshare as a second video track to…

… the call.

* feat(multi-stream-support) Add screenshare as a second video track to the call.
This feature is behind a sendMultipleVideoStreams config.js flag. sourceNameSignaling flag also needs to enabled. Sending multiple tracks is currently supported only on endpoints running in unified plan mode. However, clients with source-name signaling enabled and running in plan-b can still receive multiple streams .

* squash: check if there is an existing track before adding camera/desktop

* squash: enable multi-stream only on unified plan endpoints.

conference.js

@@ -52,7 +52,7 @@ import {
     sendLocalParticipant,
     nonParticipantMessageReceived
 } from './react/features/base/conference';
-import { getReplaceParticipant } from './react/features/base/config/functions';
+import { getReplaceParticipant, getMultipleVideoSupportFeatureFlag } from './react/features/base/config/functions';
 import {
     checkAndNotifyForNewDevice,
     getAvailableDevices,
@@ -106,6 +106,7 @@ import {
     updateSettings
 } from './react/features/base/settings';
 import {
+    addLocalTrack,
     createLocalPresenterTrack,
     createLocalTracksF,
     destroyLocalTracks,
@@ -1444,11 +1445,13 @@ export default {
      * @returns {Promise}
      */
     useVideoStream(newTrack) {
+        const state = APP.store.getState();
+
         logger.debug(`useVideoStream: ${newTrack}`);
 
         return new Promise((resolve, reject) => {
             _replaceLocalVideoTrackQueue.enqueue(onFinish => {
-                const oldTrack = getLocalJitsiVideoTrack(APP.store.getState());
+                const oldTrack = getLocalJitsiVideoTrack(state);
 
                 logger.debug(`useVideoStream: Replacing ${oldTrack} with ${newTrack}`);
 
@@ -1459,6 +1462,26 @@ export default {
                     return;
                 }
 
+                // In the multi-stream mode, add the track to the conference if there is no existing track, replace it
+                // otherwise.
+                if (getMultipleVideoSupportFeatureFlag(state)) {
+                    const trackAction = oldTrack
+                        ? replaceLocalTrack(oldTrack, newTrack, room)
+                        : addLocalTrack(newTrack);
+
+                    APP.store.dispatch(trackAction)
+                        .then(() => {
+                            this.setVideoMuteStatus();
+                        })
+                        .then(resolve)
+                        .catch(error => {
+                            logger.error(`useVideoStream failed: ${error}`);
+                            reject(error);
+                        })
+                        .then(onFinish);
+
+                    return;
+                }
                 APP.store.dispatch(
                     replaceLocalTrack(oldTrack, newTrack, room))
                     .then(() => {

react/features/av-moderation/constants.js

@@ -26,6 +26,7 @@ export const CS_MODERATION_NOTIFICATION_ID = 'screensharing-moderation';
 
 export const MODERATION_NOTIFICATIONS = {
     [MEDIA_TYPE.AUDIO]: AUDIO_MODERATION_NOTIFICATION_ID,
+    [MEDIA_TYPE.SCREENSHARE]: CS_MODERATION_NOTIFICATION_ID,
     [MEDIA_TYPE.VIDEO]: VIDEO_MODERATION_NOTIFICATION_ID,
     [MEDIA_TYPE.PRESENTER]: CS_MODERATION_NOTIFICATION_ID
 };

react/features/base/conference/middleware.web.js

@@ -1,13 +1,30 @@
 // @flow
 
+import { AUDIO_ONLY_SCREEN_SHARE_NO_TRACK } from '../../../../modules/UI/UIErrors';
+import { showNotification, NOTIFICATION_TIMEOUT_TYPE } from '../../notifications';
 import { setSkipPrejoinOnReload } from '../../prejoin';
-import { JitsiConferenceErrors } from '../lib-jitsi-meet';
+import { setScreenAudioShareState, setScreenshareAudioTrack } from '../../screen-share';
+import { AudioMixerEffect } from '../../stream-effects/audio-mixer/AudioMixerEffect';
+import { setAudioOnly } from '../audio-only';
+import { getMultipleVideoSupportFeatureFlag } from '../config/functions.any';
+import { JitsiConferenceErrors, JitsiTrackErrors } from '../lib-jitsi-meet';
+import { MEDIA_TYPE, setScreenshareMuted, VIDEO_TYPE } from '../media';
 import { MiddlewareRegistry } from '../redux';
+import {
+    addLocalTrack,
+    createLocalTracksF,
+    getLocalDesktopTrack,
+    getLocalJitsiAudioTrack,
+    replaceLocalTrack,
+    TOGGLE_SCREENSHARING
+} from '../tracks';
 
 import { CONFERENCE_FAILED, CONFERENCE_JOINED } from './actionTypes';
+import { getCurrentConference } from './functions';
 import './middleware.any';
 
-MiddlewareRegistry.register(({ dispatch, getState }) => next => action => {
+MiddlewareRegistry.register(store => next => action => {
+    const { dispatch, getState } = store;
     const { enableForcedReload } = getState()['features/base/config'];
 
     switch (action.type) {
@@ -25,7 +42,153 @@ MiddlewareRegistry.register(({ dispatch, getState }) => next => action => {
 
         break;
     }
+    case TOGGLE_SCREENSHARING: {
+        getMultipleVideoSupportFeatureFlag(getState()) && _toggleScreenSharing(action, store);
+
+        break;
+    }
     }
 
     return next(action);
 });
+
+/**
+ * Displays a UI notification for screensharing failure based on the error passed.
+ *
+ * @private
+ * @param {Object} error - The error.
+ * @param {Object} store - The redux store.
+ * @returns {void}
+ */
+function _handleScreensharingError(error, { dispatch }) {
+    if (error.name === JitsiTrackErrors.SCREENSHARING_USER_CANCELED) {
+        return;
+    }
+    let descriptionKey, titleKey;
+
+    if (error.name === JitsiTrackErrors.PERMISSION_DENIED) {
+        descriptionKey = 'dialog.screenSharingPermissionDeniedError';
+        titleKey = 'dialog.screenSharingFailedTitle';
+    } else if (error.name === JitsiTrackErrors.CONSTRAINT_FAILED) {
+        descriptionKey = 'dialog.cameraConstraintFailedError';
+        titleKey = 'deviceError.cameraError';
+    } else if (error.name === JitsiTrackErrors.SCREENSHARING_GENERIC_ERROR) {
+        descriptionKey = 'dialog.screenSharingFailed';
+        titleKey = 'dialog.screenSharingFailedTitle';
+    } else if (error === AUDIO_ONLY_SCREEN_SHARE_NO_TRACK) {
+        descriptionKey = 'notify.screenShareNoAudio';
+        titleKey = 'notify.screenShareNoAudioTitle';
+    }
+
+    dispatch(showNotification({
+        titleKey,
+        descriptionKey
+    }, NOTIFICATION_TIMEOUT_TYPE.MEDIUM));
+}
+
+/**
+ * Applies the AudioMixer effect on the local audio track if applicable. If there is no local audio track, the desktop
+ * audio track is added to the conference.
+ *
+ * @private
+ * @param {JitsiLocalTrack} desktopAudioTrack - The audio track to be added to the conference.
+ * @param {*} state - The redux state.
+ * @returns {void}
+ */
+async function _maybeApplyAudioMixerEffect(desktopAudioTrack, state) {
+    const localAudio = getLocalJitsiAudioTrack(state);
+    const conference = getCurrentConference(state);
+
+    if (localAudio) {
+        // If there is a localAudio stream, mix in the desktop audio stream captured by the screen sharing API.
+        const mixerEffect = new AudioMixerEffect(desktopAudioTrack);
+
+        await localAudio.setEffect(mixerEffect);
+    } else {
+        // If no local stream is present ( i.e. no input audio devices) we use the screen share audio
+        // stream as we would use a regular stream.
+        await conference.replaceTrack(null, desktopAudioTrack);
+    }
+}
+
+/**
+ * Toggles screen sharing.
+ *
+ * @private
+ * @param {boolean} enabled - The state to toggle screen sharing to.
+ * @param {Store} store - The redux store.
+ * @returns {void}
+ */
+async function _toggleScreenSharing({ enabled, audioOnly = false }, store) {
+    const { dispatch, getState } = store;
+    const state = getState();
+    const conference = getCurrentConference(state);
+    const localAudio = getLocalJitsiAudioTrack(state);
+    const localScreenshare = getLocalDesktopTrack(state['features/base/tracks']);
+
+    if (enabled) {
+        let tracks;
+
+        try {
+            tracks = await createLocalTracksF({ devices: [ VIDEO_TYPE.DESKTOP ] });
+        } catch (error) {
+            _handleScreensharingError(error, store);
+
+            return;
+        }
+        const desktopAudioTrack = tracks.find(track => track.getType() === MEDIA_TYPE.AUDIO);
+        const desktopVideoTrack = tracks.find(track => track.getType() === MEDIA_TYPE.VIDEO);
+
+        // Dispose the desktop track for audio-only screensharing.
+        if (audioOnly) {
+            desktopVideoTrack.dispose();
+
+            if (!desktopAudioTrack) {
+                _handleScreensharingError(AUDIO_ONLY_SCREEN_SHARE_NO_TRACK, store);
+
+                return;
+            }
+        } else if (desktopVideoTrack) {
+            if (localScreenshare) {
+                await dispatch(replaceLocalTrack(localScreenshare.jitsiTrack, desktopVideoTrack, conference));
+            } else {
+                await dispatch(addLocalTrack(desktopVideoTrack));
+            }
+        }
+
+        // Apply the AudioMixer effect if there is a local audio track, add the desktop track to the conference
+        // otherwise without unmuting the microphone.
+        if (desktopAudioTrack) {
+            _maybeApplyAudioMixerEffect(desktopAudioTrack, state);
+            dispatch(setScreenshareAudioTrack(desktopAudioTrack));
+        }
+
+        // Disable audio-only or best performance mode if the user starts screensharing. This doesn't apply to
+        // audio-only screensharing.
+        const { enabled: bestPerformanceMode } = state['features/base/audio-only'];
+
+        if (bestPerformanceMode && !audioOnly) {
+            dispatch(setAudioOnly(false));
+        }
+    } else {
+        const { desktopAudioTrack } = state['features/screen-share'];
+
+        // Mute the desktop track instead of removing it from the conference since we don't want the client to signal
+        // a source-remove to the remote peer for the screenshare track. Later when screenshare is enabled again, the
+        // same sender will be re-used without the need for signaling a new ssrc through source-add.
+        dispatch(setScreenshareMuted(true));
+        if (desktopAudioTrack) {
+            if (localAudio) {
+                localAudio.setEffect(undefined);
+            } else {
+                await conference.replaceTrack(desktopAudioTrack, null);
+            }
+            desktopAudioTrack.dispose();
+            dispatch(setScreenshareAudioTrack(null));
+        }
+    }
+
+    if (audioOnly) {
+        dispatch(setScreenAudioShareState(enabled));
+    }
+}

react/features/base/config/constants.js

@@ -67,5 +67,6 @@ export const THIRD_PARTY_PREJOIN_BUTTONS = [ 'microphone', 'camera', 'select-bac
  */
 
 export const FEATURE_FLAGS = {
+    MULTIPLE_VIDEO_STREAMS_SUPPORT: 'sendMultipleVideoStreams',
     SOURCE_NAME_SIGNALING: 'sourceNameSignaling'
 };

react/features/base/config/functions.any.js

@@ -4,6 +4,7 @@ import Bourne from '@hapi/bourne';
 import { jitsiLocalStorage } from '@jitsi/js-utils';
 import _ from 'lodash';
 
+import { browser } from '../lib-jitsi-meet';
 import { parseURLParams } from '../util';
 
 import CONFIG_WHITELIST from './configWhitelist';
@@ -49,6 +50,18 @@ export function getMeetingRegion(state: Object) {
     return state['features/base/config']?.deploymentInfo?.region || '';
 }
 
+/**
+ * Selector used to get the sendMultipleVideoStreams feature flag.
+ *
+ * @param {Object} state - The global state.
+ * @returns {boolean}
+ */
+export function getMultipleVideoSupportFeatureFlag(state: Object) {
+    return getFeatureFlag(state, FEATURE_FLAGS.MULTIPLE_VIDEO_STREAMS_SUPPORT)
+        && getSourceNameSignalingFeatureFlag(state)
+        && isUnifiedPlanEnabled(state);
+}
+
 /**
  * Selector used to get the sourceNameSignaling feature flag.
  *
@@ -196,6 +209,19 @@ export function isDisplayNameVisible(state: Object): boolean {
     return !state['features/base/config'].hideDisplayName;
 }
 
+/**
+ * Selector for determining if Unified plan support is enabled.
+ *
+ * @param {Object} state - The state of the app.
+ * @returns {boolean}
+ */
+export function isUnifiedPlanEnabled(state: Object): boolean {
+    const { enableUnifiedOnChrome = true } = state['features/base/config'];
+
+    return browser.supportsUnifiedPlan()
+        && (!browser.isChromiumBased() || (browser.isChromiumBased() && enableUnifiedOnChrome));
+}
+
 /**
  * Restores a Jitsi Meet config.js from {@code localStorage} if it was
  * previously downloaded from a specific {@code baseURL} and stored with

react/features/base/media/actionTypes.js

@@ -40,6 +40,16 @@ export const SET_AUDIO_UNMUTE_PERMISSIONS = 'SET_AUDIO_UNMUTE_PERMISSIONS';
  */
 export const SET_CAMERA_FACING_MODE = 'SET_CAMERA_FACING_MODE';
 
+/**
+ * The type of (redux) action to set the muted state of the local screenshare.
+ *
+ * {
+ *     type: SET_SCREENSHARE_MUTED,
+ *     muted: boolean
+ * }
+ */
+export const SET_SCREENSHARE_MUTED = 'SET_SCREENSHARE_MUTED';
+
 /**
  * The type of (redux) action to adjust the availability of the local video.
  *

react/features/base/media/actions.js

@@ -11,6 +11,7 @@ import {
     SET_AUDIO_AVAILABLE,
     SET_AUDIO_UNMUTE_PERMISSIONS,
     SET_CAMERA_FACING_MODE,
+    SET_SCREENSHARE_MUTED,
     SET_VIDEO_AVAILABLE,
     SET_VIDEO_MUTED,
     SET_VIDEO_UNMUTE_PERMISSIONS,
@@ -20,6 +21,7 @@ import {
 import {
     MEDIA_TYPE,
     type MediaType,
+    SCREENSHARE_MUTISM_AUTHORITY,
     VIDEO_MUTISM_AUTHORITY
 } from './constants';
 
@@ -92,6 +94,47 @@ export function setCameraFacingMode(cameraFacingMode: string) {
     };
 }
 
+/**
+ * Action to set the muted state of the local screenshare.
+ *
+ * @param {boolean} muted - True if the local screenshare is to be enabled or false otherwise.
+ * @param {MEDIA_TYPE} mediaType - The type of media.
+ * @param {number} authority - The {@link SCREENSHARE_MUTISM_AUTHORITY} which is muting/unmuting the local screenshare.
+ * @param {boolean} ensureTrack - True if we want to ensure that a new track is created if missing.
+ * @returns {Function}
+ */
+export function setScreenshareMuted(
+        muted: boolean,
+        mediaType: MediaType = MEDIA_TYPE.SCREENSHARE,
+        authority: number = SCREENSHARE_MUTISM_AUTHORITY.USER,
+        ensureTrack: boolean = false) {
+    return (dispatch: Dispatch<any>, getState: Function) => {
+        const state = getState();
+
+        // check for A/V Moderation when trying to unmute
+        if (!muted && shouldShowModeratedNotification(MEDIA_TYPE.SCREENSHARE, state)) {
+            if (!isModerationNotificationDisplayed(MEDIA_TYPE.SCREENSHARE, state)) {
+                ensureTrack && dispatch(showModeratedNotification(MEDIA_TYPE.SCREENSHARE));
+            }
+
+            return;
+        }
+
+        const oldValue = state['features/base/media'].screenshare.muted;
+
+        // eslint-disable-next-line no-bitwise
+        const newValue = muted ? oldValue | authority : oldValue & ~authority;
+
+        return dispatch({
+            type: SET_SCREENSHARE_MUTED,
+            authority,
+            mediaType,
+            ensureTrack,
+            muted: newValue
+        });
+    };
+}
+
 /**
  * Action to adjust the availability of the local video.
  *