refactor(multi-stream) refactor virtual screenshare creation and supp…

…ort plan-b clients (jitsi#11445)

* fix(multi-stream) update selector to find ss track by videoType or mediaType

* ref(multi-stream) move fake ss creation logic and support video type changed

* refactor(multi-stream) decouple sending and receiving multiple screenshare streams

* fix(multi-stream) fix receiver constraints with signaling and without multi-stream

* fix(mutli-stream) ensure plan b original SS thumbnail displays avatar

* fix(multi-stream) show fake SS for plan b sender

* refactor(multi-stream) poc for moving SS creation to state listener

* remove reference to fake SS creation

* fix lint errors

* rename to virtual screenshare participants

* fix minor bugs

* rename participant subscriber to specify web support only

conference.js

@@ -56,8 +56,7 @@ import {
 } from './react/features/base/conference';
 import {
     getReplaceParticipant,
-    getMultipleVideoSupportFeatureFlag,
-    getSourceNameSignalingFeatureFlag
+    getMultipleVideoSendingSupportFeatureFlag
 } from './react/features/base/config/functions';
 import {
     checkAndNotifyForNewDevice,
@@ -97,7 +96,7 @@ import {
     dominantSpeakerChanged,
     getLocalParticipant,
     getNormalizedDisplayName,
-    getScreenshareParticipantByOwnerId,
+    getVirtualScreenshareParticipantByOwnerId,
     localParticipantAudioLevelChanged,
     localParticipantConnectionStatusChanged,
     localParticipantRoleChanged,
@@ -1472,7 +1471,7 @@ export default {
 
                 // In the multi-stream mode, add the track to the conference if there is no existing track, replace it
                 // otherwise.
-                if (getMultipleVideoSupportFeatureFlag(state)) {
+                if (getMultipleVideoSendingSupportFeatureFlag(state)) {
                     const trackAction = oldTrack
                         ? replaceLocalTrack(oldTrack, newTrack, room)
                         : addLocalTrack(newTrack);
@@ -2265,14 +2264,12 @@ export default {
                     name: formattedDisplayName
                 }));
 
-                if (getSourceNameSignalingFeatureFlag(state)) {
-                    const screenshareParticipantId = getScreenshareParticipantByOwnerId(state, id)?.id;
+                const virtualScreenshareParticipantId = getVirtualScreenshareParticipantByOwnerId(state, id)?.id;
 
-                    if (screenshareParticipantId) {
-                        APP.store.dispatch(
-                            screenshareParticipantDisplayNameChanged(screenshareParticipantId, formattedDisplayName)
-                        );
-                    }
+                if (virtualScreenshareParticipantId) {
+                    APP.store.dispatch(
+                        screenshareParticipantDisplayNameChanged(virtualScreenshareParticipantId, formattedDisplayName)
+                    );
                 }
 
                 APP.API.notifyDisplayNameChanged(id, {

modules/UI/videolayout/LargeVideoManager.js

@@ -9,7 +9,10 @@ import { Provider } from 'react-redux';
 import { createScreenSharingIssueEvent, sendAnalytics } from '../../../react/features/analytics';
 import { Avatar } from '../../../react/features/base/avatar';
 import theme from '../../../react/features/base/components/themes/participantsPaneTheme.json';
-import { getSourceNameSignalingFeatureFlag } from '../../../react/features/base/config';
+import {
+    getMultipleVideoSupportFeatureFlag,
+    getSourceNameSignalingFeatureFlag
+} from '../../../react/features/base/config';
 import { i18next } from '../../../react/features/base/i18n';
 import { JitsiTrackEvents } from '../../../react/features/base/lib-jitsi-meet';
 import { VIDEO_TYPE } from '../../../react/features/base/media';
@@ -283,9 +286,18 @@ export default class LargeVideoManager {
             }
 
             const isAudioOnly = APP.conference.isAudioOnly();
+
+            // Multi-stream is not supported on plan-b endpoints even if its is enabled via config.js. A virtual
+            // screenshare tile is still created when a remote endpoint starts screenshare to keep the behavior
+            // consistent and an avatar is displayed on the original participant thumbnail as long as screenshare is in
+            // progress.
+            const legacyScreenshare = getMultipleVideoSupportFeatureFlag(state)
+                                        && videoType === VIDEO_TYPE.DESKTOP
+                                        && !participant.isVirtualScreenshareParticipant;
+
             const showAvatar
                 = isVideoContainer
-                    && ((isAudioOnly && videoType !== VIDEO_TYPE.DESKTOP) || !isVideoRenderable);
+                    && ((isAudioOnly && videoType !== VIDEO_TYPE.DESKTOP) || !isVideoRenderable || legacyScreenshare);
 
             let promise;
 

modules/UI/videolayout/VideoLayout.js

@@ -10,7 +10,7 @@ import {
 } from '../../../react/features/base/participants';
 import {
     getTrackByMediaTypeAndParticipant,
-    getFakeScreenshareParticipantTrack
+    getVirtualScreenshareParticipantTrack
 } from '../../../react/features/base/tracks';
 
 import LargeVideoManager from './LargeVideoManager';
@@ -95,7 +95,7 @@ const VideoLayout = {
             return VIDEO_TYPE.CAMERA;
         }
 
-        if (getSourceNameSignalingFeatureFlag(state) && participant?.isFakeScreenShareParticipant) {
+        if (getSourceNameSignalingFeatureFlag(state) && participant?.isVirtualScreenshareParticipant) {
             return VIDEO_TYPE.DESKTOP;
         }
 
@@ -190,8 +190,8 @@ const VideoLayout = {
 
         let videoTrack;
 
-        if (getSourceNameSignalingFeatureFlag(state) && participant?.isFakeScreenShareParticipant) {
-            videoTrack = getFakeScreenshareParticipantTrack(tracks, id);
+        if (getSourceNameSignalingFeatureFlag(state) && participant?.isVirtualScreenshareParticipant) {
+            videoTrack = getVirtualScreenshareParticipantTrack(tracks, id);
         } else {
             videoTrack = getTrackByMediaTypeAndParticipant(tracks, MEDIA_TYPE.VIDEO, id);
         }

react/features/base/conference/middleware.web.js

@@ -9,7 +9,7 @@ import {
 import { setScreenAudioShareState, setScreenshareAudioTrack } from '../../screen-share';
 import { AudioMixerEffect } from '../../stream-effects/audio-mixer/AudioMixerEffect';
 import { setAudioOnly } from '../audio-only';
-import { getMultipleVideoSupportFeatureFlag } from '../config/functions.any';
+import { getMultipleVideoSendingSupportFeatureFlag } from '../config/functions.any';
 import { JitsiConferenceErrors, JitsiTrackErrors } from '../lib-jitsi-meet';
 import { MEDIA_TYPE, setScreenshareMuted, VIDEO_TYPE } from '../media';
 import { MiddlewareRegistry } from '../redux';
@@ -51,7 +51,7 @@ MiddlewareRegistry.register(store => next => action => {
         break;
     }
     case TOGGLE_SCREENSHARING: {
-        getMultipleVideoSupportFeatureFlag(getState()) && _toggleScreenSharing(action, store);
+        getMultipleVideoSendingSupportFeatureFlag(getState()) && _toggleScreenSharing(action, store);
 
         break;
     }

react/features/base/config/functions.any.js

@@ -51,15 +51,24 @@ export function getMeetingRegion(state: Object) {
 }
 
 /**
- * Selector used to get the sendMultipleVideoStreams feature flag.
+ * Selector for determining if receiving multiple stream support is enabled.
  *
  * @param {Object} state - The global state.
  * @returns {boolean}
  */
 export function getMultipleVideoSupportFeatureFlag(state: Object) {
     return getFeatureFlag(state, FEATURE_FLAGS.MULTIPLE_VIDEO_STREAMS_SUPPORT)
-        && getSourceNameSignalingFeatureFlag(state)
-        && isUnifiedPlanEnabled(state);
+        && getSourceNameSignalingFeatureFlag(state);
+}
+
+/**
+ * Selector for determining if sending multiple stream support is enabled.
+ *
+ * @param {Object} state - The global state.
+ * @returns {boolean}
+ */
+export function getMultipleVideoSendingSupportFeatureFlag(state: Object) {
+    return getMultipleVideoSupportFeatureFlag(state) && isUnifiedPlanEnabled(state);
 }
 
 /**

react/features/base/lastn/middleware.js

@@ -6,9 +6,9 @@ import { SET_FILMSTRIP_ENABLED } from '../../filmstrip/actionTypes';
 import { SELECT_LARGE_VIDEO_PARTICIPANT } from '../../large-video/actionTypes';
 import { APP_STATE_CHANGED } from '../../mobile/background/actionTypes';
 import {
-    FAKE_SCREEN_SHARE_REMOTE_PARTICIPANTS_UPDATED,
     SCREEN_SHARE_REMOTE_PARTICIPANTS_UPDATED,
-    SET_TILE_VIEW
+    SET_TILE_VIEW,
+    VIRTUAL_SCREENSHARE_REMOTE_PARTICIPANTS_UPDATED
 } from '../../video-layout/actionTypes';
 import { SET_AUDIO_ONLY } from '../audio-only/actionTypes';
 import { CONFERENCE_JOINED } from '../conference/actionTypes';
@@ -95,7 +95,6 @@ MiddlewareRegistry.register(store => next => action => {
     switch (action.type) {
     case APP_STATE_CHANGED:
     case CONFERENCE_JOINED:
-    case FAKE_SCREEN_SHARE_REMOTE_PARTICIPANTS_UPDATED:
     case PARTICIPANT_JOINED:
     case PARTICIPANT_KICKED:
     case PARTICIPANT_LEFT:
@@ -104,6 +103,7 @@ MiddlewareRegistry.register(store => next => action => {
     case SET_AUDIO_ONLY:
     case SET_FILMSTRIP_ENABLED:
     case SET_TILE_VIEW:
+    case VIRTUAL_SCREENSHARE_REMOTE_PARTICIPANTS_UPDATED:
         _updateLastN(store);
         break;
     }

react/features/base/media/middleware.js

@@ -16,7 +16,7 @@ import { isForceMuted } from '../../participants-pane/functions';
 import { isScreenMediaShared } from '../../screen-share/functions';
 import { SET_AUDIO_ONLY, setAudioOnly } from '../audio-only';
 import { isRoomValid, SET_ROOM } from '../conference';
-import { getMultipleVideoSupportFeatureFlag } from '../config';
+import { getMultipleVideoSendingSupportFeatureFlag } from '../config';
 import { getLocalParticipant } from '../participants';
 import { MiddlewareRegistry } from '../redux';
 import { getPropertyValue } from '../settings';
@@ -192,7 +192,7 @@ function _setAudioOnly({ dispatch, getState }, next, action) {
 
     // Make sure we mute both the desktop and video tracks.
     dispatch(setVideoMuted(audioOnly, MEDIA_TYPE.VIDEO, VIDEO_MUTISM_AUTHORITY.AUDIO_ONLY, ensureVideoTrack));
-    if (getMultipleVideoSupportFeatureFlag(state)) {
+    if (getMultipleVideoSendingSupportFeatureFlag(state)) {
         dispatch(setScreenshareMuted(audioOnly, MEDIA_TYPE.SCREENSHARE, SCREENSHARE_MUTISM_AUTHORITY.AUDIO_ONLY));
     } else if (navigator.product !== 'ReactNative') {
         dispatch(setVideoMuted(audioOnly, MEDIA_TYPE.PRESENTER, VIDEO_MUTISM_AUTHORITY.AUDIO_ONLY, ensureVideoTrack));

react/features/base/participants/actions.js

@@ -25,6 +25,7 @@ import {
 } from './constants';
 import {
     getLocalParticipant,
+    getVirtualScreenshareParticipantOwnerId,
     getNormalizedDisplayName,
     getParticipantDisplayName,
     getParticipantById
@@ -503,6 +504,36 @@ export function participantMutedUs(participant, track) {
     };
 }
 
+/**
+ * Action to create a virtual screenshare participant.
+ *
+ * @param {(string)} sourceName - JitsiTrack instance.
+ * @param {(boolean)} local - JitsiTrack instance.
+ * @returns {Function}
+ */
+export function createVirtualScreenshareParticipant(sourceName, local) {
+    return (dispatch, getState) => {
+        const state = getState();
+        const ownerId = getVirtualScreenshareParticipantOwnerId(sourceName);
+        const owner = getParticipantById(state, ownerId);
+        const ownerName = owner.name;
+
+        if (!ownerName) {
+            logger.error(`Failed to create a screenshare participant for sourceName: ${sourceName}`);
+
+            return;
+        }
+
+        dispatch(participantJoined({
+            conference: state['features/base/conference'].conference,
+            id: sourceName,
+            isVirtualScreenshareParticipant: true,
+            isLocalScreenShare: local,
+            name: ownerName
+        }));
+    };
+}
+
 /**
  * Action to signal that a participant had been kicked.
  *

react/features/base/participants/functions.js

@@ -6,21 +6,16 @@ import type { Store } from 'redux';
 import { i18next } from '../../base/i18n';
 import { isStageFilmstripAvailable } from '../../filmstrip/functions';
 import { GRAVATAR_BASE_URL, isCORSAvatarURL } from '../avatar';
-import { getSourceNameSignalingFeatureFlag } from '../config';
+import { getMultipleVideoSupportFeatureFlag, getSourceNameSignalingFeatureFlag } from '../config';
 import { JitsiParticipantConnectionStatus } from '../lib-jitsi-meet';
 import { MEDIA_TYPE, shouldRenderVideoTrack } from '../media';
 import { toState } from '../redux';
-import { getTrackByMediaTypeAndParticipant } from '../tracks';
+import { getScreenShareTrack, getTrackByMediaTypeAndParticipant } from '../tracks';
 import { createDeferred } from '../util';
 
-import {
-    JIGASI_PARTICIPANT_ICON,
-    MAX_DISPLAY_NAME_LENGTH,
-    PARTICIPANT_ROLE
-} from './constants';
+import { JIGASI_PARTICIPANT_ICON, MAX_DISPLAY_NAME_LENGTH, PARTICIPANT_ROLE } from './constants';
 import { preloadImage } from './preloadImage';
 
-
 /**
  * Temp structures for avatar urls to be checked/preloaded.
  */
@@ -114,12 +109,16 @@ export function getLocalScreenShareParticipant(stateful: Object | Function) {
  * @param {string} id - The owner ID of the screenshare participant to retrieve.
  * @returns {(Participant|undefined)}
  */
-export function getScreenshareParticipantByOwnerId(stateful: Object | Function, id: string) {
-    const track = getTrackByMediaTypeAndParticipant(
-        toState(stateful)['features/base/tracks'], MEDIA_TYPE.SCREENSHARE, id
-    );
+export function getVirtualScreenshareParticipantByOwnerId(stateful: Object | Function, id: string) {
+    const state = toState(stateful);
+
+    if (getMultipleVideoSupportFeatureFlag(state)) {
+        const track = getScreenShareTrack(state['features/base/tracks'], id);
+
+        return getParticipantById(stateful, track?.jitsiTrack.getSourceName());
+    }
 
-    return getParticipantById(stateful, track?.jitsiTrack.getSourceName());
+    return;
 }
 
 /**
@@ -186,25 +185,25 @@ export function getParticipantCount(stateful: Object | Function) {
         local,
         remote,
         fakeParticipants,
-        sortedRemoteFakeScreenShareParticipants
+        sortedRemoteVirtualScreenshareParticipants
     } = state['features/base/participants'];
 
     if (getSourceNameSignalingFeatureFlag(state)) {
-        return remote.size - fakeParticipants.size - sortedRemoteFakeScreenShareParticipants.size + (local ? 1 : 0);
+        return remote.size - fakeParticipants.size - sortedRemoteVirtualScreenshareParticipants.size + (local ? 1 : 0);
     }
 
     return remote.size - fakeParticipants.size + (local ? 1 : 0);
 
 }
 
 /**
- * Returns participant ID of the owner of a fake screenshare participant.
+ * Returns participant ID of the owner of a virtual screenshare participant.
  *
- * @param {string} id - The ID of the fake screenshare participant.
+ * @param {string} id - The ID of the virtual screenshare participant.
  * @private
  * @returns {(string|undefined)}
  */
-export function getFakeScreenShareParticipantOwnerId(id: string) {
+export function getVirtualScreenshareParticipantOwnerId(id: string) {
     return id.split('-')[0];
 }
 
@@ -232,7 +231,7 @@ export function getRemoteParticipantCount(stateful: Object | Function) {
     const state = toState(stateful)['features/base/participants'];
 
     if (getSourceNameSignalingFeatureFlag(state)) {
-        return state.remote.size - state.sortedRemoteFakeScreenShareParticipants.size;
+        return state.remote.size - state.sortedRemoteVirtualScreenshareParticipants.size;
     }
 
     return state.remote.size;
@@ -274,7 +273,7 @@ export function getParticipantDisplayName(stateful: Object | Function, id: strin
     } = toState(stateful)['features/base/config'];
 
     if (participant) {
-        if (participant.isFakeScreenShareParticipant) {
+        if (participant.isVirtualScreenshareParticipant) {
             return getScreenshareParticipantDisplayName(stateful, id);
         }
 
@@ -299,7 +298,7 @@ export function getParticipantDisplayName(stateful: Object | Function, id: strin
  * @returns {string}
  */
 export function getScreenshareParticipantDisplayName(stateful: Object | Function, id: string) {
-    const owner = getParticipantById(stateful, getFakeScreenShareParticipantOwnerId(id));
+    const owner = getParticipantById(stateful, getVirtualScreenshareParticipantOwnerId(id));
     const name = owner.name;
 
     return i18next.t('screenshareDisplayName', { name });

react/features/base/participants/middleware.js

@@ -69,6 +69,7 @@ import {
     isLocalParticipantModerator
 } from './functions';
 import { PARTICIPANT_JOINED_FILE, PARTICIPANT_LEFT_FILE } from './sounds';
+import './subscriber';
 
 declare var APP: Object;
 
@@ -210,19 +211,19 @@ MiddlewareRegistry.register(store => next => action => {
     }
 
     case PARTICIPANT_JOINED: {
-        const { isFakeScreenShareParticipant } = action.participant;
+        const { isVirtualScreenshareParticipant } = action.participant;
 
-        // Do not play sounds when a fake participant tile is created for screenshare.
-        !isFakeScreenShareParticipant && _maybePlaySounds(store, action);
+        // Do not play sounds when a virtual participant tile is created for screenshare.
+        !isVirtualScreenshareParticipant && _maybePlaySounds(store, action);
 
         return _participantJoinedOrUpdated(store, next, action);
     }
 
     case PARTICIPANT_LEFT: {
-        const { isFakeScreenShareParticipant } = action.participant;
+        const { isVirtualScreenshareParticipant } = action.participant;
 
         // Do not play sounds when a tile for screenshare is removed.
-        !isFakeScreenShareParticipant && _maybePlaySounds(store, action);
+        !isVirtualScreenshareParticipant && _maybePlaySounds(store, action);
 
         break;
     }