Skip to content

Commit

Permalink
feat: Handle dominant speaker silence
Browse files Browse the repository at this point in the history
  • Loading branch information
hristoterezov committed Sep 14, 2022
1 parent 81f5e68 commit f0a45a9
Show file tree
Hide file tree
Showing 10 changed files with 96 additions and 37 deletions.
4 changes: 3 additions & 1 deletion conference.js
Original file line number Diff line number Diff line change
Expand Up @@ -2291,7 +2291,9 @@ export default {

room.on(
JitsiConferenceEvents.DOMINANT_SPEAKER_CHANGED,
(dominant, previous) => APP.store.dispatch(dominantSpeakerChanged(dominant, previous, room)));
(dominant, previous, silence) => {
APP.store.dispatch(dominantSpeakerChanged(dominant, previous, Boolean(silence), room));
});

room.on(
JitsiConferenceEvents.CONFERENCE_CREATED_TIMESTAMP,
Expand Down
4 changes: 3 additions & 1 deletion react/features/base/conference/actions.js
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,9 @@ function _addConferenceListeners(conference, dispatch, state) {

conference.on(
JitsiConferenceEvents.DOMINANT_SPEAKER_CHANGED,
(dominant, previous) => dispatch(dominantSpeakerChanged(dominant, previous, conference)));
(dominant, previous, silence) => {
dispatch(dominantSpeakerChanged(dominant, previous, Boolean(silence), conference));
});

conference.on(
JitsiConferenceEvents.ENDPOINT_MESSAGE_RECEIVED,
Expand Down
3 changes: 2 additions & 1 deletion react/features/base/participants/actionTypes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
* participant: {
* conference: JitsiConference,
* id: string,
* previousSpeakers: Array<string>
* previousSpeakers: Array<string>,
* silence: boolean
* }
* }
*/
Expand Down
10 changes: 7 additions & 3 deletions react/features/base/participants/actions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import { Participant } from './types';
*
* @param {string} dominantSpeaker - Participant ID of the dominant speaker.
* @param {Array<string>} previousSpeakers - Participant IDs of the previous speakers.
* @param {boolean} silence - Whether the dominant speaker is silent or not.
* @param {JitsiConference} conference - The {@code JitsiConference} associated
* with the participant identified by the specified {@code id}. Only the local
* participant is allowed to not specify an associated {@code JitsiConference}
Expand All @@ -54,17 +55,20 @@ import { Participant } from './types';
* participant: {
* conference: JitsiConference,
* id: string,
* previousSpeakers: Array<string>
* previousSpeakers: Array<string>,
* silence: boolean
* }
* }}
*/
export function dominantSpeakerChanged(dominantSpeaker: string, previousSpeakers: string[], conference: any) {
export function dominantSpeakerChanged(
dominantSpeaker: string, previousSpeakers: string[], silence: boolean, conference: any) {
return {
type: DOMINANT_SPEAKER_CHANGED,
participant: {
conference,
id: dominantSpeaker,
previousSpeakers
previousSpeakers,
silence
}
};
}
Expand Down
5 changes: 4 additions & 1 deletion react/features/base/participants/middleware.js
Original file line number Diff line number Diff line change
Expand Up @@ -111,9 +111,12 @@ MiddlewareRegistry.register(store => next => action => {
const { id } = action.participant;
const state = store.getState();
const participant = getLocalParticipant(state);
const dominantSpeaker = getDominantSpeakerParticipant(state);
const isLocal = participant && participant.id === id;

if (isLocal && hasRaisedHand(participant) && !getDisableRemoveRaisedHandOnFocus(state)) {
if (isLocal && dominantSpeaker?.id !== id
&& hasRaisedHand(participant)
&& !getDisableRemoveRaisedHandOnFocus(state)) {
store.dispatch(raiseHand(false));
}

Expand Down
20 changes: 15 additions & 5 deletions react/features/external-api/middleware.js
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ import {
PARTICIPANT_ROLE_CHANGED,
SET_LOADABLE_AVATAR_URL,
getLocalParticipant,
getParticipantById
getParticipantById,
getDominantSpeakerParticipant
} from '../base/participants';
import { MiddlewareRegistry } from '../base/redux';
import { getBaseUrl } from '../base/util';
Expand All @@ -38,6 +39,19 @@ declare var APP: Object;
MiddlewareRegistry.register(store => next => action => {
// We need to do these before executing the rest of the middelware chain
switch (action.type) {
case DOMINANT_SPEAKER_CHANGED: {
const dominantSpeaker = getDominantSpeakerParticipant(store.getState());

if (dominantSpeaker?.id !== action.participant.id) {
const result = next(action);

APP.API.notifyDominantSpeakerChanged(action.participant.id);

return result;
}

break;
}
case SET_LOADABLE_AVATAR_URL: {
const { id, loadableAvatarUrl } = action.participant;
const participant = getParticipantById(
Expand Down Expand Up @@ -115,10 +129,6 @@ MiddlewareRegistry.register(store => next => action => {
APP.API.notifyDataChannelOpened();
break;

case DOMINANT_SPEAKER_CHANGED:
APP.API.notifyDominantSpeakerChanged(action.participant.id);
break;

case KICKED_OUT:
APP.API.notifyKickedOut(
{
Expand Down
3 changes: 2 additions & 1 deletion react/features/filmstrip/middleware.web.js
Original file line number Diff line number Diff line change
Expand Up @@ -236,8 +236,9 @@ MiddlewareRegistry.register(store => next => action => {
const stageFilmstrip = isStageFilmstripAvailable(state);
const local = getLocalParticipant(state);
const currentLayout = getCurrentLayout(state);
const dominantSpeaker = getDominantSpeakerParticipant(state);

if (id === local.id || currentLayout === LAYOUTS.TILE_VIEW) {
if (dominantSpeaker?.id === id || id === local.id || currentLayout === LAYOUTS.TILE_VIEW) {
break;
}

Expand Down
24 changes: 18 additions & 6 deletions react/features/large-video/middleware.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@ import {
PARTICIPANT_JOINED,
PARTICIPANT_LEFT,
PIN_PARTICIPANT,
getLocalParticipant
getLocalParticipant,
getDominantSpeakerParticipant
} from '../base/participants';
import { MiddlewareRegistry } from '../base/redux';
import { isTestModeEnabled } from '../base/testing';
Expand All @@ -28,12 +29,18 @@ import './subscriber';
* @returns {Function}
*/
MiddlewareRegistry.register(store => next => action => {
const result = next(action);

switch (action.type) {
case DOMINANT_SPEAKER_CHANGED: {
const state = store.getState();
const localParticipant = getLocalParticipant(state);
const dominantSpeaker = getDominantSpeakerParticipant(state);


if (dominantSpeaker?.id === action.participant.id) {
return next(action);
}

const result = next(action);

if (isTestModeEnabled(state)) {
logger.info(`Dominant speaker changed event for: ${action.participant.id}`);
Expand All @@ -43,17 +50,22 @@ MiddlewareRegistry.register(store => next => action => {
store.dispatch(selectParticipantInLargeVideo());
}

break;
return result;
}
case PARTICIPANT_JOINED:
case PARTICIPANT_LEFT:
case PIN_PARTICIPANT:
case TOGGLE_DOCUMENT_EDITING:
case TRACK_ADDED:
case TRACK_REMOVED:
case TRACK_REMOVED: {
const result = next(action);

store.dispatch(selectParticipantInLargeVideo());
break;

return result;
}
}
const result = next(action);

return result;
});
8 changes: 5 additions & 3 deletions react/features/rtcstats/middleware.ts
Original file line number Diff line number Diff line change
Expand Up @@ -143,10 +143,12 @@ MiddlewareRegistry.register((store: IStore) => (next: Function) => (action: any)
}
case DOMINANT_SPEAKER_CHANGED: {
if (canSendRtcstatsData(state)) {
const { id, previousSpeakers } = action.participant;
const { id, previousSpeakers, silence } = action.participant;

RTCStats.sendDominantSpeakerData({ dominantSpeakerEndpoint: id,
previousSpeakers });
RTCStats.sendDominantSpeakerData({
dominantSpeakerEndpoint: silence ? null : id,
previousSpeakers
});
}
break;
}
Expand Down
52 changes: 37 additions & 15 deletions resources/prosody-plugins/mod_speakerstats_component.lua
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ function on_message(event)
= event.stanza:get_child('speakerstats', 'http://jitsi.org/jitmeet');
if speakerStats then
local roomAddress = speakerStats.attr.room;
local silence = speakerStats.attr.silence == 'true';
local room = get_room_from_jid(room_jid_match_rewrite(roomAddress));

if not room then
Expand All @@ -85,15 +86,15 @@ function on_message(event)
local newDominantSpeaker = roomSpeakerStats[occupant.jid];
local oldDominantSpeakerId = roomSpeakerStats['dominantSpeakerId'];

if oldDominantSpeakerId then
if oldDominantSpeakerId and occupant.jid ~= oldDominantSpeakerId then
local oldDominantSpeaker = roomSpeakerStats[oldDominantSpeakerId];
if oldDominantSpeaker then
oldDominantSpeaker:setDominantSpeaker(false);
oldDominantSpeaker:setDominantSpeaker(false, false);
end
end

if newDominantSpeaker then
newDominantSpeaker:setDominantSpeaker(true);
newDominantSpeaker:setDominantSpeaker(true, silence);
end

room.speakerStats['dominantSpeakerId'] = occupant.jid;
Expand Down Expand Up @@ -136,6 +137,8 @@ function new_SpeakerStats(nick, context_user)
return setmetatable({
totalDominantSpeakerTime = 0;
_dominantSpeakerStart = 0;
_isSilent = false;
_isDominantSpeaker = false;
nick = nick;
context_user = context_user;
displayName = nil;
Expand All @@ -154,24 +157,43 @@ end
-- Changes the dominantSpeaker data for current occupant
-- saves start time if it is new dominat speaker
-- or calculates and accumulates time of speaking
function SpeakerStats:setDominantSpeaker(isNowDominantSpeaker)
function SpeakerStats:setDominantSpeaker(isNowDominantSpeaker, silence)
-- log("debug", "set isDominant %s for %s", tostring(isNowDominantSpeaker), self.nick);

if not self:isDominantSpeaker() and isNowDominantSpeaker then
self._dominantSpeakerStart = socket.gettime()*1000;
elseif self:isDominantSpeaker() and not isNowDominantSpeaker then
local now = socket.gettime()*1000;
local timeElapsed = math.floor(now - self._dominantSpeakerStart);
local now = socket.gettime()*1000;

self.totalDominantSpeakerTime
= self.totalDominantSpeakerTime + timeElapsed;
self._dominantSpeakerStart = 0;
if not self:isDominantSpeaker() and isNowDominantSpeaker and not silence then
self._dominantSpeakerStart = now;
elseif self:isDominantSpeaker() then
if not isNowDominantSpeaker then
if not self._isSilent then
local timeElapsed = math.floor(now - self._dominantSpeakerStart);

self.totalDominantSpeakerTime = self.totalDominantSpeakerTime + timeElapsed;
self._dominantSpeakerStart = 0;
end
elseif self._isSilent and not silence then
self._dominantSpeakerStart = now;
elseif not self._isSilent and silence then
local timeElapsed = math.floor(now - self._dominantSpeakerStart);

self.totalDominantSpeakerTime = self.totalDominantSpeakerTime + timeElapsed;
self._dominantSpeakerStart = 0;
end
end

self._isDominantSpeaker = isNowDominantSpeaker;
self._isSilent = silence;
end

-- Returns true if the tracked user is currently a dominant speaker.
function SpeakerStats:isDominantSpeaker()
return self._dominantSpeakerStart > 0;
return self._isDominantSpeaker;
end

-- Returns true if the tracked user is currently silent.
function SpeakerStats:isSilent()
return self._isSilent;
end
--- End SpeakerStats

Expand Down Expand Up @@ -225,7 +247,7 @@ function occupant_joined(event)
if totalDominantSpeakerTime > 0 or room:get_occupant_jid(jid) == nil or values:isDominantSpeaker()
or get_participant_expressions_count(faceExpressions) > 0 then
-- before sending we need to calculate current dominant speaker state
if values:isDominantSpeaker() then
if values:isDominantSpeaker() and not values:isSilent() then
local timeElapsed = math.floor(socket.gettime()*1000 - values._dominantSpeakerStart);
totalDominantSpeakerTime = totalDominantSpeakerTime + timeElapsed;
end
Expand Down Expand Up @@ -276,7 +298,7 @@ function occupant_leaving(event)

local speakerStatsForOccupant = room.speakerStats[occupant.jid];
if speakerStatsForOccupant then
speakerStatsForOccupant:setDominantSpeaker(false);
speakerStatsForOccupant:setDominantSpeaker(false, false);

-- set display name
local displayName = occupant:get_presence():get_child_text(
Expand Down

0 comments on commit f0a45a9

Please sign in to comment.