Skip to content

Commit

Permalink
feat(codec-selection): Use the new codec selection API (#2520)
Browse files Browse the repository at this point in the history 
* feat(codec-selection): Use the new codec selection API
w3ctag/design-reviews#836. This allows the client to seamlessly switch between the codecs without having to trigger a renegotiation.
This feature is behind the flag testing.enableCodecSelectionAPI in config.js

* fix(stats): Fix local resolution stats.
The video codec for the local video sources needs to identified differently now, from the codecs field in the RTCRtpSendParameters returned by the browser. We no longer munge the remote SDP to switch to a different codec.

* feat(stats): Feed encodeTime stats for all local SSRCs to the codec selection mechanism.

* fix(codec-selection) Continue to mumge SDP for selecting H.264.

* feat(codec-selection) Make screenshare codec configurable.
If no 'screenshareCodec' is set under videoQuality or p2p settings, AV1 will be selected as default.

* squash: Address review comments

* Update modules/RTC/CodecSelection.js

Co-authored-by: Saúl Ibarra Corretgé <[email protected]>

* fix(codec-selection) Add codec to existing stats

---------

Co-authored-by: Saúl Ibarra Corretgé <[email protected]>
  • Loading branch information
@jallamsetty1 @saghul
jallamsetty1 and saghul authored Jun 25, 2024
1 parent d05325f commit 6bcc577
Show file tree
Hide file tree
Showing 13 changed files with 314 additions and 36 deletions.
19 changes: 14 additions & 5 deletions JitsiConference.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -390,14 +390,20 @@ JitsiConference.prototype._init = function(options = {}) {
? config.videoQuality.mobileCodecPreferenceOrder
: config.videoQuality?.codecPreferenceOrder,
disabledCodec: _getCodecMimeType(config.videoQuality?.disabledCodec),
preferredCodec: _getCodecMimeType(config.videoQuality?.preferredCodec)
preferredCodec: _getCodecMimeType(config.videoQuality?.preferredCodec),
screenshareCodec: browser.isMobileDevice()
? _getCodecMimeType(config.videoQuality?.mobileScreenshareCodec)
: _getCodecMimeType(config.videoQuality?.screenshareCodec)
},
p2p: {
preferenceOrder: browser.isMobileDevice() && config.p2p?.mobileCodecPreferenceOrder
? config.p2p.mobileCodecPreferenceOrder
: config.p2p?.codecPreferenceOrder,
disabledCodec: _getCodecMimeType(config.p2p?.disabledCodec),
preferredCodec: _getCodecMimeType(config.p2p?.preferredCodec)
preferredCodec: _getCodecMimeType(config.p2p?.preferredCodec),
screenshareCodec: browser.isMobileDevice()
? _getCodecMimeType(config.p2p?.mobileScreenshareCodec)
: _getCodecMimeType(config.p2p?.screenshareCodec)
}
};

Expand Down Expand Up @@ -2198,7 +2204,8 @@ JitsiConference.prototype._acceptJvbIncomingCall = function(jingleSession, jingl
...this.options.config,
codecSettings: {
mediaType: MediaType.VIDEO,
codecList: this.codecSelection.getCodecPreferenceList('jvb')
codecList: this.codecSelection.getCodecPreferenceList('jvb'),
screenshareCodec: this.codecSelection.getScreenshareCodec('jvb')
},
enableInsertableStreams: this.isE2EEEnabled() || FeatureFlags.isRunInLiteModeEnabled()
});
Expand Down Expand Up @@ -2908,7 +2915,8 @@ JitsiConference.prototype._acceptP2PIncomingCall = function(jingleSession, jingl
...this.options.config,
codecSettings: {
mediaType: MediaType.VIDEO,
codecList: this.codecSelection.getCodecPreferenceList('p2p')
codecList: this.codecSelection.getCodecPreferenceList('p2p'),
screenshareCodec: this.codecSelection.getScreenshareCodec('p2p')
},
enableInsertableStreams: this.isE2EEEnabled() || FeatureFlags.isRunInLiteModeEnabled()
});
Expand Down Expand Up @@ -3263,7 +3271,8 @@ JitsiConference.prototype._startP2PSession = function(remoteJid) {
...this.options.config,
codecSettings: {
mediaType: MediaType.VIDEO,
codecList: this.codecSelection.getCodecPreferenceList('p2p')
codecList: this.codecSelection.getCodecPreferenceList('p2p'),
screenshareCodec: this.codecSelection.getScreenshareCodec('p2p')
},
enableInsertableStreams: this.isE2EEEnabled() || FeatureFlags.isRunInLiteModeEnabled()
});
Expand Down
5 changes: 5 additions & 0 deletions JitsiConferenceEventManager.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -734,6 +734,11 @@ JitsiConferenceEventManager.prototype.setupStatisticsListeners = function() {
JitsiConferenceEvents.BEFORE_STATISTICS_DISPOSED);
});

conference.statistics.addEncodeTimeStatsListener((tpc, stats) => {
conference.eventEmitter.emit(
JitsiConferenceEvents.ENCODE_TIME_STATS_RECEIVED, tpc, stats);
});

// if we are in startSilent mode we will not be sending/receiving so nothing to detect
if (!conference.options.config.startSilent) {
conference.statistics.addByteSentStatsListener((tpc, stats) => {
Expand Down
3 changes: 3 additions & 0 deletions JitsiConferenceEvents.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ describe( "/JitsiConferenceEvents members", () => {
E2EE_VERIFICATION_AVAILABLE,
E2EE_VERIFICATION_READY,
E2EE_VERIFICATION_COMPLETED,
ENCODE_TIME_STATS_RECEIVED,
ENDPOINT_MESSAGE_RECEIVED,
ENDPOINT_STATS_RECEIVED,
JVB121_STATUS,
Expand Down Expand Up @@ -166,6 +167,7 @@ describe( "/JitsiConferenceEvents members", () => {
expect( BREAKOUT_ROOMS_MOVE_TO_ROOM ).toBe( 'conference.breakout-rooms.move-to-room' );
expect( BREAKOUT_ROOMS_UPDATED ).toBe( 'conference.breakout-rooms.updated' );
expect( METADATA_UPDATED ).toBe( 'conference.metadata.updated' );
expect( ENCODE_TIME_STATS_RECEIVED ).toBe( 'conference.encode_time_stats_received' );

expect( JitsiConferenceEvents ).toBeDefined();

Expand All @@ -188,6 +190,7 @@ describe( "/JitsiConferenceEvents members", () => {
expect( JitsiConferenceEvents.DOMINANT_SPEAKER_CHANGED ).toBe( 'conference.dominantSpeaker' );
expect( JitsiConferenceEvents.CONFERENCE_CREATED_TIMESTAMP ).toBe( 'conference.createdTimestamp' );
expect( JitsiConferenceEvents.DTMF_SUPPORT_CHANGED ).toBe( 'conference.dtmfSupportChanged' );
expect( JitsiConferenceEvents.ENCODE_TIME_STATS_RECEIVED ).toBe( 'conference.encode_time_stats_received' );
expect( JitsiConferenceEvents.ENDPOINT_MESSAGE_RECEIVED ).toBe( 'conference.endpoint_message_received' );
expect( JitsiConferenceEvents.ENDPOINT_STATS_RECEIVED ).toBe( 'conference.endpoint_stats_received' );
expect( JitsiConferenceEvents.JVB121_STATUS ).toBe( 'conference.jvb121Status' );
Expand Down
6 changes: 6 additions & 0 deletions JitsiConferenceEvents.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,11 @@ export enum JitsiConferenceEvents {

E2EE_VERIFICATION_READY = 'conference.e2ee.verification.ready',

/**
* Indicates that the encode time stats for the local video sources has been received.
*/
ENCODE_TIME_STATS_RECEIVED = 'conference.encode_time_stats_received',

/**
* Indicates that a message from another participant is received on data
* channel.
Expand Down Expand Up @@ -518,6 +523,7 @@ export const DTMF_SUPPORT_CHANGED = JitsiConferenceEvents.DTMF_SUPPORT_CHANGED;
export const E2EE_VERIFICATION_AVAILABLE = JitsiConferenceEvents.E2EE_VERIFICATION_AVAILABLE;
export const E2EE_VERIFICATION_COMPLETED = JitsiConferenceEvents.E2EE_VERIFICATION_COMPLETED;
export const E2EE_VERIFICATION_READY = JitsiConferenceEvents.E2EE_VERIFICATION_READY;
export const ENCODE_TIME_STATS_RECEIVED = JitsiConferenceEvents.ENCODE_TIME_STATS_RECEIVED;
export const ENDPOINT_MESSAGE_RECEIVED = JitsiConferenceEvents.ENDPOINT_MESSAGE_RECEIVED;
export const ENDPOINT_STATS_RECEIVED = JitsiConferenceEvents.ENDPOINT_STATS_RECEIVED;
export const FORWARDED_SOURCES_CHANGED = JitsiConferenceEvents.FORWARDED_SOURCES_CHANGED;
Expand Down
109 changes: 101 additions & 8 deletions modules/RTC/CodecSelection.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,16 @@ export class CodecSelection {
* @param {string} options.p2p settings (codec list, preferred and disabled) for the p2p connection.
*/
constructor(conference, options) {
this.codecPreferenceOrder = {};
this.conference = conference;
this.encodeTimeStats = new Map();
this.options = options;
this.codecPreferenceOrder = {};
this.screenshareCodec = {};
this.visitorCodecs = [];

for (const connectionType of Object.keys(options)) {
// eslint-disable-next-line prefer-const
let { disabledCodec, preferredCodec, preferenceOrder } = options[connectionType];
let { disabledCodec, preferredCodec, preferenceOrder, screenshareCodec } = options[connectionType];
const supportedCodecs = new Set(this._getSupportedVideoCodecs(connectionType));

// Default preference codec order when no codec preferences are set in config.js
Expand Down Expand Up @@ -89,6 +91,11 @@ export class CodecSelection {

logger.info(`Codec preference order for ${connectionType} connection is ${selectedOrder}`);
this.codecPreferenceOrder[connectionType] = selectedOrder;

// Set the preferred screenshare codec.
if (screenshareCodec && supportedCodecs.has(screenshareCodec)) {
this.screenshareCodec[connectionType] = screenshareCodec;
}
}

this.conference.on(
Expand All @@ -103,6 +110,9 @@ export class CodecSelection {
this.conference.on(
JitsiConferenceEvents.USER_LEFT,
() => this._selectPreferredCodec());
this.conference.on(
JitsiConferenceEvents.ENCODE_TIME_STATS_RECEIVED,
(tpc, stats) => this._processEncodeTimeStats(tpc, stats));
}

/**
Expand All @@ -127,6 +137,84 @@ export class CodecSelection {
return supportedCodecs;
}

/**
* Processes the encode time stats received for all the local video sources.
*
* @param {TraceablePeerConnection} tpc - the peerconnection for which stats were gathered.
* @param {Object} stats - the encode time stats for local video sources.
* @returns {void}
*/
_processEncodeTimeStats(tpc, stats) {
const activeSession = this.conference.getActiveMediaSession();

// Process stats only for the active media session.
if (activeSession.peerconnection !== tpc) {
return;
}

const statsPerTrack = new Map();

for (const ssrc of stats.keys()) {
const { codec, encodeTime, qualityLimitationReason, resolution, timestamp } = stats.get(ssrc);
const track = tpc.getTrackBySSRC(ssrc);
let existingStats = statsPerTrack.get(track.rtcId);
const encodeResolution = Math.min(resolution.height, resolution.width);
const ssrcStats = {
encodeResolution,
encodeTime,
qualityLimitationReason
};

if (existingStats) {
existingStats.codec = codec;
existingStats.timestamp = timestamp;
existingStats.trackStats.push(ssrcStats);
} else {
existingStats = {
codec,
timestamp,
trackStats: [ ssrcStats ]
};

statsPerTrack.set(track.rtcId, existingStats);
}
}

// Aggregate the stats for multiple simulcast streams with different SSRCs but for the same video stream.
for (const trackId of statsPerTrack.keys()) {
const { codec, timestamp, trackStats } = statsPerTrack.get(trackId);
const totalEncodeTime = trackStats
.map(stat => stat.encodeTime)
.reduce((totalValue, currentValue) => totalValue + currentValue, 0);
const avgEncodeTime = totalEncodeTime / trackStats.length;
const { qualityLimitationReason = 'none' }
= trackStats.find(stat => stat.qualityLimitationReason !== 'none') ?? {};
const encodeResolution = trackStats
.map(stat => stat.encodeResolution)
.reduce((resolution, currentValue) => Math.max(resolution, currentValue), 0);
const localTrack = this.conference.getLocalVideoTracks().find(t => t.rtcId === trackId);

const exisitingStats = this.encodeTimeStats.get(trackId);
const sourceStats = {
avgEncodeTime,
codec,
encodeResolution,
qualityLimitationReason,
localTrack,
timestamp
};

if (exisitingStats) {
exisitingStats.push(sourceStats);
} else {
this.encodeTimeStats.set(trackId, [ sourceStats ]);
}

logger.debug(`Encode stats for ${localTrack}: codec=${codec}, time=${avgEncodeTime},`
+ `resolution=${encodeResolution}, qualityLimitationReason=${qualityLimitationReason}`);
}
}

/**
* Sets the codec on the media session based on the codec preference order configured in config.js and the supported
* codecs published by the remote participants in their presence.
Expand All @@ -139,9 +227,7 @@ export class CodecSelection {
if (!session) {
return;
}
const currentCodecOrder = session.peerconnection.getConfiguredVideoCodecs();
const isJvbSession = session === this.conference.jvbJingleSession;

let localPreferredCodecOrder = isJvbSession ? this.codecPreferenceOrder.jvb : this.codecPreferenceOrder.p2p;

// E2EE is curently supported only for VP8 codec.
Expand Down Expand Up @@ -195,10 +281,7 @@ export class CodecSelection {
return;
}

// Reconfigure the codecs on the media session.
if (!selectedCodecOrder.every((val, index) => val === currentCodecOrder[index])) {
session.setVideoCodecs(selectedCodecOrder);
}
session.setVideoCodecs(selectedCodecOrder);
}

/**
Expand All @@ -225,4 +308,14 @@ export class CodecSelection {
getCodecPreferenceList(connectionType) {
return this.codecPreferenceOrder[connectionType];
}

/**
* Returns the preferred screenshare codec for the given connection type.
*
* @param {String} connectionType The media connection type, 'p2p' or 'jvb'.
* @returns CodecMimeType
*/
getScreenshareCodec(connectionType) {
return this.screenshareCodec[connectionType];
}
}
12 changes: 6 additions & 6 deletions modules/RTC/CodecSelection.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ describe('Codec Selection', () => {
participant1 = new MockParticipant('remote-1');
conference.addParticipant(participant1, [ 'vp9', 'vp8' ]);

expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(0);
expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(1);

// Add a third user joining the call with a subset of codecs.
participant2 = new MockParticipant('remote-2');
Expand All @@ -145,7 +145,7 @@ describe('Codec Selection', () => {

// Make p2 leave the call
conference.removeParticipant(participant2);
expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(1);
expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(3);
});

it('and remote endpoints use the old codec selection logic (RN)', () => {
Expand All @@ -163,7 +163,7 @@ describe('Codec Selection', () => {

// Make p1 leave the call
conference.removeParticipant(participant1);
expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(2);
expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(3);
});
});

Expand All @@ -185,7 +185,7 @@ describe('Codec Selection', () => {
participant1 = new MockParticipant('remote-1');
conference.addParticipant(participant1, [ 'vp9', 'vp8', 'h264' ]);

expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(0);
expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(1);

// Add a third user joining the call with a subset of codecs.
participant2 = new MockParticipant('remote-2');
Expand All @@ -195,7 +195,7 @@ describe('Codec Selection', () => {

// Make p2 leave the call
conference.removeParticipant(participant2);
expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(1);
expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(3);
});

it('and remote endpoint prefers a codec that is locally disabled', () => {
Expand All @@ -221,7 +221,7 @@ describe('Codec Selection', () => {

// Make p1 leave the call
conference.removeParticipant(participant1);
expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(2);
expect(jingleSession.setVideoCodecs).toHaveBeenCalledTimes(3);
});
});
});
Loading

0 comments on commit 6bcc577

Please sign in to comment.