feat(webcodecs): reset the decoder on every keyframe (#826)

Author: yume-chan

libraries/scrcpy-decoder-webcodecs/src/video/codec/av1.ts

@@ -9,6 +9,9 @@ export class Av1Codec implements CodecDecoder {
     #updateSize: (width: number, height: number) => void;
     #options: CodecDecoderOptions | undefined;
 
+    #config: VideoDecoderConfig | undefined;
+    #configured = false;
+
     constructor(
         decoder: VideoDecoder,
         updateSize: (width: number, height: number) => void,
@@ -19,7 +22,7 @@ export class Av1Codec implements CodecDecoder {
         this.#options = options;
     }
 
-    #configure(data: Uint8Array) {
+    #parseConfig(data: Uint8Array) {
         const parser = new Av1(data);
         const sequenceHeader = parser.searchSequenceHeaderObu();
 
@@ -80,24 +83,49 @@ export class Av1Codec implements CodecDecoder {
             decimalTwoDigits(matrixCoefficients),
             colorRange ? "1" : "0",
         ].join(".");
-        this.#decoder.configure({
+        this.#config = {
             codec,
             hardwareAcceleration:
                 this.#options?.hardwareAcceleration ?? "no-preference",
             optimizeForLatency: true,
-        });
+        };
+        this.#configured = false;
     }
 
     decode(packet: ScrcpyMediaStreamPacket): void {
         if (packet.type === "configuration") {
             return;
         }
 
-        this.#configure(packet.data);
+        this.#parseConfig(packet.data);
+
+        if (!this.#config) {
+            throw new Error("Decoder not configured");
+        }
+
+        if (packet.keyframe) {
+            if (this.#decoder.decodeQueueSize) {
+                // If the device is too slow to decode all frames,
+                // discard queued frames when next keyframe arrives.
+                // (can only do this for keyframes because decoding must start from a keyframe)
+                // This limits the maximum latency to 1 keyframe interval
+                // (60 frames by default).
+                this.#decoder.reset();
+
+                // `reset` also resets the decoder configuration
+                // so we need to re-configure it again.
+                this.#decoder.configure(this.#config);
+                this.#configured = true;
+            } else if (!this.#configured) {
+                this.#decoder.configure(this.#config);
+                this.#configured = true;
+            }
+        }
+
         this.#decoder.decode(
             new EncodedVideoChunk({
-                // Treat `undefined` as `key`, otherwise it won't decode.
-                type: packet.keyframe === false ? "delta" : "key",
+                // AV1 requires Scrcpy 2.0 where `keyframe` flag must be set
+                type: packet.keyframe! ? "key" : "delta",
                 timestamp: 0,
                 data: packet.data,
             }),

libraries/scrcpy-decoder-webcodecs/src/video/codec/h264.ts

@@ -5,7 +5,6 @@ import type { CodecDecoderOptions } from "./type.js";
 import { hexTwoDigits } from "./utils.js";
 
 export class H264Decoder extends H26xDecoder {
-    #decoder: VideoDecoder;
     #updateSize: (width: number, height: number) => void;
     #options: CodecDecoderOptions | undefined;
 
@@ -15,12 +14,11 @@ export class H264Decoder extends H26xDecoder {
         options?: CodecDecoderOptions,
     ) {
         super(decoder);
-        this.#decoder = decoder;
         this.#updateSize = updateSize;
         this.#options = options;
     }
 
-    override configure(data: Uint8Array): void {
+    override configure(data: Uint8Array): VideoDecoderConfig {
         const {
             profileIndex,
             constraintSet,
@@ -38,11 +36,11 @@ export class H264Decoder extends H26xDecoder {
             hexTwoDigits(profileIndex) +
             hexTwoDigits(constraintSet) +
             hexTwoDigits(levelIndex);
-        this.#decoder.configure({
+        return {
             codec: codec,
             hardwareAcceleration:
                 this.#options?.hardwareAcceleration ?? "no-preference",
             optimizeForLatency: true,
-        });
+        };
     }
 }

libraries/scrcpy-decoder-webcodecs/src/video/codec/h265.ts

@@ -6,7 +6,6 @@ import type { CodecDecoderOptions } from "./type.js";
 import { hexDigits } from "./utils.js";
 
 export class H265Decoder extends H26xDecoder {
-    #decoder: VideoDecoder;
     #updateSize: (width: number, height: number) => void;
     #options: CodecDecoderOptions | undefined;
 
@@ -16,12 +15,11 @@ export class H265Decoder extends H26xDecoder {
         options?: CodecDecoderOptions,
     ) {
         super(decoder);
-        this.#decoder = decoder;
         this.#updateSize = updateSize;
         this.#options = options;
     }
 
-    override configure(data: Uint8Array): void {
+    override configure(data: Uint8Array): VideoDecoderConfig {
         const {
             generalProfileSpace,
             generalProfileIndex,
@@ -43,14 +41,14 @@ export class H265Decoder extends H26xDecoder {
             (generalTierFlag ? "H" : "L") + generalLevelIndex.toString(),
             ...Array.from(generalConstraintSet, hexDigits),
         ].join(".");
-        this.#decoder.configure({
+        return {
             codec,
             // Microsoft Edge requires explicit size to work
             codedWidth: croppedWidth,
             codedHeight: croppedHeight,
             hardwareAcceleration:
                 this.#options?.hardwareAcceleration ?? "no-preference",
             optimizeForLatency: true,
-        });
+        };
     }
 }

libraries/scrcpy-decoder-webcodecs/src/video/codec/h26x.ts

@@ -1,44 +1,100 @@
-import type { ScrcpyMediaStreamPacket } from "@yume-chan/scrcpy";
+import type {
+    ScrcpyMediaStreamDataPacket,
+    ScrcpyMediaStreamPacket,
+} from "@yume-chan/scrcpy";
 
 import type { CodecDecoder } from "./type.js";
 
 export abstract class H26xDecoder implements CodecDecoder {
-    #config: Uint8Array | undefined;
     #decoder: VideoDecoder;
 
+    #config: (VideoDecoderConfig & { raw: Uint8Array }) | undefined;
+    #configured = false;
+
     constructor(decoder: VideoDecoder) {
         this.#decoder = decoder;
     }
 
-    abstract configure(data: Uint8Array): void;
+    abstract configure(data: Uint8Array): VideoDecoderConfig;
 
-    decode(packet: ScrcpyMediaStreamPacket): void {
-        if (packet.type === "configuration") {
-            this.#config = packet.data;
-            this.configure(packet.data);
-            return;
-        }
+    #configureAndDecodeFirstKeyframe(
+        config: VideoDecoderConfig & { raw: Uint8Array },
+        packet: ScrcpyMediaStreamDataPacket,
+    ) {
+        this.#decoder.configure(config);
+        this.#configured = true;
 
         // For H.264 and H.265, when the stream is in Annex B format
         // (which Scrcpy uses, as Android MediaCodec produces),
         // configuration data needs to be combined with the first frame data.
         // https://www.w3.org/TR/webcodecs-avc-codec-registration/#encodedvideochunk-type
-        let data: Uint8Array;
-        if (this.#config !== undefined) {
-            data = new Uint8Array(this.#config.length + packet.data.length);
-            data.set(this.#config, 0);
-            data.set(packet.data, this.#config.length);
-            this.#config = undefined;
-        } else {
-            data = packet.data;
+        const { raw } = config;
+        const data = new Uint8Array(raw.length + packet.data.length);
+        data.set(raw, 0);
+        data.set(packet.data, raw.length);
+
+        this.#decoder.decode(
+            new EncodedVideoChunk({
+                type: "key",
+                timestamp: 0,
+                data,
+            }),
+        );
+    }
+
+    decode(packet: ScrcpyMediaStreamPacket): void {
+        if (packet.type === "configuration") {
+            this.#config = {
+                ...this.configure(packet.data),
+                raw: packet.data,
+            };
+            this.#configured = false;
+            return;
+        }
+
+        if (!this.#config) {
+            throw new Error("Decoder not configured");
+        }
+
+        if (packet.keyframe) {
+            if (this.#decoder.decodeQueueSize) {
+                // If the device is too slow to decode all frames,
+                // discard queued frames when next keyframe arrives.
+                // (can only do this for keyframes because decoding must start from a keyframe)
+                // This limits the maximum latency to 1 keyframe interval
+                // (60 frames by default).
+                this.#decoder.reset();
+
+                // `reset` also resets the decoder configuration
+                // so we need to re-configure it again.
+                this.#configureAndDecodeFirstKeyframe(this.#config, packet);
+                return;
+            }
+
+            if (!this.#configured) {
+                this.#configureAndDecodeFirstKeyframe(this.#config, packet);
+                return;
+            }
+        }
+
+        if (!this.#configured) {
+            if (packet.keyframe === undefined) {
+                // Scrcpy <1.23 doesn't send `keyframe` flag
+                // Infer the first frame after configuration as keyframe
+                // (`VideoDecoder` will throw error if it's not)
+                this.#configureAndDecodeFirstKeyframe(this.#config, packet);
+                return;
+            }
+
+            throw new Error("Expect a keyframe but got a delta frame");
         }
 
         this.#decoder.decode(
             new EncodedVideoChunk({
                 // Treat `undefined` as `key`, otherwise won't decode.
                 type: packet.keyframe === false ? "delta" : "key",
                 timestamp: 0,
-                data,
+                data: packet.data,
             }),
         );
     }