feat(webcodecs): reset the decoder on every keyframe

libraries/scrcpy-decoder-webcodecs/src/video/codec/av1.ts

@@ -6,6 +6,7 @@ import { decimalTwoDigits } from "./utils.js";
 
 export class Av1Codec implements CodecDecoder {
     #decoder: VideoDecoder;
+    #parsedConfig: VideoDecoderConfig | undefined;
     #updateSize: (width: number, height: number) => void;
     #options: CodecDecoderOptions | undefined;
 
@@ -80,12 +81,12 @@ export class Av1Codec implements CodecDecoder {
             decimalTwoDigits(matrixCoefficients),
             colorRange ? "1" : "0",
         ].join(".");
-        this.#decoder.configure({
+        this.#parsedConfig = {
             codec,
             hardwareAcceleration:
                 this.#options?.hardwareAcceleration ?? "no-preference",
             optimizeForLatency: true,
-        });
+        };
     }
 
     decode(packet: ScrcpyMediaStreamPacket): void {
@@ -94,6 +95,12 @@ export class Av1Codec implements CodecDecoder {
         }
 
         this.#configure(packet.data);
+
+        if (packet.keyframe) {
+            this.#decoder.reset();
+            this.#decoder.configure(this.#parsedConfig!);
+        }
+
         this.#decoder.decode(
             new EncodedVideoChunk({
                 // Treat `undefined` as `key`, otherwise it won't decode.

libraries/scrcpy-decoder-webcodecs/src/video/codec/h264.ts

@@ -5,7 +5,6 @@ import type { CodecDecoderOptions } from "./type.js";
 import { hexTwoDigits } from "./utils.js";
 
 export class H264Decoder extends H26xDecoder {
-    #decoder: VideoDecoder;
     #updateSize: (width: number, height: number) => void;
     #options: CodecDecoderOptions | undefined;
 
@@ -15,12 +14,11 @@ export class H264Decoder extends H26xDecoder {
         options?: CodecDecoderOptions,
     ) {
         super(decoder);
-        this.#decoder = decoder;
         this.#updateSize = updateSize;
         this.#options = options;
     }
 
-    override configure(data: Uint8Array): void {
+    override configure(data: Uint8Array): VideoDecoderConfig {
         const {
             profileIndex,
             constraintSet,
@@ -38,11 +36,11 @@ export class H264Decoder extends H26xDecoder {
             hexTwoDigits(profileIndex) +
             hexTwoDigits(constraintSet) +
             hexTwoDigits(levelIndex);
-        this.#decoder.configure({
+        return {
             codec: codec,
             hardwareAcceleration:
                 this.#options?.hardwareAcceleration ?? "no-preference",
             optimizeForLatency: true,
-        });
+        };
     }
 }

libraries/scrcpy-decoder-webcodecs/src/video/codec/h265.ts

@@ -6,7 +6,6 @@ import type { CodecDecoderOptions } from "./type.js";
 import { hexDigits } from "./utils.js";
 
 export class H265Decoder extends H26xDecoder {
-    #decoder: VideoDecoder;
     #updateSize: (width: number, height: number) => void;
     #options: CodecDecoderOptions | undefined;
 
@@ -16,12 +15,11 @@ export class H265Decoder extends H26xDecoder {
         options?: CodecDecoderOptions,
     ) {
         super(decoder);
-        this.#decoder = decoder;
         this.#updateSize = updateSize;
         this.#options = options;
     }
 
-    override configure(data: Uint8Array): void {
+    override configure(data: Uint8Array): VideoDecoderConfig {
         const {
             generalProfileSpace,
             generalProfileIndex,
@@ -43,14 +41,14 @@ export class H265Decoder extends H26xDecoder {
             (generalTierFlag ? "H" : "L") + generalLevelIndex.toString(),
             ...Array.from(generalConstraintSet, hexDigits),
         ].join(".");
-        this.#decoder.configure({
+        return {
             codec,
             // Microsoft Edge requires explicit size to work
             codedWidth: croppedWidth,
             codedHeight: croppedHeight,
             hardwareAcceleration:
                 this.#options?.hardwareAcceleration ?? "no-preference",
             optimizeForLatency: true,
-        });
+        };
     }
 }

libraries/scrcpy-decoder-webcodecs/src/video/codec/h26x.ts

@@ -4,31 +4,34 @@ import type { CodecDecoder } from "./type.js";
 
 export abstract class H26xDecoder implements CodecDecoder {
     #config: Uint8Array | undefined;
+    #parsedConfig: VideoDecoderConfig | undefined;
     #decoder: VideoDecoder;
 
     constructor(decoder: VideoDecoder) {
         this.#decoder = decoder;
     }
 
-    abstract configure(data: Uint8Array): void;
+    abstract configure(data: Uint8Array): VideoDecoderConfig;
 
     decode(packet: ScrcpyMediaStreamPacket): void {
         if (packet.type === "configuration") {
             this.#config = packet.data;
-            this.configure(packet.data);
+            this.#parsedConfig = this.configure(packet.data);
             return;
         }
 
-        // For H.264 and H.265, when the stream is in Annex B format
-        // (which Scrcpy uses, as Android MediaCodec produces),
-        // configuration data needs to be combined with the first frame data.
-        // https://www.w3.org/TR/webcodecs-avc-codec-registration/#encodedvideochunk-type
         let data: Uint8Array;
-        if (this.#config !== undefined) {
-            data = new Uint8Array(this.#config.length + packet.data.length);
-            data.set(this.#config, 0);
-            data.set(packet.data, this.#config.length);
-            this.#config = undefined;
+        if (packet.keyframe) {
+            this.#decoder.reset();
+            this.#decoder.configure(this.#parsedConfig!);
+
+            // For H.264 and H.265, when the stream is in Annex B format
+            // (which Scrcpy uses, as Android MediaCodec produces),
+            // configuration data needs to be combined with the first frame data.
+            // https://www.w3.org/TR/webcodecs-avc-codec-registration/#encodedvideochunk-type
+            data = new Uint8Array(this.#config!.length + packet.data.length);
+            data.set(this.#config!, 0);
+            data.set(packet.data, this.#config!.length);
         } else {
             data = packet.data;
         }