diff --git a/src/resources/README.md b/src/resources/README.md index a1ed0604170c..1db940714778 100644 --- a/src/resources/README.md +++ b/src/resources/README.md @@ -1,93 +1,98 @@ Always use `getResourcePath()` to get the appropriate path to these resources depending on the context (WPT, standalone, worker, etc.) - The test video files were generated with by ffmpeg cmds below: ``` // Generate four-colors-vp8-bt601.webm, mimeType: 'video/webm; codecs=vp8' -ffmpeg.exe -loop 1 -i .\four-colors.png -c:v libvpx -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv four-colors-vp8-bt601.webm +ffmpeg -loop 1 -i four-colors.png -c:v libvpx -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv four-colors-vp8-bt601.webm // Generate four-colors-h264-bt601.mp4, mimeType: 'video/mp4; codecs=avc1.4d400c' -ffmpeg.exe -loop 1 -i .\four-colors.png -c:v libx264 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv four-colors-h264-bt601.mp4 +ffmpeg -loop 1 -i four-colors.png -c:v libx264 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv four-colors-h264-bt601.mp4 // Generate four-colors-vp9-bt601.webm, mimeType: 'video/webm; codecs=vp9' -ffmpeg.exe -loop 1 -i .\four-colors.png -c:v libvpx-vp9 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv four-colors-vp9-bt601.webm +ffmpeg -loop 1 -i four-colors.png -c:v libvpx-vp9 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv four-colors-vp9-bt601.webm // Generate four-colors-vp9-bt709.webm, mimeType: 'video/webm; codecs=vp9' -ffmpeg.exe -loop 1 -i .\four-colors.png -c:v libvpx-vp9 -pix_fmt yuv420p -frames 50 -colorspace bt709 -color_primaries bt709 -color_trc bt709 -color_range tv -vf scale=out_color_matrix=bt709:out_range=tv four-colors-vp9-bt709.webm +ffmpeg -loop 1 -i four-colors.png -c:v libvpx-vp9 -pix_fmt yuv420p -frames 50 -colorspace bt709 -color_primaries bt709 -color_trc bt709 -color_range tv -vf scale=out_color_matrix=bt709:out_range=tv four-colors-vp9-bt709.webm // Generate four-colors-vp9-bt601.mp4, mimeType: 'video/mp4; codecs=vp9' -ffmpeg.exe -loop 1 -i .\four-colors.png -c:v libvpx-vp9 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv four-colors-vp9-bt601.mp4 +ffmpeg -loop 1 -i four-colors.png -c:v libvpx-vp9 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv four-colors-vp9-bt601.mp4 ``` Generate video files to test rotation behaviour. Use ffmepg to rotate video content x degrees in cw direction (by using `transpose`) and update transform matrix in metadata through `display_rotation` to x degrees to apply ccw direction rotation. H264 rotated video files are generated by ffmpeg cmds below: + ``` // Generate four-colors-h264-bt601-rotate-90.mp4, mimeType: 'video/mp4; codecs=avc1.4d400c' -ffmpeg.exe -loop 1 -i .\four-colors.png -c:v libx264 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv -vf transpose=2 temp.mp4 +ffmpeg -loop 1 -i four-colors.png -c:v libx264 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv -vf transpose=2 temp.mp4 ffmpeg -display_rotation 270 -i temp.mp4 -c copy four-colors-h264-bt601-rotate-90.mp4 rm temp.mp4 // Generate four-colors-h264-bt601-rotate-180.mp4, mimeType: 'video/mp4; codecs=avc1.4d400c' -ffmpeg.exe -loop 1 -i .\four-colors.png -c:v libx264 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv -vf transpose=2,transpose=2 temp.mp4 +ffmpeg -loop 1 -i four-colors.png -c:v libx264 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv -vf transpose=2,transpose=2 temp.mp4 ffmpeg -display_rotation 180 -i temp.mp4 -c copy four-colors-h264-bt601-rotate-180.mp4 rm temp.mp4 // Generate four-colors-h264-bt601-rotate-270.mp4, mimeType: 'video/mp4; codecs=avc1.4d400c' -ffmpeg.exe -loop 1 -i .\four-colors.png -c:v libx264 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv -vf transpose=1 temp.mp4 +ffmpeg -loop 1 -i four-colors.png -c:v libx264 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv -vf transpose=1 temp.mp4 ffmpeg -display_rotation 90 -i temp.mp4 -c copy four-colors-h264-bt601-rotate-270.mp4 rm temp.mp4 - ``` Vp9 rotated video files are generated by ffmpeg cmds below: + ``` // Generate four-colors-vp9-bt601-rotate-90.mp4, mimeType: 'video/mp4; codecs=vp9' -ffmpeg.exe -loop 1 -i .\four-colors.png -c:v libvpx-vp9 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv -vf transpose=2 temp.mp4 +ffmpeg -loop 1 -i four-colors.png -c:v libvpx-vp9 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv -vf transpose=2 temp.mp4 ffmpeg -display_rotation 270 -i temp.mp4 -c copy four-colors-vp9-bt601-rotate-90.mp4 rm temp.mp4 // Generate four-colors-vp9-bt601-rotate-180.mp4, mimeType: 'video/mp4; codecs=vp9' -ffmpeg.exe -loop 1 -i .\four-colors.png -c:v libvpx-vp9 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv -vf transpose=2,transpose=2 temp.mp4 +ffmpeg -loop 1 -i four-colors.png -c:v libvpx-vp9 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv -vf transpose=2,transpose=2 temp.mp4 ffmpeg -display_rotation 180 -i temp.mp4 -c copy four-colors-vp9-bt601-rotate-180.mp4 rm temp.mp4 // Generate four-colors-vp9-bt601-rotate-270.mp4, mimeType: 'video/mp4; codecs=vp9' -ffmpeg.exe -loop 1 -i .\four-colors.png -c:v libvpx-vp9 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv -vf transpose=1 temp.mp4 +ffmpeg -loop 1 -i four-colors.png -c:v libvpx-vp9 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv -vf transpose=1 temp.mp4 ffmpeg -display_rotation 90 -i temp.mp4 -c copy four-colors-vp9-bt601-rotate-270.mp4 rm temp.mp4 - ``` Generate video files to test flip behaviour. Use ffmpeg to flip video content. Using `display_hflip` to do horizontal flip and `display_vflip` to do vertical flip. H264 flip video files are generated by ffmpeg cmds below: + ``` // Generate four-colors-h264-bt601-hflip.mp4, mimeType: 'video/mp4; codecs=avc1.4d400c' -ffmpeg.exe -loop 1 -i .\four-colors.png -c:v libx264 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv temp.mp4 +ffmpeg -loop 1 -i four-colors.png -c:v libx264 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv temp.mp4 ffmpeg -display_hflip -i temp.mp4 -c copy four-colors-h264-bt601-hflip.mp4 rm temp.mp4 // Generate four-colors-h264-bt601-vflip.mp4, mimeType: 'video/mp4; codecs=avc1.4d400c' -ffmpeg.exe -loop 1 -i .\four-colors.png -c:v libx264 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv temp.mp4 +ffmpeg -loop 1 -i four-colors.png -c:v libx264 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv temp.mp4 ffmpeg -display_vflip -i temp.mp4 -c copy four-colors-h264-bt601-vflip.mp4 rm temp.mp4 - ``` Vp9 flip video files are generated by ffmpeg cmds below: + ``` // Generate four-colors-vp9-bt601-hflip.mp4, mimeType: 'video/mp4; codecs=vp09.00.10.08' -ffmpeg.exe -loop 1 -i .\four-colors.png -c:v libvpx-vp9 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv temp.mp4 +ffmpeg -loop 1 -i four-colors.png -c:v libvpx-vp9 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv temp.mp4 ffmpeg -display_hflip -i temp.mp4 -c copy four-colors-vp9-bt601-hflip.mp4 rm temp.mp4 // Generate four-colors-vp9-bt601-vflip.mp4, mimeType: 'video/mp4; codecs=vp09.00.10.08' -ffmpeg.exe -loop 1 -i .\four-colors.png -c:v libvpx-vp9 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv temp.mp4 +ffmpeg -loop 1 -i four-colors.png -c:v libvpx-vp9 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv temp.mp4 ffmpeg -display_vflip -i temp.mp4 -c copy four-colors-vp9-bt601-vflip.mp4 rm temp.mp4 +``` +Scaled 10x1 video files are generated by ffmpeg cmds below: ``` +// Generate four-colors-h264-bt601-scaled-10x1.mp4, mimeType: 'video/mp4; codecs=avc1.4d400c' +ffmpeg -loop 1 -i four-colors.png -c:v libx264 -pix_fmt yuv420p -frames 50 -colorspace smpte170m -color_primaries smpte170m -color_trc smpte170m -color_range tv -vf transpose=2 temp.mp4 +``` \ No newline at end of file diff --git a/src/resources/four-colors-h264-bt601-scaled-10x1.mp4 b/src/resources/four-colors-h264-bt601-scaled-10x1.mp4 new file mode 100644 index 000000000000..f78540e33898 Binary files /dev/null and b/src/resources/four-colors-h264-bt601-scaled-10x1.mp4 differ diff --git a/src/resources/four-colors-vp9-bt601-scaled-10x1.mp4 b/src/resources/four-colors-vp9-bt601-scaled-10x1.mp4 new file mode 100644 index 000000000000..4c461dbc7513 Binary files /dev/null and b/src/resources/four-colors-vp9-bt601-scaled-10x1.mp4 differ diff --git a/src/webgpu/web_platform/external_texture/video.spec.ts b/src/webgpu/web_platform/external_texture/video.spec.ts index b5796f0a1800..e6da48166f05 100644 --- a/src/webgpu/web_platform/external_texture/video.spec.ts +++ b/src/webgpu/web_platform/external_texture/video.spec.ts @@ -252,6 +252,152 @@ for several combinations of video format, video color spaces and dst color space }); }); +g.test('importExternalTexture,loadAndDimensions') + .desc( + ` +Tests that we can import an HTMLVideoElement/VideoFrame into a GPUExternalTexture, textureLoad from it +for several combinations of video format, video color spaces and dst color spaces, and get the correct textureDimensions() +in the shader. +` + ) + .params(u => + u // + .combine('videoName', kVideoNames) + .combine('sourceType', ['VideoElement', 'VideoFrame'] as const) + ) + .fn(async t => { + const { videoName, sourceType } = t.params; + const kDstColorSpace = 'srgb'; + + if (sourceType === 'VideoFrame' && typeof VideoFrame === 'undefined') { + t.skip('WebCodec is not supported'); + } + + const videoElement = getVideoElement(t, videoName); + + await startPlayingAndWaitForVideo(videoElement, async () => { + // Get the video source and its size + let source: HTMLVideoElement | VideoFrame = videoElement; + let expectedWidth = videoElement.videoWidth; + let expectedHeight = videoElement.videoHeight; + if (sourceType === 'VideoFrame') { + const frame = await getVideoFrameFromVideoElement(t, source); + source = frame; + expectedWidth = frame.displayWidth; + expectedHeight = frame.displayHeight; + } + + // Create the pipeline + const module = t.device.createShaderModule({ + code: ` + @vertex fn vs(@builtin(vertex_index) i : u32) -> @builtin(position) vec4f { + const pos = array( + vec2(2, 0), + vec2(-1, 3), + vec2(-1, -3), + ); + return vec4f(vec2f(pos[i]), 0, 1); + } + + @group(0) @binding(0) var dimension : vec2u; + @group(0) @binding(1) var t : texture_external; + @fragment fn fs(@builtin(position) pos : vec4f) -> @location(0) vec4f { + dimension = textureDimensions(t); + let normCoord = pos.xy / vec2f(${kWidth}, ${kHeight}); + let loadCoord = normCoord * vec2f(dimension - vec2u(1, 1)); + return textureLoad(t, vec2u(loadCoord)); + } + `, + }); + const pipeline = t.device.createRenderPipeline({ + layout: 'auto', + vertex: { module }, + fragment: { module, targets: [{ format: kFormat }] }, + }); + + // Create the test resources + const dimensionsBuffer = t.device.createBufferTracked({ + size: 8, + usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC, + }); + + const bindGroup = t.device.createBindGroup({ + layout: pipeline.getBindGroupLayout(0), + entries: [ + { + binding: 0, + resource: { buffer: dimensionsBuffer }, + }, + { + binding: 1, + resource: t.device.importExternalTexture({ source, colorSpace: kDstColorSpace }), + }, + ], + }); + + // Run the test + const colorAttachment = t.createTextureTracked({ + format: kFormat, + size: { width: kWidth, height: kHeight, depthOrArrayLayers: 1 }, + usage: GPUTextureUsage.COPY_SRC | GPUTextureUsage.RENDER_ATTACHMENT, + }); + + const commandEncoder = t.device.createCommandEncoder(); + const passEncoder = commandEncoder.beginRenderPass({ + colorAttachments: [ + { + view: colorAttachment.createView(), + clearValue: { r: 0.0, g: 0.0, b: 0.0, a: 1.0 }, + loadOp: 'clear', + storeOp: 'store', + }, + ], + }); + passEncoder.setPipeline(pipeline); + passEncoder.setBindGroup(0, bindGroup); + passEncoder.draw(6); + passEncoder.end(); + t.device.queue.submit([commandEncoder.finish()]); + + const srcColorSpace = kVideoInfo[videoName].colorSpace; + const presentColors = kVideoExpectedColors[srcColorSpace][kDstColorSpace]; + + // visible rect is whole frame, no clipping. + const expect = kVideoInfo[videoName].display; + + // For validation, we sample a few pixels away from the edges to avoid compression + // artifacts. + t.expectSinglePixelComparisonsAreOkInTexture({ texture: colorAttachment }, [ + // Top-left. + { + coord: { x: kWidth * 0.25, y: kHeight * 0.25 }, + exp: convertToUnorm8(presentColors[expect.topLeftColor]), + }, + // Top-right. + { + coord: { x: kWidth * 0.75, y: kHeight * 0.25 }, + exp: convertToUnorm8(presentColors[expect.topRightColor]), + }, + // Bottom-left. + { + coord: { x: kWidth * 0.25, y: kHeight * 0.75 }, + exp: convertToUnorm8(presentColors[expect.bottomLeftColor]), + }, + // Bottom-right. + { + coord: { x: kWidth * 0.75, y: kHeight * 0.75 }, + exp: convertToUnorm8(presentColors[expect.bottomRightColor]), + }, + ]); + + // Check that we got the correct values reflected in textureDimensions + t.expectGPUBufferValuesEqual( + dimensionsBuffer, + new Uint32Array([expectedWidth, expectedHeight]) + ); + }); + }); + g.test('importExternalTexture,sample_non_YUV_video_frame') .desc( ` diff --git a/src/webgpu/web_platform/util.ts b/src/webgpu/web_platform/util.ts index f000d80e56b9..0eb0a3cc7eb1 100644 --- a/src/webgpu/web_platform/util.ts +++ b/src/webgpu/web_platform/util.ts @@ -343,6 +343,38 @@ export const kVideoInfo = makeTable({ bottomRightColor: 'red', }, }, + 'four-colors-h264-bt601-scaled-10x1.mp4': { + mimeType: 'video/mp4; codecs=avc1.4d400c', + colorSpace: 'bt601', + coded: { + topLeftColor: 'yellow', + topRightColor: 'red', + bottomLeftColor: 'blue', + bottomRightColor: 'green', + }, + display: { + topLeftColor: 'yellow', + topRightColor: 'red', + bottomLeftColor: 'blue', + bottomRightColor: 'green', + }, + }, + 'four-colors-vp9-bt601-scaled-10x1.mp4': { + mimeType: 'video/webm; codecs=vp9', + colorSpace: 'bt601', + coded: { + topLeftColor: 'yellow', + topRightColor: 'red', + bottomLeftColor: 'blue', + bottomRightColor: 'green', + }, + display: { + topLeftColor: 'yellow', + topRightColor: 'red', + bottomLeftColor: 'blue', + bottomRightColor: 'green', + }, + }, }, } as const);