From 8e13851e7be8b32077a97653990ef3d1bd8223a0 Mon Sep 17 00:00:00 2001 From: Andrei Volykhin Date: Fri, 22 Sep 2017 18:38:43 +0300 Subject: [PATCH] Add new object and tokens to WEBGL_video_texture * Add new HTMLElementTexture object and new tokens for TEXTURE_VIDEO_WEBGL texture target. --- .../WEBGL_video_texture/extension.xml | 293 +++++++++++++++--- 1 file changed, 242 insertions(+), 51 deletions(-) diff --git a/extensions/proposals/WEBGL_video_texture/extension.xml b/extensions/proposals/WEBGL_video_texture/extension.xml index 764e6044af..9af4797694 100644 --- a/extensions/proposals/WEBGL_video_texture/extension.xml +++ b/extensions/proposals/WEBGL_video_texture/extension.xml @@ -6,8 +6,12 @@ working group (public_webgl 'at' khronos.org) - Byungseon Shin (sun.shin 'at' lge.com) - Andrey Volykhin (andrey.volykhin 'at' lge.com) + Byungseon Shin, LG Electronics + + Andrei Volykhin, LG Electronics + + Mark Callow, Edgewise Consulting + Members of the WebGL working group @@ -18,19 +22,47 @@ - - Defines a new texture target TEXTURE_VIDEO_IMAGE. - Provides a mechanism for binding HTMLVideoElement stream to video texture targets. - Provides time of frame, texture width and height of HTMLVideoElement's texture binding. - +

This extension defines a new object, the HTMLVideoTexture, + that can be used to efficiently transfer a sequence of image frames from + a producer which is outside the control of WebGL into a WebGL texture. + This is done via a new texture target, TEXTURE_VIDEO_WEBGL + which can only be specified as being the consumer of an image stream from a + producer element.

- - Add support for WEBGL_video_texture - binding of HTMLVideoElement. +

There is no support for most of the functions that manipulate other + texture targets (e.g. you cannot use *[Tt]ex*Image*() + functions with TEXTURE_VIDEO_WEBGL). Also, + TEXTURE_VIDEO_WEBGL targets never have more than a single + level of detail. Because of these restrictions, it is possible to + allow sources which have internal formats not otherwise supported by WebGL, + such as planar or interleaved YUV data to be WebGL texture target siblings.

+ +

The extension extends GLSL ES with a new samplerVideoWebGL type + and matching sampling functions that provide a place for an implementation + to inject code for sampling non-RGB data when necessary without degrading performance + for other texture targets. Sampling a TEXTURE_VIDEO_WEBGL via a sampler of + type samplerVideoWebGL always returns RGBA data.

+ +

Sampling a WebGL texture which is not associated with any + HTMLVideoTexture object will return a sample value of (0,0,0,1).

+ +

Each TEXTURE_VIDEO_WEBGL texture object may require up + to 3 texture image units for each texture unit to which it is bound. + The number of texture image units required by a bound texture object + can be queried using getTexParameter with target set + to the texture target in question, value set to + REQUIRED_TEXTURE_VIDEO_IMAGE_UNITS_WEBGL, and activeTexture + set to the texture unit to which the texture object is bound.

+ +

HTMLVideoTexture provides a commands for latching an + image frame into the consuming texture as its contents and retrieving additional + information as a timestamp or a transformation matrix.

+ + + @@ -39,87 +71,243 @@ + + + + + + + + + + + [NoInterfaceObject] -interface WebGLVideoFrameInfo { - readonly attribute double currentTime; - readonly attribute unsigned long textureWidth; - readonly attribute unsigned long textureHeight; +interface HTMLVideoTexture : EventTarget { + void updateTexImage(); + + double getTimestamp(); + void getTransformMatrix(Float32Array matrix); + + double getLastAvailableTimestamp(); + + attribute EventHandler onframeavailable; + attribute EventHandler onerror; }; [NoInterfaceObject] interface WEBGL_video_texture { - const GLenum TEXTURE_VIDEO_IMAGE = 0x851D; - const GLenum SAMPLER_VIDEO_IMAGE = 0x8B61; + const GLenum TEXTURE_VIDEO_WEBGL = 0x9248; + const GLenum SAMPLER_VIDEO_WEBGL = 0x9249; + const GLenum TEXTURE_BINDING_VIDEO_WEBGL = 0x924A; + const GLenum REQUIRED_TEXTURE_VIDEO_IMAGE_UNITS_WEBGL = 0x924B; - [RaisesException] WebGLVideoFrameInfo VideoElementTargetVideoTexture( - GLenum target, HTMLVideoElement video); + [RaisesException] HTMLVideoTexture? createVideoTexture(WebGLTexture? texture, HTMLVideoElement video); }; - + + + +

On HTMLVideoTexture:

+ + + Release the previously-held frame, if any, and update the texture image + to the most recent frame from the image stream. + This may cause some frames of the stream to be skipped. + Sampling the WebGL texture, that is the HTMLVideoTexture's + consumer, will return values from the latched image. + The image data is guaranteed not to change as long as the image is latched. + It will implicitly binds associated WebGL texture to the active texture unit's + TEXTURE_VIDEO_WEBGL texture target. + + + + Retrieve the timestamp associated with the texture image set by the most + recent call to updateTexImage. This timestamp represent + the time of frame relative to start of the producer timeline, the time + when the frame was produced (the Media Stream Counter in OpenML), + not the time when frame was received by application. + This timestamp is in seconds, and is normally monotonically increasing. + It is equivalent of HTMLMediaElement's currentTime property. + + + + + Retrieve the 4x4 texture coordinate transform matrix associated with + the texture image set by the most recent call to updateTexImage. + This transform matrix maps 2D homogeneous texture coordinates + of the form (s, t, 0, 1) with s and t in the inclusive range [0, 1] + to the texture coordinate that should be used to sample that location + from the texture. Sampling the texture outside of the range + of this transform is undefined. + The matrix is stored in column-major order. + + + + Retrieve the timestamp associated with the most recent texture image + which could be provided by a producer. + + + +
+
onframeavailable of type + EventHandler
+ +
The onframeavailable handler is executed when a new frame + could be latched from image stream.
+ +
onerror of type + EventHandler
+ +
The onerror handler is executed when this object + will be in "incomplete" state in case if a image stream's consumer or + producer will be deleted or due internal errors.
+
-

This a fragment shader that samples a video texture.

+ + + +

On WEBGL_video_texture:

+ +

The meaning and use of these tokens is similar as described in OES_EGL_image_external.

+ + + + + TEXTURE_VIDEO_WEBGL is accepted as a target by the + target parameter of bindTexture + + + + + + SAMPLER_VIDEO_WEBGL can be returned in the + type field of the WebGLActiveInfo returned by + getActiveUniform + + + + + TEXTURE_BINDING_VIDEO_WEBGL is accepted by + the pname parameter of getParameter + + + + + + REQUIRED_TEXTURE_VIDEO_IMAGE_UNITS_WEBGL is accepted + as the pname parameter of getTexParameter* + + + + +

New HTMLVideoTexture object creation

+ + + + + Create a new HTMLVideoTexture> object that can be used to + transfer a sequence of image frames from a HTMLVideoElement + into a WebGL texture. + WebGL texture must be defined as a texture with TEXTURE_VIDEO_WEBGL + target before passing it as texture parameter + (Note: to create a TEXTURE_VIDEO_WEBGL texture and bind it, + call bindTexture with target set to + TEXTURE_VIDEO_WEBGL and texture + set to the name of the new created texture). + If this function is called with HTMLVideoElement + which origin differs from the origin of the containing Document, + a SECURITY_ERR exception must be thrown. + If the WebGL texture is later deleted, connected to a different + HTMLVideoTexture, then this HTMLVideoTexture + will be in "incomplete" state and release any associated resources. + If the HTMLVideoElement or HTMLVideoTexture + is later destroyed then the consuming texture will be "incomplete" and + sampling from it will return a sample value of (0,0,0,1). + + + + + + The error INVALID_OPERATION is generated by calling + createVideoTexture with a texture parameter + that does not identify a VIDEO_WEBGL texture. + + + + +

This a fragment shader that samples a video texture.

+    #version 100 es
     #extension GL_WEBGL_video_texture : require
+
     precision mediump float;
-    varying vec2 v_texCoord;
 
     uniform samplerVideoWEBGL uSampler;
+    varying vec2 v_texCoord;
 
     void main(void) {
       gl_FragColor = texture2D(uSampler, v_texCoord);
     }
-

This shows application that renders video using proposed extension.

+

This shows application that renders an HTMLVideoElement using proposed extension.

-    var videoElement = document.getElementById("video");
-    var videoTexture = gl.createTexture();
+    var gl = document.createElement('canvas').getContext('webgl');
+    var video = document.getElementById("video");
 
-    function update() {
-        var ext = gl.getExtension('WEBGL_video_texture');
-        if(ext !=== null){
-            gl.bindTexture(ext.TEXTURE_VIDEO_IMAGE, videoTexture);
-            ext.VideoElementTargetVideoTexture(ext.TEXTURE_VIDEO_IMAGE, videoElement);
-            gl.bindTexture(ext.TEXTURE_VIDEO_IMAGE, null);
-        }
-    }
-
-    function render() {
-        gl.clearColor(0.0, 0.0, 1.0, 1.0);
-        gl.clear(gl.COLOR_BUFFER_BIT);
+    var ext = gl.getExtension('WEBGL_video_texture');
 
-        gl.bindBuffer(gl.ARRAY_BUFFER, squareVerticesBuffer);
-        gl.vertexAttribPointer(vertexPositionAttribute, 3, gl.FLOAT, false, 0, 0);
+    var texture = gl.createTexture();
+    gl.bindTexture(ext.TEXTURE_VIDEO_WEBGL, texture); // explicit texture binding
 
-        gl.activeTexture(gl.TEXTURE0);
-        gl.bindTexture(ext.TEXTURE_VIDEO_IMAGE, videoTexture);
-        gl.uniform1i(gl.getUniformLocation(shaderProgram, "uSampler"), 0);
+    var videoTexture = ext.createVideoTexture(texture, video);
+    videoTexture.onframeavailable = function() { ... };
+    videoTexture.onerror = function() { ... };
 
-        gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4);
+    function update() {
+      videoTexture.updateTexImage(); // implicit texture binding
     }
-
-

Application renders each video frames into WebGL canvas based on game-loop pattern.

- 
+    function render() {
+        // ... do something
+        gl.drawArrays(...);
+    }
 
     while (true) {
-       update();
-       processInput();
-       render();
+      update();
+      processInput();
+      render();
     }
- - + +
    +
  1. +

    In which colorspace should be the RGB value returned by sampler?

    + +

    A. In linear RGB colorspace.

    + +

    B. In extended sRGB (scRGB-non-linear) colorspace kCGColorSpaceExtendedSRGB

    + +

    C. It will be allowed to choose by passed parameter at a + HTMLVideoTexture object creation time.

    + +

    UNRESOLVED.

    +
    2. +
+ @@ -135,5 +323,8 @@ interface WEBGL_video_texture { Define new sampler and texture type, TEXTURE_VIDEO_IMAGE and SAMPLER_VIDEO_IMAGE. Change EGLImageTargetTexture2DOES to VideoElementTargetVideoTexture. + + Add new HTMLVideoTexture object and new tokens. +