feat: Allow additional images in Materials (#2892)

This PR adds the ability to load additional images into `ex.Material`s!
  ```typescript
  const noise = new ex.ImageSource('./noise.avif');
  loader.addResource(noise);

  var waterMaterial = game.graphicsContext.createMaterial({
    name: 'water',
    fragmentSource: waterFrag,
    color: ex.Color.fromRGB(55, 0, 200, .6),
    images: {
      u_noise: noise
    }
  });
  ```

CHANGELOG.md

@@ -16,6 +16,20 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ### Added
 
+- Added ability to load additional images into `ex.Material`s!
+  ```typescript
+  const noise = new ex.ImageSource('./noise.avif');
+  loader.addResource(noise);
+
+  var waterMaterial = game.graphicsContext.createMaterial({
+    name: 'water',
+    fragmentSource: waterFrag,
+    color: ex.Color.fromRGB(55, 0, 200, .6),
+    images: {
+      u_noise: noise
+    }
+  });
+  ```
 - Scene Transition & Loader API, this gives you the ability to have first class support for individual scene resource loading and scene transitions.
   * Add or remove scenes by constructor
   * Add loaders by constructor

sandbox/tests/material/index.ts

@@ -232,6 +232,7 @@ uniform float u_time_ms;
 uniform vec4 u_color;
 uniform sampler2D u_graphic;
 uniform sampler2D u_screen_texture;
+uniform sampler2D u_noise;
 
 uniform vec2 u_resolution; // screen resolution
 uniform vec2 u_graphic_resolution; // graphic resolution
@@ -251,8 +252,8 @@ void main() {
         cos(0.2 * v_uv.x * scale.x /wave_period + time_sec * wave_speed) *
         wave_amplitude - wave_amplitude;
 
-  
-  float distortion = noise(v_uv*scale*vec2(2.1, 1.05) + time_sec * 0.12) * .25 - .125;
+  float distortion = (texture(u_noise, v_uv)).x;
+  //float distortion = noise(v_uv*scale*vec2(2.1, 1.05) + time_sec * 0.12) * .25 - .125;
 
   vec2 reflected_screenuv = vec2(v_screenuv.x - distortion, v_screenuv.y);
   vec4 screen_color = texture(u_screen_texture, reflected_screenuv);
@@ -264,13 +265,19 @@ void main() {
   vec3 mixColor = (u_color.rgb * u_color.a); // pre-multiplied alpha
   
   fragColor.rgb = mix(screen_color.rgb, mixColor, u_color.a)*fragColor.a + (wave_crest_color.rgb * wave_crest);
+  fragColor.rgb = texture(u_noise, v_uv).rgb * fragColor.a;
 }`;
 
+const noise = new ex.ImageSource('./noise.avif', false, ex.ImageFiltering.Pixel);
+loader.addResource(noise);
 
 var waterMaterial = game.graphicsContext.createMaterial({
   name: 'water',
   fragmentSource: waterFrag,
-  color: ex.Color.fromRGB(55, 0, 200, .6)
+  color: ex.Color.fromRGB(55, 0, 200, .6),
+  images: {
+    u_noise: noise
+  }
 });
 var reflection = new ex.Actor({
   x: 0,

site/docs/04-graphics/4.2-material.mdx

@@ -116,6 +116,30 @@ game.input.pointers.primary.on('move', evt => {
   code={SimpleMaterial}
 />
 
+## Additional Textures in Materials 🧪
+
+:::warning
+
+This is currently an alpha feature, and will be released in the next version! API might change/fluctuate until it lands in a supported version.
+
+:::
+
+You can now specify as many additional textures into your [[Material]] as your GPU has texture slots! You may provide a dictionary of uniform names to excalibur [[ImageSource]] and it will be loaded into the material.
+
+```typescript
+const noise = new ex.ImageSource('./noise.avif', false, ex.ImageFiltering.Pixel);
+loader.addResource(noise);
+
+var waterMaterial = game.graphicsContext.createMaterial({
+  name: 'water',
+  fragmentSource: waterFrag,
+  color: ex.Color.fromRGB(55, 0, 200, .6),
+  images: {
+    u_noise: noise
+  }
+});
+```
+
 
 ## Material Built In Uniforms
 

src/engine/Graphics/Context/material-renderer/material-renderer.ts

@@ -181,6 +181,9 @@ export class MaterialRenderer implements RendererPlugin {
     gl.bindTexture(gl.TEXTURE_2D, this._context.materialScreenTexture);
     shader.trySetUniformInt('u_screen_texture', 1);
 
+    // bind any additional textures in the material
+    material.uploadAndBind(gl);
+
     // bind quad index buffer
     this._quads.bind();
 

src/engine/Graphics/Context/material.ts

@@ -3,6 +3,8 @@ import { ExcaliburGraphicsContext } from './ExcaliburGraphicsContext';
 import { ExcaliburGraphicsContextWebGL } from './ExcaliburGraphicsContextWebGL';
 import { Shader } from './shader';
 import { Logger } from '../../Util/Log';
+import { ImageSource } from '../ImageSource';
+import { ImageFiltering } from '../Filtering';
 
 export interface MaterialOptions {
   /**
@@ -64,6 +66,13 @@ export interface MaterialOptions {
    * Add custom color, by default ex.Color.Transparent
    */
   color?: Color,
+
+  /**
+   * Add additional images to the material, you are limited by the GPU's maximum texture slots
+   *
+   * Specify a dictionary of uniform sampler names to ImageSource
+   */
+  images?: Record<string, ImageSource>
 }
 
 const defaultVertexSource = `#version 300 es
@@ -88,6 +97,11 @@ void main() {
 }
 `;
 
+export interface MaterialImageOptions {
+  filtering?: ImageFiltering,
+
+}
+
 export class Material {
   private _logger = Logger.getInstance();
   private _name: string;
@@ -96,27 +110,42 @@ export class Material {
   private _initialized = false;
   private _fragmentSource: string;
   private _vertexSource: string;
+
+  private _images = new Map<string, ImageSource>();
+  private _textures = new Map<ImageSource, WebGLTexture>();
+  private _maxTextureSlots: number;
+  private _graphicsContext: ExcaliburGraphicsContextWebGL;
+
   constructor(options: MaterialOptions) {
-    const { color, name, vertexSource, fragmentSource, graphicsContext } = options;
-    this._name = name;
+    const { color, name, vertexSource, fragmentSource, graphicsContext, images } = options;
+    this._name = name ?? 'anonymous material';
     this._vertexSource = vertexSource ?? defaultVertexSource;
     this._fragmentSource = fragmentSource;
     this._color = color ?? this._color;
     if (!graphicsContext) {
       throw Error(`Material ${name} must be provided an excalibur webgl graphics context`);
     }
     if (graphicsContext instanceof ExcaliburGraphicsContextWebGL) {
+      this._graphicsContext = graphicsContext;
       this._initialize(graphicsContext);
     } else {
       this._logger.warn(`Material ${name} was created in 2D Canvas mode, currently only WebGL is supported`);
     }
+
+    if (images) {
+      for (const key in images) {
+        this.addImageSource(key, images[key]);
+      }
+    }
   }
 
   private _initialize(graphicsContextWebGL: ExcaliburGraphicsContextWebGL) {
     if (this._initialized) {
       return;
     }
-
+    const gl = graphicsContextWebGL.__gl;
+    // max texture slots - 2 for the graphic texture and screen texture
+    this._maxTextureSlots = gl.getParameter(gl.MAX_TEXTURE_IMAGE_UNITS) - 2;
     this._shader = graphicsContextWebGL.createShader({
       vertexSource: this._vertexSource,
       fragmentSource: this._fragmentSource
@@ -144,12 +173,65 @@ export class Material {
     return this._shader;
   }
 
+  addImageSource(textureUniformName: string, image: ImageSource) {
+    if (this._images.size < this._maxTextureSlots) {
+      this._images.set(textureUniformName, image);
+    } else {
+      this._logger.warn(`Max number texture slots ${this._maxTextureSlots} have been reached for material "${this.name}", `+
+      `no more textures will be uploaded due to hardware constraints.`);
+    }
+  }
+
+  removeImageSource(textureName: string) {
+    const image = this._images.get(textureName);
+    this._graphicsContext.textureLoader.delete(image.image);
+    this._images.delete(textureName);
+  }
+
+  private _loadImageSource(image: ImageSource) {
+    const imageElement = image.image;
+    const maybeFiltering = imageElement.getAttribute('filtering');
+    let filtering: ImageFiltering = null;
+    if (maybeFiltering === ImageFiltering.Blended ||
+        maybeFiltering === ImageFiltering.Pixel) {
+      filtering = maybeFiltering;
+    }
+
+    const force = imageElement.getAttribute('forceUpload') === 'true' ? true : false;
+    const texture = this._graphicsContext.textureLoader.load(imageElement, filtering, force);
+    // remove force attribute after upload
+    imageElement.removeAttribute('forceUpload');
+    if (!this._textures.has(image)) {
+      this._textures.set(image, texture);
+    }
+
+    return texture;
+  }
+
+  uploadAndBind(gl: WebGL2RenderingContext, startingTextureSlot: number = 2) {
+
+    let textureSlot = startingTextureSlot;
+    for (const [textureName, image] of this._images.entries()) {
+      if (!image.isLoaded()) {
+        continue;
+      } // skip unloaded images, maybe warn
+      const texture = this._loadImageSource(image);
+
+      gl.activeTexture(gl.TEXTURE0 + textureSlot);
+      gl.bindTexture(gl.TEXTURE_2D, texture);
+      this._shader.trySetUniformInt(textureName, textureSlot);
+
+      textureSlot++;
+    }
+  }
+
   use() {
     if (this._initialized) {
       // bind the shader
       this._shader.use();
       // Apply standard uniforms
       this._shader.trySetUniformFloatColor('u_color', this._color);
+
     } else {
       throw Error(`Material ${this.name} not yet initialized, use the ExcaliburGraphicsContext.createMaterial() to work around this.`);
     }

src/spec/MaterialRendererSpec.ts

@@ -311,4 +311,135 @@ describe('A Material', () => {
     await expectAsync(TestUtils.flushWebGLCanvasTo2D(engine.canvas))
       .toEqualImage('src/spec/images/MaterialRendererSpec/multi-mat.png');
   });
+
+
+  it('will allow addition images', async () => {
+    const canvas = document.createElement('canvas');
+    canvas.width = 100;
+    canvas.height = 100;
+    const graphicsContext = new ex.ExcaliburGraphicsContextWebGL({
+      canvasElement: canvas,
+      backgroundColor: ex.Color.Black,
+      smoothing: false,
+      snapToPixel: true
+    });
+
+    const stars = new ex.ImageSource('src/spec/images/MaterialRendererSpec/stars.png');
+    await stars.load();
+
+    const material = new ex.Material({
+      name: 'test',
+      graphicsContext,
+      fragmentSource: `#version 300 es
+      precision mediump float;
+      // UV coord
+      in vec2 v_uv;
+      uniform sampler2D u_graphic;
+      uniform sampler2D u_additional;
+      uniform vec4 u_color;
+      uniform float u_opacity;
+      out vec4 fragColor;
+      void main() {
+        vec4 color = u_color;
+        color = mix(texture(u_additional, v_uv), texture(u_graphic, v_uv), .5);
+        color.rgb = color.rgb * u_opacity;
+        color.a = color.a * u_opacity;
+        fragColor = color * u_color;
+      }`,
+      images: {
+        u_additional: stars
+      }
+    });
+
+    const tex = new ex.ImageSource('src/spec/images/MaterialRendererSpec/sword.png');
+    await tex.load();
+
+    graphicsContext.clear();
+    graphicsContext.save();
+    graphicsContext.material = material;
+    graphicsContext.drawImage(tex.image, 0, 0);
+    graphicsContext.flush();
+    graphicsContext.restore();
+
+    expect(graphicsContext.material).toBe(null);
+    await expectAsync(TestUtils.flushWebGLCanvasTo2D(canvas))
+      .toEqualImage('src/spec/images/MaterialRendererSpec/additional.png');
+  });
+
+  it('will log a warning if you exceed you texture slots', () => {
+    const logger = ex.Logger.getInstance();
+    spyOn(logger, 'warn');
+
+    const canvas = document.createElement('canvas');
+    canvas.width = 100;
+    canvas.height = 100;
+    const graphicsContext = new ex.ExcaliburGraphicsContextWebGL({
+      canvasElement: canvas,
+      backgroundColor: ex.Color.Black,
+      smoothing: false,
+      snapToPixel: true
+    });
+
+    const stars = new ex.ImageSource('src/spec/images/MaterialRendererSpec/stars.png');
+
+    const material = new ex.Material({
+      name: 'test',
+      graphicsContext,
+      fragmentSource: `#version 300 es
+      precision mediump float;
+      // UV coord
+      in vec2 v_uv;
+      uniform sampler2D u_graphic;
+      uniform sampler2D u_additional;
+      uniform vec4 u_color;
+      uniform float u_opacity;
+      out vec4 fragColor;
+      void main() {
+        vec4 color = u_color;
+        color = mix(texture(u_additional, v_uv), texture(u_graphic, v_uv), .5);
+        color.rgb = color.rgb * u_opacity;
+        color.a = color.a * u_opacity;
+        fragColor = color * u_color;
+      }`,
+      images: {
+        u_additional: stars,
+        u_additional1: stars,
+        u_additional2: stars,
+        u_additional3: stars,
+        u_additional4: stars,
+        u_additional5: stars,
+        u_additional6: stars,
+        u_additional7: stars,
+        u_additional8: stars,
+        u_additional9: stars,
+        u_additional10: stars,
+        u_additional11: stars,
+        u_additional12: stars,
+        u_additional13: stars,
+        u_additional14: stars,
+        u_additional15: stars,
+        u_additional16: stars,
+        u_additional17: stars,
+        u_additional18: stars,
+        u_additional19: stars,
+        u_additional20: stars,
+        u_additional21: stars,
+        u_additional22: stars,
+        u_additional23: stars,
+        u_additional24: stars,
+        u_additional25: stars,
+        u_additional26: stars,
+        u_additional27: stars,
+        u_additional28: stars,
+        u_additional29: stars,
+        u_additional30: stars,
+        u_additional31: stars,
+        u_additional32: stars
+      }
+    });
+
+    expect(material).toBeDefined();
+    expect(logger.warn).toHaveBeenCalledWith(
+      'Max number texture slots 30 have been reached for material "test", no more textures will be uploaded due to hardware constraints.');
+  });
 });