Image x/y extend, alpha, nearest neighbor sampling (#766)

This adds support for image extend modes, alpha and nearest neighbor sampling to fine and plumbs support for all through the pipeline. Extend modes with bilinear filtering (bottom right is x repeat and y reflect): <img width="785" alt="image_extend_bilinear" src="https://github.com/user-attachments/assets/3ba8a07e-b876-4b10-8c01-767c69974242" /> Extend modes with nearest neighbor filtering: <img width="785" alt="image_extend_nearest" src="https://github.com/user-attachments/assets/cda97485-e049-4721-b0c2-2c0a34c99683" /> Image alpha: <img width="785" alt="image_alpha" src="https://github.com/user-attachments/assets/b1222040-1548-4dcd-bfa0-30b5572b516b" /> --------- Co-authored-by: Bruce Mitchener <[email protected]>
linebender · Jan 17, 2025 · 0e79178 · 0e79178
1 parent c7bce5e
commit 0e79178
Show file tree

Hide file tree

Showing 13 changed files with 168 additions and 82 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -34,6 +34,7 @@ This is the first step towards providing richer color functionality, better hand
 - Offset in image rendering, and sampling outside correct atlas area ([#722][] by [@dfrg])
 - Inference conflict when using Kurbo's `schemars` feature ([#733][] by [@ratmice][])
 - Detection of PNG format bitmap fonts, primarily for Apple systems ([#740][] by [@LaurenzV])
+- Support image extend modes, nearest-neighbor sampling and alpha ([#766][] by [@dfrg])
 
 ## [0.3.0][] - 2024-10-04
 
@@ -218,6 +219,7 @@ This release has an [MSRV][] of 1.75.
 [#747]: https://github.com/linebender/vello/pull/747
 [#754]: https://github.com/linebender/vello/pull/754
 [#756]: https://github.com/linebender/vello/pull/756
+[#766]: https://github.com/linebender/vello/pull/766
 [#757]: https://github.com/linebender/vello/pull/757
 
 [Unreleased]: https://github.com/linebender/vello/compare/v0.3.0...HEAD

diff --git a/examples/scenes/src/test_scenes.rs b/examples/scenes/src/test_scenes.rs
@@ -2,7 +2,10 @@
 // SPDX-License-Identifier: Apache-2.0 OR MIT
 
 use crate::{ExampleScene, SceneConfig, SceneSet};
-use vello::kurbo::{Affine, Cap};
+use vello::{
+    kurbo::{Affine, Cap},
+    peniko::ImageQuality,
+};
 
 /// All of the test scenes supported by Vello.
 pub fn test_scenes() -> SceneSet {
@@ -80,7 +83,8 @@ export_scenes!(
     many_draw_objects(many_draw_objects),
     blurred_rounded_rect(blurred_rounded_rect),
     image_sampling(image_sampling),
-    image_extend_modes(image_extend_modes)
+    image_extend_modes_bilinear(impls::image_extend_modes(ImageQuality::Medium), "image_extend_modes (bilinear)", false),
+    image_extend_modes_nearest_neighbor(impls::image_extend_modes(ImageQuality::Low), "image_extend_modes (nearest neighbor)", false),
 );
 
 /// Implementations for the test scenes.
@@ -659,7 +663,8 @@ mod impls {
         let piet_logo = params
             .images
             .from_bytes(FLOWER_IMAGE.as_ptr() as usize, FLOWER_IMAGE)
-            .unwrap();
+            .unwrap()
+            .with_alpha(((params.time * 0.5 + 200.0).sin() as f32 + 1.0) * 0.5);
 
         use PathEl::*;
         let rect = Rect::from_origin_size(Point::new(0.0, 0.0), (1000.0, 1000.0));
@@ -1807,46 +1812,61 @@ mod impls {
         );
     }
 
-    pub(super) fn image_extend_modes(scene: &mut Scene, params: &mut SceneParams<'_>) {
-        params.resolution = Some(Vec2::new(1500., 1500.));
-        params.base_color = Some(palette::css::WHITE);
-        let mut blob: Vec<u8> = Vec::new();
-        [
-            palette::css::RED,
-            palette::css::BLUE,
-            palette::css::CYAN,
-            palette::css::MAGENTA,
-        ]
-        .iter()
-        .for_each(|c| {
-            blob.extend(c.premultiply().to_rgba8().to_u8_array());
-        });
-        let data = Blob::new(Arc::new(blob));
-        let image = Image::new(data, ImageFormat::Rgba8, 2, 2);
-        let image = image.with_extend(Extend::Pad);
-        // Pad extend mode
-        scene.fill(
-            Fill::NonZero,
-            Affine::scale(100.).then_translate((100., 100.).into()),
-            &image,
-            Some(Affine::translate((2., 2.)).then_scale(100.)),
-            &Rect::new(0., 0., 6., 6.),
-        );
-        let image = image.with_extend(Extend::Reflect);
-        scene.fill(
-            Fill::NonZero,
-            Affine::scale(100.).then_translate((100., 800.).into()),
-            &image,
-            Some(Affine::translate((2., 2.))),
-            &Rect::new(0., 0., 6., 6.),
-        );
-        let image = image.with_extend(Extend::Repeat);
-        scene.fill(
-            Fill::NonZero,
-            Affine::scale(100.).then_translate((800., 100.).into()),
-            &image,
-            Some(Affine::translate((2., 2.))),
-            &Rect::new(0., 0., 6., 6.),
-        );
+    pub(super) fn image_extend_modes(
+        quality: ImageQuality,
+    ) -> impl FnMut(&mut Scene, &mut SceneParams<'_>) {
+        move |scene, params| {
+            params.resolution = Some(Vec2::new(1500., 1500.));
+            params.base_color = Some(palette::css::WHITE);
+            let mut blob: Vec<u8> = Vec::new();
+            [
+                palette::css::RED,
+                palette::css::BLUE,
+                palette::css::CYAN,
+                palette::css::MAGENTA,
+            ]
+            .iter()
+            .for_each(|c| {
+                blob.extend(c.premultiply().to_rgba8().to_u8_array());
+            });
+            let data = Blob::new(Arc::new(blob));
+            let image = Image::new(data, ImageFormat::Rgba8, 2, 2).with_quality(quality);
+            let brush_offset = Some(Affine::translate((2., 2.)));
+            // Pad extend mode
+            let image = image.with_extend(Extend::Pad);
+            scene.fill(
+                Fill::NonZero,
+                Affine::scale(100.).then_translate((100., 100.).into()),
+                &image,
+                brush_offset,
+                &Rect::new(0., 0., 6., 6.),
+            );
+            let image = image.with_extend(Extend::Reflect);
+            scene.fill(
+                Fill::NonZero,
+                Affine::scale(100.).then_translate((100., 800.).into()),
+                &image,
+                brush_offset,
+                &Rect::new(0., 0., 6., 6.),
+            );
+            let image = image.with_extend(Extend::Repeat);
+            scene.fill(
+                Fill::NonZero,
+                Affine::scale(100.).then_translate((800., 100.).into()),
+                &image,
+                brush_offset,
+                &Rect::new(0., 0., 6., 6.),
+            );
+            let image = image
+                .with_x_extend(Extend::Repeat)
+                .with_y_extend(Extend::Reflect);
+            scene.fill(
+                Fill::NonZero,
+                Affine::scale(100.).then_translate((800., 800.).into()),
+                &image,
+                brush_offset,
+                &Rect::new(0., 0., 6., 6.),
+            );
+        }
     }
 }
diff --git a/vello_encoding/src/draw.rs b/vello_encoding/src/draw.rs
@@ -31,7 +31,7 @@ impl DrawTag {
     pub const SWEEP_GRADIENT: Self = Self(0x254);
 
     /// Image fill.
-    pub const IMAGE: Self = Self(0x248);
+    pub const IMAGE: Self = Self(0x28C); // info: 10, scene: 3
 
     /// Blurred rounded rectangle.
     pub const BLUR_RECT: Self = Self(0x2d4); // info: 11, scene: 5 (DrawBlurRoundedRect)
@@ -164,6 +164,9 @@ pub struct DrawImage {
     pub xy: u32,
     /// Packed image dimensions.
     pub width_height: u32,
+    /// Packed quality, extend mode and 8-bit alpha (bits `qqxxyyaaaaaaaa`,
+    /// 18 unused prefix bits).
+    pub sample_alpha: u32,
 }
 
 /// Draw data for a blurred rounded rectangle.

diff --git a/vello_encoding/src/encoding.rs b/vello_encoding/src/encoding.rs
@@ -402,7 +402,7 @@ impl Encoding {
 
     /// Encodes an image brush.
     pub fn encode_image(&mut self, image: &Image, alpha: f32) {
-        let _alpha = alpha * image.alpha;
+        let alpha = (alpha * image.alpha * 255.0).round() as u8;
         // TODO: feed the alpha multiplier through the full pipeline for consistency
         // with other brushes?
         // Tracked in https://github.com/linebender/vello/issues/692
@@ -415,6 +415,10 @@ impl Encoding {
             .extend_from_slice(bytemuck::bytes_of(&DrawImage {
                 xy: 0,
                 width_height: (image.width << 16) | (image.height & 0xFFFF),
+                sample_alpha: ((image.quality as u32) << 12)
+                    | ((image.x_extend as u32) << 10)
+                    | ((image.y_extend as u32) << 8)
+                    | alpha as u32,
             }));
     }
 
@@ -569,3 +573,30 @@ impl StreamOffsets {
         self.styles += other.styles;
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use peniko::{Extend, ImageQuality};
+
+    #[test]
+    fn ensure_image_quality_values() {
+        assert_eq!(ImageQuality::Low as u32, 0);
+        assert_eq!(ImageQuality::Medium as u32, 1);
+        assert_eq!(ImageQuality::High as u32, 2);
+        // exhaustive match to catch new variants
+        match ImageQuality::Low {
+            ImageQuality::Low | ImageQuality::Medium | ImageQuality::High => {}
+        }
+    }
+
+    #[test]
+    fn ensure_extend_values() {
+        assert_eq!(Extend::Pad as u32, 0);
+        assert_eq!(Extend::Repeat as u32, 1);
+        assert_eq!(Extend::Reflect as u32, 2);
+        // exhaustive match to catch new variants
+        match Extend::Pad {
+            Extend::Pad | Extend::Repeat | Extend::Reflect => {}
+        }
+    }
+}
diff --git a/vello_shaders/shader/draw_leaf.wgsl b/vello_shaders/shader/draw_leaf.wgsl
@@ -253,6 +253,7 @@ fn main(
                     info[di + 6u] = bitcast<u32>(inv.translate.y);
                     info[di + 7u] = scene[dd];
                     info[di + 8u] = scene[dd + 1u];
+                    info[di + 9u] = scene[dd + 2u];
                 }
                 case DRAWTAG_BLURRED_ROUNDED_RECT: {
                     info[di] = draw_flags;

diff --git a/vello_shaders/shader/fine.wgsl b/vello_shaders/shader/fine.wgsl
@@ -25,6 +25,10 @@ var<storage> segments: array<Segment>;
 
 const GRADIENT_WIDTH = 512;
 
+const IMAGE_QUALITY_LOW = 0u;
+const IMAGE_QUALITY_MEDIUM = 1u;
+const IMAGE_QUALITY_HIGH = 2u;
+
 @group(0) @binding(2)
 var<storage> ptcl: array<u32>;
 
@@ -805,12 +809,17 @@ fn read_image(cmd_ix: u32) -> CmdImage {
     let xlat = vec2(bitcast<f32>(info[info_offset + 4u]), bitcast<f32>(info[info_offset + 5u]));
     let xy = info[info_offset + 6u];
     let width_height = info[info_offset + 7u];
+    let sample_alpha = info[info_offset + 8u];
+    let alpha = f32(sample_alpha & 0xFFu) / 255.0;
+    let quality = sample_alpha >> 12u;
+    let x_extend = (sample_alpha >> 10u) & 0x3u;
+    let y_extend = (sample_alpha >> 8u) & 0x3u;
     // The following are not intended to be bitcasts
     let x = f32(xy >> 16u);
     let y = f32(xy & 0xffffu);
     let width = f32(width_height >> 16u);
     let height = f32(width_height & 0xffffu);
-    return CmdImage(matrx, xlat, vec2(x, y), vec2(width, height));
+    return CmdImage(matrx, xlat, vec2(x, y), vec2(width, height), x_extend, y_extend, quality, alpha);
 }
 
 fn read_end_clip(cmd_ix: u32) -> CmdEndClip {
@@ -1146,26 +1155,52 @@ fn main(
             case CMD_IMAGE: {
                 let image = read_image(cmd_ix);
                 let atlas_max = image.atlas_offset + image.extents - vec2(1.0);
-                for (var i = 0u; i < PIXELS_PER_THREAD; i += 1u) {
-                    // We only need to load from the textures if the value will be used.
-                    if area[i] != 0.0 {
-                        let my_xy = vec2(xy.x + f32(i), xy.y);
-                        let atlas_uv = image.matrx.xy * my_xy.x + image.matrx.zw * my_xy.y + image.xlat + image.atlas_offset - vec2(0.5);
-                        // This currently only implements the Pad extend mode
-                        // TODO: Support repeat and reflect
-                        // TODO: If the image couldn't be added to the atlas (i.e. was too big), this isn't robust
-                        let atlas_uv_clamped = clamp(atlas_uv, image.atlas_offset, atlas_max);
-                        // We know that the floor and ceil are within the atlas area because atlas_max and
-                        // atlas_offset are integers
-                        let uv_quad = vec4(floor(atlas_uv_clamped), ceil(atlas_uv_clamped));
-                        let uv_frac = fract(atlas_uv);
-                        let a = premul_alpha(textureLoad(image_atlas, vec2<i32>(uv_quad.xy), 0));
-                        let b = premul_alpha(textureLoad(image_atlas, vec2<i32>(uv_quad.xw), 0));
-                        let c = premul_alpha(textureLoad(image_atlas, vec2<i32>(uv_quad.zy), 0));
-                        let d = premul_alpha(textureLoad(image_atlas, vec2<i32>(uv_quad.zw), 0));
-                        let fg_rgba = mix(mix(a, b, uv_frac.y), mix(c, d, uv_frac.y), uv_frac.x);
-                        let fg_i = fg_rgba * area[i];
-                        rgba[i] = rgba[i] * (1.0 - fg_i.a) + fg_i;
+                let extents_inv = vec2(1.0) / image.extents;
+                switch image.quality {
+                    case IMAGE_QUALITY_LOW: {
+                        for (var i = 0u; i < PIXELS_PER_THREAD; i += 1u) {
+                            // We only need to load from the textures if the value will be used.
+                            if area[i] != 0.0 {
+                                let my_xy = vec2(xy.x + f32(i), xy.y);
+                                var atlas_uv = image.matrx.xy * my_xy.x + image.matrx.zw * my_xy.y + image.xlat;
+                                atlas_uv.x = extend_mode(atlas_uv.x * extents_inv.x, image.x_extend_mode) * image.extents.x;
+                                atlas_uv.y = extend_mode(atlas_uv.y * extents_inv.y, image.y_extend_mode) * image.extents.y;
+                                atlas_uv = atlas_uv + image.atlas_offset;
+                                // TODO: If the image couldn't be added to the atlas (i.e. was too big), this isn't robust
+                                let atlas_uv_clamped = clamp(atlas_uv, image.atlas_offset, atlas_max);
+                                // Nearest neighbor sampling
+                                let fg_rgba = premul_alpha(textureLoad(image_atlas, vec2<i32>(atlas_uv_clamped), 0));
+                                let fg_i = fg_rgba * area[i] * image.alpha;
+                                rgba[i] = rgba[i] * (1.0 - fg_i.a) + fg_i;
+                            }
+                        }
+                    }
+                    case IMAGE_QUALITY_MEDIUM, default: {
+                        // We don't have an implementation for `IMAGE_QUALITY_HIGH` yet, just use the same as medium                        
+                        for (var i = 0u; i < PIXELS_PER_THREAD; i += 1u) {
+                            // We only need to load from the textures if the value will be used.
+                            if area[i] != 0.0 {
+                                let my_xy = vec2(xy.x + f32(i), xy.y);
+                                var atlas_uv = image.matrx.xy * my_xy.x + image.matrx.zw * my_xy.y + image.xlat;
+                                atlas_uv.x = extend_mode(atlas_uv.x * extents_inv.x, image.x_extend_mode) * image.extents.x;
+                                atlas_uv.y = extend_mode(atlas_uv.y * extents_inv.y, image.y_extend_mode) * image.extents.y;
+                                atlas_uv = atlas_uv + image.atlas_offset - vec2(0.5);
+                                // TODO: If the image couldn't be added to the atlas (i.e. was too big), this isn't robust
+                                let atlas_uv_clamped = clamp(atlas_uv, image.atlas_offset, atlas_max);
+                                // We know that the floor and ceil are within the atlas area because atlas_max and
+                                // atlas_offset are integers
+                                let uv_quad = vec4(floor(atlas_uv_clamped), ceil(atlas_uv_clamped));
+                                let uv_frac = fract(atlas_uv);
+                                let a = premul_alpha(textureLoad(image_atlas, vec2<i32>(uv_quad.xy), 0));
+                                let b = premul_alpha(textureLoad(image_atlas, vec2<i32>(uv_quad.xw), 0));
+                                let c = premul_alpha(textureLoad(image_atlas, vec2<i32>(uv_quad.zy), 0));
+                                let d = premul_alpha(textureLoad(image_atlas, vec2<i32>(uv_quad.zw), 0));
+                                // Bilinear sampling
+                                let fg_rgba = mix(mix(a, b, uv_frac.y), mix(c, d, uv_frac.y), uv_frac.x);
+                                let fg_i = fg_rgba * area[i] * image.alpha;
+                                rgba[i] = rgba[i] * (1.0 - fg_i.a) + fg_i;
+                            }
+                        }
                     }
                 }
                 cmd_ix += 2u;

diff --git a/vello_shaders/shader/shared/drawtag.wgsl b/vello_shaders/shader/shared/drawtag.wgsl
@@ -21,7 +21,7 @@ const DRAWTAG_FILL_COLOR = 0x44u;
 const DRAWTAG_FILL_LIN_GRADIENT = 0x114u;
 const DRAWTAG_FILL_RAD_GRADIENT = 0x29cu;
 const DRAWTAG_FILL_SWEEP_GRADIENT = 0x254u;
-const DRAWTAG_FILL_IMAGE = 0x248u;
+const DRAWTAG_FILL_IMAGE = 0x28Cu;
 const DRAWTAG_BLURRED_ROUNDED_RECT = 0x2d4u;
 const DRAWTAG_BEGIN_CLIP = 0x9u;
 const DRAWTAG_END_CLIP = 0x21u;

diff --git a/vello_shaders/shader/shared/ptcl.wgsl b/vello_shaders/shader/shared/ptcl.wgsl
@@ -97,6 +97,10 @@ struct CmdImage {
     xlat: vec2<f32>,
     atlas_offset: vec2<f32>,
     extents: vec2<f32>,
+    x_extend_mode: u32,
+    y_extend_mode: u32,
+    quality: u32,
+    alpha: f32,
 }
 
 struct CmdEndClip {

diff --git a/vello_shaders/src/cpu/draw_leaf.rs b/vello_shaders/src/cpu/draw_leaf.rs
@@ -175,6 +175,7 @@ fn draw_leaf_main(
                         info[di + 6] = f32::to_bits(xform.0[5]);
                         info[di + 7] = scene[dd as usize];
                         info[di + 8] = scene[dd as usize + 1];
+                        info[di + 9] = scene[dd as usize + 2];
                     }
                     DrawTag::BLUR_RECT => {
                         info[di] = draw_flags;

diff --git a/vello_tests/snapshots/big_colr.png b/vello_tests/snapshots/big_colr.png
diff --git a/vello_tests/snapshots/image_extend_modes.png b/vello_tests/snapshots/image_extend_modes.png
diff --git a/vello_tests/tests/emoji.rs b/vello_tests/tests/emoji.rs
@@ -59,7 +59,7 @@ fn big_colr() {
     );
     snapshot_test_sync(scene, &params)
         .unwrap()
-        .assert_mean_less_than(0.001);
+        .assert_mean_less_than(0.002);
 }
 
 #[test]

diff --git a/vello_tests/tests/snapshot_test_scenes.rs b/vello_tests/tests/snapshot_test_scenes.rs
@@ -121,11 +121,3 @@ fn snapshot_image_sampling() {
     let params = TestParams::new("image_sampling", 400, 400);
     snapshot_test_scene(test_scene, params);
 }
-
-#[test]
-#[cfg_attr(skip_gpu_tests, ignore)]
-fn snapshot_image_extend_modes() {
-    let test_scene = test_scenes::image_extend_modes();
-    let params = TestParams::new("image_extend_modes", 375, 375);
-    snapshot_test_scene(test_scene, params);
-}