Refactor (#17)

* split reactor on app side into sink and stream

* add effect: setup_frame_listener

* add effect to setup video stream

* add scheduler and sub workers

* use scheduler without additional web worker

* replace round robin with fifo

* add box-sizing for frames

* rm init sleep

Author: cospectrum

webdemo/Cargo.lock

@@ -11,6 +11,7 @@ path = "src/bin/main.rs"
 name = "worker"
 path = "src/bin/worker.rs"
 
+
 [dependencies]
 leptos = { version = "0.7.7", features = ["csr"] }
 leptos-use = { version = "0.15.6", default-features = false, features = [
@@ -22,17 +23,20 @@ web-sys = { version = "0.3", features = [
     "HtmlCanvasElement",
     "HtmlVideoElement",
     "ImageData",
+    "WorkerGlobalScope",
+    "WorkerNavigator",
 ] }
 wasm-bindgen = "0.2"
 anyhow = "1.0.98"
 imageproc = { version = "0.25.0", default-features = false }
 web-time = "1.1.0"
 gloo-worker = { version = "0.5.0", features = ["futures"] }
+gloo-timers = { version = "0.3.0", features = ["futures"] }
 futures = "0.3.31"
 serde = "1.0.219"
 base64 = "0.22.1"
 console_error_panic_hook = "0.1.7"
-gloo-timers = { version = "0.3.0", features = ["futures"] }
+wasm-bindgen-futures = "0.4.50"
 
 [profile.dev]
 opt-level = 1

webdemo/Cargo.toml

@@ -24,6 +24,7 @@
     }
 
     .frame {
+      box-sizing: border-box;
       height: 100%;
       width: 100%;
       object-fit: contain;

webdemo/index.html

@@ -1,9 +1,6 @@
-use crate::{canvas::RenderState, hooks, reactor::ModelReactor};
-use gloo_worker::Spawnable;
-use leptos::{logging::*, prelude::*};
+use crate::{canvas::RenderState, effects};
+use leptos::prelude::*;
 use leptos_use::{use_user_media, UseUserMediaReturn};
-use wasm_bindgen::{prelude::Closure, JsCast};
-use web_sys::Event;
 
 #[component]
 pub fn App() -> impl IntoView {
@@ -13,40 +10,17 @@ pub fn App() -> impl IntoView {
     let UseUserMediaReturn { stream, start, .. } = use_user_media();
     start();
 
-    let frame_listener = Closure::<dyn Fn(Event)>::wrap(Box::new(move |_: Event| {
-        log!("entered frame listener");
-        let reactor = ModelReactor::spawner().spawn("./worker.js");
+    Effect::new(move |_| {
+        effects::setup_video_stream(video_ref, &stream);
+    });
+
+    Effect::new(move |_| {
         let state = RenderState {
             video: video_ref,
             canvas: canvas_ref,
             temporary_canvas: temporary_canvas_ref,
         };
-        hooks::on_video_play(state, reactor);
-    }));
-
-    Effect::new(move |_| {
-        match stream.get() {
-            Some(Ok(stream)) => {
-                video_ref.with(|video| {
-                    if let Some(video) = video {
-                        video.set_src_object(Some(&stream))
-                    }
-                });
-            }
-            Some(Err(e)) => error!("failed to get media stream: {:?}", e),
-            None => log!("no stream yet"),
-        };
-        video_ref.with(|video| {
-            let Some(video) = video else {
-                log!("no video yet");
-                return;
-            };
-            let _ = video
-                .add_event_listener_with_callback("play", frame_listener.as_ref().unchecked_ref())
-                .inspect_err(|e| {
-                    error!("failed to attach frame listener: {:?}", e);
-                });
-        })
+        effects::setup_frame_listener(state);
     });
 
     view! {

webdemo/src/app.rs

@@ -1,8 +1,7 @@
 use gloo_worker::Registrable;
-use webdemo::ModelReactor;
 
 fn main() {
     console_error_panic_hook::set_once();
 
-    ModelReactor::registrar().register();
+    webdemo::Worker::registrar().register();
 }

webdemo/src/bin/worker.rs

@@ -0,0 +1,47 @@
+use crate::{canvas::RenderState, on_video_play};
+use leptos::{logging::*, prelude::*};
+use wasm_bindgen::{prelude::Closure, JsCast, JsValue};
+use web_sys::{Event, MediaStream};
+
+pub fn setup_video_stream(
+    video_ref: NodeRef<leptos::html::Video>,
+    stream: &Signal<Option<Result<MediaStream, JsValue>>, LocalStorage>,
+) {
+    let Some(stream) = stream.get() else {
+        log!("setup_video_stream: stream is not ready yet");
+        return;
+    };
+    let Some(video) = video_ref.get() else {
+        log!("setup_video_stream: video is not ready yet");
+        return;
+    };
+
+    let stream = match stream {
+        Ok(stream) => stream,
+        Err(e) => {
+            error!("failed to get media stream: {:?}", e);
+            return;
+        }
+    };
+    video.set_src_object(Some(&stream))
+}
+
+pub fn setup_frame_listener(state: RenderState) {
+    let Some(video) = state.video.get() else {
+        log!("setup_frame_listener: video is not ready yet");
+        return;
+    };
+
+    let frame_listener = Closure::<dyn Fn(Event)>::wrap(Box::new(move |_: Event| {
+        log!("entered frame listener");
+        on_video_play(state);
+    }));
+
+    let _ = video
+        .add_event_listener_with_callback("play", frame_listener.as_ref().unchecked_ref())
+        .inspect_err(|e| {
+            error!("failed to attach frame listener: {:?}", e);
+        });
+
+    frame_listener.forget();
+}

webdemo/src/effects.rs

@@ -0,0 +1,49 @@
+use image::{DynamicImage, Rgba, RgbaImage};
+use imageproc::image;
+use leptos::logging::log;
+
+pub fn process_image_with_model(
+    model: &scrfd::Scrfd,
+    mut inout: RgbaImage,
+) -> anyhow::Result<RgbaImage> {
+    let img = DynamicImage::ImageRgba8(inout.clone()).into_rgb8();
+    let threshold = scrfd::Threshold {
+        score: 0.5,
+        iou: 0.6,
+    };
+    let clock = web_time::Instant::now();
+    let faces = model.detect_with_threshold(&img, threshold)?;
+    log!("face detection took: {:?}", clock.elapsed());
+
+    draw_faces(&mut inout, &faces);
+    Ok(inout)
+}
+
+fn draw_faces(img: &mut RgbaImage, faces: &[scrfd::Face]) {
+    for &face in faces {
+        draw_face(img, face);
+    }
+}
+
+fn draw_face(img: &mut RgbaImage, face: scrfd::Face) {
+    let points = [
+        face.keypoints.left_eye,
+        face.keypoints.right_eye,
+        face.keypoints.nose,
+        face.keypoints.left_mouth,
+        face.keypoints.right_mouth,
+    ];
+    for point in points {
+        let radius = 4;
+        let color = Rgba([255, 0, 0, 255]);
+        let center = (point.x as i32, point.y as i32);
+        imageproc::drawing::draw_filled_circle_mut(img, center, radius, color);
+    }
+
+    let color = Rgba([255, 0, 0, 255]);
+    let x = face.bbox.upper_left.x as i32;
+    let y = face.bbox.upper_left.y as i32;
+    let (width, height) = (face.bbox.width() as u32, face.bbox.height() as u32);
+    let rect = imageproc::rect::Rect::at(x, y).of_size(width, height);
+    imageproc::drawing::draw_hollow_rect_mut(img, rect, color);
+}

webdemo/src/hooks.rs

@@ -1,7 +1,12 @@
 mod app;
 mod canvas;
-mod hooks;
-mod reactor;
+mod effects;
+mod image_processing;
+mod video_scheduler;
+mod worker;
+
+use image_processing::process_image_with_model;
+use video_scheduler::on_video_play;
 
 pub use app::App;
-pub use reactor::ModelReactor;
+pub use worker::Worker;