Avoid copying the image buffer when calling the JS callback to render an image update. The Rust side can provide a pointer to the JS runtime which bindgen converts to a number. This number represents an index into the wasm module's memory which the JS runtime has read-only access to in the form of an ArrayBuffer. One additional change of significance is that the render callback no longer registers itself with 'requestAnimationFrame', and instead writes the update out synchronously with 'putImageData'.

Author: rhammonds-teleport

web/packages/shared/components/CanvasRenderer.tsx

@@ -27,6 +27,7 @@ import React, {
 import { Logger } from 'design/logger';
 import { BitmapFrame, PngFrame } from 'shared/libs/tdp';
 import { debounce } from 'shared/utils/highbar';
+import {wasm} from 'shared/libs/tdp/client'
 
 const logger = new Logger('CanvasRenderer');
 
@@ -231,21 +232,9 @@ function makeBitmapFrameRenderer(
 ): (frame: BitmapFrame) => void {
   const ctx = canvas.getContext('2d');
 
-  // Buffered rendering logic
-  let bitmapBuffer: BitmapFrame[] = [];
-  const renderBuffer = () => {
-    if (bitmapBuffer.length) {
-      for (let i = 0; i < bitmapBuffer.length; i++) {
-        if (bitmapBuffer[i].image_data.data.length != 0) {
-          const bmpFrame = bitmapBuffer[i];
-          ctx.putImageData(bmpFrame.image_data, bmpFrame.left, bmpFrame.top, bmpFrame.dirty_x, bmpFrame.dirty_y, bmpFrame.dirty_width, bmpFrame.dirty_height);
-        }
-      }
-      bitmapBuffer = [];
-    }
-    requestAnimationFrame(renderBuffer);
-  };
-  requestAnimationFrame(renderBuffer);
-
-  return frame => bitmapBuffer.push(frame);
+  return (frame) => {
+    let buf = new Uint8ClampedArray(wasm.memory.buffer, frame.image_buffer_ptr, frame.image_buffer_length);
+    let imageData = new ImageData(buf, frame.image_width,frame.image_height)
+    ctx.putImageData(imageData, 0, 0, frame.dirty_x, frame.dirty_y, frame.dirty_width, frame.dirty_height);
+  }
 }

web/packages/shared/libs/ironrdp/src/lib.rs

@@ -71,26 +71,26 @@ pub fn init_wasm_log(log_level: &str) {
 
 #[wasm_bindgen]
 pub struct BitmapFrame {
-    top: u16,
-    left: u16,
-
+    image_width: u16,
+    image_height: u16,
     dirty_x: u16,
     dirty_y: u16,
     dirty_width: u16,
     dirty_height: u16,
-    image_data: ImageData,
+    image_buffer_ptr: *const u8,
+    image_buffer_length: usize,
 }
 
 #[wasm_bindgen]
 impl BitmapFrame {
     #[wasm_bindgen(getter)]
-    pub fn top(&self) -> u16 {
-        self.top
+    pub fn image_height(&self) -> u16 {
+        self.image_height
     }
 
     #[wasm_bindgen(getter)]
-    pub fn left(&self) -> u16 {
-        self.left
+    pub fn image_width(&self) -> u16 {
+        self.image_width
     }
 
     #[wasm_bindgen(getter)]
@@ -113,9 +113,14 @@ impl BitmapFrame {
         self.dirty_height
     }
 
+     #[wasm_bindgen(getter)]
+    pub fn image_buffer_ptr(&self) -> *const u8 {
+        self.image_buffer_ptr
+    }
+
     #[wasm_bindgen(getter)]
-    pub fn image_data(&self) -> ImageData {
-        self.image_data.clone() // TODO(isaiah): can we remove this clone?
+    pub fn image_buffer_length(&self) -> usize {
+        self.image_buffer_length
     }
 }
 
@@ -310,28 +315,18 @@ impl FastPathProcessor {
         image_width: u16,
         image_height: u16,
     ) -> Result<(), JsValue> {
-        let image_data = create_image_data_from_image_and_region(&image_data,
-             &InclusiveRectangle {
-                left: 0,
-                top: 0,
-                right: image_width - 1,
-                bottom: image_height - 1,
-            })?;
-
-        let bitmap_frame = BitmapFrame {
-            top: 0,
-            left: 0,
+
+        // TODO(isaiah): return this?
+        let _ret = callback.call1(cb_context, &JsValue::from(BitmapFrame {
+            image_height: image_height,
+            image_width: image_width,
             dirty_x: image_location.left,
             dirty_y: image_location.top,
-            dirty_width: image_location.width(),
             dirty_height: image_location.height(),
-            image_data,
-        };
-
-        let bitmap_frame = &JsValue::from(bitmap_frame);
-
-        // TODO(isaiah): return this?
-        let _ret = callback.call1(cb_context, bitmap_frame)?;
+            dirty_width: image_location.width(),
+            image_buffer_ptr: image_data.as_ptr(),
+            image_buffer_length: image_data.len(),
+        }))?;
         Ok(())
     }
 }

web/packages/shared/libs/tdp/client.ts

@@ -57,6 +57,11 @@ import {
   type FileOrDirInfo,
 } from './sharedDirectoryAccess';
 
+import { InitOutput } from 'shared/libs/ironrdp/pkg/ironrdp';
+
+let wasm: InitOutput;
+export {wasm};
+
 export enum TdpClientEvent {
   TDP_CLIENT_SCREEN_SPEC = 'tdp client screen spec',
   TDP_PNG_FRAME = 'tdp png frame',
@@ -323,7 +328,7 @@ export class TdpClient extends EventEmitter<EventMap> {
       wasmLogLevel = LogType.TRACE;
     }
 
-    await init();
+    wasm = await init();
     init_wasm_log(wasmLogLevel);
   }
 
@@ -887,13 +892,14 @@ export class TdpClient extends EventEmitter<EventMap> {
 
 // Mimics the BitmapFrame struct in rust.
 export type BitmapFrame = {
-  top: number;
-  left: number;
-  dirty_x: number;
-  dirty_y: number;
-  dirty_width: number;
-  dirty_height: number;
-  image_data: ImageData;
+  image_height: number,
+  image_width: number,
+  dirty_x: number,
+  dirty_y: number,
+  dirty_width: number,
+  dirty_height: number,
+  image_buffer_ptr: number,
+  image_buffer_length: number,
 };
 
 export function useListener<T extends any[]>(