diff --git a/Apps/UnitTests/Android/app/src/main/cpp/CMakeLists.txt b/Apps/UnitTests/Android/app/src/main/cpp/CMakeLists.txt index aac3d46ba..e8049cf63 100644 --- a/Apps/UnitTests/Android/app/src/main/cpp/CMakeLists.txt +++ b/Apps/UnitTests/Android/app/src/main/cpp/CMakeLists.txt @@ -37,6 +37,7 @@ target_link_libraries(UnitTestsJNI PRIVATE NativeEngine PRIVATE NativeInput PRIVATE NativeOptimizations + PRIVATE NativeEncoding PRIVATE ScriptLoader PRIVATE XMLHttpRequest PRIVATE gtest_main diff --git a/Apps/UnitTests/CMakeLists.txt b/Apps/UnitTests/CMakeLists.txt index 1bcea9698..3b4d70a89 100644 --- a/Apps/UnitTests/CMakeLists.txt +++ b/Apps/UnitTests/CMakeLists.txt @@ -36,6 +36,7 @@ target_link_libraries(UnitTests PRIVATE Console PRIVATE GraphicsDevice PRIVATE NativeEngine + PRIVATE NativeEncoding PRIVATE ScriptLoader PRIVATE UrlLib PRIVATE Window diff --git a/Apps/UnitTests/Scripts/tests.ts b/Apps/UnitTests/Scripts/tests.ts index 5020d57b0..b9f004325 100644 --- a/Apps/UnitTests/Scripts/tests.ts +++ b/Apps/UnitTests/Scripts/tests.ts @@ -292,6 +292,48 @@ describe("PostProcesses", function () { });*/ }); +describe("NativeEncoding", function () { + this.timeout(0); + + function expectValidPNG(arrayBuffer: ArrayBuffer) { + expect(arrayBuffer).to.be.instanceOf(ArrayBuffer); + expect(arrayBuffer.byteLength).to.be.greaterThan(0); + + const pngSignature = new Uint8Array(arrayBuffer.slice(0, 4)); + expect(pngSignature[0]).to.equal(137); // PNG signature bytes + expect(pngSignature[1]).to.equal(80); // 'P' + expect(pngSignature[2]).to.equal(78); // 'N' + expect(pngSignature[3]).to.equal(71); // 'G' + } + + it("should encode a PNG", async function () { + const pixelData = new Uint8Array(4).fill(255); + const result = await _native.EncodeImageAsync(pixelData, 1, 1, "image/png", false); + expectValidPNG(result); + }); + + it("should handle multiple concurrent encoding tasks", async function () { + const pixelDatas = []; + for (let i = 0; i < 10; i++) { + pixelDatas.push(new Uint8Array(4).fill(255)); + } + const results = await Promise.all(pixelDatas.map((pixelData) => + _native.EncodeImageAsync(pixelData, 1, 1, "image/png", false) + )); + results.forEach(expectValidPNG); + }); + + it("should reject if MIME type not supported", async function () { + const pixelData = new Uint8Array([255, 0, 0, 255]); + try { + await _native.EncodeImageAsync(pixelData, 1, 1, "bad-mimetype", false); + expect.fail("Expected promise to reject with unsupported mime type"); + } catch (error) { + expect(error).to.exist; + } + }); +}); + mocha.run((failures) => { // Test program will wait for code to be set before exiting if (failures > 0) { diff --git a/Apps/UnitTests/Shared/Shared.cpp b/Apps/UnitTests/Shared/Shared.cpp index 7a0540539..7ab708136 100644 --- a/Apps/UnitTests/Shared/Shared.cpp +++ b/Apps/UnitTests/Shared/Shared.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -74,6 +75,7 @@ TEST(JavaScript, All) Babylon::Polyfills::Window::Initialize(env); nativeCanvas.emplace(Babylon::Polyfills::Canvas::Initialize(env)); Babylon::Plugins::NativeEngine::Initialize(env); + Babylon::Plugins::NativeEncoding::Initialize(env); auto setExitCodeCallback = Napi::Function::New( env, [&exitCodePromise](const Napi::CallbackInfo& info) { diff --git a/CMakeLists.txt b/CMakeLists.txt index 624a9c9f2..4a7773e78 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -94,6 +94,7 @@ option(BABYLON_NATIVE_CHECK_THREAD_AFFINITY "Checks thread safety in the graphic option(BABYLON_NATIVE_PLUGIN_EXTERNALTEXTURE "Include Babylon Native Plugin ExternalTexture." ON) option(BABYLON_NATIVE_PLUGIN_NATIVECAMERA "Include Babylon Native Plugin NativeCamera." ON) option(BABYLON_NATIVE_PLUGIN_NATIVECAPTURE "Include Babylon Native Plugin NativeCapture." ON) +option(BABYLON_NATIVE_PLUGIN_NATIVEENCODING "Include Babylon Native Plugin NativeEncoding." ON) option(BABYLON_NATIVE_PLUGIN_NATIVEENGINE "Include Babylon Native Plugin NativeEngine." ON) option(BABYLON_NATIVE_PLUGIN_NATIVEENGINE_WEBP "Include Babylon Native Plugin NativeEngine - WebP." ON) option(BABYLON_NATIVE_PLUGIN_NATIVEINPUT "Include Babylon Native Plugin NativeInput." ON) diff --git a/Install/Install.cmake b/Install/Install.cmake index c57a25526..459e3d8cc 100644 --- a/Install/Install.cmake +++ b/Install/Install.cmake @@ -159,6 +159,11 @@ if(TARGET NativeXr) install_include_for_targets(NativeXr) endif() +if(TARGET NativeEncoding) + install_targets(NativeEncoding) + install_include_for_targets(NativeEncoding) +endif() + # ---------------- # Polyfills # ---------------- diff --git a/Install/Test/CMakeLists.txt b/Install/Test/CMakeLists.txt index a4753c689..ca24bb2a0 100644 --- a/Install/Test/CMakeLists.txt +++ b/Install/Test/CMakeLists.txt @@ -162,6 +162,7 @@ target_link_libraries(TestInstall NativeInput NativeOptimizations NativeTracing + NativeEncoding OGLCompiler OSDependent ScriptLoader diff --git a/Plugins/CMakeLists.txt b/Plugins/CMakeLists.txt index 93f48d358..5ee02d811 100644 --- a/Plugins/CMakeLists.txt +++ b/Plugins/CMakeLists.txt @@ -33,3 +33,7 @@ endif() if(BABYLON_NATIVE_PLUGIN_NATIVEXR AND (ANDROID OR IOS)) add_subdirectory(NativeXr) endif() + +if(BABYLON_NATIVE_PLUGIN_NATIVEENCODING) + add_subdirectory(NativeEncoding) +endif() diff --git a/Plugins/NativeEncoding/CMakeLists.txt b/Plugins/NativeEncoding/CMakeLists.txt new file mode 100644 index 000000000..5d204a7a3 --- /dev/null +++ b/Plugins/NativeEncoding/CMakeLists.txt @@ -0,0 +1,18 @@ +set(SOURCES + "Include/Babylon/Plugins/NativeEncoding.h" + "Source/NativeEncoding.cpp") + +add_library(NativeEncoding ${SOURCES}) +warnings_as_errors(NativeEncoding) + +target_include_directories(NativeEncoding + PUBLIC "Include") + +target_link_libraries(NativeEncoding + PUBLIC napi + PRIVATE GraphicsDevice + PRIVATE GraphicsDeviceContext + PRIVATE JsRuntimeInternal) + +set_property(TARGET NativeEncoding PROPERTY FOLDER Plugins) +source_group(TREE ${CMAKE_CURRENT_SOURCE_DIR} FILES ${SOURCES}) diff --git a/Plugins/NativeEncoding/Include/Babylon/Plugins/NativeEncoding.h b/Plugins/NativeEncoding/Include/Babylon/Plugins/NativeEncoding.h new file mode 100644 index 000000000..4dd11c3c6 --- /dev/null +++ b/Plugins/NativeEncoding/Include/Babylon/Plugins/NativeEncoding.h @@ -0,0 +1,9 @@ +#pragma once + +#include +#include + +namespace Babylon::Plugins::NativeEncoding +{ + void BABYLON_API Initialize(Napi::Env env); +} diff --git a/Plugins/NativeEncoding/README.md b/Plugins/NativeEncoding/README.md new file mode 100644 index 000000000..211559273 --- /dev/null +++ b/Plugins/NativeEncoding/README.md @@ -0,0 +1,29 @@ +# NativeEncoding + +> ⚠️ **This plugin is experimental and subject to change.** + +The NativeEncoding plugin provides native image encoding capabilities to Babylon, allowing raw pixel data to be encoded into standard image formats (PNG, JPEG, WebP, etc.). + +## Limitations + +Currently, **only PNG encoding** is supported. + +## Design + +Unlike a traditional polyfill which would implement Canvas's `toBlob()` or `toDataURL()` methods, NativeEncoding exists as a plugin because: +1. **No standard Web API exists** for general-purpose image encoding separate from Canvas +2. **Simplicity** - Exposes only what Babylon actually needs: direct pixel-to-bytes encoding +3. **Efficiency** - Avoids extra routing through the Canvas API via intermediate data structures +4. **Modularity** - Image encoding is a separate concern from 2D canvas rendering +5. **Extensibility** - New codecs can be added in the future without bloating other components + +An encoding function is exposed on the `_native` global object, similar to NativeOptimizations. + +```typescript +interface INative { + EncodeImageAsync: (pixelData: Uint8Array, width: number, height: number, mimeType: string, invertY: boolean) => Promise; +} +``` + +It should be wrapped by higher-level Babylon.js APIs (e.g., DumpTools) for common workflows like asset exports and screenshots. + diff --git a/Plugins/NativeEncoding/Source/NativeEncoding.cpp b/Plugins/NativeEncoding/Source/NativeEncoding.cpp new file mode 100644 index 000000000..88b7d4730 --- /dev/null +++ b/Plugins/NativeEncoding/Source/NativeEncoding.cpp @@ -0,0 +1,102 @@ +#include +#include +#include +#include + +#include + +#include +#include + +#include +#include + +namespace Babylon::Plugins +{ + namespace + { + std::shared_ptr> EncodePNG(const std::vector& pixelData, uint32_t width, uint32_t height, bool invertY) + { + auto memoryBlock{bx::MemoryBlock(&Graphics::DeviceContext::GetDefaultAllocator())}; + auto writer{bx::MemoryWriter(&memoryBlock)}; + auto err{bx::Error()}; + + bimg::imageWritePng(&writer, width, height, width * 4, pixelData.data(), bimg::TextureFormat::RGBA8, !invertY, &err); + + auto byteLength{memoryBlock.getSize()}; + + if (!err.isOk()) + { + throw std::runtime_error("Failed to encode PNG image: " + std::string(err.getMessage().getCPtr())); + } + + if (byteLength == 0) + { + throw std::runtime_error("Failed to encode PNG image: output is empty"); + } + + auto result{std::make_shared>(byteLength)}; + std::memcpy(result->data(), memoryBlock.more(0), byteLength); + + return result; + } + + Napi::Value EncodeImageAsync(const Napi::CallbackInfo& info) + { + auto buffer{info[0].As()}; + auto width{info[1].As().Uint32Value()}; + auto height{info[2].As().Uint32Value()}; + auto mimeType{info[3].As().Utf8Value()}; + auto invertY{info[4].As().Value()}; + + auto env{info.Env()}; + auto deferred{Napi::Promise::Deferred::New(env)}; + + if (mimeType != "image/png") + { + deferred.Reject(Napi::Error::New(env, "Unsupported mime type: " + mimeType + ". Only image/png is currently supported.").Value()); + return deferred.Promise(); + } + + if (buffer.ByteLength() != width * height * 4) + { + deferred.Reject(Napi::Error::New(env, "Buffer byte length does not match RGBA8 format (4 bytes per pixel) of provided dimensions.").Value()); + return deferred.Promise(); + } + + auto runtimeScheduler{std::make_shared(JsRuntime::GetFromJavaScript(env))}; + auto pixelData{std::vector(buffer.Data(), buffer.Data() + buffer.ByteLength())}; + + arcana::make_task(arcana::threadpool_scheduler, arcana::cancellation_source::none(), + [pixelData{std::move(pixelData)}, width, height, invertY]() { + return EncodePNG(pixelData, width, height, invertY); + }) + .then(*runtimeScheduler, arcana::cancellation_source::none(), + [runtimeScheduler, deferred, env](const arcana::expected>, std::exception_ptr>& result) { + // TODO: Crash risk on JS teardown - this async work isn't tied to any JS object lifetime, + // unlike other plugins that cancel / clean up pending work in their destructors. + if (result.has_error()) + { + deferred.Reject(Napi::Error::New(env, result.error()).Value()); + return; + } + + auto& imageData = result.value(); + auto arrayBuffer{Napi::ArrayBuffer::New(env, imageData->data(), imageData->size(), [imageData](Napi::Env, void*) {})}; + + deferred.Resolve(arrayBuffer); + }); + + return deferred.Promise(); + } + } +} + +namespace Babylon::Plugins::NativeEncoding +{ + void BABYLON_API Initialize(Napi::Env env) + { + auto native{JsRuntime::NativeObject::GetFromJavaScript(env)}; + native.Set("EncodeImageAsync", Napi::Function::New(env, EncodeImageAsync, "EncodeImageAsync")); + } +} \ No newline at end of file