Sb/adjust detail (#43)

* View: Adjust detail bias if there is enough memory when scene is resolved

* Adjust triangle limits

* Cr

* updated values. Added last update check

* Make sure to update scene resolved if the last node is aborted

* Added scene resolved as a public function on view

* Fix

* move abort scene resolved

* fix

* fix

core3d/context.ts

@@ -54,6 +54,7 @@ export class RenderContext {
     private readonly defaultResourceBin;
     private readonly iblResourceBin;
     private pickBuffersValid = false;
+    private sceneResolved = false;
     private currentPick: Uint32Array | undefined;
     private activeTimers = new Set<Timer>();
     private currentFrameTime = 0;
@@ -339,6 +340,18 @@ export class RenderContext {
         return this.pickBuffersValid;
     }
 
+    /** 
+     * The scene is considered resolved when all currently split nodes are done downloading.
+     */
+    isSceneResolved() {
+        return this.sceneResolved;
+    }
+
+    /** @internal */
+    setSceneResolved(resolved: boolean) {
+        this.sceneResolved = resolved;
+    }
+
     /** Query whether the underlying WebGL render context is currently lost.
      * @remarks
      * This could occur when too many resources are allocated or when browser window is dragged across screens.

core3d/modules/octree/context.ts

@@ -67,10 +67,14 @@ export class OctreeModuleContext implements RenderModuleContext, OctreeContext {
         return this.highlight.indices;
     }
 
+    getAdjustedProjectedSizeSplitThreshold(state: DerivedRenderState) {
+        return this.projectedSizeSplitThreshold / (state.quality.detail.activeBias * state.quality.detail.downloadBias);
+    }
+
     update(state: DerivedRenderState) {
         // const beginTime = performance.now();
 
-        const { renderContext, resources, uniforms, projectedSizeSplitThreshold, module, currentProgramFlags } = this;
+        const { renderContext, resources, uniforms, module, currentProgramFlags } = this;
         const { gl, deviceProfile } = renderContext;
         const { scene, localSpaceTranslation, highlights, points, terrain, pick, output, clipping, outlines } = state;
         const { values } = uniforms.scene;
@@ -367,6 +371,8 @@ export class OctreeModuleContext implements RenderModuleContext, OctreeContext {
             recompile();
         }
 
+        const detailAdjustment = state.quality.detail.downloadBias;
+
         if (!this.suspendUpdates) {
             const nodes: OctreeNode[] = [];
             for (const rootNode of Object.values(rootNodes)) {
@@ -375,7 +381,7 @@ export class OctreeModuleContext implements RenderModuleContext, OctreeContext {
                 // collapse nodes
                 const preCollapseNodes = [...iterateNodes(rootNode)];
                 for (const node of preCollapseNodes) {
-                    if (!node.shouldSplit(projectedSizeSplitThreshold * 0.98)) { // add a little "slack" before collapsing back again
+                    if (!node.shouldSplit((this.projectedSizeSplitThreshold / detailAdjustment) * 0.98)) { // add a little "slack" before collapsing back again
                         if (node.state != NodeState.collapsed) {
                             node.dispose(); // collapse node
                         }
@@ -404,7 +410,7 @@ export class OctreeModuleContext implements RenderModuleContext, OctreeContext {
 
             // split nodes based on camera orientation
             for (const node of nodes) {
-                if (node.shouldSplit(projectedSizeSplitThreshold)) {
+                if (node.shouldSplit(this.projectedSizeSplitThreshold / detailAdjustment)) {
                     if (node.state == NodeState.collapsed) {
                         if (primitives + node.data.primitivesDelta <= maxPrimitives && gpuBytes + node.data.gpuBytes <= maxGPUBytes) {
                             node.state = NodeState.requestDownload;
@@ -478,7 +484,7 @@ export class OctreeModuleContext implements RenderModuleContext, OctreeContext {
         const { programs } = resources;
         const { gl } = renderContext;
         for (const rootNode of Object.values(this.rootNodes)) {
-            const renderNodes = this.getRenderNodes(this.projectedSizeSplitThreshold / state.quality.detail, rootNode);
+            const renderNodes = this.getRenderNodes(this.getAdjustedProjectedSizeSplitThreshold(state), rootNode);
             glState(gl, {
                 program: programs.pre,
                 depth: { test: true },
@@ -499,6 +505,9 @@ export class OctreeModuleContext implements RenderModuleContext, OctreeContext {
         const { programs, sceneUniforms, samplerNearest, materialTexture, highlightTexture, gradientsTexture } = resources;
         const { gl, iblTextures, lut_ggx, cameraUniforms, clippingUniforms, deviceProfile } = renderContext;
 
+
+        const projectedSizeSplitThreshold = this.getAdjustedProjectedSizeSplitThreshold(state);
+
         // glClear(gl, { kind: "DEPTH_STENCIL", depth: 1.0, stencil: 0 });
 
         if (state.debug.wireframe) {
@@ -531,7 +540,7 @@ export class OctreeModuleContext implements RenderModuleContext, OctreeContext {
         gl.activeTexture(gl.TEXTURE0);
 
         for (const rootNode of Object.values(this.rootNodes)) {
-            const renderNodes = this.getRenderNodes(this.projectedSizeSplitThreshold / state.quality.detail, rootNode);
+            const renderNodes = this.getRenderNodes(projectedSizeSplitThreshold, rootNode);
             const meshState: MeshState = {};
             for (const { mask, node } of renderNodes) {
                 this.renderNode(node, mask, meshState, ShaderPass.color);
@@ -548,7 +557,7 @@ export class OctreeModuleContext implements RenderModuleContext, OctreeContext {
                 const [x, y, z, offset] = plane;
                 const p = vec4.fromValues(x, y, z, -offset);
                 renderContext.updateOutlinesUniforms(state.outlines, p, planeIndex);
-                const renderNodes = this.getRenderNodes(this.projectedSizeSplitThreshold / state.quality.detail,
+                const renderNodes = this.getRenderNodes(projectedSizeSplitThreshold,
                     this.rootNodes[NodeGeometryKind.triangles], this.rootNodes[NodeGeometryKind.terrain]);
 
                 this.renderNodeClippingOutline(plane, state, renderNodes, state.outlines.hidden, color);
@@ -570,7 +579,7 @@ export class OctreeModuleContext implements RenderModuleContext, OctreeContext {
 
         if (debug) {
             for (const rootNode of Object.values(this.rootNodes)) {
-                const renderNodes = this.getRenderNodes(this.projectedSizeSplitThreshold / state.quality.detail, rootNode);
+                const renderNodes = this.getRenderNodes(projectedSizeSplitThreshold, rootNode);
                 glState(gl, {
                     program: programs.debug,
                     uniformBuffers: [cameraUniforms, clippingUniforms, sceneUniforms, null],
@@ -603,6 +612,7 @@ export class OctreeModuleContext implements RenderModuleContext, OctreeContext {
                 }
             }
         }
+        renderContext.setSceneResolved(this.loader.activeDownloads == 0);
     }
 
     pick() {
@@ -611,9 +621,10 @@ export class OctreeModuleContext implements RenderModuleContext, OctreeContext {
         const { programs, sceneUniforms, samplerNearest, materialTexture, highlightTexture, gradientsTexture } = resources;
         const { diffuse, specular } = iblTextures;
         const state = currentState!;
+        const projectedSizeSplitThreshold = this.getAdjustedProjectedSizeSplitThreshold(state);
 
         for (const rootNode of Object.values(this.rootNodes)) {
-            const renderNodes = this.getRenderNodes(this.projectedSizeSplitThreshold / state.quality.detail, rootNode);
+            const renderNodes = this.getRenderNodes(projectedSizeSplitThreshold, rootNode);
             glState(gl, {
                 program: programs.pick,
                 uniformBuffers: [cameraUniforms, clippingUniforms, sceneUniforms, null],
@@ -642,7 +653,7 @@ export class OctreeModuleContext implements RenderModuleContext, OctreeContext {
         }
 
         if (deviceProfile.features.outline && state.outlines.on) {
-            const renderNodes = this.getRenderNodes(this.projectedSizeSplitThreshold / state.quality.detail,
+            const renderNodes = this.getRenderNodes(projectedSizeSplitThreshold,
                 this.rootNodes[NodeGeometryKind.triangles],
                 state.terrain.asBackground ? undefined : this.rootNodes[NodeGeometryKind.terrain]);
 

core3d/modules/octree/loader.ts

@@ -1,6 +1,7 @@
 import type { AbortAllMessage, AbortMessage, InitMessage, CloseMessage, LoadMessage, MessageRequest, MessageResponse, NodePayload, ParseMessage, ParseConfig } from "./worker";
 import { OctreeNode } from "./node.js";
 import type { DeviceProfile } from "core3d/device.js";
+import type { RenderContext } from "core3d/context";
 
 interface PayloadPromiseMethods { readonly resolve: (value: NodePayload | undefined) => void, readonly reject: (reason: string) => void };
 
@@ -13,7 +14,7 @@ export class NodeLoader {
     private resolveBuffer: (() => void) | undefined;
     aborted = false;
 
-    constructor(readonly worker: Worker) {
+    constructor(readonly worker: Worker, readonly renderContext: RenderContext) {
         worker.onmessage = e => {
             this.receive(e.data as MessageResponse);
         }
@@ -37,6 +38,7 @@ export class NodeLoader {
     }
 
     private receive(msg: MessageResponse) {
+        const { renderContext } = this;
         if (msg.kind == "buffer") {
             const { resolveBuffer } = this;
             this.resolveBuffer = undefined;
@@ -47,6 +49,7 @@ export class NodeLoader {
             const { resolveAbortAll } = this;
             this.resolveAbortAll = undefined;
             resolveAbortAll?.();
+            renderContext.setSceneResolved(true);
             return;
         }
         const { id } = msg;
@@ -60,9 +63,11 @@ export class NodeLoader {
                     resolve(msg);
                     break;
                 case "aborted":
+                    renderContext.setSceneResolved(payloadPromises.size == 0);
                     resolve(undefined);
                     break;
                 case "error":
+                    renderContext.setSceneResolved(payloadPromises.size == 0);
                     reject(msg.error);
                     break;
             }

core3d/modules/octree/module.ts

@@ -68,7 +68,7 @@ export class OctreeModule implements RenderModule {
         const uniforms = this.createUniforms();
         const resources = await this.createResources(context, uniforms);
 
-        const loader = new NodeLoader(context.imports.loaderWorker);
+        const loader = new NodeLoader(context.imports.loaderWorker, context);
         const maxObjects = 25_000_000;// TODO: Get from device profile?
         const maxByteLength = maxObjects + 4; // add four bytes for mutex
         const buffer = new SharedArrayBuffer(maxByteLength);

core3d/state/default.ts

@@ -20,7 +20,10 @@ export function defaultRenderState(): RenderState {
             far: 10000,
         },
         quality: {
-            detail: 1,
+            detail: {
+                activeBias: 1,
+                downloadBias: 1,
+            },
         },
         debug: {
             wireframe: false,

core3d/state/index.ts

@@ -277,7 +277,16 @@ export interface RenderStateQuality {
      * Higher values will increase geometric detail, at the cost of performance and memory consumption.
      * At some point the amount of detail is capped by device specific constraints.
      */
-    readonly detail: number;
+    readonly detail: {
+        /**
+         * Active bias will only work for lowering the current level of detail. It will keep the current nodes in memory but not render.
+         */
+        readonly activeBias: number,
+        /**
+         * Download bias will affect the downloaded nodes.
+         */
+        readonly downloadBias: number,
+    };
 }
 
 /** Debug related state.

core3d/state/validate.ts

@@ -193,7 +193,8 @@ class ValidationContext {
 
     quality(state: RenderStateQuality) {
         const { detail } = state;
-        this.numeric(detail).positive().report("quality.detail");
+        this.numeric(detail.activeBias).positive().report("quality.detail.activeBias");
+        this.numeric(detail.downloadBias).positive().report("quality.detail.downloadBias");
     }
 
     scene(state: RenderStateScene) {

web_app/view.ts

@@ -72,9 +72,14 @@ export class View<
     private drsHighInterval = 50;
     private drsLowInterval = 100;
     private lastDrsAdjustTime = 0;
+    private lastDetailAdjustTime = 0;
     private resolutionTier: 0 | 1 | 2 = 2;
 
-    private currentDetailBias: number = 1;
+    private currentActiveDetailBias: number = 1;
+
+    //Set to 0.5 to account for the first frame increase.
+    private currentDownloadDetailBias: number = 0.75;
+    private lastDetailChangeAfterUpdate: "positive" | "negative" | "none" = "none";
 
     /**
      * @param canvas The HtmlCanvasElement used for rendering.
@@ -221,6 +226,14 @@ export class View<
         return Math.abs(mat[8]) > 0.98;
     }
 
+    /**
+     * The scene is considered resolved when it is done loading all nodes for the current render state.
+     * @returns if the scene is currently resolved
+     */
+    isSceneResolved() {
+        return this._renderContext?.isSceneResolved() ?? false;
+    }
+
     /**
      * Convert 2D pixel position to 3D position.
      * @param x Pixel x coordinate, in CSS pixels.
@@ -857,18 +870,22 @@ export class View<
         const frameIntervals: number[] = [];
         let possibleChanges = false;
         while (this._run && !(abortSignal?.aborted ?? false)) {
-            const { _renderContext, _activeController, deviceProfile } = this;
+            const { _renderContext, _activeController, deviceProfile, _stateChanges } = this;
             const renderTime = await RenderContext.nextFrame(_renderContext);
             const frameTime = renderTime - prevRenderTime;
             const cameraChanges = _activeController.renderStateChanges(this.renderStateCad.camera, renderTime - prevRenderTime);
             if (cameraChanges) {
                 this.modifyRenderState(cameraChanges);
             }
+            if (cameraChanges || _stateChanges?.highlights) {
+                this.lastDetailChangeAfterUpdate = "none";
+            }
             const { moving } = _activeController;
 
             const isIdleFrame = idleFrameTime > 500 && !moving;
             if (_renderContext && !_renderContext.isContextLost()) {
                 _renderContext.poll(); // poll for events, such as async reads and shader linking
+                const activeDetailDownloadModifier = this.dynamicDetailAdjustment();
 
                 if (isIdleFrame) { // increase resolution and detail bias on idle frame
                     if (deviceProfile.tier > 0 && this.renderState.toonOutline.on == false) {
@@ -883,8 +900,8 @@ export class View<
                         // set max quality and resolution when the camera stops moving
                         this.resolutionModifier = Math.min(1, this.baseRenderResolution * 2);
                         this.resize();
-                        this.modifyRenderState({ quality: { detail: 1 } });
-                        this.currentDetailBias = 1;
+                        this.modifyRenderState({ quality: { detail: { activeBias: 1 } } });
+                        this.currentActiveDetailBias = 1;
                         wasIdle = true;
                         // if pick is not already rendered then start to make the pick buffer available when the camera stops
                     }
@@ -902,12 +919,17 @@ export class View<
                         this.dynamicQualityAdjustment(frameIntervals);
                     }
                     const activeDetailModifier = 0.5;
-                    if (this.renderStateGL.quality.detail != activeDetailModifier) {
-                        this.currentDetailBias = activeDetailModifier;
-                        this.modifyRenderState({ quality: { detail: activeDetailModifier } });
+                    if (this.renderStateGL.quality.detail.activeBias != activeDetailModifier) {
+                        this.currentActiveDetailBias = activeDetailModifier;
+                        this.modifyRenderState({ quality: { detail: { activeBias: activeDetailModifier } } });
                     }
                 }
 
+                if (this.currentDownloadDetailBias != activeDetailDownloadModifier) {
+                    this.modifyRenderState({ quality: { detail: { downloadBias: activeDetailDownloadModifier } } });
+                    this.currentDownloadDetailBias = activeDetailDownloadModifier;
+                }
+
                 this.animate?.(renderTime);
 
                 if (this._stateChanges) {
@@ -920,7 +942,7 @@ export class View<
                     prevState = renderStateGL;
                     const statsPromise = _renderContext.render(renderStateGL);
                     statsPromise.then((stats) => {
-                        this._statistics = { render: stats, view: { resolution: this.resolutionModifier, detailBias: deviceProfile.detailBias * this.currentDetailBias, fps: stats.frameInterval ? 1000 / stats.frameInterval : undefined } };
+                        this._statistics = { render: stats, view: { resolution: this.resolutionModifier, detailBias: deviceProfile.detailBias * this.currentActiveDetailBias * this.currentDownloadDetailBias, fps: stats.frameInterval ? 1000 / stats.frameInterval : undefined } };
                         this.render?.({ isIdleFrame, cameraMoved: moving });
                         possibleChanges = true;
                     });
@@ -1118,6 +1140,39 @@ export class View<
         return clone;
     }
 
+    private dynamicDetailAdjustment() {
+        const { _renderContext, deviceProfile, statistics, currentDownloadDetailBias } = this;
+        if (!_renderContext || !statistics) {
+            return 1;
+        }
+        const now = performance.now();
+        const adjustUpCooldown = 500;
+        const adjustDownCooldown = 100;
+        const maxDetail = 10;
+        const minDetail = 1.0;
+        const increaseMemorySize = deviceProfile.limits.maxGPUBytes * 0.75;
+        const decreaseMemorySize = deviceProfile.limits.maxGPUBytes * 0.90;
+        const increasePrimitiveSize = deviceProfile.limits.maxPrimitives * 0.80;
+        const decreasePrimitiveSize = deviceProfile.limits.maxPrimitives * 0.95;
+        const detailStep = 0.25;
+        if (statistics.render.bufferBytes < increaseMemorySize && statistics.render.primitives < increasePrimitiveSize) {
+            if (_renderContext.isSceneResolved() && this.lastDetailChangeAfterUpdate != "negative") {
+                this.lastDetailChangeAfterUpdate = "positive";
+                if (now > this.lastDetailAdjustTime + adjustUpCooldown) {
+                    this.lastDetailAdjustTime = now;
+                    return Math.min(currentDownloadDetailBias + detailStep, maxDetail);
+                }
+            }
+        } else if (statistics.render.bufferBytes > decreaseMemorySize || statistics.render.primitives > decreasePrimitiveSize) {
+            if (now > this.lastDetailAdjustTime + adjustDownCooldown) {
+                this.lastDetailChangeAfterUpdate = "negative";
+                this.lastDetailAdjustTime = now;
+                return Math.max(currentDownloadDetailBias - detailStep, minDetail);
+            }
+        }
+        return currentDownloadDetailBias;
+    }
+
 
     //Dynamically change the quality of rendering based on the last 9 frames 
     private dynamicQualityAdjustment(frameIntervals: number[]) {