From 5a9fe4404c2950969fe1d4c2b386c26ed0bd114c Mon Sep 17 00:00:00 2001 From: jmschrack Date: Fri, 16 Dec 2022 10:41:40 -0600 Subject: [PATCH 1/9] Added FilterInterface. ImageTracking Controller can now take a custom filter. (Defaults to OneEuroFilter) Added SmoothDamp implementation. --- .../assets/card-example/axes.gltf | 420 ++++++++++++++++++ examples/image-tracking/three-filter.html | 138 ++++++ src/image-target/controller.js | 256 +++++------ src/image-target/index.js | 8 +- src/image-target/three.js | 28 +- src/libs/filter-interface.js | 14 + src/libs/one-euro-filter.js | 19 +- src/libs/smooth-damp-filter.js | 103 +++++ 8 files changed, 845 insertions(+), 141 deletions(-) create mode 100644 examples/image-tracking/assets/card-example/axes.gltf create mode 100644 examples/image-tracking/three-filter.html create mode 100644 src/libs/filter-interface.js create mode 100644 src/libs/smooth-damp-filter.js diff --git a/examples/image-tracking/assets/card-example/axes.gltf b/examples/image-tracking/assets/card-example/axes.gltf new file mode 100644 index 00000000..c0a8bb48 --- /dev/null +++ b/examples/image-tracking/assets/card-example/axes.gltf @@ -0,0 +1,420 @@ +{ + "asset": { + "version": "2.0", + "generator": "THREE.GLTFExporter" + }, + "scenes": [ + { + "name": "Scene", + "nodes": [ + 3 + ] + } + ], + "scene": 0, + "nodes": [ + { + "matrix": [ + 0.25, + 0, + 0, + 0, + 0, + 0.25, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0.49896340070573575, + 1 + ], + "name": "Box", + "mesh": 0 + }, + { + "matrix": [ + 0.25, + 0, + 0, + 0, + 0, + 1, + 0, + 0, + 0, + 0, + 0.25, + 0, + 0, + 0.49446752579591347, + 0, + 1 + ], + "name": "Box", + "mesh": 1 + }, + { + "matrix": [ + 1, + 0, + 0, + 0, + 0, + 0.25, + 0, + 0, + 0, + 0, + 0.25, + 0, + 0.5, + 0, + 0, + 1 + ], + "name": "Box", + "mesh": 2 + }, + { + "name": "Group", + "children": [ + 0, + 1, + 2 + ] + } + ], + "bufferViews": [ + { + "buffer": 0, + "byteOffset": 0, + "byteLength": 288, + "target": 34962, + "byteStride": 12 + }, + { + "buffer": 0, + "byteOffset": 288, + "byteLength": 288, + "target": 34962, + "byteStride": 12 + }, + { + "buffer": 0, + "byteOffset": 576, + "byteLength": 192, + "target": 34962, + "byteStride": 8 + }, + { + "buffer": 0, + "byteOffset": 768, + "byteLength": 72, + "target": 34963 + }, + { + "buffer": 0, + "byteOffset": 840, + "byteLength": 288, + "target": 34962, + "byteStride": 12 + }, + { + "buffer": 0, + "byteOffset": 1128, + "byteLength": 288, + "target": 34962, + "byteStride": 12 + }, + { + "buffer": 0, + "byteOffset": 1416, + "byteLength": 192, + "target": 34962, + "byteStride": 8 + }, + { + "buffer": 0, + "byteOffset": 1608, + "byteLength": 72, + "target": 34963 + }, + { + "buffer": 0, + "byteOffset": 1680, + "byteLength": 288, + "target": 34962, + "byteStride": 12 + }, + { + "buffer": 0, + "byteOffset": 1968, + "byteLength": 288, + "target": 34962, + "byteStride": 12 + }, + { + "buffer": 0, + "byteOffset": 2256, + "byteLength": 192, + "target": 34962, + "byteStride": 8 + }, + { + "buffer": 0, + "byteOffset": 2448, + "byteLength": 72, + "target": 34963 + } + ], + "buffers": [ + { + "byteLength": 2520, + "uri": "data:application/octet-stream;base64,AAAAPwAAAD8AAAA/AAAAPwAAAD8AAAC/AAAAPwAAAL8AAAA/AAAAPwAAAL8AAAC/AAAAvwAAAD8AAAC/AAAAvwAAAD8AAAA/AAAAvwAAAL8AAAC/AAAAvwAAAL8AAAA/AAAAvwAAAD8AAAC/AAAAPwAAAD8AAAC/AAAAvwAAAD8AAAA/AAAAPwAAAD8AAAA/AAAAvwAAAL8AAAA/AAAAPwAAAL8AAAA/AAAAvwAAAL8AAAC/AAAAPwAAAL8AAAC/AAAAvwAAAD8AAAA/AAAAPwAAAD8AAAA/AAAAvwAAAL8AAAA/AAAAPwAAAL8AAAA/AAAAPwAAAD8AAAC/AAAAvwAAAD8AAAC/AAAAPwAAAL8AAAC/AAAAvwAAAL8AAAC/AACAPwAAAAAAAAAAAACAPwAAAAAAAAAAAACAPwAAAAAAAAAAAACAPwAAAAAAAAAAAACAvwAAAAAAAAAAAACAvwAAAAAAAAAAAACAvwAAAAAAAAAAAACAvwAAAAAAAAAAAAAAAAAAgD8AAAAAAAAAAAAAgD8AAAAAAAAAAAAAgD8AAAAAAAAAAAAAgD8AAAAAAAAAAAAAgL8AAAAAAAAAAAAAgL8AAAAAAAAAAAAAgL8AAAAAAAAAAAAAgL8AAAAAAAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIC/AAAAAAAAAAAAAIC/AAAAAAAAAAAAAIC/AAAAAAAAAAAAAIC/AAAAAAAAgD8AAIA/AACAPwAAAAAAAAAAAACAPwAAAAAAAAAAAACAPwAAgD8AAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AACAPwAAgD8AAAAAAAAAAAAAgD8AAAAAAAAAAAAAgD8AAIA/AACAPwAAAAAAAAAAAACAPwAAAAAAAAAAAACAPwAAgD8AAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AACAPwAAgD8AAAAAAAAAAAAAgD8AAAAAAAACAAEAAgADAAEABAAGAAUABgAHAAUACAAKAAkACgALAAkADAAOAA0ADgAPAA0AEAASABEAEgATABEAFAAWABUAFgAXABUAAAAAPwAAAD8AAAA/AAAAPwAAAD8AAAC/AAAAPwAAAL8AAAA/AAAAPwAAAL8AAAC/AAAAvwAAAD8AAAC/AAAAvwAAAD8AAAA/AAAAvwAAAL8AAAC/AAAAvwAAAL8AAAA/AAAAvwAAAD8AAAC/AAAAPwAAAD8AAAC/AAAAvwAAAD8AAAA/AAAAPwAAAD8AAAA/AAAAvwAAAL8AAAA/AAAAPwAAAL8AAAA/AAAAvwAAAL8AAAC/AAAAPwAAAL8AAAC/AAAAvwAAAD8AAAA/AAAAPwAAAD8AAAA/AAAAvwAAAL8AAAA/AAAAPwAAAL8AAAA/AAAAPwAAAD8AAAC/AAAAvwAAAD8AAAC/AAAAPwAAAL8AAAC/AAAAvwAAAL8AAAC/AACAPwAAAAAAAAAAAACAPwAAAAAAAAAAAACAPwAAAAAAAAAAAACAPwAAAAAAAAAAAACAvwAAAAAAAAAAAACAvwAAAAAAAAAAAACAvwAAAAAAAAAAAACAvwAAAAAAAAAAAAAAAAAAgD8AAAAAAAAAAAAAgD8AAAAAAAAAAAAAgD8AAAAAAAAAAAAAgD8AAAAAAAAAAAAAgL8AAAAAAAAAAAAAgL8AAAAAAAAAAAAAgL8AAAAAAAAAAAAAgL8AAAAAAAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIC/AAAAAAAAAAAAAIC/AAAAAAAAAAAAAIC/AAAAAAAAAAAAAIC/AAAAAAAAgD8AAIA/AACAPwAAAAAAAAAAAACAPwAAAAAAAAAAAACAPwAAgD8AAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AACAPwAAgD8AAAAAAAAAAAAAgD8AAAAAAAAAAAAAgD8AAIA/AACAPwAAAAAAAAAAAACAPwAAAAAAAAAAAACAPwAAgD8AAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AACAPwAAgD8AAAAAAAAAAAAAgD8AAAAAAAACAAEAAgADAAEABAAGAAUABgAHAAUACAAKAAkACgALAAkADAAOAA0ADgAPAA0AEAASABEAEgATABEAFAAWABUAFgAXABUAAAAAPwAAAD8AAAA/AAAAPwAAAD8AAAC/AAAAPwAAAL8AAAA/AAAAPwAAAL8AAAC/AAAAvwAAAD8AAAC/AAAAvwAAAD8AAAA/AAAAvwAAAL8AAAC/AAAAvwAAAL8AAAA/AAAAvwAAAD8AAAC/AAAAPwAAAD8AAAC/AAAAvwAAAD8AAAA/AAAAPwAAAD8AAAA/AAAAvwAAAL8AAAA/AAAAPwAAAL8AAAA/AAAAvwAAAL8AAAC/AAAAPwAAAL8AAAC/AAAAvwAAAD8AAAA/AAAAPwAAAD8AAAA/AAAAvwAAAL8AAAA/AAAAPwAAAL8AAAA/AAAAPwAAAD8AAAC/AAAAvwAAAD8AAAC/AAAAPwAAAL8AAAC/AAAAvwAAAL8AAAC/AACAPwAAAAAAAAAAAACAPwAAAAAAAAAAAACAPwAAAAAAAAAAAACAPwAAAAAAAAAAAACAvwAAAAAAAAAAAACAvwAAAAAAAAAAAACAvwAAAAAAAAAAAACAvwAAAAAAAAAAAAAAAAAAgD8AAAAAAAAAAAAAgD8AAAAAAAAAAAAAgD8AAAAAAAAAAAAAgD8AAAAAAAAAAAAAgL8AAAAAAAAAAAAAgL8AAAAAAAAAAAAAgL8AAAAAAAAAAAAAgL8AAAAAAAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIC/AAAAAAAAAAAAAIC/AAAAAAAAAAAAAIC/AAAAAAAAAAAAAIC/AAAAAAAAgD8AAIA/AACAPwAAAAAAAAAAAACAPwAAAAAAAAAAAACAPwAAgD8AAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AACAPwAAgD8AAAAAAAAAAAAAgD8AAAAAAAAAAAAAgD8AAIA/AACAPwAAAAAAAAAAAACAPwAAAAAAAAAAAACAPwAAgD8AAIA/AAAAAAAAAAAAAIA/AAAAAAAAAAAAAIA/AACAPwAAgD8AAAAAAAAAAAAAgD8AAAAAAAACAAEAAgADAAEABAAGAAUABgAHAAUACAAKAAkACgALAAkADAAOAA0ADgAPAA0AEAASABEAEgATABEAFAAWABUAFgAXABUA" + } + ], + "accessors": [ + { + "bufferView": 0, + "componentType": 5126, + "count": 24, + "max": [ + 0.5, + 0.5, + 0.5 + ], + "min": [ + -0.5, + -0.5, + -0.5 + ], + "type": "VEC3" + }, + { + "bufferView": 1, + "componentType": 5126, + "count": 24, + "max": [ + 1, + 1, + 1 + ], + "min": [ + -1, + -1, + -1 + ], + "type": "VEC3" + }, + { + "bufferView": 2, + "componentType": 5126, + "count": 24, + "max": [ + 1, + 1 + ], + "min": [ + 0, + 0 + ], + "type": "VEC2" + }, + { + "bufferView": 3, + "componentType": 5123, + "count": 36, + "max": [ + 23 + ], + "min": [ + 0 + ], + "type": "SCALAR" + }, + { + "bufferView": 4, + "componentType": 5126, + "count": 24, + "max": [ + 0.5, + 0.5, + 0.5 + ], + "min": [ + -0.5, + -0.5, + -0.5 + ], + "type": "VEC3" + }, + { + "bufferView": 5, + "componentType": 5126, + "count": 24, + "max": [ + 1, + 1, + 1 + ], + "min": [ + -1, + -1, + -1 + ], + "type": "VEC3" + }, + { + "bufferView": 6, + "componentType": 5126, + "count": 24, + "max": [ + 1, + 1 + ], + "min": [ + 0, + 0 + ], + "type": "VEC2" + }, + { + "bufferView": 7, + "componentType": 5123, + "count": 36, + "max": [ + 23 + ], + "min": [ + 0 + ], + "type": "SCALAR" + }, + { + "bufferView": 8, + "componentType": 5126, + "count": 24, + "max": [ + 0.5, + 0.5, + 0.5 + ], + "min": [ + -0.5, + -0.5, + -0.5 + ], + "type": "VEC3" + }, + { + "bufferView": 9, + "componentType": 5126, + "count": 24, + "max": [ + 1, + 1, + 1 + ], + "min": [ + -1, + -1, + -1 + ], + "type": "VEC3" + }, + { + "bufferView": 10, + "componentType": 5126, + "count": 24, + "max": [ + 1, + 1 + ], + "min": [ + 0, + 0 + ], + "type": "VEC2" + }, + { + "bufferView": 11, + "componentType": 5123, + "count": 36, + "max": [ + 23 + ], + "min": [ + 0 + ], + "type": "SCALAR" + } + ], + "materials": [ + { + "pbrMetallicRoughness": { + "metallicFactor": 0, + "roughnessFactor": 1 + } + }, + { + "pbrMetallicRoughness": { + "metallicFactor": 0, + "roughnessFactor": 1 + } + }, + { + "pbrMetallicRoughness": { + "metallicFactor": 0, + "roughnessFactor": 1 + } + } + ], + "meshes": [ + { + "primitives": [ + { + "mode": 4, + "attributes": { + "POSITION": 0, + "NORMAL": 1, + "TEXCOORD_0": 2 + }, + "indices": 3, + "material": 0 + } + ] + }, + { + "primitives": [ + { + "mode": 4, + "attributes": { + "POSITION": 4, + "NORMAL": 5, + "TEXCOORD_0": 6 + }, + "indices": 7, + "material": 1 + } + ] + }, + { + "primitives": [ + { + "mode": 4, + "attributes": { + "POSITION": 8, + "NORMAL": 9, + "TEXCOORD_0": 10 + }, + "indices": 11, + "material": 2 + } + ] + } + ] +} \ No newline at end of file diff --git a/examples/image-tracking/three-filter.html b/examples/image-tracking/three-filter.html new file mode 100644 index 00000000..ca7829b4 --- /dev/null +++ b/examples/image-tracking/three-filter.html @@ -0,0 +1,138 @@ + + + + + + + + + + + + + + + +
+ + + +
+ +
+
+ + + \ No newline at end of file diff --git a/src/image-target/controller.js b/src/image-target/controller.js index 91156cc9..bd903cc2 100644 --- a/src/image-target/controller.js +++ b/src/image-target/controller.js @@ -1,28 +1,31 @@ -import {memory,nextFrame} from '@tensorflow/tfjs'; -const tf = {memory,nextFrame}; -import ControllerWorker from "./controller.worker.js?worker&inline"; -import {Tracker} from './tracker/tracker.js'; -import {CropDetector} from './detector/crop-detector.js'; -import {Compiler} from './compiler.js'; -import {InputLoader} from './input-loader.js'; -import {OneEuroFilter} from '../libs/one-euro-filter.js'; +import { memory, nextFrame } from '@tensorflow/tfjs'; +const tf = { memory, nextFrame }; +import ControllerWorker from "./controller.worker.js?worker&inline"; +import { Tracker } from './tracker/tracker.js'; +import { CropDetector } from './detector/crop-detector.js'; +import { Compiler } from './compiler.js'; +import { InputLoader } from './input-loader.js'; +import { OneEuroFilter } from '../libs/one-euro-filter.js'; const DEFAULT_FILTER_CUTOFF = 0.001; // 1Hz. time period in milliseconds const DEFAULT_FILTER_BETA = 1000; const DEFAULT_WARMUP_TOLERANCE = 5; const DEFAULT_MISS_TOLERANCE = 5; +const DEFAULT_FILTER={filter:OneEuroFilter,opts:{minCutOff: DEFAULT_FILTER_CUTOFF, beta: DEFAULT_FILTER_BETA}} + class Controller { - constructor({inputWidth, inputHeight, onUpdate=null, debugMode=false, maxTrack=1, - warmupTolerance=null, missTolerance=null, filterMinCF=null, filterBeta=null}) { + constructor({ inputWidth, inputHeight, onUpdate = null, debugMode = false, maxTrack = 1, + warmupTolerance = null, missTolerance = null, customFilter=null}) { this.inputWidth = inputWidth; this.inputHeight = inputHeight; this.maxTrack = maxTrack; - this.filterMinCF = filterMinCF === null? DEFAULT_FILTER_CUTOFF: filterMinCF; - this.filterBeta = filterBeta === null? DEFAULT_FILTER_BETA: filterBeta; - this.warmupTolerance = warmupTolerance === null? DEFAULT_WARMUP_TOLERANCE: warmupTolerance; - this.missTolerance = missTolerance === null? DEFAULT_MISS_TOLERANCE: missTolerance; + this.customFilter=customFilter===null?DEFAULT_FILTER:customFilter; + if(!this.customFilter.hasOwnProperty("filter")) customFilter.filter=DEFAULT_FILTER.filter; + if(!this.customFilter.hasOwnProperty("opts")) customFilter.opts={} + this.warmupTolerance = warmupTolerance === null ? DEFAULT_WARMUP_TOLERANCE : warmupTolerance; + this.missTolerance = missTolerance === null ? DEFAULT_MISS_TOLERANCE : missTolerance; this.cropDetector = new CropDetector(this.inputWidth, this.inputHeight, debugMode); this.inputLoader = new InputLoader(this.inputWidth, this.inputHeight); this.markerDimensions = null; @@ -35,7 +38,7 @@ class Controller { const near = 10; const far = 100000; const fovy = 45.0 * Math.PI / 180; // 45 in radian. field of view vertical - const f = (this.inputHeight/2) / Math.tan(fovy/2); + const f = (this.inputHeight / 2) / Math.tan(fovy / 2); // [fx s cx] // K = [ 0 fx cy] // [ 0 0 1] @@ -107,7 +110,7 @@ class Controller { this.markerDimensions = dimensions; - return {dimensions: dimensions, matchingDataList, trackingDataList}; + return { dimensions: dimensions, matchingDataList, trackingDataList }; } // warm up gpu - build kernels is slow @@ -127,14 +130,14 @@ class Controller { } async _detectAndMatch(inputT, targetIndexes) { - const {featurePoints} = this.cropDetector.detectMoving(inputT); - const {targetIndex: matchedTargetIndex, modelViewTransform} = await this._workerMatch(featurePoints, targetIndexes); - return {targetIndex: matchedTargetIndex, modelViewTransform} + const { featurePoints } = this.cropDetector.detectMoving(inputT); + const { targetIndex: matchedTargetIndex, modelViewTransform } = await this._workerMatch(featurePoints, targetIndexes); + return { targetIndex: matchedTargetIndex, modelViewTransform } } async _trackAndUpdate(inputT, lastModelViewTransform, targetIndex) { - const {worldCoords, screenCoords} = this.tracker.track(inputT, lastModelViewTransform, targetIndex); + const { worldCoords, screenCoords } = this.tracker.track(inputT, lastModelViewTransform, targetIndex); if (worldCoords.length < 4) return null; - const modelViewTransform = await this._workerTrackUpdate(lastModelViewTransform, {worldCoords, screenCoords}); + const modelViewTransform = await this._workerTrackUpdate(lastModelViewTransform, { worldCoords, screenCoords }); return modelViewTransform; } @@ -144,106 +147,107 @@ class Controller { this.processingVideo = true; this.trackingStates = []; + for (let i = 0; i < this.markerDimensions.length; i++) { this.trackingStates.push({ - showing: false, - isTracking: false, - currentModelViewTransform: null, - trackCount: 0, - trackMiss: 0, - filter: new OneEuroFilter({minCutOff: this.filterMinCF, beta: this.filterBeta}) + showing: false, + isTracking: false, + currentModelViewTransform: null, + trackCount: 0, + trackMiss: 0, + filter: new this.customFilter.filter(this.customFilter.opts),/* { minCutOff: this.filterMinCF, beta: this.filterBeta } )*/ }); //console.log("filterMinCF", this.filterMinCF, this.filterBeta); } - const startProcessing = async() => { + const startProcessing = async () => { while (true) { - if (!this.processingVideo) break; - - const inputT = this.inputLoader.loadInput(input); - - const nTracking = this.trackingStates.reduce((acc, s) => { - return acc + (!!s.isTracking? 1: 0); - }, 0); - - // detect and match only if less then maxTrack - if (nTracking < this.maxTrack) { - - const matchingIndexes = []; - for (let i = 0; i < this.trackingStates.length; i++) { - const trackingState = this.trackingStates[i]; - if (trackingState.isTracking === true) continue; - if (this.interestedTargetIndex !== -1 && this.interestedTargetIndex !== i) continue; - - matchingIndexes.push(i); - } - - const {targetIndex: matchedTargetIndex, modelViewTransform} = await this._detectAndMatch(inputT, matchingIndexes); - - if (matchedTargetIndex !== -1) { - this.trackingStates[matchedTargetIndex].isTracking = true; - this.trackingStates[matchedTargetIndex].currentModelViewTransform = modelViewTransform; - } - } - - // tracking update - for (let i = 0; i < this.trackingStates.length; i++) { - const trackingState = this.trackingStates[i]; - - if (trackingState.isTracking) { - let modelViewTransform = await this._trackAndUpdate(inputT, trackingState.currentModelViewTransform, i); - if (modelViewTransform === null) { - trackingState.isTracking = false; - } else { - trackingState.currentModelViewTransform = modelViewTransform; - } - } - - // if not showing, then show it once it reaches warmup number of frames - if (!trackingState.showing) { - if (trackingState.isTracking) { - trackingState.trackMiss = 0; - trackingState.trackCount += 1; - if (trackingState.trackCount > this.warmupTolerance) { - trackingState.showing = true; - trackingState.trackingMatrix = null; - trackingState.filter.reset(); - } - } - } - - // if showing, then count miss, and hide it when reaches tolerance - if (trackingState.showing) { - if (!trackingState.isTracking) { - trackingState.trackCount = 0; - trackingState.trackMiss += 1; - - if (trackingState.trackMiss > this.missTolerance) { - trackingState.showing = false; - trackingState.trackingMatrix = null; - this.onUpdate && this.onUpdate({type: 'updateMatrix', targetIndex: i, worldMatrix: null}); - } - } else { - trackingState.trackMiss = 0; - } - } - - // if showing, then call onUpdate, with world matrix - if (trackingState.showing) { - const worldMatrix = this._glModelViewMatrix(trackingState.currentModelViewTransform, i); - trackingState.trackingMatrix = trackingState.filter.filter(Date.now(), worldMatrix); - - const clone = []; - for (let j = 0; j < trackingState.trackingMatrix.length; j++) { - clone[j] = trackingState.trackingMatrix[j]; - } - this.onUpdate && this.onUpdate({type: 'updateMatrix', targetIndex: i, worldMatrix: clone}); - } - } - - inputT.dispose(); - this.onUpdate && this.onUpdate({type: 'processDone'}); - await tf.nextFrame(); + if (!this.processingVideo) break; + + const inputT = this.inputLoader.loadInput(input); + + const nTracking = this.trackingStates.reduce((acc, s) => { + return acc + (!!s.isTracking ? 1 : 0); + }, 0); + + // detect and match only if less then maxTrack + if (nTracking < this.maxTrack) { + + const matchingIndexes = []; + for (let i = 0; i < this.trackingStates.length; i++) { + const trackingState = this.trackingStates[i]; + if (trackingState.isTracking === true) continue; + if (this.interestedTargetIndex !== -1 && this.interestedTargetIndex !== i) continue; + + matchingIndexes.push(i); + } + + const { targetIndex: matchedTargetIndex, modelViewTransform } = await this._detectAndMatch(inputT, matchingIndexes); + + if (matchedTargetIndex !== -1) { + this.trackingStates[matchedTargetIndex].isTracking = true; + this.trackingStates[matchedTargetIndex].currentModelViewTransform = modelViewTransform; + } + } + + // tracking update + for (let i = 0; i < this.trackingStates.length; i++) { + const trackingState = this.trackingStates[i]; + + if (trackingState.isTracking) { + let modelViewTransform = await this._trackAndUpdate(inputT, trackingState.currentModelViewTransform, i); + if (modelViewTransform === null) { + trackingState.isTracking = false; + } else { + trackingState.currentModelViewTransform = modelViewTransform; + } + } + + // if not showing, then show it once it reaches warmup number of frames + if (!trackingState.showing) { + if (trackingState.isTracking) { + trackingState.trackMiss = 0; + trackingState.trackCount += 1; + if (trackingState.trackCount > this.warmupTolerance) { + trackingState.showing = true; + trackingState.trackingMatrix = null; + trackingState.filter.reset(); + } + } + } + + // if showing, then count miss, and hide it when reaches tolerance + if (trackingState.showing) { + if (!trackingState.isTracking) { + trackingState.trackCount = 0; + trackingState.trackMiss += 1; + + if (trackingState.trackMiss > this.missTolerance) { + trackingState.showing = false; + trackingState.trackingMatrix = null; + this.onUpdate && this.onUpdate({ type: 'updateMatrix', targetIndex: i, worldMatrix: null }); + } + } else { + trackingState.trackMiss = 0; + } + } + + // if showing, then call onUpdate, with world matrix + if (trackingState.showing) { + const worldMatrix = this._glModelViewMatrix(trackingState.currentModelViewTransform, i); + trackingState.trackingMatrix = trackingState.filter.filter(Date.now(), worldMatrix); + + const clone = []; + for (let j = 0; j < trackingState.trackingMatrix.length; j++) { + clone[j] = trackingState.trackingMatrix[j]; + } + this.onUpdate && this.onUpdate({ type: 'updateMatrix', targetIndex: i, worldMatrix: clone }); + } + } + + inputT.dispose(); + this.onUpdate && this.onUpdate({ type: 'processDone' }); + await tf.nextFrame(); } } startProcessing(); @@ -255,14 +259,14 @@ class Controller { async detect(input) { const inputT = this.inputLoader.loadInput(input); - const {featurePoints, debugExtra} = await this.cropDetector.detect(inputT); + const { featurePoints, debugExtra } = await this.cropDetector.detect(inputT); inputT.dispose(); - return {featurePoints, debugExtra}; + return { featurePoints, debugExtra }; } async match(featurePoints, targetIndex) { - const {modelViewTransform, debugExtra} = await this._workerMatch(featurePoints, [targetIndex]); - return {modelViewTransform, debugExtra}; + const { modelViewTransform, debugExtra } = await this._workerMatch(featurePoints, [targetIndex]); + return { modelViewTransform, debugExtra }; } async track(input, modelViewTransform, targetIndex) { @@ -273,7 +277,7 @@ class Controller { } async trackUpdate(modelViewTransform, trackFeatures) { - if (trackFeatures.worldCoords.length < 4 ) return null; + if (trackFeatures.worldCoords.length < 4) return null; const modelViewTransform2 = await this._workerTrackUpdate(modelViewTransform, trackFeatures); return modelViewTransform2; } @@ -281,9 +285,9 @@ class Controller { _workerMatch(featurePoints, targetIndexes) { return new Promise(async (resolve, reject) => { this.workerMatchDone = (data) => { - resolve({targetIndex: data.targetIndex, modelViewTransform: data.modelViewTransform, debugExtra: data.debugExtra}); + resolve({ targetIndex: data.targetIndex, modelViewTransform: data.modelViewTransform, debugExtra: data.debugExtra }); } - this.worker.postMessage({type: 'match', featurePoints: featurePoints, targetIndexes}); + this.worker.postMessage({ type: 'match', featurePoints: featurePoints, targetIndexes }); }); } @@ -292,8 +296,8 @@ class Controller { this.workerTrackDone = (data) => { resolve(data.modelViewTransform); } - const {worldCoords, screenCoords} = trackingFeatures; - this.worker.postMessage({type: 'trackUpdate', modelViewTransform, worldCoords, screenCoords}); + const { worldCoords, screenCoords } = trackingFeatures; + this.worker.postMessage({ type: 'trackUpdate', modelViewTransform, worldCoords, screenCoords }); }); } @@ -346,7 +350,7 @@ class Controller { // build openGL projection matrix // ref: https://strawlab.org/2011/11/05/augmented-reality-with-OpenGL/ - _glProjectionMatrix({projectionTransform, width, height, near, far}) { + _glProjectionMatrix({ projectionTransform, width, height, near, far }) { const proj = [ [2 * projectionTransform[0][0] / width, 0, -(2 * projectionTransform[0][2] / width - 1), 0], [0, 2 * projectionTransform[1][1] / height, -(2 * projectionTransform[1][2] / height - 1), 0], @@ -356,7 +360,7 @@ class Controller { const projMatrix = []; for (let i = 0; i < 4; i++) { for (let j = 0; j < 4; j++) { - projMatrix.push(proj[j][i]); + projMatrix.push(proj[j][i]); } } return projMatrix; @@ -364,5 +368,5 @@ class Controller { } export { - Controller + Controller } diff --git a/src/image-target/index.js b/src/image-target/index.js index 13d7fb3c..c4241f33 100644 --- a/src/image-target/index.js +++ b/src/image-target/index.js @@ -1,11 +1,17 @@ import {Controller} from './controller.js'; import {Compiler} from './compiler.js'; import {UI} from '../ui/ui.js'; +import {FilterInterface} from '../libs/filter-interface.js'; +import {OneEuroFilter} from '../libs/one-euro-filter.js' +import {SmoothDampFilter} from '../libs/smooth-damp-filter.js' export { Controller, Compiler, - UI + UI, + FilterInterface, + OneEuroFilter, + SmoothDampFilter } if (!window.MINDAR) { diff --git a/src/image-target/three.js b/src/image-target/three.js index 7942a119..ab5d5b26 100644 --- a/src/image-target/three.js +++ b/src/image-target/three.js @@ -1,9 +1,13 @@ import { Matrix4, Vector3, Quaternion, Scene, WebGLRenderer, PerspectiveCamera, Group, sRGBEncoding } from "three"; import * as tf from '@tensorflow/tfjs'; -//import { CSS3DRenderer } from '../libs/CSS3DRenderer.js'; -import {CSS3DRenderer} from 'three/addons/renderers/CSS3DRenderer.js' +import { CSS3DRenderer } from 'three/addons/renderers/CSS3DRenderer.js' import { Controller } from "./controller.js"; import { UI } from "../ui/ui.js"; +import { FilterInterface } from '../libs/filter-interface.js'; +import { OneEuroFilter } from '../libs/one-euro-filter.js' +import { SmoothDampFilter } from '../libs/smooth-damp-filter.js' + +export { FilterInterface, OneEuroFilter, SmoothDampFilter }; const cssScaleDownMatrix = new Matrix4(); cssScaleDownMatrix.compose(new Vector3(), new Quaternion(), new Vector3(0.001, 0.001, 0.001)); @@ -11,13 +15,14 @@ cssScaleDownMatrix.compose(new Vector3(), new Quaternion(), new Vector3(0.001, 0 export class MindARThree { constructor({ container, imageTargetSrc, maxTrack, uiLoading = "yes", uiScanning = "yes", uiError = "yes", - filterMinCF = null, filterBeta = null, warmupTolerance = null, missTolerance = null + filterMinCF = null, filterBeta = null, warmupTolerance = null, missTolerance = null, customFilter = null }) { this.container = container; this.imageTargetSrc = imageTargetSrc; this.maxTrack = maxTrack; this.filterMinCF = filterMinCF; this.filterBeta = filterBeta; + this.customFilter = customFilter; this.warmupTolerance = warmupTolerance; this.missTolerance = missTolerance; this.ui = new UI({ uiLoading, uiScanning, uiError }); @@ -115,19 +120,21 @@ export class MindARThree { return new Promise(async (resolve, reject) => { const video = this.video; const container = this.container; - + this.isScanning = true; + this.scanState = true; this.controller = new Controller({ inputWidth: video.videoWidth, inputHeight: video.videoHeight, filterMinCF: this.filterMinCF, filterBeta: this.filterBeta, + customFilter: this.customFilter, warmupTolerance: this.warmupTolerance, missTolerance: this.missTolerance, maxTrack: this.maxTrack, onUpdate: (data) => { if (data.type === 'updateMatrix') { const { targetIndex, worldMatrix } = data; - + this.isScanning = true; for (let i = 0; i < this.anchors.length; i++) { if (this.anchors[i].targetIndex === targetIndex) { if (this.anchors[i].css) { @@ -163,10 +170,18 @@ export class MindARThree { } if (worldMatrix !== null) { - this.ui.hideScanning(); + //this.ui.hideScanning(); + this.isScanning = false; } } } + if (this.isScanning != this.scanState) { + if (this.isScanning) + this.ui.showScanning(); + else + this.ui.hideScanning(); + this.scanState = this.isScanning; + } } } }); @@ -259,5 +274,4 @@ if (!window.MINDAR.IMAGE) { } window.MINDAR.IMAGE.MindARThree = MindARThree; -//window.MINDAR.IMAGE.THREE = THREE; window.MINDAR.IMAGE.tf = tf; diff --git a/src/libs/filter-interface.js b/src/libs/filter-interface.js new file mode 100644 index 00000000..8aea09fb --- /dev/null +++ b/src/libs/filter-interface.js @@ -0,0 +1,14 @@ + +export class FilterInterface{ + constructor(opts){} + /** + * Called when tracking is lost + */ + reset(){} + /** + * + * @param {number} timestamp Passed in every frame. Usually: Date.now() + * @param {number[]} x Matrix + */ + filter(timestamp, x){} +} \ No newline at end of file diff --git a/src/libs/one-euro-filter.js b/src/libs/one-euro-filter.js index 818f1e0a..fafde244 100644 --- a/src/libs/one-euro-filter.js +++ b/src/libs/one-euro-filter.js @@ -1,16 +1,16 @@ // Ref: https://jaantollander.com/post/noise-filtering-using-one-euro-filter/#mjx-eqn%3A1 -const smoothingFactor = (te, cutoff) => { +const smoothingFactor = (te, cutoff) => {//5 const r = 2 * Math.PI * cutoff * te; - return r / (r+1); + return r / (r + 1); } -const exponentialSmoothing = (a, x, xPrev) => { +const exponentialSmoothing = (a, x, xPrev) => {//4 return a * x + (1 - a) * xPrev; } class OneEuroFilter { - constructor({minCutOff, beta}) { + constructor({ minCutOff, beta }) { this.minCutOff = minCutOff; this.beta = beta; this.dCutOff = 0.001; // period in milliseconds, so default to 0.001 = 1Hz @@ -24,7 +24,12 @@ class OneEuroFilter { reset() { this.initialized = false; } - + /** + * + * @param {number} t + * @param {number[]} x + * @returns + */ filter(t, x) { if (!this.initialized) { this.initialized = true; @@ -34,7 +39,7 @@ class OneEuroFilter { return x; } - const {xPrev, tPrev, dxPrev} = this; + const { xPrev, tPrev, dxPrev } = this; //console.log("filter", x, xPrev, x.map((xx, i) => x[i] - xPrev[i])); @@ -57,7 +62,7 @@ class OneEuroFilter { } // update prev - this.xPrev = xHat; + this.xPrev = xHat; this.dxPrev = dxHat; this.tPrev = t; diff --git a/src/libs/smooth-damp-filter.js b/src/libs/smooth-damp-filter.js new file mode 100644 index 00000000..9bf30838 --- /dev/null +++ b/src/libs/smooth-damp-filter.js @@ -0,0 +1,103 @@ +/* +* Critically Damped Ease-In/Ease-Out Smoothing +* reference: "Game Programming Gems 4" Chapter 1.10 (ISBN-13: 978-1584502951) +* https://archive.org/details/game-programming-gems-4/mode/2up (page 96) +* Originally named "SmoothCD", but Unity calls this "SmoothDamp" so we'll stick with that for easier time with google +* I've modified this to run on an array, it makes 2 assumptions: +* 1) Every value uses the same smoothTime and maxSpeed +* 2) Every value can be modified independently of each other +*/ +import { FilterInterface } from './filter-interface.js' +export class SmoothDampFilter extends FilterInterface { + constructor({ smoothTime = 0.1, maxSpeed = Number.MAX_VALUE }) { + super(); + this.current = []; + this.currentVelocity = []; + console.log("SmoothDampFilter: smoothTime",smoothTime," maxSpeed:",maxSpeed); + this.smoothTime = Math.max(smoothTime, 0.0001); + this.maxSpeed = maxSpeed; + this.maxChange = this.maxSpeed * this.smoothTime; + this.omega = 2.0 / smoothTime; + this.lastTime = 0; + this.initialized = false; + } + reset() { + this.initialized = false; + } + /** + * + * @param {number} timestamp + * @param {Array} target + * @returns + */ + filter(timestamp, target) { + //this.current is updated with the new matrix everytime + //but we copy to output as a safe reference + const output = [] + if (!this.initialized) { + this.currentVelocity= target.map(()=>0);//this.currentVelocity.fill(0, 0, target.length); + this.current=target.map((value)=>{return value;}); + output.push(...target); + this.initialized = true; + } else { + const deltaTime = (timestamp - this.lastTime)*0.001; + + const x = this.omega * deltaTime; + const exp = 1.0 / (1.0 + x + 0.48 * x * x + 0.235 * x * x * x); + + for (let i = 0; i < target.length; i++) { + let change = this.current[i] - target[i]; + const originalTo = target[i]; + change = Math.min(Math.max(change, -1 * this.maxChange), this.maxChange); + const _target = this.current[i] - change; + + const temp = (this.currentVelocity[i] + this.omega * change) * deltaTime; + this.currentVelocity[i] = (this.currentVelocity[i] - this.omega * temp) * exp; + let out = _target + (change + temp) * exp; + + if (originalTo - this.current[i] > 0.0 == out > originalTo) { + out = originalTo; + this.currentVelocity[i] = (out - originalTo) / deltaTime; + } + this.current[i] = out; + output.push(out); + } + } + this.lastTime = timestamp; + return output; + } +} + +/* + +ublic static float SmoothDamp(float current, float target, ref float currentVelocity, float smoothTime, [uei.DefaultValue("Mathf.Infinity")] float maxSpeed, [uei.DefaultValue("Time.deltaTime")] float deltaTime) + { + // Based on Game Programming Gems 4 Chapter 1.10 + smoothTime = Mathf.Max(0.0001F, smoothTime); + float omega = 2F / smoothTime; + + float x = omega * deltaTime; + float exp = 1F / (1F + x + 0.48F * x * x + 0.235F * x * x * x); + float change = current - target; + float originalTo = target; + + // Clamp maximum speed + float maxChange = maxSpeed * smoothTime; + change = Mathf.Clamp(change, -maxChange, maxChange); + target = current - change; + + float temp = (currentVelocity + omega * change) * deltaTime; + currentVelocity = (currentVelocity - omega * temp) * exp; + float output = target + (change + temp) * exp; + + // Prevent overshooting + if (originalTo - current > 0.0F == output > originalTo) + { + output = originalTo; + currentVelocity = (output - originalTo) / deltaTime; + } + + return output; + } + +*/ \ No newline at end of file From ca8296d72ffd35e7ac96e602cf03c304adaec49f Mon Sep 17 00:00:00 2001 From: jmschrack Date: Wed, 25 Jan 2023 20:29:21 -0600 Subject: [PATCH 2/9] Cleaned up old comments from smooth-damp-filter --- src/libs/smooth-damp-filter.js | 53 +++++++--------------------------- 1 file changed, 10 insertions(+), 43 deletions(-) diff --git a/src/libs/smooth-damp-filter.js b/src/libs/smooth-damp-filter.js index 9bf30838..328688c0 100644 --- a/src/libs/smooth-damp-filter.js +++ b/src/libs/smooth-damp-filter.js @@ -17,6 +17,7 @@ export class SmoothDampFilter extends FilterInterface { this.smoothTime = Math.max(smoothTime, 0.0001); this.maxSpeed = maxSpeed; this.maxChange = this.maxSpeed * this.smoothTime; + this.negMaxChange=-1.0*this.maxChange this.omega = 2.0 / smoothTime; this.lastTime = 0; this.initialized = false; @@ -32,12 +33,10 @@ export class SmoothDampFilter extends FilterInterface { */ filter(timestamp, target) { //this.current is updated with the new matrix everytime - //but we copy to output as a safe reference - const output = [] if (!this.initialized) { this.currentVelocity= target.map(()=>0);//this.currentVelocity.fill(0, 0, target.length); this.current=target.map((value)=>{return value;}); - output.push(...target); + //output.push(...target); this.initialized = true; } else { const deltaTime = (timestamp - this.lastTime)*0.001; @@ -45,10 +44,12 @@ export class SmoothDampFilter extends FilterInterface { const x = this.omega * deltaTime; const exp = 1.0 / (1.0 + x + 0.48 * x * x + 0.235 * x * x * x); - for (let i = 0; i < target.length; i++) { - let change = this.current[i] - target[i]; - const originalTo = target[i]; - change = Math.min(Math.max(change, -1 * this.maxChange), this.maxChange); + //for (let i = 0; i < target.length; i++) { + target.forEach((value,i)=>{ + const originalTo = value; + //clamp the change between negative and positive max change + const change = Math.min(Math.max(this.current[i] - value, this.negMaxChange), this.maxChange); + const _target = this.current[i] - change; const temp = (this.currentVelocity[i] + this.omega * change) * deltaTime; @@ -60,44 +61,10 @@ export class SmoothDampFilter extends FilterInterface { this.currentVelocity[i] = (out - originalTo) / deltaTime; } this.current[i] = out; - output.push(out); - } + }); } this.lastTime = timestamp; - return output; + return this.current; } } -/* - -ublic static float SmoothDamp(float current, float target, ref float currentVelocity, float smoothTime, [uei.DefaultValue("Mathf.Infinity")] float maxSpeed, [uei.DefaultValue("Time.deltaTime")] float deltaTime) - { - // Based on Game Programming Gems 4 Chapter 1.10 - smoothTime = Mathf.Max(0.0001F, smoothTime); - float omega = 2F / smoothTime; - - float x = omega * deltaTime; - float exp = 1F / (1F + x + 0.48F * x * x + 0.235F * x * x * x); - float change = current - target; - float originalTo = target; - - // Clamp maximum speed - float maxChange = maxSpeed * smoothTime; - change = Mathf.Clamp(change, -maxChange, maxChange); - target = current - change; - - float temp = (currentVelocity + omega * change) * deltaTime; - currentVelocity = (currentVelocity - omega * temp) * exp; - float output = target + (change + temp) * exp; - - // Prevent overshooting - if (originalTo - current > 0.0F == output > originalTo) - { - output = originalTo; - currentVelocity = (output - originalTo) / deltaTime; - } - - return output; - } - -*/ \ No newline at end of file From f1a19ae4d875253aa751c68229284ea1103a9de7 Mon Sep 17 00:00:00 2001 From: jmschrack Date: Wed, 25 Jan 2023 20:43:28 -0600 Subject: [PATCH 3/9] Set SmoothDamp filter to be default filter --- src/image-target/controller.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/image-target/controller.js b/src/image-target/controller.js index 1feaef25..ea23081a 100644 --- a/src/image-target/controller.js +++ b/src/image-target/controller.js @@ -6,13 +6,14 @@ import { CropDetector } from './detector/crop-detector.js'; import { Compiler } from './compiler.js'; import { InputLoader } from './input-loader.js'; import { OneEuroFilter } from '../libs/one-euro-filter.js'; +import { SmoothDampFilter } from '../libs/smooth-damp-filter.js' const DEFAULT_FILTER_CUTOFF = 0.001; // 1Hz. time period in milliseconds const DEFAULT_FILTER_BETA = 1000; const DEFAULT_WARMUP_TOLERANCE = 5; const DEFAULT_MISS_TOLERANCE = 5; -const DEFAULT_FILTER={filter:OneEuroFilter,opts:{minCutOff: DEFAULT_FILTER_CUTOFF, beta: DEFAULT_FILTER_BETA}} +const DEFAULT_FILTER={filter:SmoothDampFilter} //,opts:{minCutOff: DEFAULT_FILTER_CUTOFF, beta: DEFAULT_FILTER_BETA} //old settings from One Euro Filter class Controller { constructor({ inputWidth, inputHeight, onUpdate = null, debugMode = false, maxTrack = 1, From 9c808fedce0c9d203a4cf33e9728c082bdc5df54 Mon Sep 17 00:00:00 2001 From: jmschrack Date: Wed, 25 Jan 2023 20:54:39 -0600 Subject: [PATCH 4/9] Updated Example three-filter --- examples/image-tracking/three-filter.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/image-tracking/three-filter.html b/examples/image-tracking/three-filter.html index ca7829b4..eb29964f 100644 --- a/examples/image-tracking/three-filter.html +++ b/examples/image-tracking/three-filter.html @@ -17,7 +17,7 @@