From e4716c998cbe06e53f31ef5b93d6c2573c72955d Mon Sep 17 00:00:00 2001 From: Gregg Tavares Date: Sat, 27 Apr 2024 08:59:54 +0200 Subject: [PATCH] DataEffect --- editor/index.js | 11 +- editor/visualizers/effects/DataEffect.js | 80 ++++++++++--- src/ByteBeatNode.js | 136 +++++++++++++++++------ 3 files changed, 176 insertions(+), 51 deletions(-) diff --git a/editor/index.js b/editor/index.js index b88731c..80bc69f 100644 --- a/editor/index.js +++ b/editor/index.js @@ -10,7 +10,7 @@ import WebGLVisualizer from './visualizers/WebGLVisualizer.js'; import CanvasVisualizer from './visualizers/CanvasVisualizer.js'; import NullVisualizer from './visualizers/NullVisualizer.js'; -//import DataEffect from './visualizers/effects/DataEffect.js'; +import DataEffect from './visualizers/effects/DataEffect.js'; import FFTEffect from './visualizers/effects/FFTEffect.js'; //import SampleEffect from './visualizers/effects/SampleEffect.js'; import VSAEffect from './visualizers/effects/VSAEffect.js'; @@ -252,7 +252,7 @@ async function main() { g_vsaVisualizer = new WebGLVisualizer(gl, [g_vsaEffect]); const effects = [ - //new DataEffect(gl), + new DataEffect(gl), // ...(showSample ? [new SampleEffect(gl)] : []), //new WaveEffect(gl), new FFTEffect(gl), @@ -599,7 +599,7 @@ async function setExpressions(expressions, resetToZero) { setURL(); } -function compile(text, resetToZero) { +async function compile(text, resetToZero) { const sections = splitBySections(text); if (sections.default || sections.channel1) { const expressions = [sections.default?.body || sections.channel1?.body]; @@ -609,7 +609,10 @@ function compile(text, resetToZero) { if (resetToZero) { g_visualizer.reset(); } - setExpressions(expressions, resetToZero); + await setExpressions(expressions, resetToZero); + if (resetToZero) { + g_visualizer.reset(); + } } if (sections.vsa) { g_vsaEffect.setURL(sections.vsa.argString); diff --git a/editor/visualizers/effects/DataEffect.js b/editor/visualizers/effects/DataEffect.js index 4b3473e..7c3ef0c 100644 --- a/editor/visualizers/effects/DataEffect.js +++ b/editor/visualizers/effects/DataEffect.js @@ -1,10 +1,11 @@ import * as twgl from '../../../js/twgl-full.module.js'; -import ByteBeatNode from '../../../src/ByteBeatNode.js'; import { drawEffect } from './effect-utils.js'; const colorBlue = new Float32Array([0, 0, 1, 1]); const colorGray = new Float32Array([0.25, 0.25, 0.25, 1]); +const kChunkSize = 1024; + export default class DataEffect { constructor(gl) { this.programInfo = twgl.createProgramInfo(gl, [ @@ -52,24 +53,17 @@ export default class DataEffect { wrap: gl.CLAMP_TO_EDGE, }), ]; + this.dataCursor = 0; + this.data = []; } reset(gl) { - this.dataTime = 0; - this.dataPos = 0; for (let i = 0; i < this.dataWidth; ++i) { this.dataBuf[i] = 0; } - for (const tex of this.dataTex) { - gl.bindTexture(gl.TEXTURE_2D, tex); - gl.texImage2D( - gl.TEXTURE_2D, 0, gl.LUMINANCE, this.dataWidth, 1, 0, - gl.LUMINANCE, gl.UNSIGNED_BYTE, this.dataBuf); - } + this.resize(gl); } - resize(gl) { - this.dataContext = ByteBeatNode.createContext(); - this.dataStack = ByteBeatNode.createStack(); + async resize(gl) { this.dataWidth = gl.drawingBufferWidth; const dataBuf = new Uint8Array(this.dataWidth); this.dataPos = 0; @@ -82,15 +76,74 @@ export default class DataEffect { } this.dataBuf = dataBuf; this.dataTime = 0; + this.oldDataTime = 0; + this.data = new Map(); + this.state = 'init'; + } + async #getData(byteBeat) { + this.updating = true; + const start = Math.ceil(this.dataTime / kChunkSize) * kChunkSize; + const numChannels = byteBeat.getNumChannels(); + const dataP = []; + for (let channel = 0; channel < numChannels; ++channel) { + dataP.push(byteBeat.getSamplesForTimeRange(start, start + kChunkSize, 1, this.dataContext, this.dataStack, channel)); + } + const data = await Promise.all(dataP); + const chunkId = start / kChunkSize; + this.data.set(chunkId, data); + this.updating = false; } + + #update(byteBeat) { + const noData = this.data.length === 0; + const passingHalfWayPoint = (this.oldDataTime % kChunkSize) < kChunkSize / 2 && (this.dataTime % kChunkSize) >= kChunkSize / 2; + const passingChunk = (this.oldDataTime % kChunkSize) === kChunkSize - 1 && this.dataTime % kChunkSize === 0; + const oldChunkId = this.oldDataTime / kChunkSize | 0; + this.oldDataTime = this.dataTime; + if (passingChunk) { + this.data.delete(oldChunkId); + } + if (!this.updating && (noData || passingHalfWayPoint)) { + this.#getData(byteBeat); + } + } + + async #init(byteBeat) { + if (this.dataContext) { + byteBeat.destroyContext(this.dataContext); + byteBeat.destroyStack(this.dataStack); + } + this.dataContext = await byteBeat.createContext(); + this.dataStack = await byteBeat.createStack(); + await this.#getData(byteBeat); + this.state = 'running'; + } + render(gl, commonUniforms, byteBeat) { + if (this.state === 'init') { + this.state = 'initializing'; + this.#init(byteBeat); + } + if (this.state !== 'running') { + return; + } + this.#update(byteBeat); const numChannels = byteBeat.getNumChannels(); const {uniforms, programInfo, bufferInfo} = this; + const chunkId = this.dataTime / kChunkSize | 0; + const chunk = this.data.get(chunkId); + const ndx = this.dataTime % kChunkSize; for (let channel = 0; channel < numChannels; ++channel) { - this.dataPixel[0] = Math.round(byteBeat.getSampleForTime(this.dataTime++, this.dataContext, this.dataStack, channel) * 127) + 127; + try { + const ch = chunk[channel]; + const sample = ch[ndx]; + this.dataPixel[0] = Math.round(sample * 127) + 127; + } catch { + // + } gl.bindTexture(gl.TEXTURE_2D, this.dataTex[channel]); gl.texSubImage2D(gl.TEXTURE_2D, 0, this.dataPos, 0, 1, 1, gl.LUMINANCE, gl.UNSIGNED_BYTE, this.dataPixel); this.dataPos = (this.dataPos + 1) % this.dataWidth; @@ -107,5 +160,6 @@ export default class DataEffect { gl.disable(gl.BLEND); } } + ++this.dataTime; } } \ No newline at end of file diff --git a/src/ByteBeatNode.js b/src/ByteBeatNode.js index def76cd..25ea232 100644 --- a/src/ByteBeatNode.js +++ b/src/ByteBeatNode.js @@ -33,6 +33,18 @@ class BeatWorkletProcessor extends AudioWorkletProcessor { }; this.expressions = []; this.functions = []; + this.nextObjId = 1; + this.idToObj = new Map(); + } + + #registerObj(obj) { + const id = this.nextObjId++; + this.idToObj.set(id, obj); + return id; + } + + #deregisterObj(id) { + this.idToObj.delete(id); } // TODO: replace @@ -47,8 +59,12 @@ class BeatWorkletProcessor extends AudioWorkletProcessor { callAsync({fn, msgId, args}) { let result; let error; + const transferables = []; try { result = this[fn].call(this, ...args); + if (result instanceof Float32Array) { + //transferables.push(result); + } } catch (e) { error = e; } @@ -59,7 +75,7 @@ class BeatWorkletProcessor extends AudioWorkletProcessor { error, result, }, - }); + }, transferables); } setExpressions(expressions, resetToZero) { @@ -116,6 +132,7 @@ class BeatWorkletProcessor extends AudioWorkletProcessor { } return { numChannels: this.byteBeat.getNumChannels(), + expressions: exp, }; } @@ -135,6 +152,31 @@ class BeatWorkletProcessor extends AudioWorkletProcessor { //} return true; } + + createStack() { + return this.#registerObj(new WrappingStack()); + } + createContext() { + return this.#registerObj(ByteBeatCompiler.makeContext()); + } + destroyStack(id) { + this.#deregisterObj(id); + } + destroyContext(id) { + this.#deregisterObj(id); + } + + getSamplesForTimeRange(start, end, step, contextId, stackId, channel = 0) { + const context = this.idToObj.get(contextId); + const stack = this.idToObj.get(stackId); + const len = Math.ceil((end - start) / step); + const data = new Float32Array(len); + let cursor = 0; + for (let time = start; time < end; time += step) { + data[cursor++] = this.byteBeat.getSampleForTime(time, context, stack, channel); + } + return data; + } } registerProcessor('bytebeat-processor', BeatWorkletProcessor); @@ -165,19 +207,19 @@ export default class ByteBeatNode extends AudioWorkletNode { static async setup(context) { return await context.audioWorklet.addModule(workerURL); } - static createStack() { - return new WrappingStack(); - } - static createContext() { - return ByteBeatCompiler.makeContext(); - } + #startTime = 0; // time since the song started playing + #pauseTime = 0; // time since the song was paused + #connected = false; + #expressionType = 0; + #expressions = []; #msgIdToResolveMap = new Map(); #nextId = 0; #type; #numChannels = 1; #desiredSampleRate; #actualSampleRate; + #busyPromise; constructor(context) { super(context, 'bytebeat-processor', { outputChannelCount: [2] }); @@ -209,15 +251,9 @@ export default class ByteBeatNode extends AudioWorkletNode { }, false); } } - - // This is the previous expressions so we don't double compile - this.expressions = []; - - this.extra = ByteBeatCompiler.makeExtra(); - this.time = 0; - this.startTime = performance.now(); // time since the song started playing - this.pauseTime = this.startTime; // time since the song was paused - this.connected = false; // whether or not we're playing the bytebeat + this.#startTime = performance.now(); // time since the song started playing + this.#pauseTime = this.#startTime; // time since the song was paused + this.#connected = false; // whether or not we're playing the bytebeat this.#actualSampleRate = context.sampleRate; this.#callFunc('setActualSampleRate', context.sampleRate); @@ -282,17 +318,17 @@ export default class ByteBeatNode extends AudioWorkletNode { connect(dest) { super.connect(dest); - if (!this.connected) { - this.connected = true; - const elapsedPauseTime = performance.now() - this.pauseTime; - this.startTime += elapsedPauseTime; + if (!this.#connected) { + this.#connected = true; + const elapsedPauseTime = performance.now() - this.#pauseTime; + this.#startTime += elapsedPauseTime; } } disconnect() { - if (this.connected) { - this.connected = false; - this.pauseTime = performance.now(); + if (this.#connected) { + this.#connected = false; + this.#pauseTime = performance.now(); super.disconnect(); } } @@ -304,23 +340,34 @@ export default class ByteBeatNode extends AudioWorkletNode { reset() { this.#callFunc('reset'); - this.startTime = performance.now(); - this.pauseTime = this.startTime; + this.#startTime = performance.now(); + this.#pauseTime = this.#startTime; } isRunning() { - return this.connected; + return this.#connected; } getTime() { - const time = this.connected ? performance.now() : this.pauseTime; - return (time - this.startTime) * 0.001 * this.getDesiredSampleRate() | 0; + const time = this.#connected ? performance.now() : this.#pauseTime; + return (time - this.#startTime) * 0.001 * this.getDesiredSampleRate() | 0; } async setExpressions(expressions, resetToZero) { - const data = await this.#callAsync('setExpressions', expressions, resetToZero); - this.#numChannels = data.numChannels; - return; + if (this.#busyPromise) { + await this.#busyPromise; + } + let resolve; + this.#busyPromise = new Promise(r => { + resolve = r; + }); + try { + const data = await this.#callAsync('setExpressions', expressions, resetToZero); + this.#numChannels = data.numChannels; + this.#expressions = data.expressions; + } finally { + resolve(); + } } convertToDesiredSampleRate(rate) { @@ -337,16 +384,16 @@ export default class ByteBeatNode extends AudioWorkletNode { } setExpressionType(type) { - this.expressionType = type; + this.#expressionType = type; this.#callFunc('setExpressionType', type); } getExpressions() { - return this.expressions.slice(); + return this.#expressions.slice(); } getExpressionType() { - return this.expressionType; + return this.#expressionType; } setType(type) { @@ -361,4 +408,25 @@ export default class ByteBeatNode extends AudioWorkletNode { getNumChannels() { return this.#numChannels; } + + async createStack() { + return await this.#callAsync('createStack'); + } + async createContext() { + return await this.#callAsync('createContext'); + } + + destroyStack(id) { + return this.#callAsync('destroyStack', id); + } + async destroyContext(id) { + return await this.#callAsync('destroyContext', id); + } + + async getSamplesForTimeRange(start, end, step, contextId, stackId, channel) { + if (this.#busyPromise) { + await this.#busyPromise; + } + return await this.#callAsync('getSamplesForTimeRange', start, end, step, contextId, stackId, channel); + } }