diff --git a/public/assets/gltf/whale.glb b/public/assets/gltf/whale.glb new file mode 100644 index 00000000..4d361020 Binary files /dev/null and b/public/assets/gltf/whale.glb differ diff --git a/src/pages/samples/[slug].tsx b/src/pages/samples/[slug].tsx index 079d501d..84831874 100644 --- a/src/pages/samples/[slug].tsx +++ b/src/pages/samples/[slug].tsx @@ -49,6 +49,7 @@ export const pages: PageComponentType = { 'A-buffer': dynamic(() => import('../../sample/a-buffer/main')), bitonicSort: dynamic(() => import('../../sample/bitonicSort/main')), normalMap: dynamic(() => import('../../sample/normalMap/main')), + skinnedMesh: dynamic(() => import('../../sample/skinnedMesh/main')), }; function Page({ slug }: Props): JSX.Element { diff --git a/src/sample/skinnedMesh/glbUtils.ts b/src/sample/skinnedMesh/glbUtils.ts new file mode 100644 index 00000000..96d4cdec --- /dev/null +++ b/src/sample/skinnedMesh/glbUtils.ts @@ -0,0 +1,1017 @@ +import { Quat } from 'wgpu-matrix/dist/2.x/quat'; +import { Accessor, BufferView, GlTf, Scene } from './gltf'; +import { Mat4, Vec3, mat4 } from 'wgpu-matrix'; + +//NOTE: GLTF code is not generally extensible to all gltf models +// Modified from Will Usher code found at this link https://www.willusher.io/graphics/2023/05/16/0-to-gltf-first-mesh + +// Associates the mode paramete of a gltf primitive object with the primitive's intended render mode +enum GLTFRenderMode { + POINTS = 0, + LINE = 1, + LINE_LOOP = 2, + LINE_STRIP = 3, + TRIANGLES = 4, + TRIANGLE_STRIP = 5, + TRIANGLE_FAN = 6, +} + +// Determines how to interpret each element of the structure that is accessed from our accessor +enum GLTFDataComponentType { + BYTE = 5120, + UNSIGNED_BYTE = 5121, + SHORT = 5122, + UNSIGNED_SHORT = 5123, + INT = 5124, + UNSIGNED_INT = 5125, + FLOAT = 5126, + DOUBLE = 5130, +} + +// Determines how to interpret the structure of the values accessed by an accessor +enum GLTFDataStructureType { + SCALAR = 0, + VEC2 = 1, + VEC3 = 2, + VEC4 = 3, + MAT2 = 4, + MAT3 = 5, + MAT4 = 6, +} + +export const alignTo = (val: number, align: number): number => { + return Math.floor((val + align - 1) / align) * align; +}; + +const parseGltfDataStructureType = (type: string) => { + switch (type) { + case 'SCALAR': + return GLTFDataStructureType.SCALAR; + case 'VEC2': + return GLTFDataStructureType.VEC2; + case 'VEC3': + return GLTFDataStructureType.VEC3; + case 'VEC4': + return GLTFDataStructureType.VEC4; + case 'MAT2': + return GLTFDataStructureType.MAT2; + case 'MAT3': + return GLTFDataStructureType.MAT3; + case 'MAT4': + return GLTFDataStructureType.MAT4; + default: + throw Error(`Unhandled glTF Type ${type}`); + } +}; + +const gltfDataStructureTypeNumComponents = (type: GLTFDataStructureType) => { + switch (type) { + case GLTFDataStructureType.SCALAR: + return 1; + case GLTFDataStructureType.VEC2: + return 2; + case GLTFDataStructureType.VEC3: + return 3; + case GLTFDataStructureType.VEC4: + case GLTFDataStructureType.MAT2: + return 4; + case GLTFDataStructureType.MAT3: + return 9; + case GLTFDataStructureType.MAT4: + return 16; + default: + throw Error(`Invalid glTF Type ${type}`); + } +}; + +// Note: only returns non-normalized type names, +// so byte/ubyte = sint8/uint8, not snorm8/unorm8, same for ushort +const gltfVertexType = ( + componentType: GLTFDataComponentType, + type: GLTFDataStructureType +) => { + let typeStr = null; + switch (componentType) { + case GLTFDataComponentType.BYTE: + typeStr = 'sint8'; + break; + case GLTFDataComponentType.UNSIGNED_BYTE: + typeStr = 'uint8'; + break; + case GLTFDataComponentType.SHORT: + typeStr = 'sint16'; + break; + case GLTFDataComponentType.UNSIGNED_SHORT: + typeStr = 'uint16'; + break; + case GLTFDataComponentType.INT: + typeStr = 'int32'; + break; + case GLTFDataComponentType.UNSIGNED_INT: + typeStr = 'uint32'; + break; + case GLTFDataComponentType.FLOAT: + typeStr = 'float32'; + break; + default: + throw Error(`Unrecognized or unsupported glTF type ${componentType}`); + } + + switch (gltfDataStructureTypeNumComponents(type)) { + case 1: + return typeStr; + case 2: + return typeStr + 'x2'; + case 3: + return typeStr + 'x3'; + case 4: + return typeStr + 'x4'; + // Vertex attributes should never be a matrix type, so we should not hit this + // unless we're passed an improperly created gltf file + default: + throw Error(`Invalid number of components for gltfType: ${type}`); + } +}; + +const gltfElementSize = ( + componentType: GLTFDataComponentType, + type: GLTFDataStructureType +) => { + let componentSize = 0; + switch (componentType) { + case GLTFDataComponentType.BYTE: + componentSize = 1; + break; + case GLTFDataComponentType.UNSIGNED_BYTE: + componentSize = 1; + break; + case GLTFDataComponentType.SHORT: + componentSize = 2; + break; + case GLTFDataComponentType.UNSIGNED_SHORT: + componentSize = 2; + break; + case GLTFDataComponentType.INT: + componentSize = 4; + break; + case GLTFDataComponentType.UNSIGNED_INT: + componentSize = 4; + break; + case GLTFDataComponentType.FLOAT: + componentSize = 4; + break; + case GLTFDataComponentType.DOUBLE: + componentSize = 8; + break; + default: + throw Error('Unrecognized GLTF Component Type?'); + } + return gltfDataStructureTypeNumComponents(type) * componentSize; +}; + +// Convert differently depending on if the shader is a vertex or compute shader +const convertGPUVertexFormatToWGSLFormat = (vertexFormat: GPUVertexFormat) => { + switch (vertexFormat) { + case 'float32': { + return 'f32'; + } + case 'float32x2': { + return 'vec2'; + } + case 'float32x3': { + return 'vec3'; + } + case 'float32x4': { + return 'vec4'; + } + case 'uint32': { + return 'u32'; + } + case 'uint32x2': { + return 'vec2'; + } + case 'uint32x3': { + return 'vec3'; + } + case 'uint32x4': { + return 'vec4'; + } + case 'uint8x2': { + return 'vec2'; + } + case 'uint8x4': { + return 'vec4'; + } + case 'uint16x4': { + return 'vec4'; + } + case 'uint16x2': { + return 'vec2'; + } + default: { + return 'f32'; + } + } +}; + +export class GLTFBuffer { + buffer: Uint8Array; + constructor(buffer: ArrayBuffer, offset: number, size: number) { + this.buffer = new Uint8Array(buffer, offset, size); + } +} + +export class GLTFBufferView { + byteLength: number; + byteStride: number; + view: Uint8Array; + needsUpload: boolean; + gpuBuffer: GPUBuffer; + usage: number; + constructor(buffer: GLTFBuffer, view: BufferView) { + this.byteLength = view['byteLength']; + this.byteStride = 0; + if (view['byteStride'] !== undefined) { + this.byteStride = view['byteStride']; + } + // Create the buffer view. Note that subarray creates a new typed + // view over the same array buffer, we do not make a copy here. + let viewOffset = 0; + if (view['byteOffset'] !== undefined) { + viewOffset = view['byteOffset']; + } + // NOTE: This creates a uint8array view into the buffer! + // When we call .buffer on this view, it will give us back the original array buffer + // Accordingly, when converting our buffer from a uint8array to a float32array representation + // we need to apply the byte offset of our view when creating our buffer + // ie new Float32Array(this.view.buffer, this.view.byteOffset, this.view.byteLength) + this.view = buffer.buffer.subarray( + viewOffset, + viewOffset + this.byteLength + ); + + this.needsUpload = false; + this.gpuBuffer = null; + this.usage = 0; + } + + addUsage(usage: number) { + this.usage = this.usage | usage; + } + + upload(device: GPUDevice) { + // Note: must align to 4 byte size when mapped at creation is true + const buf: GPUBuffer = device.createBuffer({ + size: alignTo(this.view.byteLength, 4), + usage: this.usage, + mappedAtCreation: true, + }); + new Uint8Array(buf.getMappedRange()).set(this.view); + buf.unmap(); + this.gpuBuffer = buf; + this.needsUpload = false; + } +} + +export class GLTFAccessor { + count: number; + componentType: GLTFDataComponentType; + structureType: GLTFDataStructureType; + view: GLTFBufferView; + byteOffset: number; + constructor(view: GLTFBufferView, accessor: Accessor) { + this.count = accessor['count']; + this.componentType = accessor['componentType']; + this.structureType = parseGltfDataStructureType(accessor['type']); + this.view = view; + this.byteOffset = 0; + if (accessor['byteOffset'] !== undefined) { + this.byteOffset = accessor['byteOffset']; + } + } + + get byteStride() { + const elementSize = gltfElementSize(this.componentType, this.structureType); + return Math.max(elementSize, this.view.byteStride); + } + + get byteLength() { + return this.count * this.byteStride; + } + + // Get the vertex attribute type for accessors that are used as vertex attributes + get vertexType() { + return gltfVertexType(this.componentType, this.structureType); + } +} + +interface AttributeMapInterface { + [key: string]: GLTFAccessor; +} + +export class GLTFPrimitive { + topology: GLTFRenderMode; + renderPipeline: GPURenderPipeline; + private attributeMap: AttributeMapInterface; + private attributes: string[] = []; + constructor( + topology: GLTFRenderMode, + attributeMap: AttributeMapInterface, + attributes: string[] + ) { + this.topology = topology; + this.renderPipeline = null; + // Maps attribute names to accessors + this.attributeMap = attributeMap; + this.attributes = attributes; + + for (const key in this.attributeMap) { + this.attributeMap[key].view.needsUpload = true; + if (key === 'INDICES') { + this.attributeMap['INDICES'].view.addUsage(GPUBufferUsage.INDEX); + continue; + } + this.attributeMap[key].view.addUsage(GPUBufferUsage.VERTEX); + } + } + + buildRenderPipeline( + device: GPUDevice, + vertexShader: string, + fragmentShader: string, + colorFormat: GPUTextureFormat, + depthFormat: GPUTextureFormat, + bgLayouts: GPUBindGroupLayout[], + label: string + ) { + // For now, just check if the attributeMap contains a given attribute using map.has(), and add it if it does + // POSITION, NORMAL, TEXCOORD_0, JOINTS_0, WEIGHTS_0 for order + // Vertex attribute state and shader stage + let VertexInputShaderString = `struct VertexInput {\n`; + const vertexBuffers: GPUVertexBufferLayout[] = this.attributes.map( + (attr, idx) => { + const vertexFormat: GPUVertexFormat = + this.attributeMap[attr].vertexType; + const attrString = attr.toLowerCase().replace(/_0$/, ''); + VertexInputShaderString += `\t@location(${idx}) ${attrString}: ${convertGPUVertexFormatToWGSLFormat( + vertexFormat + )},\n`; + return { + arrayStride: this.attributeMap[attr].byteStride, + attributes: [ + { + format: this.attributeMap[attr].vertexType, + offset: this.attributeMap[attr].byteOffset, + shaderLocation: idx, + }, + ], + } as GPUVertexBufferLayout; + } + ); + VertexInputShaderString += '}'; + + const vertexState: GPUVertexState = { + // Shader stage info + module: device.createShaderModule({ + code: VertexInputShaderString + vertexShader, + }), + entryPoint: 'vertexMain', + buffers: vertexBuffers, + }; + + const fragmentState: GPUFragmentState = { + // Shader info + module: device.createShaderModule({ + code: VertexInputShaderString + fragmentShader, + }), + entryPoint: 'fragmentMain', + // Output render target info + targets: [{ format: colorFormat }], + }; + + // Our loader only supports triangle lists and strips, so by default we set + // the primitive topology to triangle list, and check if it's instead a triangle strip + const primitive: GPUPrimitiveState = { topology: 'triangle-list' }; + if (this.topology == GLTFRenderMode.TRIANGLE_STRIP) { + primitive.topology = 'triangle-strip'; + primitive.stripIndexFormat = this.attributeMap['INDICES'].vertexType; + } + + const layout: GPUPipelineLayout = device.createPipelineLayout({ + bindGroupLayouts: bgLayouts, + label: `${label}.pipelineLayout`, + }); + + const rpDescript: GPURenderPipelineDescriptor = { + layout: layout, + label: `${label}.pipeline`, + vertex: vertexState, + fragment: fragmentState, + primitive: primitive, + depthStencil: { + format: depthFormat, + depthWriteEnabled: true, + depthCompare: 'less', + }, + }; + + this.renderPipeline = device.createRenderPipeline(rpDescript); + } + + render(renderPassEncoder: GPURenderPassEncoder, bindGroups: GPUBindGroup[]) { + renderPassEncoder.setPipeline(this.renderPipeline); + bindGroups.forEach((bg, idx) => { + renderPassEncoder.setBindGroup(idx, bg); + }); + + //if skin do something with bone bind group + this.attributes.map((attr, idx) => { + renderPassEncoder.setVertexBuffer( + idx, + this.attributeMap[attr].view.gpuBuffer, + this.attributeMap[attr].byteOffset, + this.attributeMap[attr].byteLength + ); + }); + + if (this.attributeMap['INDICES']) { + renderPassEncoder.setIndexBuffer( + this.attributeMap['INDICES'].view.gpuBuffer, + this.attributeMap['INDICES'].vertexType, + this.attributeMap['INDICES'].byteOffset, + this.attributeMap['INDICES'].byteLength + ); + renderPassEncoder.drawIndexed(this.attributeMap['INDICES'].count); + } else { + renderPassEncoder.draw(this.attributeMap['POSITION'].count); + } + } +} + +export class GLTFMesh { + name: string; + primitives: GLTFPrimitive[]; + constructor(name: string, primitives: GLTFPrimitive[]) { + this.name = name; + this.primitives = primitives; + } + + buildRenderPipeline( + device: GPUDevice, + vertexShader: string, + fragmentShader: string, + colorFormat: GPUTextureFormat, + depthFormat: GPUTextureFormat, + bgLayouts: GPUBindGroupLayout[] + ) { + // We take a pretty simple approach to start. Just loop through all the primitives and + // build their respective render pipelines + for (let i = 0; i < this.primitives.length; ++i) { + this.primitives[i].buildRenderPipeline( + device, + vertexShader, + fragmentShader, + colorFormat, + depthFormat, + bgLayouts, + `PrimitivePipeline${i}` + ); + } + } + + render(renderPassEncoder: GPURenderPassEncoder, bindGroups: GPUBindGroup[]) { + // We take a pretty simple approach to start. Just loop through all the primitives and + // call their individual draw methods + for (let i = 0; i < this.primitives.length; ++i) { + this.primitives[i].render(renderPassEncoder, bindGroups); + } + } +} + +export const validateGLBHeader = (header: DataView) => { + if (header.getUint32(0, true) != 0x46546c67) { + throw Error('Provided file is not a glB file'); + } + if (header.getUint32(4, true) != 2) { + throw Error('Provided file is glTF 2.0 file'); + } +}; + +export const validateBinaryHeader = (header: Uint32Array) => { + if (header[1] != 0x004e4942) { + throw Error( + 'Invalid glB: The second chunk of the glB file is not a binary chunk!' + ); + } +}; + +type TempReturn = { + meshes: GLTFMesh[]; + nodes: GLTFNode[]; + scenes: GLTFScene[]; + skins: GLTFSkin[]; +}; + +export class BaseTransformation { + position: Vec3; + rotation: Quat; + scale: Vec3; + constructor( + // Identity translation vec3 + position = [0, 0, 0], + // Identity quaternion + rotation = [0, 0, 0, 1], + // Identity scale vec3 + scale = [1, 1, 1] + ) { + this.position = position; + this.rotation = rotation; + this.scale = scale; + } + getMatrix(): Mat4 { + // Analagous to let transformationMatrix: mat4x4f = translation * rotation * scale; + const dst = mat4.identity(); + // Scale the transformation Matrix + mat4.scale(dst, this.scale, dst); + // Calculate the rotationMatrix from the quaternion + const rotationMatrix = mat4.fromQuat(this.rotation); + // Apply the rotation Matrix to the scaleMatrix (rotMat * scaleMat) + mat4.multiply(rotationMatrix, dst, dst); + // Translate the transformationMatrix + mat4.translate(dst, this.position, dst); + return dst; + } +} + +export class GLTFNode { + name: string; + source: BaseTransformation; + parent: GLTFNode | null; + children: GLTFNode[]; + // Transforms all node's children in the node's local space, with node itself acting as the origin + localMatrix: Mat4; + worldMatrix: Mat4; + // List of Meshes associated with this node + drawables: GLTFMesh[]; + test = 0; + skin?: GLTFSkin; + private nodeTransformGPUBuffer: GPUBuffer; + private nodeTransformBindGroup: GPUBindGroup; + + constructor( + device: GPUDevice, + bgLayout: GPUBindGroupLayout, + source: BaseTransformation, + name?: string, + skin?: GLTFSkin + ) { + this.name = name + ? name + : `node_${source.position} ${source.rotation} ${source.scale}`; + this.source = source; + this.parent = null; + this.children = []; + this.localMatrix = mat4.identity(); + this.worldMatrix = mat4.identity(); + this.drawables = []; + this.nodeTransformGPUBuffer = device.createBuffer({ + size: Float32Array.BYTES_PER_ELEMENT * 16, + usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, + }); + this.nodeTransformBindGroup = device.createBindGroup({ + layout: bgLayout, + entries: [ + { + binding: 0, + resource: { + buffer: this.nodeTransformGPUBuffer, + }, + }, + ], + }); + this.skin = skin; + } + + setParent(parent: GLTFNode) { + if (this.parent) { + this.parent.removeChild(this); + this.parent = null; + } + parent.addChild(this); + this.parent = parent; + } + + updateWorldMatrix(device: GPUDevice, parentWorldMatrix?: Mat4) { + // Get local transform of this particular node, and if the node has a parent, + // multiply it against the parent's transform matrix to get transformMatrix relative to world. + this.localMatrix = this.source.getMatrix(); + if (parentWorldMatrix) { + mat4.multiply(parentWorldMatrix, this.localMatrix, this.worldMatrix); + } else { + mat4.copy(this.localMatrix, this.worldMatrix); + } + const worldMatrix = this.worldMatrix as Float32Array; + device.queue.writeBuffer( + this.nodeTransformGPUBuffer, + 0, + worldMatrix.buffer, + worldMatrix.byteOffset, + worldMatrix.byteLength + ); + for (const child of this.children) { + child.updateWorldMatrix(device, worldMatrix); + } + } + + traverse(fn: (n: GLTFNode, ...args) => void) { + fn(this); + for (const child of this.children) { + child.traverse(fn); + } + } + + renderDrawables( + passEncoder: GPURenderPassEncoder, + bindGroups: GPUBindGroup[] + ) { + if (this.drawables !== undefined) { + for (const drawable of this.drawables) { + if (this.skin) { + drawable.render(passEncoder, [ + ...bindGroups, + this.nodeTransformBindGroup, + this.skin.skinBindGroup, + ]); + } else { + drawable.render(passEncoder, [ + ...bindGroups, + this.nodeTransformBindGroup, + ]); + } + } + } + // Render any of its children + for (const child of this.children) { + child.renderDrawables(passEncoder, bindGroups); + } + } + + private addChild(child: GLTFNode) { + this.children.push(child); + } + + private removeChild(child: GLTFNode) { + const ndx = this.children.indexOf(child); + this.children.splice(ndx, 1); + } +} + +export class GLTFScene { + nodes?: number[]; + name?: any; + extensions?: any; + extras?: any; + [k: string]: any; + root: GLTFNode; + + constructor( + device: GPUDevice, + nodeTransformBGL: GPUBindGroupLayout, + baseScene: Scene + ) { + this.nodes = baseScene.nodes; + this.name = baseScene.name; + this.root = new GLTFNode( + device, + nodeTransformBGL, + new BaseTransformation(), + baseScene.name + ); + } +} + +export class GLTFSkin { + // Nodes of the skin's joints + // [5, 2, 3] means our joint info is at nodes 5, 2, and 3 + joints: number[]; + // Bind Group for this skin's uniform buffer + skinBindGroup: GPUBindGroup; + // Static bindGroupLayout shared across all skins + // In a larger shader with more properties, certain bind groups + // would likely have to be combined due to device limitations in the number of bind groups + // allowed within a shader + // Inverse bind matrices parsed from the accessor + private inverseBindMatrices: Float32Array; + private jointMatricesUniformBuffer: GPUBuffer; + private inverseBindMatricesUniformBuffer: GPUBuffer; + static skinBindGroupLayout: GPUBindGroupLayout; + + static createSharedBindGroupLayout(device: GPUDevice) { + this.skinBindGroupLayout = device.createBindGroupLayout({ + label: 'StaticGLTFSkin.bindGroupLayout', + entries: [ + // Holds the initial joint matrices buffer + { + binding: 0, + buffer: { + type: 'read-only-storage', + }, + visibility: GPUShaderStage.VERTEX, + }, + // Holds the inverse bind matrices buffer + { + binding: 1, + buffer: { + type: 'read-only-storage', + }, + visibility: GPUShaderStage.VERTEX, + }, + ], + }); + } + + // For the sake of simplicity and easier debugging, we're going to convert our skin gpu accessor to a + // float32array, which should be performant enough for this example since there is only one skin (again, this) + // is not a comprehensive gltf parser + constructor( + device: GPUDevice, + inverseBindMatricesAccessor: GLTFAccessor, + joints: number[] + ) { + if ( + inverseBindMatricesAccessor.componentType !== + GLTFDataComponentType.FLOAT || + inverseBindMatricesAccessor.byteStride !== 64 + ) { + throw Error( + `This skin's provided accessor does not access a mat4x4 matrix, or does not access the provided mat4x4 data correctly` + ); + } + // NOTE: Come back to this uint8array to float32array conversion in case it is incorrect + this.inverseBindMatrices = new Float32Array( + inverseBindMatricesAccessor.view.view.buffer, + inverseBindMatricesAccessor.view.view.byteOffset, + inverseBindMatricesAccessor.view.view.byteLength / 4 + ); + this.joints = joints; + const skinGPUBufferUsage: GPUBufferDescriptor = { + size: Float32Array.BYTES_PER_ELEMENT * 16 * joints.length, + usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST, + }; + this.jointMatricesUniformBuffer = device.createBuffer(skinGPUBufferUsage); + this.inverseBindMatricesUniformBuffer = + device.createBuffer(skinGPUBufferUsage); + device.queue.writeBuffer( + this.inverseBindMatricesUniformBuffer, + 0, + this.inverseBindMatrices + ); + this.skinBindGroup = device.createBindGroup({ + layout: GLTFSkin.skinBindGroupLayout, + label: 'StaticGLTFSkin.bindGroup', + entries: [ + { + binding: 0, + resource: { + buffer: this.jointMatricesUniformBuffer, + }, + }, + { + binding: 1, + resource: { + buffer: this.inverseBindMatricesUniformBuffer, + }, + }, + ], + }); + } + + update(device: GPUDevice, currentNodeIndex: number, nodes: GLTFNode[]) { + const globalWorldInverse = mat4.inverse( + nodes[currentNodeIndex].worldMatrix + ); + for (let j = 0; j < this.joints.length; j++) { + const joint = this.joints[j]; + const dstMatrix: Mat4 = mat4.identity(); + mat4.multiply(globalWorldInverse, nodes[joint].worldMatrix, dstMatrix); + const toWrite = dstMatrix as Float32Array; + device.queue.writeBuffer( + this.jointMatricesUniformBuffer, + j * 64, + toWrite.buffer, + toWrite.byteOffset, + toWrite.byteLength + ); + } + } +} + +// Upload a GLB model, parse its JSON and Binary components, and create the requisite GPU resources +// to render them. NOTE: Not extensible to all GLTF contexts at this point in time +export const convertGLBToJSONAndBinary = async ( + buffer: ArrayBuffer, + device: GPUDevice +): Promise => { + // Binary GLTF layout: https://cdn.willusher.io/webgpu-0-to-gltf/glb-layout.svg + const jsonHeader = new DataView(buffer, 0, 20); + validateGLBHeader(jsonHeader); + + // Length of the jsonChunk found at jsonHeader[12 - 15] + const jsonChunkLength = jsonHeader.getUint32(12, true); + + // Parse the JSON chunk of the glB file to a JSON object + const jsonChunk: GlTf = JSON.parse( + new TextDecoder('utf-8').decode(new Uint8Array(buffer, 20, jsonChunkLength)) + ); + + console.log(jsonChunk); + // Binary data located after jsonChunk + const binaryHeader = new Uint32Array(buffer, 20 + jsonChunkLength, 2); + validateBinaryHeader(binaryHeader); + + const binaryChunk = new GLTFBuffer( + buffer, + 28 + jsonChunkLength, + binaryHeader[0] + ); + + //Const populate missing properties of jsonChunk + for (const accessor of jsonChunk.accessors) { + accessor.byteOffset = accessor.byteOffset ?? 0; + accessor.normalized = accessor.normalized ?? false; + } + + for (const bufferView of jsonChunk.bufferViews) { + bufferView.byteOffset = bufferView.byteOffset ?? 0; + } + + if (jsonChunk.samplers) { + for (const sampler of jsonChunk.samplers) { + sampler.wrapS = sampler.wrapS ?? 10497; //GL.REPEAT + sampler.wrapT = sampler.wrapT ?? 10947; //GL.REPEAT + } + } + + //Mark each accessor with its intended usage within the vertexShader. + //Often necessary due to infrequencey with which the BufferView target field is populated. + for (const mesh of jsonChunk.meshes) { + for (const primitive of mesh.primitives) { + if ('indices' in primitive) { + const accessor = jsonChunk.accessors[primitive.indices]; + jsonChunk.accessors[primitive.indices].bufferViewUsage |= + GPUBufferUsage.INDEX; + jsonChunk.bufferViews[accessor.bufferView].usage |= + GPUBufferUsage.INDEX; + } + for (const attribute of Object.values(primitive.attributes)) { + const accessor = jsonChunk.accessors[attribute]; + jsonChunk.accessors[attribute].bufferViewUsage |= GPUBufferUsage.VERTEX; + jsonChunk.bufferViews[accessor.bufferView].usage |= + GPUBufferUsage.VERTEX; + } + } + } + + // Create GLTFBufferView objects for all the buffer views in the glTF file + const bufferViews: GLTFBufferView[] = []; + for (let i = 0; i < jsonChunk.bufferViews.length; ++i) { + bufferViews.push(new GLTFBufferView(binaryChunk, jsonChunk.bufferViews[i])); + } + + const accessors: GLTFAccessor[] = []; + for (let i = 0; i < jsonChunk.accessors.length; ++i) { + const accessorInfo = jsonChunk.accessors[i]; + const viewID = accessorInfo['bufferView']; + accessors.push(new GLTFAccessor(bufferViews[viewID], accessorInfo)); + } + // Load the first mesh + const meshes: GLTFMesh[] = []; + for (let i = 0; i < jsonChunk.meshes.length; i++) { + const mesh = jsonChunk.meshes[i]; + const meshPrimitives: GLTFPrimitive[] = []; + for (let j = 0; j < mesh.primitives.length; ++j) { + const prim = mesh.primitives[j]; + let topology = prim['mode']; + // Default is triangles if mode specified + if (topology === undefined) { + topology = GLTFRenderMode.TRIANGLES; + } + if ( + topology != GLTFRenderMode.TRIANGLES && + topology != GLTFRenderMode.TRIANGLE_STRIP + ) { + throw Error(`Unsupported primitive mode ${prim['mode']}`); + } + + const primitiveAttributeMap = {}; + const attributes = []; + if (jsonChunk['accessors'][prim['indices']] !== undefined) { + const indices = accessors[prim['indices']]; + primitiveAttributeMap['INDICES'] = indices; + } + + // Loop through all the attributes and store within our attributeMap + for (const attr in prim['attributes']) { + const accessor = accessors[prim['attributes'][attr]]; + primitiveAttributeMap[attr] = accessor; + if (accessor.structureType > 3) { + throw Error( + 'Vertex attribute accessor accessed an unsupported data type for vertex attribute' + ); + } + attributes.push(attr); + } + meshPrimitives.push( + new GLTFPrimitive(topology, primitiveAttributeMap, attributes) + ); + } + meshes.push(new GLTFMesh(mesh.name, meshPrimitives)); + } + + const skins: GLTFSkin[] = []; + for (const skin of jsonChunk.skins) { + const inverseBindMatrixAccessor = accessors[skin.inverseBindMatrices]; + inverseBindMatrixAccessor.view.addUsage( + GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST + ); + inverseBindMatrixAccessor.view.needsUpload = true; + } + + // Upload the buffer views used by mesh + for (let i = 0; i < bufferViews.length; ++i) { + if (bufferViews[i].needsUpload) { + bufferViews[i].upload(device); + } + } + + GLTFSkin.createSharedBindGroupLayout(device); + for (const skin of jsonChunk.skins) { + const inverseBindMatrixAccessor = accessors[skin.inverseBindMatrices]; + const joints = skin.joints; + skins.push(new GLTFSkin(device, inverseBindMatrixAccessor, joints)); + } + + const nodes: GLTFNode[] = []; + + // Access each node. If node references a mesh, add mesh to that node + const nodeUniformsBindGroupLayout = device.createBindGroupLayout({ + label: 'NodeUniforms.bindGroupLayout', + entries: [ + { + binding: 0, + buffer: { + type: 'uniform', + }, + visibility: GPUShaderStage.VERTEX, + }, + ], + }); + for (const currNode of jsonChunk.nodes) { + const baseTransformation = new BaseTransformation( + currNode.translation, + currNode.rotation, + currNode.scale + ); + const nodeToCreate = new GLTFNode( + device, + nodeUniformsBindGroupLayout, + baseTransformation, + currNode.name, + skins[currNode.skin] + ); + const meshToAdd = meshes[currNode.mesh]; + if (meshToAdd) { + nodeToCreate.drawables.push(meshToAdd); + } + nodes.push(nodeToCreate); + } + + // Assign each node its children + nodes.forEach((node, idx) => { + const children = jsonChunk.nodes[idx].children; + if (children) { + children.forEach((childIdx) => { + const child = nodes[childIdx]; + child.setParent(node); + }); + } + }); + + const scenes: GLTFScene[] = []; + + for (const jsonScene of jsonChunk.scenes) { + const scene = new GLTFScene(device, nodeUniformsBindGroupLayout, jsonScene); + const sceneChildren = scene.nodes; + sceneChildren.forEach((childIdx) => { + const child = nodes[childIdx]; + child.setParent(scene.root); + }); + scenes.push(scene); + } + return { + meshes, + nodes, + scenes, + skins, + }; +}; diff --git a/src/sample/skinnedMesh/gltf.ts b/src/sample/skinnedMesh/gltf.ts new file mode 100644 index 00000000..e5cd3d52 --- /dev/null +++ b/src/sample/skinnedMesh/gltf.ts @@ -0,0 +1,224 @@ +import { Mat4 } from 'wgpu-matrix'; +import { GLTFNode } from './glbUtils'; + +/* Sourced from https://github.com/bwasty/gltf-loader-ts/blob/master/source/gltf.ts */ +/* License for use can be found here: https://github.com/bwasty/gltf-loader-ts/blob/master/LICENSE */ +/* Comments and types have been excluded from original source for sake of cleanliness and brevity */ +export type GlTfId = number; + +export interface AccessorSparseIndices { + bufferView: GlTfId; + byteOffset?: number; + componentType: 5121 | 5123 | 5125 | number; +} + +export interface AccessorSparseValues { + bufferView: GlTfId; + byteOffset?: number; +} + +export interface AccessorSparse { + count: number; + indices: AccessorSparseIndices; + values: AccessorSparseValues; +} + +export interface Accessor { + bufferView?: GlTfId; + bufferViewUsage?: 34962 | 34963 | number; + byteOffset?: number; + componentType: 5120 | 5121 | 5122 | 5123 | 5125 | 5126 | number; + normalized?: boolean; + count: number; + type: 'SCALAR' | 'VEC2' | 'VEC3' | 'VEC4' | 'MAT2' | 'MAT3' | 'MAT4' | string; + max?: number[]; + min?: number[]; + sparse?: AccessorSparse; + name?: string; +} + +export interface AnimationChannelTarget { + node?: GlTfId; + path: 'translation' | 'rotation' | 'scale' | 'weights' | string; +} + +export interface AnimationChannel { + sampler: GlTfId; + target: AnimationChannelTarget; +} + +export interface AnimationSampler { + input: GlTfId; + interpolation?: 'LINEAR' | 'STEP' | 'CUBICSPLINE' | string; + output: GlTfId; +} + +export interface Animation { + channels: AnimationChannel[]; + samplers: AnimationSampler[]; + name?: string; +} + +export interface Asset { + copyright?: string; + generator?: string; + version: string; + minVersion?: string; +} + +export interface Buffer { + uri?: string; + byteLength: number; + name?: string; +} + +export interface BufferView { + buffer: GlTfId; + byteOffset?: number; + byteLength: number; + byteStride?: number; + target?: 34962 | 34963 | number; + name?: string; + usage?: number; +} + +export interface CameraOrthographic { + xmag: number; + ymag: number; + zfar: number; + znear: number; +} + +export interface CameraPerspective { + aspectRatio?: number; + yfov: number; + zfar?: number; + znear: number; +} + +export interface Camera { + orthographic?: CameraOrthographic; + perspective?: CameraPerspective; + type: 'perspective' | 'orthographic' | string; + name?: any; +} + +export interface Image { + uri?: string; + mimeType?: 'image/jpeg' | 'image/png' | string; + bufferView?: GlTfId; + name?: string; +} + +export interface TextureInfo { + index: GlTfId; + texCoord?: number; +} + +export interface MaterialPbrMetallicRoughness { + baseColorFactor?: number[]; + baseColorTexture?: TextureInfo; + metallicFactor?: number; + roughnessFactor?: number; + metallicRoughnessTexture?: TextureInfo; +} +export interface MaterialNormalTextureInfo { + index?: any; + texCoord?: any; + scale?: number; +} +export interface MaterialOcclusionTextureInfo { + index?: any; + texCoord?: any; + strength?: number; +} + +export interface Material { + name?: string; + pbrMetallicRoughness?: MaterialPbrMetallicRoughness; + normalTexture?: MaterialNormalTextureInfo; + occlusionTexture?: MaterialOcclusionTextureInfo; + emissiveTexture?: TextureInfo; + emissiveFactor?: number[]; + alphaMode?: 'OPAQUE' | 'MASK' | 'BLEND' | string; + alphaCutoff?: number; + doubleSided?: boolean; +} + +export interface MeshPrimitive { + attributes: { + [k: string]: GlTfId; + }; + indices?: GlTfId; + material?: GlTfId; + mode?: 0 | 1 | 2 | 3 | 4 | 5 | 6 | number; + targets?: { + [k: string]: GlTfId; + }[]; +} + +export interface Mesh { + primitives: MeshPrimitive[]; + weights?: number[]; + name?: string; +} + +export interface Node { + camera?: GlTfId; + children?: GlTfId[]; + skin?: GlTfId; + matrix?: number[]; + worldTransformationMatrix?: Mat4; + mesh?: GlTfId; + rotation?: number[]; + scale?: number[]; + translation?: number[]; + weights?: number[]; + name?: string; +} + +export interface Sampler { + magFilter?: 9728 | 9729 | number; + minFilter?: 9728 | 9729 | 9984 | 9985 | 9986 | 9987 | number; + wrapS?: 33071 | 33648 | 10497 | number; + wrapT?: 33071 | 33648 | 10497 | number; + name?: string; +} + +export interface Scene { + nodes?: GlTfId[]; + name?: any; + root?: GLTFNode; +} +export interface Skin { + inverseBindMatrices?: GlTfId; + skeleton?: GlTfId; + joints: GlTfId[]; + name?: string; +} + +export interface Texture { + sampler?: GlTfId; + source?: GlTfId; + name?: string; +} + +export interface GlTf { + extensionsUsed?: string[]; + extensionsRequired?: string[]; + accessors?: Accessor[]; + animations?: Animation[]; + asset: Asset; + buffers?: Buffer[]; + bufferViews?: BufferView[]; + cameras?: Camera[]; + images?: Image[]; + materials?: Material[]; + meshes?: Mesh[]; + nodes?: Node[]; + samplers?: Sampler[]; + scene?: GlTfId; + scenes?: Scene[]; + skins?: Skin[]; + textures?: Texture[]; +} diff --git a/src/sample/skinnedMesh/gltf.wgsl b/src/sample/skinnedMesh/gltf.wgsl new file mode 100644 index 00000000..0310a2a3 --- /dev/null +++ b/src/sample/skinnedMesh/gltf.wgsl @@ -0,0 +1,80 @@ +// Whale.glb Vertex attributes +// Read in VertexInput from attributes +// f32x3 f32x3 f32x2 u8x4 f32x4 +struct VertexOutput { + @builtin(position) Position: vec4, + @location(0) normal: vec3, + @location(1) joints: vec4, + @location(2) weights: vec4, +} + +struct CameraUniforms { + proj_matrix: mat4x4f, + view_matrix: mat4x4f, + model_matrix: mat4x4f, +} + +struct GeneralUniforms { + render_mode: u32, + skin_mode: u32, +} + +struct NodeUniforms { + world_matrix: mat4x4f, +} + +@group(0) @binding(0) var camera_uniforms: CameraUniforms; +@group(1) @binding(0) var general_uniforms: GeneralUniforms; +@group(2) @binding(0) var node_uniforms: NodeUniforms; +@group(3) @binding(0) var joint_matrices: array>; +@group(3) @binding(1) var inverse_bind_matrices: array>; + +@vertex +fn vertexMain(input: VertexInput) -> VertexOutput { + var output: VertexOutput; + // Compute joint_matrices * inverse_bind_matrices + let joint0 = joint_matrices[input.joints[0]] * inverse_bind_matrices[input.joints[0]]; + let joint1 = joint_matrices[input.joints[1]] * inverse_bind_matrices[input.joints[1]]; + let joint2 = joint_matrices[input.joints[2]] * inverse_bind_matrices[input.joints[2]]; + let joint3 = joint_matrices[input.joints[3]] * inverse_bind_matrices[input.joints[3]]; + // Compute influence of joint based on weight + let skin_matrix = + joint0 * input.weights[0] + + joint1 * input.weights[1] + + joint2 * input.weights[2] + + joint3 * input.weights[3]; + // Position of the vertex relative to our world + let world_position = vec4(input.position.x, input.position.y, input.position.z, 1.0); + // Vertex position with model rotation, skinning, and the mesh's node transformation applied. + let skinned_position = camera_uniforms.model_matrix * skin_matrix * node_uniforms.world_matrix * world_position; + // Vertex position with only the model rotation applied. + let rotated_position = camera_uniforms.model_matrix * world_position; + // Determine which position to used based on whether skinMode is turnd on or off. + let transformed_position = select( + rotated_position, + skinned_position, + general_uniforms.skin_mode == 0 + ); + // Apply the camera and projection matrix transformations to our transformed position; + output.Position = camera_uniforms.proj_matrix * camera_uniforms.view_matrix * transformed_position; + output.normal = input.normal; + // Convert u32 joint data to f32s to prevent flat interpolation error. + output.joints = vec4(f32(input.joints[0]), f32(input.joints[1]), f32(input.joints[2]), f32(input.joints[3])); + output.weights = input.weights; + return output; +} + +@fragment +fn fragmentMain(input: VertexOutput) -> @location(0) vec4 { + switch general_uniforms.render_mode { + case 1: { + return input.joints; + } + case 2: { + return input.weights; + } + default: { + return vec4(input.normal, 1.0); + } + } +} \ No newline at end of file diff --git a/src/sample/skinnedMesh/grid.wgsl b/src/sample/skinnedMesh/grid.wgsl new file mode 100644 index 00000000..cdfc9cac --- /dev/null +++ b/src/sample/skinnedMesh/grid.wgsl @@ -0,0 +1,74 @@ +struct VertexInput { + @location(0) vert_pos: vec2, + @location(1) joints: vec4, + @location(2) weights: vec4 +} + +struct VertexOutput { + @builtin(position) Position: vec4, + @location(0) world_pos: vec3, + @location(1) joints: vec4, + @location(2) weights: vec4, +} + +struct CameraUniforms { + projMatrix: mat4x4f, + viewMatrix: mat4x4f, + modelMatrix: mat4x4f, +} + +struct GeneralUniforms { + render_mode: u32, + skin_mode: u32, +} + +@group(0) @binding(0) var camera_uniforms: CameraUniforms; +@group(1) @binding(0) var general_uniforms: GeneralUniforms; +@group(2) @binding(0) var joint_matrices: array>; +@group(2) @binding(1) var inverse_bind_matrices: array>; + +@vertex +fn vertexMain(input: VertexInput) -> VertexOutput { + var output: VertexOutput; + var bones = vec4(0.0, 0.0, 0.0, 0.0); + let position = vec4(input.vert_pos.x, input.vert_pos.y, 0.0, 1.0); + // Get relevant 4 bone matrices + let joint0 = joint_matrices[input.joints[0]] * inverse_bind_matrices[input.joints[0]]; + let joint1 = joint_matrices[input.joints[1]] * inverse_bind_matrices[input.joints[1]]; + let joint2 = joint_matrices[input.joints[2]] * inverse_bind_matrices[input.joints[2]]; + let joint3 = joint_matrices[input.joints[3]] * inverse_bind_matrices[input.joints[3]]; + // Compute influence of joint based on weight + let skin_matrix = + joint0 * input.weights[0] + + joint1 * input.weights[1] + + joint2 * input.weights[2] + + joint3 * input.weights[3]; + // Bone transformed mesh + output.Position = select( + camera_uniforms.projMatrix * camera_uniforms.viewMatrix * camera_uniforms.modelMatrix * position, + camera_uniforms.projMatrix * camera_uniforms.viewMatrix * camera_uniforms.modelMatrix * skin_matrix * position, + general_uniforms.skin_mode == 0 + ); + + //Get unadjusted world coordinates + output.world_pos = position.xyz; + output.joints = vec4(f32(input.joints.x), f32(input.joints.y), f32(input.joints.z), f32(input.joints.w)); + output.weights = input.weights; + return output; +} + + +@fragment +fn fragmentMain(input: VertexOutput) -> @location(0) vec4 { + switch general_uniforms.render_mode { + case 1: { + return input.joints; + } + case 2: { + return input.weights; + } + default: { + return vec4(255.0, 0.0, 1.0, 1.0); + } + } +} \ No newline at end of file diff --git a/src/sample/skinnedMesh/gridData.ts b/src/sample/skinnedMesh/gridData.ts new file mode 100644 index 00000000..6cb7c144 --- /dev/null +++ b/src/sample/skinnedMesh/gridData.ts @@ -0,0 +1,106 @@ +/* eslint-disable prettier/prettier */ +export const gridVertices = new Float32Array([ + // B0 + 0, 1, // 0 + 0, -1, // 1 + // CONNECTOR + 2, 1, // 2 + 2, -1, // 3 + // B1 + 4, 1, // 4 + 4, -1, // 5 + // CONNECTOR + 6, 1, // 6 + 6, -1, // 7 + // B2 + 8, 1, // 8 + 8, -1, // 9, + // CONNECTOR + 10, 1, //10 + 10, -1, //11 + // B3 + 12, 1, //12 + 12, -1, //13 +]); + +// Representing the indice of four bones that can influence each vertex +export const gridJoints = new Uint32Array([ + 0, 0, 0, 0, // Vertex 0 is influenced by bone 0 + 0, 0, 0, 0, // 1 + 0, 1, 0, 0, // 2 + 0, 1, 0, 0, // 3 + 1, 0, 0, 0, // 4 + 1, 0, 0, 0, // 5 + 1, 2, 0, 0, // Vertex 6 is influenced by bone 1 and bone 2 + 1, 2, 0, 0, // 7 + 2, 0, 0, 0, // 8 + 2, 0, 0, 0, // 9 + 1, 2, 3, 0, //10 + 1, 2, 3, 0, //11 + 2, 3, 0, 0, //12 + 2, 3, 0, 0, //13 +]) + +// The weights applied when ve +export const gridWeights = new Float32Array([ + // B0 + 1, 0, 0, 0, // 0 + 1, 0, 0, 0, // 1 + // CONNECTOR + .5,.5, 0, 0, // 2 + .5,.5, 0, 0, // 3 + // B1 + 1, 0, 0, 0, // 4 + 1, 0, 0, 0, // 5 + // CONNECTOR + .5,.5, 0, 0, // 6 + .5,.5, 0, 0, // 7 + // B2 + 1, 0, 0, 0, // 8 + 1, 0, 0, 0, // 9 + // CONNECTOR + .5,.5, 0, 0, // 10 + .5,.5, 0, 0, // 11 + // B3 + 1, 0, 0, 0, // 12 + 1, 0, 0, 0, // 13 +]); + +// Using data above... +// Vertex 0 is influenced by bone 0 with a weight of 1 +// Vertex 1 is influenced by bone 1 with a weight of 1 +// Vertex 2 is influenced by bone 0 and 1 with a weight of 0.5 each +// and so on.. +// Although a vertex can hypothetically be influenced by 4 bones, +// in this example, we stick to each vertex being infleunced by only two +// although there can be downstream effects of parent bones influencing child bones +// that influence their own children + +export const gridIndices = new Uint16Array([ + // B0 + 0, 1, + 0, 2, + 1, 3, + // CONNECTOR + 2, 3, // + 2, 4, + 3, 5, + // B1 + 4, 5, + 4, 6, + 5, 7, + // CONNECTOR + 6, 7, + 6, 8, + 7, 9, + // B2 + 8, 9, + 8, 10, + 9, 11, + // CONNECTOR + 10, 11, + 10, 12, + 11, 13, + // B3 + 12, 13, +]); \ No newline at end of file diff --git a/src/sample/skinnedMesh/gridUtils.ts b/src/sample/skinnedMesh/gridUtils.ts new file mode 100644 index 00000000..4d42af41 --- /dev/null +++ b/src/sample/skinnedMesh/gridUtils.ts @@ -0,0 +1,114 @@ +import { gridVertices, gridIndices, gridJoints, gridWeights } from './gridData'; + +// Uses constant grid data to create appropriately sized GPU Buffers for our skinned grid +export const createSkinnedGridBuffers = (device: GPUDevice) => { + // Utility function that creates GPUBuffers from data + const createBuffer = ( + data: Float32Array | Uint32Array, + type: 'f32' | 'u32' + ) => { + const buffer = device.createBuffer({ + size: data.byteLength, + usage: GPUBufferUsage.VERTEX, + mappedAtCreation: true, + }); + if (type === 'f32') { + new Float32Array(buffer.getMappedRange()).set(data); + } else { + new Uint32Array(buffer.getMappedRange()).set(data); + } + buffer.unmap(); + return buffer; + }; + const positionsBuffer = createBuffer(gridVertices, 'f32'); + const jointsBuffer = createBuffer(gridJoints, 'u32'); + const weightsBuffer = createBuffer(gridWeights, 'f32'); + const indicesBuffer = device.createBuffer({ + size: Uint16Array.BYTES_PER_ELEMENT * gridIndices.length, + usage: GPUBufferUsage.INDEX, + mappedAtCreation: true, + }); + new Uint16Array(indicesBuffer.getMappedRange()).set(gridIndices); + indicesBuffer.unmap(); + + return { + positions: positionsBuffer, + joints: jointsBuffer, + weights: weightsBuffer, + indices: indicesBuffer, + }; +}; + +export const createSkinnedGridRenderPipeline = ( + device: GPUDevice, + presentationFormat: GPUTextureFormat, + vertexShader: string, + fragmentShader: string, + bgLayouts: GPUBindGroupLayout[] +) => { + const pipeline = device.createRenderPipeline({ + label: 'SkinnedGridRenderer', + layout: device.createPipelineLayout({ + label: `SkinnedGridRenderer.pipelineLayout`, + bindGroupLayouts: bgLayouts, + }), + vertex: { + module: device.createShaderModule({ + label: `SkinnedGridRenderer.vertexShader`, + code: vertexShader, + }), + entryPoint: 'vertexMain', + buffers: [ + // Vertex Positions (positions) + { + arrayStride: Float32Array.BYTES_PER_ELEMENT * 2, + attributes: [ + { + format: 'float32x2', + offset: 0, + shaderLocation: 0, + }, + ], + }, + // Bone Indices (joints) + { + arrayStride: Uint32Array.BYTES_PER_ELEMENT * 4, + attributes: [ + { + format: 'uint32x4', + offset: 0, + shaderLocation: 1, + }, + ], + }, + // Bone Weights (weights) + { + arrayStride: Float32Array.BYTES_PER_ELEMENT * 4, + attributes: [ + { + format: 'float32x4', + offset: 0, + shaderLocation: 2, + }, + ], + }, + ], + }, + fragment: { + module: device.createShaderModule({ + label: `SkinnedGridRenderer.fragmentShader`, + code: fragmentShader, + }), + entryPoint: 'fragmentMain', + targets: [ + { + format: presentationFormat, + }, + ], + }, + primitive: { + topology: 'line-list', + }, + }); + return pipeline; +}; diff --git a/src/sample/skinnedMesh/main.ts b/src/sample/skinnedMesh/main.ts new file mode 100644 index 00000000..1bcb98e7 --- /dev/null +++ b/src/sample/skinnedMesh/main.ts @@ -0,0 +1,605 @@ +import { makeSample, SampleInit } from '../../components/SampleLayout'; +import { convertGLBToJSONAndBinary, GLTFSkin } from './glbUtils'; +import gltfWGSL from './gltf.wgsl'; +import gridWGSL from './grid.wgsl'; +import { Mat4, mat4, Quat, vec3 } from 'wgpu-matrix'; +import { createBindGroupCluster } from '../bitonicSort/utils'; +import { + createSkinnedGridBuffers, + createSkinnedGridRenderPipeline, +} from './gridUtils'; +import { gridIndices } from './gridData'; + +const MAT4X4_BYTES = 64; + +interface BoneObject { + transforms: Mat4[]; + bindPoses: Mat4[]; + bindPosesInv: Mat4[]; +} + +enum RenderMode { + NORMAL, + JOINTS, + WEIGHTS, +} + +enum SkinMode { + ON, + OFF, +} + +// Copied from toji/gl-matrix +const getRotation = (mat: Mat4): Quat => { + // Initialize our output quaternion + const out = [0, 0, 0, 0]; + // Extract the scaling factor from the final matrix transformation + // to normalize our rotation; + const scaling = mat4.getScaling(mat); + const is1 = 1 / scaling[0]; + const is2 = 1 / scaling[1]; + const is3 = 1 / scaling[2]; + + // Scale the matrix elements by the scaling factors + const sm11 = mat[0] * is1; + const sm12 = mat[1] * is2; + const sm13 = mat[2] * is3; + const sm21 = mat[4] * is1; + const sm22 = mat[5] * is2; + const sm23 = mat[6] * is3; + const sm31 = mat[8] * is1; + const sm32 = mat[9] * is2; + const sm33 = mat[10] * is3; + + // The trace of a square matrix is the sum of its diagonal entries + // While the matrix trace has many interesting mathematical properties, + // the primary purpose of the trace is to assess the characteristics of the rotation. + const trace = sm11 + sm22 + sm33; + let S = 0; + + // If all matrix elements contribute equally to the rotation. + if (trace > 0) { + S = Math.sqrt(trace + 1.0) * 2; + out[3] = 0.25 * S; + out[0] = (sm23 - sm32) / S; + out[1] = (sm31 - sm13) / S; + out[2] = (sm12 - sm21) / S; + // If the rotation is primarily around the x-axis + } else if (sm11 > sm22 && sm11 > sm33) { + S = Math.sqrt(1.0 + sm11 - sm22 - sm33) * 2; + out[3] = (sm23 - sm32) / S; + out[0] = 0.25 * S; + out[1] = (sm12 + sm21) / S; + out[2] = (sm31 + sm13) / S; + // If rotation is primarily around the y-axis + } else if (sm22 > sm33) { + S = Math.sqrt(1.0 + sm22 - sm11 - sm33) * 2; + out[3] = (sm31 - sm13) / S; + out[0] = (sm12 + sm21) / S; + out[1] = 0.25 * S; + out[2] = (sm23 + sm32) / S; + // If the rotation is primarily around the z-axis + } else { + S = Math.sqrt(1.0 + sm33 - sm11 - sm22) * 2; + out[3] = (sm12 - sm21) / S; + out[0] = (sm31 + sm13) / S; + out[1] = (sm23 + sm32) / S; + out[2] = 0.25 * S; + } + + return out; +}; + +const init: SampleInit = async ({ canvas, pageState, gui }) => { + //Normal setup + const adapter = await navigator.gpu.requestAdapter(); + const device = await adapter.requestDevice(); + + if (!pageState.active) return; + const context = canvas.getContext('webgpu') as GPUCanvasContext; + + const devicePixelRatio = window.devicePixelRatio || 1; + canvas.width = canvas.clientWidth * devicePixelRatio; + canvas.height = canvas.clientHeight * devicePixelRatio; + const presentationFormat = navigator.gpu.getPreferredCanvasFormat(); + + context.configure({ + device, + format: presentationFormat, + alphaMode: 'premultiplied', + }); + + const settings = { + cameraX: 0, + cameraY: -5.1, + cameraZ: -14.6, + objectScale: 1, + angle: 0.2, + speed: 50, + object: 'Whale', + renderMode: 'NORMAL', + skinMode: 'ON', + }; + + // Determine whether we want to render our whale or our skinned grid + gui.add(settings, 'object', ['Whale', 'Skinned Grid']).onChange(() => { + if (settings.object === 'Skinned Grid') { + settings.cameraX = -10; + settings.cameraY = 0; + settings.objectScale = 1.27; + } else { + if (settings.skinMode === 'OFF') { + settings.cameraX = 0; + settings.cameraY = 0; + settings.cameraZ = -11; + } else { + settings.cameraX = 0; + settings.cameraY = -5.1; + settings.cameraZ = -14.6; + } + } + }); + + // Output the mesh normals, its joints, or the weights that influence the movement of the joints + gui + .add(settings, 'renderMode', ['NORMAL', 'JOINTS', 'WEIGHTS']) + .onChange(() => { + device.queue.writeBuffer( + generalUniformsBuffer, + 0, + new Uint32Array([RenderMode[settings.renderMode]]) + ); + }); + // Determine whether the mesh is static or whether skinning is activated + gui.add(settings, 'skinMode', ['ON', 'OFF']).onChange(() => { + if (settings.object === 'Whale') { + if (settings.skinMode === 'OFF') { + settings.cameraX = 0; + settings.cameraY = 0; + settings.cameraZ = -11; + } else { + settings.cameraX = 0; + settings.cameraY = -5.1; + settings.cameraZ = -14.6; + } + } + device.queue.writeBuffer( + generalUniformsBuffer, + 4, + new Uint32Array([SkinMode[settings.skinMode]]) + ); + }); + const animFolder = gui.addFolder('Animation Settings'); + animFolder.add(settings, 'angle', 0.05, 0.5).step(0.05); + animFolder.add(settings, 'speed', 10, 100).step(10); + + const depthTexture = device.createTexture({ + size: [canvas.width, canvas.height], + format: 'depth24plus', + usage: GPUTextureUsage.RENDER_ATTACHMENT, + }); + + const cameraBuffer = device.createBuffer({ + size: MAT4X4_BYTES * 3, + usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, + }); + + const cameraBGCluster = createBindGroupCluster( + [0], + [GPUShaderStage.VERTEX], + ['buffer'], + [{ type: 'uniform' }], + [[{ buffer: cameraBuffer }]], + 'Camera', + device + ); + + const generalUniformsBuffer = device.createBuffer({ + size: Uint32Array.BYTES_PER_ELEMENT * 2, + usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST, + }); + + const generalUniformsBGCLuster = createBindGroupCluster( + [0], + [GPUShaderStage.VERTEX | GPUShaderStage.FRAGMENT], + ['buffer'], + [{ type: 'uniform' }], + [[{ buffer: generalUniformsBuffer }]], + 'General', + device + ); + + // Same bindGroupLayout as in main file. + const nodeUniformsBindGroupLayout = device.createBindGroupLayout({ + label: 'NodeUniforms.bindGroupLayout', + entries: [ + { + binding: 0, + buffer: { + type: 'uniform', + }, + visibility: GPUShaderStage.VERTEX, + }, + ], + }); + + // Fetch whale resources from the glb file + const whaleScene = await fetch('../assets/gltf/whale.glb') + .then((res) => res.arrayBuffer()) + .then((buffer) => convertGLBToJSONAndBinary(buffer, device)); + + // Builds a render pipeline for our whale mesh + // Since we are building a lightweight gltf parser around a gltf scene with a known + // quantity of meshes, we only build a renderPipeline for the singular mesh present + // within our scene. A more robust gltf parser would loop through all the meshes, + // cache replicated pipelines, and perform other optimizations. + whaleScene.meshes[0].buildRenderPipeline( + device, + gltfWGSL, + gltfWGSL, + presentationFormat, + depthTexture.format, + [ + cameraBGCluster.bindGroupLayout, + generalUniformsBGCLuster.bindGroupLayout, + nodeUniformsBindGroupLayout, + GLTFSkin.skinBindGroupLayout, + ] + ); + + // Create skinned grid resources + const skinnedGridVertexBuffers = createSkinnedGridBuffers(device); + // Buffer for our uniforms, joints, and inverse bind matrices + const skinnedGridUniformBufferUsage: GPUBufferDescriptor = { + // 5 4x4 matrices, one for each bone + size: MAT4X4_BYTES * 5, + usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST, + }; + const skinnedGridJointUniformBuffer = device.createBuffer( + skinnedGridUniformBufferUsage + ); + const skinnedGridInverseBindUniformBuffer = device.createBuffer( + skinnedGridUniformBufferUsage + ); + const skinnedGridBoneBGCluster = createBindGroupCluster( + [0, 1], + [GPUShaderStage.VERTEX, GPUShaderStage.VERTEX], + ['buffer', 'buffer'], + [{ type: 'read-only-storage' }, { type: 'read-only-storage' }], + [ + [ + { buffer: skinnedGridJointUniformBuffer }, + { buffer: skinnedGridInverseBindUniformBuffer }, + ], + ], + 'SkinnedGridJointUniforms', + device + ); + const skinnedGridPipeline = createSkinnedGridRenderPipeline( + device, + presentationFormat, + gridWGSL, + gridWGSL, + [ + cameraBGCluster.bindGroupLayout, + generalUniformsBGCLuster.bindGroupLayout, + skinnedGridBoneBGCluster.bindGroupLayout, + ] + ); + + // Global Calc + const aspect = canvas.width / canvas.height; + const perspectiveProjection = mat4.perspective( + (2 * Math.PI) / 5, + aspect, + 0.1, + 100.0 + ); + + const orthographicProjection = mat4.ortho(-20, 20, -10, 10, -100, 100); + + function getProjectionMatrix() { + if (settings.object !== 'Skinned Grid') { + return perspectiveProjection as Float32Array; + } + return orthographicProjection as Float32Array; + } + + function getViewMatrix() { + const viewMatrix = mat4.identity(); + if (settings.object === 'Skinned Grid') { + mat4.translate( + viewMatrix, + vec3.fromValues( + settings.cameraX * settings.objectScale, + settings.cameraY * settings.objectScale, + settings.cameraZ + ), + viewMatrix + ); + } else { + mat4.translate( + viewMatrix, + vec3.fromValues(settings.cameraX, settings.cameraY, settings.cameraZ), + viewMatrix + ); + } + return viewMatrix as Float32Array; + } + + function getModelMatrix() { + const modelMatrix = mat4.identity(); + const scaleVector = vec3.fromValues( + settings.objectScale, + settings.objectScale, + settings.objectScale + ); + mat4.scale(modelMatrix, scaleVector, modelMatrix); + if (settings.object === 'Whale') { + mat4.rotateY(modelMatrix, (Date.now() / 1000) * 0.5, modelMatrix); + } + return modelMatrix as Float32Array; + } + + // Pass Descriptor for GLTFs + const gltfRenderPassDescriptor: GPURenderPassDescriptor = { + colorAttachments: [ + { + view: undefined, // Assigned later + + clearValue: { r: 0.3, g: 0.3, b: 0.3, a: 1.0 }, + loadOp: 'clear', + storeOp: 'store', + }, + ], + depthStencilAttachment: { + view: depthTexture.createView(), + depthLoadOp: 'clear', + depthClearValue: 1.0, + depthStoreOp: 'store', + }, + }; + + // Pass descriptor for grid with no depth testing + const skinnedGridRenderPassDescriptor: GPURenderPassDescriptor = { + colorAttachments: [ + { + view: undefined, // Assigned later + + clearValue: { r: 0.3, g: 0.3, b: 0.3, a: 1.0 }, + loadOp: 'clear', + storeOp: 'store', + }, + ], + }; + + const animSkinnedGrid = (boneTransforms: Mat4[], angle: number) => { + const m = mat4.identity(); + mat4.rotateZ(m, angle, boneTransforms[0]); + mat4.translate(boneTransforms[0], vec3.create(4, 0, 0), m); + mat4.rotateZ(m, angle, boneTransforms[1]); + mat4.translate(boneTransforms[1], vec3.create(4, 0, 0), m); + mat4.rotateZ(m, angle, boneTransforms[2]); + }; + + // Create a group of bones + // Each index associates an actual bone to its transforms, bindPoses, uniforms, etc + const createBoneCollection = (numBones: number): BoneObject => { + // Initial bone transformation + const transforms: Mat4[] = []; + // Bone bind poses, an extra matrix per joint/bone that represents the starting point + // of the bone before any transformations are applied + const bindPoses: Mat4[] = []; + // Create a transform, bind pose, and inverse bind pose for each bone + for (let i = 0; i < numBones; i++) { + transforms.push(mat4.identity()); + bindPoses.push(mat4.identity()); + } + + // Get initial bind pose positions + animSkinnedGrid(bindPoses, 0); + const bindPosesInv = bindPoses.map((bindPose) => { + return mat4.inverse(bindPose); + }); + + return { + transforms, + bindPoses, + bindPosesInv, + }; + }; + + // Create bones of the skinned grid and write the inverse bind positions to + // the skinned grid's inverse bind matrix array + const gridBoneCollection = createBoneCollection(5); + for (let i = 0; i < gridBoneCollection.bindPosesInv.length; i++) { + device.queue.writeBuffer( + skinnedGridInverseBindUniformBuffer, + i * 64, + gridBoneCollection.bindPosesInv[i] as Float32Array + ); + } + + // A map that maps a joint index to the original matrix transformation of a bone + const origMatrices = new Map(); + const animWhaleSkin = (skin: GLTFSkin, angle: number) => { + for (let i = 0; i < skin.joints.length; i++) { + // Index into the current joint + const joint = skin.joints[i]; + // If our map does + if (!origMatrices.has(joint)) { + origMatrices.set(joint, whaleScene.nodes[joint].source.getMatrix()); + } + // Get the original position, rotation, and scale of the current joint + const origMatrix = origMatrices.get(joint); + let m = mat4.create(); + // Depending on which bone we are accessing, apply a specific rotation to the bone's original + // transformation to animate it + if (joint === 1 || joint === 0) { + m = mat4.rotateY(origMatrix, -angle); + } else if (joint === 3 || joint === 4) { + m = mat4.rotateX(origMatrix, joint === 3 ? angle : -angle); + } else { + m = mat4.rotateZ(origMatrix, angle); + } + // Apply the current transformation to the transform values within the relevant nodes + // (these nodes, of course, each being nodes that represent joints/bones) + whaleScene.nodes[joint].source.position = mat4.getTranslation(m); + whaleScene.nodes[joint].source.scale = mat4.getScaling(m); + whaleScene.nodes[joint].source.rotation = getRotation(m); + } + }; + + function frame() { + // Sample is no longer the active page. + if (!pageState.active) return; + + // Calculate camera matrices + const projectionMatrix = getProjectionMatrix(); + const viewMatrix = getViewMatrix(); + const modelMatrix = getModelMatrix(); + + // Calculate bone transformation + const t = (Date.now() / 20000) * settings.speed; + const angle = Math.sin(t) * settings.angle; + // Compute Transforms when angle is applied + animSkinnedGrid(gridBoneCollection.transforms, angle); + + // Write to mvp to camera buffer + device.queue.writeBuffer( + cameraBuffer, + 0, + projectionMatrix.buffer, + projectionMatrix.byteOffset, + projectionMatrix.byteLength + ); + + device.queue.writeBuffer( + cameraBuffer, + 64, + viewMatrix.buffer, + viewMatrix.byteOffset, + viewMatrix.byteLength + ); + + device.queue.writeBuffer( + cameraBuffer, + 128, + modelMatrix.buffer, + modelMatrix.byteOffset, + modelMatrix.byteLength + ); + + // Write to skinned grid bone uniform buffer + for (let i = 0; i < gridBoneCollection.transforms.length; i++) { + device.queue.writeBuffer( + skinnedGridJointUniformBuffer, + i * 64, + gridBoneCollection.transforms[i] as Float32Array + ); + } + + // Difference between these two render passes is just the presence of depthTexture + gltfRenderPassDescriptor.colorAttachments[0].view = context + .getCurrentTexture() + .createView(); + + skinnedGridRenderPassDescriptor.colorAttachments[0].view = context + .getCurrentTexture() + .createView(); + + // Update node matrixes + for (const scene of whaleScene.scenes) { + scene.root.updateWorldMatrix(device); + } + + // Updates skins (we index into skins in the renderer, which is not the best approach but hey) + animWhaleSkin(whaleScene.skins[0], Math.sin(t) * settings.angle); + // Node 6 should be the only node with a drawable mesh so hopefully this works fine + whaleScene.skins[0].update(device, 6, whaleScene.nodes); + + const commandEncoder = device.createCommandEncoder(); + if (settings.object === 'Whale') { + const passEncoder = commandEncoder.beginRenderPass( + gltfRenderPassDescriptor + ); + for (const scene of whaleScene.scenes) { + scene.root.renderDrawables(passEncoder, [ + cameraBGCluster.bindGroups[0], + generalUniformsBGCLuster.bindGroups[0], + ]); + } + passEncoder.end(); + } else { + // Our skinned grid isn't checking for depth, so we pass it + // a separate render descriptor that does not take in a depth texture + const passEncoder = commandEncoder.beginRenderPass( + skinnedGridRenderPassDescriptor + ); + passEncoder.setPipeline(skinnedGridPipeline); + passEncoder.setBindGroup(0, cameraBGCluster.bindGroups[0]); + passEncoder.setBindGroup(1, generalUniformsBGCLuster.bindGroups[0]); + passEncoder.setBindGroup(2, skinnedGridBoneBGCluster.bindGroups[0]); + // Pass in vertex and index buffers generated from our static skinned grid + // data at ./gridData.ts + passEncoder.setVertexBuffer(0, skinnedGridVertexBuffers.positions); + passEncoder.setVertexBuffer(1, skinnedGridVertexBuffers.joints); + passEncoder.setVertexBuffer(2, skinnedGridVertexBuffers.weights); + passEncoder.setIndexBuffer(skinnedGridVertexBuffers.indices, 'uint16'); + passEncoder.drawIndexed(gridIndices.length, 1); + passEncoder.end(); + } + + device.queue.submit([commandEncoder.finish()]); + + requestAnimationFrame(frame); + } + requestAnimationFrame(frame); +}; + +const skinnedMesh: () => JSX.Element = () => + makeSample({ + name: 'Skinned Mesh', + description: + 'A demonstration of basic gltf loading and mesh skinning, ported from https://webgl2fundamentals.org/webgl/lessons/webgl-skinning.html. Mesh data, per vertex attributes, and skin inverseBindMatrices are taken from the json parsed from the binary output of the .glb file. Animations are generated progrmatically, with animated joint matrices updated and passed to shaders per frame via uniform buffers.', + init, + gui: true, + sources: [ + { + name: __filename.substring(__dirname.length + 1), + contents: __SOURCE__, + }, + { + name: './gridData.ts', + // eslint-disable-next-line @typescript-eslint/no-var-requires + contents: require('!!raw-loader!./gridData.ts').default, + }, + { + name: './gridUtils.ts', + // eslint-disable-next-line @typescript-eslint/no-var-requires + contents: require('!!raw-loader!./gridUtils.ts').default, + }, + { + name: './grid.wgsl', + // eslint-disable-next-line @typescript-eslint/no-var-requires + contents: require('!!raw-loader!./grid.wgsl').default, + }, + { + name: './gltf.ts', + // eslint-disable-next-line @typescript-eslint/no-var-requires + contents: require('!!raw-loader!./gltf.ts').default, + }, + { + name: './glbUtils.ts', + // eslint-disable-next-line @typescript-eslint/no-var-requires + contents: require('!!raw-loader!./glbUtils.ts').default, + }, + { + name: './gltf.wgsl', + contents: gltfWGSL, + }, + ], + filename: __filename, + }); + +export default skinnedMesh;