diff --git a/package.json b/package.json index 484ca3554..6f82710c5 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "start": "pnpm run build:rust && cross-env NODE_ENV=development rspack dev", "build": "pnpm run build:rust && cross-env NODE_ENV=production rspack build", "build:rust": "wasm-pack build -t web rust", + "build:rust-dev": "wasm-pack build --dev -t web rust", "build:ZeldaWindWaker": "cd src/ZeldaWindWaker/tools && tsx --experimental-wasm-modules zww_extractor.ts", "build:ztp": "cd src/ZeldaTwilightPrincess/tools && tsx ztp_extractor.ts", "build:dk64": "cd src/DonkeyKong64/tools && tsx extractor.ts", @@ -47,7 +48,7 @@ "build:TheWitness": "cd src/TheWitness/tools && tsx extractor.ts", "test:DeBlob2": "cd src/DeBlob2/tools && tsx systest.ts", "typecheck": "tsc -w --noEmit", - "watch:rust": "pnpm run build:rust && onchange rust/**/*.rs -- pnpm run build:rust" + "watch:rust": "pnpm run build:rust && onchange rust/**/*.rs -- pnpm run build:rust-dev" }, "bin": { "nc-bcsvtool": "./src/tools/bcsvtool.ts", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 6208e0c37..0120f1447 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -12,6 +12,8 @@ crate-type = ["cdylib", "rlib"] [package.metadata.wasm-pack.profile.profiling] wasm-opt = ['-Os', '--debuginfo'] +demangle-name-section = true +dwarf-debug-info = true [profile.release] lto = true diff --git a/rust/src/unity/asset_manager.rs b/rust/src/unity/asset_manager.rs deleted file mode 100644 index e69de29bb..000000000 diff --git a/rust/src/unity/mod.rs b/rust/src/unity/mod.rs index d27640481..802530ba2 100644 --- a/rust/src/unity/mod.rs +++ b/rust/src/unity/mod.rs @@ -2,7 +2,5 @@ mod version; mod serialized_file; -mod common; -mod class_id; mod types; -mod asset_manager; +mod util; diff --git a/rust/src/unity/serialized_file.rs b/rust/src/unity/serialized_file.rs index 3d19ff2c1..9377294df 100644 --- a/rust/src/unity/serialized_file.rs +++ b/rust/src/unity/serialized_file.rs @@ -1,10 +1,10 @@ use deku::{bitvec::{BitSlice, BitVec}, prelude::*}; use wasm_bindgen::prelude::*; -use crate::unity::common::NullTerminatedAsciiString; -use super::types::object::WasmFriendlyPPtr; +use crate::unity::types::common::{NullTerminatedAsciiString, UnityArray}; +use super::types::wasm::WasmFriendlyPPtr; -use super::{class_id::ClassID, common::UnityArray}; +use crate::unity::types::class_id::ClassID; #[wasm_bindgen(js_name = "UnityAssetFile")] pub struct AssetFile { @@ -60,12 +60,18 @@ impl AssetFile { let metadata = self.get_metadata(); let mut result = Vec::new(); for obj in &metadata.objects { - let byte_start = match obj.small_file_byte_start { - Some(start) => start as u64, - None => obj.large_file_byte_start.unwrap() as u64, + let byte_start = obj.get_byte_start(); + let class_id = if obj.serialized_type_index >= 0 { + match metadata.type_tree.get(obj.serialized_type_index as usize) { + Some(obj_type) => obj_type.header.raw_type_id, + None => { + println!("bogus type: index {}, len {}", obj.serialized_type_index, metadata.type_tree.len()); + ClassID::UnknownType + } + } + } else { + ClassID::MonoBehavior }; - let obj_type = &metadata.type_tree[obj.serialized_type_index as usize]; - let class_id = obj_type.header.raw_type_id; result.push(AssetFileObject { file_id:obj.file_id, byte_start, @@ -81,14 +87,14 @@ impl AssetFile { let metadata = self.get_metadata(); metadata.externals.values .get(idx) - .map(|external_file| (&external_file.asset_path_ascii).into()) + .map(|external_file| (&external_file.path_name_ascii).into()) } } #[wasm_bindgen(js_name = "UnityAssetFileObject")] pub struct AssetFileObject { pub file_id: i64, - pub byte_start: u64, + pub byte_start: i64, pub byte_size: usize, pub class_id: ClassID, } @@ -150,6 +156,15 @@ struct ObjectInfo { pub serialized_type_index: i32, } +impl ObjectInfo { + pub fn get_byte_start(&self) -> i64 { + match self.small_file_byte_start { + Some(v) => v as i64, + None => self.large_file_byte_start.unwrap(), + } + } +} + #[derive(DekuRead, Clone, Debug)] struct LocalSerializedObjectIdentifier { pub local_serialized_file_index: i32, @@ -240,9 +255,11 @@ struct FileIdentifier { #[cfg(test)] mod tests { use std::collections::HashMap; + use std::path::PathBuf; + use std::str::FromStr; - use crate::unity::types::object::{GameObject, Mesh, MeshFilter, MeshRenderer, Transform, UnityVersion}; - use crate::unity::class_id::ClassID; + use crate::unity::types::common::UnityVersion; + use crate::unity::types::wasm::{GameObject, Mesh, MeshFilter, MeshRenderer, Transform}; use super::*; use env_logger; @@ -250,13 +267,30 @@ mod tests { #[test] fn test() { - let data = std::fs::read("M:\\Development\\NITRO_BMD\\data\\AShortHike\\level2").unwrap(); - let version = UnityVersion::V2019_4_39f1; + let mut base_path = PathBuf::from_str("C:\\Users\\ifnsp\\dev\\noclip.website\\data\\AShortHike").unwrap(); + let data = std::fs::read(&base_path.join("level2")).unwrap(); + let version = UnityVersion::V2021_3_27f1; let mut asset_file = AssetFile::initialize_with_header_chunk(&data).unwrap(); dbg!(&asset_file.header); asset_file.append_metadata_chunk(&data).unwrap(); let metadata = asset_file.metadata.as_ref().unwrap(); + let mut ext_files = Vec::new(); + for ext in &metadata.externals.values { + let ext_name: String = ext.path_name_ascii.clone().into(); + let ext_path = base_path.join(&ext_name); + let Ok(ext_data) = std::fs::read(&ext_path) else { + println!("couldn't read {:?}", ext_path); + continue; + }; + let mut ext_file = AssetFile::initialize_with_header_chunk(&ext_data).unwrap(); + ext_file.append_metadata_chunk(&ext_data).unwrap(); + println!("{:?}: v{}", ext_path, ext_file.header.version); + ext_file.get_objects(); + println!("successfully read {:?}", ext_path); + ext_files.push(ext_file); + } + let mut objects = HashMap::new(); let mut object_infos = HashMap::new(); for obj in &metadata.objects { @@ -265,10 +299,7 @@ mod tests { if !matches!(obj_type.header.raw_type_id, ClassID::GameObject) { continue; } - let byte_start = asset_file.get_data_offset() as usize + match obj.small_file_byte_start { - Some(start) => start as usize, - None => obj.large_file_byte_start.unwrap() as usize, - }; + let byte_start = asset_file.get_data_offset() as usize + obj.get_byte_start() as usize; let byte_size = obj.byte_size as usize; let data = &data[byte_start..byte_start + byte_size]; let game_object = GameObject::create(version, data).unwrap(); @@ -283,10 +314,7 @@ mod tests { } let obj: &&ObjectInfo = object_infos.get(&component_ptr.path_id).unwrap(); let obj_type = &metadata.type_tree[obj.serialized_type_index as usize]; - let byte_start = asset_file.get_data_offset() as usize + match obj.small_file_byte_start { - Some(start) => start as usize, - None => obj.large_file_byte_start.unwrap() as usize, - }; + let byte_start = asset_file.get_data_offset() as usize + obj.get_byte_start() as usize; let byte_size = obj.byte_size as usize; let bigdata = &data; let data = &data[byte_start..byte_start + byte_size]; @@ -296,20 +324,18 @@ mod tests { ClassID::RectTransform => {Transform::create(version, data).unwrap();}, ClassID::MeshFilter => { let filter = MeshFilter::create(version, data).unwrap(); + if filter.mesh.path_id == 0 || filter.mesh.file_index != 0 { + continue; + } let obj = object_infos.get(&filter.mesh.path_id).unwrap(); - let byte_start = asset_file.get_data_offset() as usize + match obj.small_file_byte_start { - Some(start) => start as usize, - None => obj.large_file_byte_start.unwrap() as usize, - }; + let byte_start = asset_file.get_data_offset() as usize + obj.get_byte_start() as usize; let byte_size = obj.byte_size as usize; let data = &bigdata[byte_start..byte_start + byte_size]; - if filter.mesh.path_id == 42 { - let mut mesh = Mesh::create(version, data).unwrap(); - mesh.submeshes.clear(); - mesh.index_buffer.clear(); - dbg!("JJJ BBB", mesh); - } + println!("reading mesh {}", obj.file_id); + let mut mesh = Mesh::create(version, data).unwrap(); + mesh.submeshes.clear(); + mesh.index_buffer.clear(); }, ClassID::MeshRenderer => {MeshRenderer::create(version, data).unwrap();}, s => {}, diff --git a/rust/src/unity/types/v2019_4_39f1.rs b/rust/src/unity/types/binary.rs similarity index 84% rename from rust/src/unity/types/v2019_4_39f1.rs rename to rust/src/unity/types/binary.rs index 1cacb881c..f74825a30 100644 --- a/rust/src/unity/types/v2019_4_39f1.rs +++ b/rust/src/unity/types/binary.rs @@ -3,9 +3,10 @@ use deku::{bitvec::{BitSlice, Msb0}, prelude::*}; // https://github.com/AssetRipper/TypeTreeDumps/blob/main/StructsDump/release/2019.4.39f1.dump // e.g. Outer Wilds -use crate::unity::common::{CharArray, ColorRGBA, Map, Matrix4x4, PPtr, Packedf32Vec, Packedi32Vec, Quaternion, UnityArray, Vec2, Vec3, Vec4, AABB}; +use super::common::{CharArray, ColorRGBA, Map, Matrix4x4, PPtr, Packedf32Vec, Packedi32Vec, Quaternion, UnityArray, Vec2, Vec3, Vec4, AABB, UnityVersion}; #[derive(DekuRead, Clone, Debug)] +#[deku(ctx = "_version: UnityVersion")] pub struct GameObject { pub components: UnityArray>, pub layer: u32, @@ -20,6 +21,7 @@ pub struct Component { } #[derive(DekuRead, Clone, Debug)] +#[deku(ctx = "_version: UnityVersion")] pub struct Transform { pub game_object: PPtr, pub local_rotation: Quaternion, @@ -30,6 +32,7 @@ pub struct Transform { } #[derive(DekuRead, Clone, Debug)] +#[deku(ctx = "_version: UnityVersion")] pub struct Material { pub name: CharArray, pub shader: PPtr<()>, @@ -60,16 +63,22 @@ pub struct Texture { } #[derive(DekuRead, Clone, Debug)] +#[deku(ctx = "version: UnityVersion")] pub struct MeshRenderer { pub game_object: PPtr, pub enabled: u8, pub cast_shadows: u8, pub receive_shadows: u8, pub dynamic_occludee: u8, + #[deku(cond = "version > UnityVersion::V2021_3_27f1")] + pub static_shadow_caster: Option, pub motion_vectors: u8, pub light_probe_usage: u8, pub reflection_probe_usage: u8, pub ray_tracing_mode: u8, + #[deku(cond = "version > UnityVersion::V2021_3_27f1")] + pub ray_trace_procedural: Option, + #[deku(count = "(4 - deku::byte_offset % 4) % 4")] _alignment: Vec, pub rendering_layer_mask: u32, pub renderer_priority: i32, pub lightmap_index: u16, @@ -85,9 +94,12 @@ pub struct MeshRenderer { pub sorting_layer: i16, pub sorting_order: i16, pub additional_vertex_streams: PPtr, + #[deku(cond = "version > UnityVersion::V2021_3_27f1")] + pub enlighten_vertex_streams: Option>, } #[derive(DekuRead, Clone, Debug)] +#[deku(ctx = "version: UnityVersion")] pub struct Mesh { pub name: CharArray, pub submeshes: UnityArray, @@ -104,6 +116,7 @@ pub struct Mesh { pub index_format: IndexFormat, pub index_buffer: UnityArray, #[deku(count = "(4 - deku::byte_offset % 4) % 4")] _alignment2: Vec, + #[deku(ctx = "version")] pub vertex_data: VertexData, #[deku(count = "(4 - deku::byte_offset % 4) % 4")] _alignment3: Vec, pub compressed_mesh: CompressedMesh, @@ -114,6 +127,7 @@ pub struct Mesh { pub baked_triangle_collision_mesh: UnityArray, #[deku(count = "(4 - deku::byte_offset % 4) % 4")] _alignment5: Vec, pub mesh_metrics: [f32; 2], + #[deku(ctx = "version")] pub streaming_info: StreamingInfo, } @@ -134,12 +148,32 @@ pub enum MeshCompression { } #[derive(DekuRead, Clone, Debug)] +#[deku(ctx = "version: UnityVersion")] pub struct StreamingInfo { - pub offset: u32, + #[deku(ctx = "version")] + pub offset: StreamingInfoOffset, pub size: u32, pub path: CharArray, } +#[derive(DekuRead, Clone, Debug)] +#[deku(ctx = "version: UnityVersion", id = "version")] +pub enum StreamingInfoOffset { + #[deku(id_pat = "UnityVersion::V2019_4_39f1")] + Small(u32), + #[deku(id_pat = "_")] + Big(u64), +} + +impl From for u64 { + fn from(value: StreamingInfoOffset) -> Self { + match value { + StreamingInfoOffset::Small(v) => v as u64, + StreamingInfoOffset::Big(v) => v, + } + } +} + #[derive(DekuRead, Clone, Debug)] pub struct SubMesh { pub first_byte: u32, @@ -152,6 +186,7 @@ pub struct SubMesh { } #[derive(DekuRead, Clone, Debug)] +#[deku(ctx = "_version: UnityVersion")] pub struct VertexData { pub vertex_count: u32, pub channels: UnityArray, @@ -239,6 +274,7 @@ pub struct StaticBatchInfo { } #[derive(DekuRead, Clone, Debug)] +#[deku(ctx = "version: UnityVersion")] pub struct Texture2D { pub name: CharArray, pub forced_fallback_format: i32, @@ -259,6 +295,7 @@ pub struct Texture2D { pub lightmap_format: i32, pub color_space: ColorSpace, pub data: UnityArray, + #[deku(ctx = "version")] pub streaming_info: StreamingInfo, } @@ -313,6 +350,7 @@ pub enum ColorSpace { } #[derive(DekuRead, Clone, Debug)] +#[deku(ctx = "_version: UnityVersion")] pub struct MeshFilter { pub game_object: PPtr, pub mesh: PPtr, diff --git a/rust/src/unity/class_id.rs b/rust/src/unity/types/class_id.rs similarity index 100% rename from rust/src/unity/class_id.rs rename to rust/src/unity/types/class_id.rs diff --git a/rust/src/unity/common.rs b/rust/src/unity/types/common.rs similarity index 78% rename from rust/src/unity/common.rs rename to rust/src/unity/types/common.rs index 8711404d6..824b0d4f6 100644 --- a/rust/src/unity/common.rs +++ b/rust/src/unity/types/common.rs @@ -3,11 +3,41 @@ use std::{collections::HashMap, fmt::Debug, hash::Hash, marker::PhantomData}; use wasm_bindgen::prelude::*; use deku::{bitvec::{BitSlice, Msb0}, ctx::BitSize, prelude::*}; +// Important: these must be ordered by chronological release date, so +// PartialOrd can correctly compare them. +#[wasm_bindgen(js_name = "UnityVersion")] +#[derive(Debug, Clone, Copy, PartialEq, PartialOrd)] +pub enum UnityVersion { + V2019_4_39f1, + V2020_3_16f1, + V2021_3_27f1, +} + #[derive(Clone, Debug)] pub struct UnityArray { pub values: Vec, } +impl<'a, T> DekuRead<'a, UnityVersion> for UnityArray where T: DekuRead<'a, UnityVersion> { + fn read( + input: &'a deku::bitvec::BitSlice, + version: UnityVersion, + ) -> Result<(&'a deku::bitvec::BitSlice, Self), DekuError> + where + Self: Sized { + let (mut rest, count) = i32::read(input, ())?; + let mut values = Vec::new(); + for _ in 0..count { + let (new_rest, value) = T::read(rest, version)?; + rest = new_rest; + values.push(value); + } + Ok((rest, UnityArray { + values, + })) + } +} + impl<'a, T> DekuRead<'a> for UnityArray where T: DekuRead<'a> { fn read( input: &'a deku::bitvec::BitSlice, @@ -16,7 +46,7 @@ impl<'a, T> DekuRead<'a> for UnityArray where T: DekuRead<'a> { where Self: Sized { let (mut rest, count) = i32::read(input, ctx)?; - let mut values = Vec::with_capacity(count as usize); + let mut values = Vec::new(); for _ in 0..count { let (new_rest, value) = T::read(rest, ctx)?; rest = new_rest; @@ -50,8 +80,8 @@ impl<'a, K, V> DekuRead<'a> for Map where Self: Sized { let (mut rest, count) = i32::read(input, ctx)?; - let mut keys = Vec::with_capacity(count as usize); - let mut values = Vec::with_capacity(count as usize); + let mut keys = Vec::new(); + let mut values = Vec::new(); for _ in 0..count { let (new_rest, key) = K::read(rest, ctx)?; rest = new_rest; @@ -67,6 +97,33 @@ impl<'a, K, V> DekuRead<'a> for Map } } +impl<'a, K, V> DekuRead<'a, UnityVersion> for Map + where K: DekuRead<'a>, V: DekuRead<'a, UnityVersion> +{ + fn read( + input: &'a deku::bitvec::BitSlice, + version: UnityVersion, + ) -> Result<(&'a deku::bitvec::BitSlice, Self), DekuError> + where + Self: Sized { + let (mut rest, count) = i32::read(input, ())?; + let mut keys = Vec::new(); + let mut values = Vec::new(); + for _ in 0..count { + let (new_rest, key) = K::read(rest, ())?; + rest = new_rest; + let (new_rest, value) = V::read(rest, version)?; + rest = new_rest; + keys.push(key); + values.push(value); + } + Ok((rest, Map { + keys, + values, + })) + } +} + impl From> for HashMap where ResK: Hash + Eq, ResK: From, @@ -87,7 +144,6 @@ pub struct CharArray { count: u32, #[deku(count = "*count")] bytes: Vec, - // align to the nearest 4 byte boundary #[deku(count = "(4 - deku::byte_offset % 4) % 4")] _alignment: Vec, } @@ -197,7 +253,7 @@ pub struct Matrix4x4 { } fn unpack_i32s(input: &BitSlice, num_items: usize, bit_size: usize) -> Result<(&BitSlice, Vec), DekuError> { - let mut result = Vec::with_capacity(num_items); + let mut result = Vec::new(); let mut rest = input; for _ in 0..num_items { let (new_rest, value) = i32::read(rest, BitSize(bit_size))?; @@ -264,7 +320,7 @@ impl<'a> DekuRead<'a> for Packedf32Vec { let max = ((1 << bit_size) as f32) - 1.0; let (_, ints) = unpack_i32s(rest, num_items as usize, bit_size as usize)?; - let mut result = Vec::with_capacity(num_items as usize); + let mut result = Vec::new(); for v in ints { result.push(start + (v as f32) * scale / max); } diff --git a/rust/src/unity/types/mod.rs b/rust/src/unity/types/mod.rs index a7c03674a..2acbdb633 100644 --- a/rust/src/unity/types/mod.rs +++ b/rust/src/unity/types/mod.rs @@ -1,4 +1,4 @@ -pub mod v2019_4_39f1; -pub mod v2020_3_16f1; -pub mod v2021_3_27f1; -pub mod object; +pub mod binary; +pub mod wasm; +pub mod common; +pub mod class_id; diff --git a/rust/src/unity/types/v2020_3_16f1.rs b/rust/src/unity/types/v2020_3_16f1.rs deleted file mode 100644 index e01535b37..000000000 --- a/rust/src/unity/types/v2020_3_16f1.rs +++ /dev/null @@ -1,6 +0,0 @@ - -// https://github.com/AssetRipper/TypeTreeDumps/blob/main/StructsDump/release/2020.3.16f1.dump -// e.g. Neon White - -// hack: just re-export all older types until we need to override one w/ changes -pub use super::v2019_4_39f1::*; diff --git a/rust/src/unity/types/v2021_3_27f1.rs b/rust/src/unity/types/v2021_3_27f1.rs deleted file mode 100644 index 787a488d0..000000000 --- a/rust/src/unity/types/v2021_3_27f1.rs +++ /dev/null @@ -1,48 +0,0 @@ -use deku::prelude::*; - -// https://github.com/AssetRipper/TypeTreeDumps/blob/main/StructsDump/release/2021.3.27f1.dump -// e.g. A Short Hike - -use crate::unity::common::{CharArray, PPtr, UnityArray, Vec4}; - -// hack: just re-export all older types until we need to override one w/ changes -pub use super::v2019_4_39f1::*; - -#[derive(DekuRead, Clone, Debug)] -pub struct MeshRenderer { - pub game_object: PPtr, - pub enabled: u8, - pub cast_shadows: u8, - pub receive_shadows: u8, - pub dynamic_occludee: u8, - pub static_shadow_caster: u8, - pub motion_vectors: u8, - pub light_probe_usage: u8, - pub reflection_probe_usage: u8, - pub ray_tracing_mode: u8, - pub ray_trace_procedural: u8, - #[deku(count = "(4 - deku::byte_offset % 4) % 4")] _alignment: Vec, - pub rendering_layer_mask: u32, - pub renderer_priority: i32, - pub lightmap_index: u16, - pub lightmap_index_dynamic: u16, - pub lightmap_tiling_offset: Vec4, - pub lightmap_tiling_offset_dynamic: Vec4, - pub materials: UnityArray>, - pub static_batch_info: StaticBatchInfo, - pub static_batch_root: PPtr, - pub probe_anchor: PPtr, - pub light_probe_volume_override: PPtr, - pub sorting_layer_id: i32, - pub sorting_layer: i16, - pub sorting_order: i16, - pub additional_vertex_streams: PPtr, - pub enlighten_vertex_streams: PPtr, -} - -#[derive(DekuRead, Clone, Debug)] -pub struct StreamingInfo { - pub offset: u64, - pub size: u32, - pub path: CharArray, -} diff --git a/rust/src/unity/types/object.rs b/rust/src/unity/types/wasm.rs similarity index 86% rename from rust/src/unity/types/object.rs rename to rust/src/unity/types/wasm.rs index 500de5bf4..7a8fbc5b9 100644 --- a/rust/src/unity/types/object.rs +++ b/rust/src/unity/types/wasm.rs @@ -1,50 +1,27 @@ use std::collections::HashMap; -use crate::unity::common::{ColorRGBA, Matrix4x4, PPtr, Quaternion, Vec2, Vec3, Vec4, AABB}; use noclip_macros::{FromStructPerField, FromEnumPerVariant, from}; - -use super::{v2019_4_39f1, v2020_3_16f1, v2021_3_27f1}; use wasm_bindgen::prelude::*; -use deku::DekuContainerRead; +use deku::{DekuRead, bitvec::BitSlice}; + +use super::common::{ColorRGBA, Matrix4x4, PPtr, Quaternion, Vec2, Vec3, Vec4, AABB, UnityVersion}; +use super::binary; macro_rules! define_create { ($t:ident, $u:expr) => { #[wasm_bindgen(js_class = $u)] impl $t { pub fn create(version: UnityVersion, data: &[u8]) -> Result<$t, String> { - match version { - UnityVersion::V2019_4_39f1 => { - match v2019_4_39f1::$t::from_bytes((data, 0)) { - Ok((_, value)) => Ok(value.into()), - Err(err) => return Err(format!("{:?}", err)), - } - }, - UnityVersion::V2020_3_16f1 => { - match v2020_3_16f1::$t::from_bytes((data, 0)) { - Ok((_, value)) => Ok(value.into()), - Err(err) => return Err(format!("{:?}", err)), - } - }, - UnityVersion::V2021_3_27f1 => { - match v2021_3_27f1::$t::from_bytes((data, 0)) { - Ok((_, value)) => Ok(value.into()), - Err(err) => return Err(format!("{:?}", err)), - } - }, + let bitslice = BitSlice::from_slice(data); + match binary::$t::read(&bitslice, version) { + Ok((_, value)) => Ok(value.into()), + Err(err) => return Err(format!("{:?}", err)), } } } }; } -#[wasm_bindgen(js_name = "UnityVersion")] -#[derive(Debug, Clone, Copy)] -pub enum UnityVersion { - V2019_4_39f1, - V2020_3_16f1, - V2021_3_27f1, -} - #[wasm_bindgen(js_name = "UnityPPtr")] #[derive(Debug, Copy, Clone)] pub struct WasmFriendlyPPtr { @@ -63,14 +40,14 @@ impl From> for WasmFriendlyPPtr { #[wasm_bindgen(js_name = "UnityComponent")] #[derive(FromStructPerField)] -#[from(v2019_4_39f1::Component)] +#[from(binary::Component)] pub struct Component { pub game_object: WasmFriendlyPPtr, } #[wasm_bindgen(js_name = "UnityGameObject", getter_with_clone)] #[derive(Debug, FromStructPerField)] -#[from(v2019_4_39f1::GameObject)] +#[from(binary::GameObject)] pub struct GameObject { pub components: Vec, pub layer: u32, @@ -81,7 +58,7 @@ pub struct GameObject { #[wasm_bindgen(js_name = "UnityTransform", getter_with_clone)] #[derive(FromStructPerField, Debug)] -#[from(v2019_4_39f1::Transform)] +#[from(binary::Transform)] pub struct Transform { pub game_object: WasmFriendlyPPtr, pub local_rotation: Quaternion, @@ -93,7 +70,7 @@ pub struct Transform { #[wasm_bindgen(js_name = "UnityMaterial", getter_with_clone)] #[derive(FromStructPerField)] -#[from(v2019_4_39f1::Material)] +#[from(binary::Material)] pub struct Material { pub name: String, pub shader: WasmFriendlyPPtr, @@ -138,7 +115,7 @@ impl Material { #[wasm_bindgen(js_name = "UnityTexEnv")] #[derive(FromStructPerField, Debug, Clone)] -#[from(v2019_4_39f1::TexEnv)] +#[from(binary::TexEnv)] pub struct TexEnv { pub texture: WasmFriendlyPPtr, pub scale: Vec2, @@ -147,7 +124,7 @@ pub struct TexEnv { #[wasm_bindgen(js_name = "UnityMesh", getter_with_clone)] #[derive(Debug, Clone, FromStructPerField)] -#[from(v2019_4_39f1::Mesh)] +#[from(binary::Mesh)] pub struct Mesh { pub name: String, pub submeshes: Vec, @@ -222,7 +199,7 @@ impl Mesh { #[wasm_bindgen(js_name = "UnityIndexFormat")] #[derive(FromEnumPerVariant, Debug, Clone, Copy)] -#[from(v2019_4_39f1::IndexFormat)] +#[from(binary::IndexFormat)] pub enum IndexFormat { UInt16, UInt32, @@ -230,7 +207,7 @@ pub enum IndexFormat { #[wasm_bindgen(js_name = "UnityMeshCompression")] #[derive(FromEnumPerVariant, Debug, Clone, Copy)] -#[from(v2019_4_39f1::MeshCompression)] +#[from(binary::MeshCompression)] pub enum MeshCompression { Off = 0, Low = 1, @@ -240,7 +217,7 @@ pub enum MeshCompression { #[wasm_bindgen(js_name = "UnityCompressedMesh", getter_with_clone)] #[derive(Clone, Debug, FromStructPerField)] -#[from(v2019_4_39f1::CompressedMesh)] +#[from(binary::CompressedMesh)] pub struct CompressedMesh { pub vertices: Vec, pub uv: Vec, @@ -281,7 +258,7 @@ impl CompressedMesh { #[wasm_bindgen(js_name = "UnityVertexData", getter_with_clone)] #[derive(Clone, Debug, FromStructPerField)] -#[from(v2019_4_39f1::VertexData)] +#[from(binary::VertexData)] pub struct VertexData { pub vertex_count: u32, pub channels: Vec, @@ -338,7 +315,7 @@ impl VertexStreamInfo { #[wasm_bindgen(js_name = "UnityChannelInfo")] #[derive(Clone, Debug, FromStructPerField)] -#[from(v2019_4_39f1::ChannelInfo)] +#[from(binary::ChannelInfo)] pub struct ChannelInfo { pub stream: u8, pub offset: u8, @@ -369,7 +346,7 @@ impl ChannelInfo { #[wasm_bindgen(js_name = "UnityVertexFormat")] #[derive(FromEnumPerVariant, Debug, Clone, Copy)] -#[from(v2019_4_39f1::VertexFormat)] +#[from(binary::VertexFormat)] pub enum VertexFormat { Float, Float16, @@ -387,7 +364,7 @@ pub enum VertexFormat { #[wasm_bindgen(js_name = "UnitySubMesh")] #[derive(Clone, Debug, FromStructPerField)] -#[from(v2019_4_39f1::SubMesh)] +#[from(binary::SubMesh)] pub struct SubMesh { pub first_byte: u32, pub index_count: u32, @@ -400,8 +377,7 @@ pub struct SubMesh { #[wasm_bindgen(js_name = "UnityStreamingInfo", getter_with_clone)] #[derive(Clone, Debug, FromStructPerField)] -#[from(v2019_4_39f1::StreamingInfo)] -#[from(v2021_3_27f1::StreamingInfo)] +#[from(binary::StreamingInfo)] pub struct StreamingInfo { pub offset: u64, pub size: u32, @@ -410,7 +386,7 @@ pub struct StreamingInfo { #[wasm_bindgen(js_name = "UnityTexture2D", getter_with_clone)] #[derive(Clone, Debug, FromStructPerField)] -#[from(v2019_4_39f1::Texture2D)] +#[from(binary::Texture2D)] pub struct Texture2D { pub name: String, pub forced_fallback_format: i32, @@ -436,7 +412,7 @@ pub struct Texture2D { #[wasm_bindgen(js_name = "UnityGLTextureSettings", getter_with_clone)] #[derive(Clone, Debug, FromStructPerField)] -#[from(v2019_4_39f1::GLTextureSettings)] +#[from(binary::GLTextureSettings)] pub struct GLTextureSettings { pub filter_mode: TextureFilterMode, pub aniso: i32, @@ -448,7 +424,7 @@ pub struct GLTextureSettings { #[wasm_bindgen(js_name = "UnityTextureFilterMode")] #[derive(FromEnumPerVariant, Clone, Copy, Debug)] -#[from(v2019_4_39f1::TextureFilterMode)] +#[from(binary::TextureFilterMode)] pub enum TextureFilterMode { Nearest = 0, Bilinear = 1, @@ -457,7 +433,7 @@ pub enum TextureFilterMode { #[wasm_bindgen(js_name = "UnityTextureWrapMode")] #[derive(FromEnumPerVariant, Clone, Copy, Debug)] -#[from(v2019_4_39f1::TextureWrapMode)] +#[from(binary::TextureWrapMode)] pub enum TextureWrapMode { Repeat = 0, Clamp = 1, @@ -467,7 +443,7 @@ pub enum TextureWrapMode { #[wasm_bindgen(js_name = "UnityTextureFormat")] #[derive(FromEnumPerVariant, Clone, Copy, Debug)] -#[from(v2019_4_39f1::TextureFormat)] +#[from(binary::TextureFormat)] pub enum TextureFormat { Alpha8 = 0x01, RGB24 = 0x03, @@ -484,7 +460,7 @@ pub enum TextureFormat { #[wasm_bindgen(js_name = "UnityTextureColorSpace")] #[derive(FromEnumPerVariant, Clone, Copy, Debug)] -#[from(v2019_4_39f1::ColorSpace)] +#[from(binary::ColorSpace)] pub enum ColorSpace { Linear = 0x00, SRGB = 0x01, @@ -492,7 +468,7 @@ pub enum ColorSpace { #[wasm_bindgen(js_name = "UnityMeshFilter", getter_with_clone)] #[derive(Clone, Debug, FromStructPerField)] -#[from(v2019_4_39f1::MeshFilter)] +#[from(binary::MeshFilter)] pub struct MeshFilter { pub game_object: WasmFriendlyPPtr, pub mesh: WasmFriendlyPPtr, @@ -500,8 +476,7 @@ pub struct MeshFilter { #[wasm_bindgen(js_name = "UnityMeshRenderer", getter_with_clone)] #[derive(Clone, Debug, FromStructPerField)] -#[from(v2019_4_39f1::MeshRenderer)] -#[from(v2021_3_27f1::MeshRenderer)] +#[from(binary::MeshRenderer)] pub struct MeshRenderer { pub game_object: WasmFriendlyPPtr, pub enabled: u8, @@ -531,7 +506,7 @@ pub struct MeshRenderer { #[wasm_bindgen(js_name = "UnityStaticBatchInfo", getter_with_clone)] #[derive(Clone, Debug, FromStructPerField)] -#[from(v2019_4_39f1::StaticBatchInfo)] +#[from(binary::StaticBatchInfo)] pub struct StaticBatchInfo { pub first_submesh: u16, pub submesh_count: u16, diff --git a/rust/src/unity/util.rs b/rust/src/unity/util.rs new file mode 100644 index 000000000..bd137d3ef --- /dev/null +++ b/rust/src/unity/util.rs @@ -0,0 +1,19 @@ +use deku::{bitvec::{BitSlice, Msb0}, prelude::*}; +use std::fmt::Debug; + +pub fn deku_peek<'a, T>(input: &'a BitSlice, byte_offset: usize, msg: &str) -> Result<(&'a BitSlice, T), DekuError> + where for<'b> T: DekuRead<'b, ()> + Debug +{ + println!("deku_peek - {}", msg); + println!(" offset: {}", byte_offset); + match T::read(input, ()) { + Ok((rest, value)) => { + println!(" value: {:?}", value); + Ok((rest, value)) + } + Err(err) => { + println!(" ERROR: {:?}", err); + Err(err) + } + } +} diff --git a/rust/src/unity/version.rs b/rust/src/unity/version.rs index e02c2bf16..7d3bdb89b 100644 --- a/rust/src/unity/version.rs +++ b/rust/src/unity/version.rs @@ -1,7 +1,7 @@ use std::convert::TryFrom; #[derive(Debug, PartialEq, PartialOrd, Default, Copy, Clone)] -pub struct UnityVersion { +pub struct ParsedUnityVersion { pub major: usize, pub minor: usize, pub build: usize, @@ -14,7 +14,7 @@ pub enum VersionParseError { InvalidString(String), } -impl TryFrom<&str> for UnityVersion { +impl TryFrom<&str> for ParsedUnityVersion { type Error = VersionParseError; fn try_from(s: &str) -> std::result::Result { let err = VersionParseError::InvalidString(s.to_string()); @@ -26,7 +26,7 @@ impl TryFrom<&str> for UnityVersion { .collect(); match numbers.len() { 3 => { - Ok(UnityVersion { + Ok(ParsedUnityVersion { major: numbers[0], minor: numbers[1], build: numbers[2], @@ -46,7 +46,7 @@ impl TryFrom<&str> for UnityVersion { }, None => return Err(err), }; - Ok(UnityVersion { + Ok(ParsedUnityVersion { major: numbers[0], minor: numbers[1], build: numbers[2], @@ -79,14 +79,14 @@ mod test { #[test] fn test_construction() { - assert_eq!(UnityVersion::try_from("2020.3.21p1"), Ok(UnityVersion { + assert_eq!(ParsedUnityVersion::try_from("2020.3.21p1"), Ok(ParsedUnityVersion { major: 2020, minor: 3, build: 21, version_type: VersionType::Patch, type_number: 1, })); - assert_eq!(UnityVersion::try_from("3.5.3"), Ok(UnityVersion { + assert_eq!(ParsedUnityVersion::try_from("3.5.3"), Ok(ParsedUnityVersion { major: 3, minor: 5, build: 3, @@ -97,9 +97,9 @@ mod test { #[test] fn test_compare() { - let v1 = UnityVersion::try_from("2020.3.21.f1").unwrap(); - let v2 = UnityVersion::try_from("2020.3.20.f1").unwrap(); - let v3 = UnityVersion::try_from("3.5.20").unwrap(); + let v1 = ParsedUnityVersion::try_from("2020.3.21.f1").unwrap(); + let v2 = ParsedUnityVersion::try_from("2020.3.20.f1").unwrap(); + let v3 = ParsedUnityVersion::try_from("3.5.20").unwrap(); assert_eq!(v1 >= v2, true); assert_eq!(v1 >= v3, true); assert_eq!(v2 >= v3, true); diff --git a/src/AShortHike/Scenes.ts b/src/AShortHike/Scenes.ts index 5eb32dc78..d5d3e0f9d 100644 --- a/src/AShortHike/Scenes.ts +++ b/src/AShortHike/Scenes.ts @@ -277,7 +277,7 @@ class AShortHikeSceneDesc implements Viewer.SceneDesc { } public async createScene(device: GfxDevice, context: SceneContext): Promise { - const runtime = await createUnityRuntime(context, `AShortHike`, UnityVersion.V2019_4_39f1); + const runtime = await createUnityRuntime(context, `AShortHike`, UnityVersion.V2021_3_27f1); runtime.materialFactory = new AShortHikeMaterialFactory(); await runtime.loadLevel(this.id); @@ -291,6 +291,7 @@ const id = 'AShortHike'; const name = 'A Short Hike'; const sceneDescs = [ + new AShortHikeSceneDesc(`level0`, "Main Menu"), new AShortHikeSceneDesc(`level2`, "The Island"), ]; diff --git a/src/Common/Unity/AssetManager.ts b/src/Common/Unity/AssetManager.ts index 4890a9c22..4aefcc7ba 100644 --- a/src/Common/Unity/AssetManager.ts +++ b/src/Common/Unity/AssetManager.ts @@ -149,6 +149,7 @@ export class AssetFile { private waitForHeaderPromise: Promise | null; private dataCache = new Map(); private promiseCache = new Map>(); + public dataOffset: bigint = BigInt(0); constructor(private path: string, public version: UnityVersion) { } @@ -189,12 +190,12 @@ export class AssetFile { })).createTypedArray(Uint8Array); this.ensureAssetFile(headerBytes); - const dataOffset = this.assetFile.get_data_offset(); - if (dataOffset > headerBytes.byteLength) { + this.dataOffset = this.assetFile.get_data_offset(); + if (this.dataOffset > headerBytes.byteLength) { // Oops, need to fetch extra bytes... const extraBytes = (await dataFetcher.fetchData(this.path, { rangeStart: headerBytes.byteLength, - rangeSize: dataOffset - BigInt(headerBytes.byteLength), + rangeSize: this.dataOffset - BigInt(headerBytes.byteLength), })).createTypedArray(Uint8Array); headerBytes = concatBufs(headerBytes, extraBytes); } @@ -228,7 +229,7 @@ export class AssetFile { await this.waitForHeaderPromise; const obj = assertExists(this.unityObjectByFileID.get(pathID)); - const byteStart = this.assetFile.get_data_offset() + obj.byte_start; + const byteStart = this.dataOffset + obj.byte_start; let buffer: ArrayBufferSlice; if (this.fetcher !== null) @@ -248,16 +249,17 @@ export class AssetFile { if (pptr.file_index === 0) { return this; } else { - const externalFilename = assertExists(this.assetFile.get_external_path(pptr)); + let externalFilename = assertExists(this.assetFile.get_external_path(pptr)); + if (externalFilename.startsWith("Library/")) { + externalFilename = externalFilename.replace("Library/", "Resources/"); + } return assetSystem.fetchAssetFile(externalFilename, true); } } private createMeshData = async (assetSystem: UnityAssetSystem, objData: AssetObjectData): Promise => { try { - debugger; const mesh = rust.UnityMesh.create(assetSystem.version, objData.data); - debugger; const streamingInfo: UnityStreamingInfo = mesh.streaming_info; if (streamingInfo.path.length !== 0) { @@ -355,8 +357,9 @@ export class UnityAssetSystem { public async fetchStreamingInfo(streamingInfo: UnityStreamingInfo): Promise { assert(streamingInfo.size !== 0); + const assetFile = assertExists(this.assetFiles.get(streamingInfo.path)); return await this.fetchBytes(streamingInfo.path, { - rangeStart: streamingInfo.offset, + rangeStart: assetFile.dataOffset + streamingInfo.offset, rangeSize: streamingInfo.size, }); }