wip

package.json

@@ -36,6 +36,7 @@
     "start": "pnpm run build:rust && cross-env NODE_ENV=development rspack dev",
     "build": "pnpm run build:rust && cross-env NODE_ENV=production rspack build",
     "build:rust": "wasm-pack build -t web rust",
+    "build:rust-dev": "wasm-pack build --dev -t web rust",
     "build:ZeldaWindWaker": "cd src/ZeldaWindWaker/tools && tsx --experimental-wasm-modules zww_extractor.ts",
     "build:ztp": "cd src/ZeldaTwilightPrincess/tools && tsx ztp_extractor.ts",
     "build:dk64": "cd src/DonkeyKong64/tools && tsx extractor.ts",
@@ -47,7 +48,7 @@
     "build:TheWitness": "cd src/TheWitness/tools && tsx extractor.ts",
     "test:DeBlob2": "cd src/DeBlob2/tools && tsx systest.ts",
     "typecheck": "tsc -w --noEmit",
-    "watch:rust": "pnpm run build:rust && onchange rust/**/*.rs -- pnpm run build:rust"
+    "watch:rust": "pnpm run build:rust && onchange rust/**/*.rs -- pnpm run build:rust-dev"
   },
   "bin": {
     "nc-bcsvtool": "./src/tools/bcsvtool.ts",

rust/Cargo.toml

@@ -12,6 +12,8 @@ crate-type = ["cdylib", "rlib"]
 
 [package.metadata.wasm-pack.profile.profiling]
 wasm-opt = ['-Os', '--debuginfo']
+demangle-name-section = true
+dwarf-debug-info = true
 
 [profile.release]
 lto = true

rust/src/unity/mod.rs

@@ -2,7 +2,5 @@
 
 mod version;
 mod serialized_file;
-mod common;
-mod class_id;
 mod types;
-mod asset_manager;
+mod util;

rust/src/unity/serialized_file.rs

@@ -1,10 +1,10 @@
 use deku::{bitvec::{BitSlice, BitVec}, prelude::*};
 use wasm_bindgen::prelude::*;
 
-use crate::unity::common::NullTerminatedAsciiString;
-use super::types::object::WasmFriendlyPPtr;
+use crate::unity::types::common::{NullTerminatedAsciiString, UnityArray};
+use super::types::wasm::WasmFriendlyPPtr;
 
-use super::{class_id::ClassID, common::UnityArray};
+use crate::unity::types::class_id::ClassID;
 
 #[wasm_bindgen(js_name = "UnityAssetFile")]
 pub struct AssetFile {
@@ -60,12 +60,18 @@ impl AssetFile {
         let metadata = self.get_metadata();
         let mut result = Vec::new();
         for obj in &metadata.objects {
-            let byte_start = match obj.small_file_byte_start {
-                Some(start) => start as u64,
-                None => obj.large_file_byte_start.unwrap() as u64,
+            let byte_start = obj.get_byte_start();
+            let class_id = if obj.serialized_type_index >= 0 {
+                match metadata.type_tree.get(obj.serialized_type_index as usize) {
+                    Some(obj_type) => obj_type.header.raw_type_id,
+                    None => {
+                        println!("bogus type: index {}, len {}", obj.serialized_type_index, metadata.type_tree.len());
+                        ClassID::UnknownType
+                    }
+                }
+            } else {
+                ClassID::MonoBehavior
             };
-            let obj_type = &metadata.type_tree[obj.serialized_type_index as usize];
-            let class_id = obj_type.header.raw_type_id;
             result.push(AssetFileObject {
                 file_id:obj.file_id,
                 byte_start,
@@ -81,14 +87,14 @@ impl AssetFile {
         let metadata = self.get_metadata();
         metadata.externals.values
             .get(idx)
-            .map(|external_file| (&external_file.asset_path_ascii).into())
+            .map(|external_file| (&external_file.path_name_ascii).into())
     }
 }
 
 #[wasm_bindgen(js_name = "UnityAssetFileObject")]
 pub struct AssetFileObject {
     pub file_id: i64,
-    pub byte_start: u64,
+    pub byte_start: i64,
     pub byte_size: usize,
     pub class_id: ClassID,
 }
@@ -150,6 +156,15 @@ struct ObjectInfo {
     pub serialized_type_index: i32,
 }
 
+impl ObjectInfo {
+    pub fn get_byte_start(&self) -> i64 {
+        match self.small_file_byte_start {
+            Some(v) => v as i64,
+            None => self.large_file_byte_start.unwrap(),
+        }
+    }
+}
+
 #[derive(DekuRead, Clone, Debug)]
 struct LocalSerializedObjectIdentifier {
     pub local_serialized_file_index: i32,
@@ -240,23 +255,42 @@ struct FileIdentifier {
 #[cfg(test)]
 mod tests {
     use std::collections::HashMap;
+    use std::path::PathBuf;
+    use std::str::FromStr;
 
-    use crate::unity::types::object::{GameObject, Mesh, MeshFilter, MeshRenderer, Transform, UnityVersion};
-    use crate::unity::class_id::ClassID;
+    use crate::unity::types::common::UnityVersion;
+    use crate::unity::types::wasm::{GameObject, Mesh, MeshFilter, MeshRenderer, Transform};
 
     use super::*;
     use env_logger;
     use deku::bitvec::BitVec;
 
     #[test]
     fn test() {
-        let data = std::fs::read("M:\\Development\\NITRO_BMD\\data\\AShortHike\\level2").unwrap();
-        let version = UnityVersion::V2019_4_39f1;
+        let mut base_path = PathBuf::from_str("C:\\Users\\ifnsp\\dev\\noclip.website\\data\\AShortHike").unwrap();
+        let data = std::fs::read(&base_path.join("level2")).unwrap();
+        let version = UnityVersion::V2021_3_27f1;
         let mut asset_file = AssetFile::initialize_with_header_chunk(&data).unwrap();
         dbg!(&asset_file.header);
         asset_file.append_metadata_chunk(&data).unwrap();
         let metadata = asset_file.metadata.as_ref().unwrap();
 
+        let mut ext_files = Vec::new();
+        for ext in &metadata.externals.values {
+            let ext_name: String = ext.path_name_ascii.clone().into();
+            let ext_path = base_path.join(&ext_name);
+            let Ok(ext_data) = std::fs::read(&ext_path) else {
+                println!("couldn't read {:?}", ext_path);
+                continue;
+            };
+            let mut ext_file = AssetFile::initialize_with_header_chunk(&ext_data).unwrap();
+            ext_file.append_metadata_chunk(&ext_data).unwrap();
+            println!("{:?}: v{}", ext_path, ext_file.header.version);
+            ext_file.get_objects();
+            println!("successfully read {:?}", ext_path);
+            ext_files.push(ext_file);
+        }
+
         let mut objects = HashMap::new();
         let mut object_infos = HashMap::new();
         for obj in &metadata.objects {
@@ -265,10 +299,7 @@ mod tests {
             if !matches!(obj_type.header.raw_type_id, ClassID::GameObject) {
                 continue;
             }
-            let byte_start = asset_file.get_data_offset() as usize + match obj.small_file_byte_start {
-                Some(start) => start as usize,
-                None => obj.large_file_byte_start.unwrap() as usize,
-            };
+            let byte_start = asset_file.get_data_offset() as usize + obj.get_byte_start() as usize;
             let byte_size = obj.byte_size as usize;
             let data = &data[byte_start..byte_start + byte_size];
             let game_object = GameObject::create(version, data).unwrap();
@@ -283,10 +314,7 @@ mod tests {
                 }
                 let obj: &&ObjectInfo = object_infos.get(&component_ptr.path_id).unwrap();
                 let obj_type = &metadata.type_tree[obj.serialized_type_index as usize];
-                let byte_start = asset_file.get_data_offset() as usize + match obj.small_file_byte_start {
-                    Some(start) => start as usize,
-                    None => obj.large_file_byte_start.unwrap() as usize,
-                };
+                let byte_start = asset_file.get_data_offset() as usize + obj.get_byte_start() as usize;
                 let byte_size = obj.byte_size as usize;
                 let bigdata = &data;
                 let data = &data[byte_start..byte_start + byte_size];
@@ -296,20 +324,18 @@ mod tests {
                     ClassID::RectTransform => {Transform::create(version, data).unwrap();},
                     ClassID::MeshFilter => {
                         let filter = MeshFilter::create(version, data).unwrap();
+                        if filter.mesh.path_id == 0 || filter.mesh.file_index != 0 {
+                            continue;
+                        }
                         let obj = object_infos.get(&filter.mesh.path_id).unwrap();
-                        let byte_start = asset_file.get_data_offset() as usize + match obj.small_file_byte_start {
-                            Some(start) => start as usize,
-                            None => obj.large_file_byte_start.unwrap() as usize,
-                        };
+                        let byte_start = asset_file.get_data_offset() as usize + obj.get_byte_start() as usize;
                         let byte_size = obj.byte_size as usize;
                         let data = &bigdata[byte_start..byte_start + byte_size];
 
-                        if filter.mesh.path_id == 42 {
-                            let mut mesh = Mesh::create(version, data).unwrap();
-                            mesh.submeshes.clear();
-                            mesh.index_buffer.clear();
-                            dbg!("JJJ BBB", mesh);
-                        }
+                        println!("reading mesh {}", obj.file_id);
+                        let mut mesh = Mesh::create(version, data).unwrap();
+                        mesh.submeshes.clear();
+                        mesh.index_buffer.clear();
                     },
                     ClassID::MeshRenderer => {MeshRenderer::create(version, data).unwrap();},
                     s => {},

rust/src/unity/types/v2019_4_39f1.rs → rust/src/unity/types/binary.rs

@@ -3,9 +3,10 @@ use deku::{bitvec::{BitSlice, Msb0}, prelude::*};
 // https://github.com/AssetRipper/TypeTreeDumps/blob/main/StructsDump/release/2019.4.39f1.dump
 // e.g. Outer Wilds
 
-use crate::unity::common::{CharArray, ColorRGBA, Map, Matrix4x4, PPtr, Packedf32Vec, Packedi32Vec, Quaternion, UnityArray, Vec2, Vec3, Vec4, AABB};
+use super::common::{CharArray, ColorRGBA, Map, Matrix4x4, PPtr, Packedf32Vec, Packedi32Vec, Quaternion, UnityArray, Vec2, Vec3, Vec4, AABB, UnityVersion};
 
 #[derive(DekuRead, Clone, Debug)]
+#[deku(ctx = "_version: UnityVersion")]
 pub struct GameObject {
     pub components: UnityArray<PPtr<Component>>,
     pub layer: u32,
@@ -20,6 +21,7 @@ pub struct Component {
 }
 
 #[derive(DekuRead, Clone, Debug)]
+#[deku(ctx = "_version: UnityVersion")]
 pub struct Transform {
     pub game_object: PPtr<GameObject>,
     pub local_rotation: Quaternion,
@@ -30,6 +32,7 @@ pub struct Transform {
 }
 
 #[derive(DekuRead, Clone, Debug)]
+#[deku(ctx = "_version: UnityVersion")]
 pub struct Material {
     pub name: CharArray,
     pub shader: PPtr<()>,
@@ -60,16 +63,22 @@ pub struct Texture {
 }
 
 #[derive(DekuRead, Clone, Debug)]
+#[deku(ctx = "version: UnityVersion")]
 pub struct MeshRenderer {
     pub game_object: PPtr<GameObject>,
     pub enabled: u8,
     pub cast_shadows: u8,
     pub receive_shadows: u8,
     pub dynamic_occludee: u8,
+    #[deku(cond = "version > UnityVersion::V2021_3_27f1")]
+    pub static_shadow_caster: Option<u8>,
     pub motion_vectors: u8,
     pub light_probe_usage: u8,
     pub reflection_probe_usage: u8,
     pub ray_tracing_mode: u8,
+    #[deku(cond = "version > UnityVersion::V2021_3_27f1")]
+    pub ray_trace_procedural: Option<u8>,
+    #[deku(count = "(4 - deku::byte_offset % 4) % 4")] _alignment: Vec<u8>,
     pub rendering_layer_mask: u32,
     pub renderer_priority: i32,
     pub lightmap_index: u16,
@@ -85,9 +94,12 @@ pub struct MeshRenderer {
     pub sorting_layer: i16,
     pub sorting_order: i16,
     pub additional_vertex_streams: PPtr<Mesh>,
+    #[deku(cond = "version > UnityVersion::V2021_3_27f1")]
+    pub enlighten_vertex_streams: Option<PPtr<Mesh>>,
 }
 
 #[derive(DekuRead, Clone, Debug)]
+#[deku(ctx = "version: UnityVersion")]
 pub struct Mesh {
     pub name: CharArray,
     pub submeshes: UnityArray<SubMesh>,
@@ -104,6 +116,7 @@ pub struct Mesh {
     pub index_format: IndexFormat,
     pub index_buffer: UnityArray<u8>,
     #[deku(count = "(4 - deku::byte_offset % 4) % 4")] _alignment2: Vec<u8>,
+    #[deku(ctx = "version")]
     pub vertex_data: VertexData,
     #[deku(count = "(4 - deku::byte_offset % 4) % 4")] _alignment3: Vec<u8>,
     pub compressed_mesh: CompressedMesh,
@@ -114,6 +127,7 @@ pub struct Mesh {
     pub baked_triangle_collision_mesh: UnityArray<u8>,
     #[deku(count = "(4 - deku::byte_offset % 4) % 4")] _alignment5: Vec<u8>,
     pub mesh_metrics: [f32; 2],
+    #[deku(ctx = "version")]
     pub streaming_info: StreamingInfo,
 }
 
@@ -134,12 +148,32 @@ pub enum MeshCompression {
 }
 
 #[derive(DekuRead, Clone, Debug)]
+#[deku(ctx = "version: UnityVersion")]
 pub struct StreamingInfo {
-    pub offset: u32,
+    #[deku(ctx = "version")]
+    pub offset: StreamingInfoOffset,
     pub size: u32,
     pub path: CharArray,
 }
 
+#[derive(DekuRead, Clone, Debug)]
+#[deku(ctx = "version: UnityVersion", id = "version")]
+pub enum StreamingInfoOffset {
+    #[deku(id_pat = "UnityVersion::V2019_4_39f1")]
+    Small(u32),
+    #[deku(id_pat = "_")]
+    Big(u64),
+}
+
+impl From<StreamingInfoOffset> for u64 {
+    fn from(value: StreamingInfoOffset) -> Self {
+        match value {
+            StreamingInfoOffset::Small(v) => v as u64,
+            StreamingInfoOffset::Big(v) => v,
+        }
+    }
+}
+
 #[derive(DekuRead, Clone, Debug)]
 pub struct SubMesh {
     pub first_byte: u32,
@@ -152,6 +186,7 @@ pub struct SubMesh {
 }
 
 #[derive(DekuRead, Clone, Debug)]
+#[deku(ctx = "_version: UnityVersion")]
 pub struct VertexData {
     pub vertex_count: u32,
     pub channels: UnityArray<ChannelInfo>,
@@ -239,6 +274,7 @@ pub struct StaticBatchInfo {
 }
 
 #[derive(DekuRead, Clone, Debug)]
+#[deku(ctx = "version: UnityVersion")]
 pub struct Texture2D {
     pub name: CharArray,
     pub forced_fallback_format: i32,
@@ -259,6 +295,7 @@ pub struct Texture2D {
     pub lightmap_format: i32,
     pub color_space: ColorSpace,
     pub data: UnityArray<u8>,
+    #[deku(ctx = "version")]
     pub streaming_info: StreamingInfo,
 }
 
@@ -313,6 +350,7 @@ pub enum ColorSpace {
 }
 
 #[derive(DekuRead, Clone, Debug)]
+#[deku(ctx = "_version: UnityVersion")]
 pub struct MeshFilter {
     pub game_object: PPtr<GameObject>,
     pub mesh: PPtr<Mesh>,