Support even older AVS, fix prop reading, better ramfs demangling

meson.build

@@ -1,4 +1,4 @@
-project('layeredfs', 'c', 'cpp', version: '3.0',
+project('layeredfs', 'c', 'cpp', version: '3.1',
     default_options: [
         'cpp_std=c++17',
         'buildtype=release',

playpen.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+meson compile -C build64 playpen && ./build64/playpen.exe --layered-verbose

src/avs.cpp

@@ -55,7 +55,7 @@ const avs_exports_t avs_exports[] = {
     x.property_insert_read                = "property_insert_read";
     x.property_mem_write                  = "property_mem_write";
     x.property_destroy                    = "property_destroy";
-    x.property_node_query_stat            = "property_node_query_stat";
+    x.property_query_size                 = "property_query_size";
     x.cstream_create                      = "cstream_create";
     x.cstream_operate                     = "cstream_operate";
     x.cstream_finish                      = "cstream_finish";
@@ -88,7 +88,7 @@ const avs_exports_t avs_exports[] = {
     x.property_insert_read                = "XC058ba5000016";
     x.property_mem_write                  = "XC058ba5000162";
     x.property_destroy                    = "XC058ba500010f";
-    x.property_node_query_stat            = "XC058ba500015e";
+    x.property_query_size                 = "XC058ba5000101";
     x.cstream_create                      = "XC058ba5000118";
     x.cstream_operate                     = "XC058ba5000078";
     x.cstream_finish                      = "XC058ba5000130";
@@ -121,7 +121,7 @@ const avs_exports_t avs_exports[] = {
     x.property_insert_read                = "XCd229cc00009a";
     x.property_mem_write                  = "XCd229cc000033";
     x.property_destroy                    = "XCd229cc00013c";
-    x.property_node_query_stat            = "XCd229cc0000b1";
+    x.property_query_size                 = "XCd229cc000032";
     x.cstream_create                      = "XCd229cc000141";
     x.cstream_operate                     = "XCd229cc00008c";
     x.cstream_finish                      = "XCd229cc000025";
@@ -154,7 +154,7 @@ const avs_exports_t avs_exports[] = {
     x.property_insert_read                = "XCnbrep7000094";
     x.property_mem_write                  = "XCnbrep70000b8";
     x.property_destroy                    = "XCnbrep7000091";
-    x.property_node_query_stat            = "XCnbrep70000c5";
+    x.property_query_size                 = "XCnbrep700009f";
     x.cstream_create                      = "XCnbrep7000130";
     x.cstream_operate                     = "XCnbrep7000132";
     x.cstream_finish                      = "XCnbrep7000133";
@@ -187,7 +187,7 @@ const avs_exports_t avs_exports[] = {
     x.property_insert_read                = "XCnbrep700007f";
     x.property_mem_write                  = "XCnbrep70000a3";
     x.property_destroy                    = "XCnbrep700007c";
-    x.property_node_query_stat            = "XCnbrep70000b0";
+    x.property_query_size                 = "XCnbrep700008a";
     x.cstream_create                      = "XCnbrep7000124";
     x.cstream_operate                     = "XCnbrep7000126";
     x.cstream_finish                      = "XCnbrep7000127";
@@ -220,7 +220,7 @@ const avs_exports_t avs_exports[] = {
     x.property_insert_read                = "XCgsqzn0000094";
     x.property_mem_write                  = "XCgsqzn00000b8";
     x.property_destroy                    = "XCgsqzn0000091";
-    x.property_node_query_stat            = "XCgsqzn00000c5";
+    x.property_query_size                 = "XCgsqzn000009f";
     x.cstream_create                      = "XCgsqzn0000130";
     x.cstream_operate                     = "XCgsqzn0000132";
     x.cstream_finish                      = "XCgsqzn0000133";
@@ -321,9 +321,13 @@ property_t prop_from_file_handle(AVS_FILE f) {
     }
 
     // MUST be 8-byte aligned so prop_free doesn't crash
-    prop_buffer = _aligned_malloc(8, (memsize + 7) & ~7);
+    prop_buffer = _aligned_malloc((memsize + 7) & ~7, 8);
+    if(!prop_buffer) {
+        log_warning("_aligned_malloc failed :(");
+        goto FAIL;
+    }
     prop = property_create(flags, prop_buffer, memsize);
-    if (prop < 0) {
+    if (!prop) {
         // double cast to squash truncation warning
         log_warning("Couldn't create prop (%s)", get_prop_error_str((int32_t)(size_t)prop));
         goto FAIL;
@@ -345,7 +349,7 @@ property_t prop_from_file_handle(AVS_FILE f) {
     if (prop)
         property_destroy(prop);
     if (prop_buffer)
-        free(prop_buffer);
+        _aligned_free(prop_buffer);
     return NULL;
 }
 
@@ -360,9 +364,7 @@ property_t prop_from_file_path(string const&path) {
 }
 
 char* prop_to_xml_string(property_t prop, rapidxml::xml_document<>& allocator) {
-    node_size dummy = { 0 };
-
-    auto prop_size = property_node_query_stat(prop, NULL, &dummy);
+    auto prop_size = property_query_size(prop);
     char* xml = allocator.allocate_string(NULL, prop_size);
 
     auto written = property_mem_write(prop, xml, prop_size);
@@ -585,7 +587,7 @@ const char* get_prop_error_str(int32_t code) {
         if (prop_error_list[i].code == (uint32_t)code)
             return prop_error_list[i].msg;
     }
-    snprintf(ret, sizeof(ret), "unknown (%d)", code);
+    snprintf(ret, sizeof(ret), "unknown (%X)", code);
     return ret;
 }
 

src/avs.h

@@ -137,13 +137,6 @@ enum compression_type {
     AVS_COMPRESS_AVSLZ = 1,
 };
 
-struct node_size {
-    int nodes;
-    int data;
-    int unk1, unk2, unk3;
-    int extra_space[16];
-};
-
 struct property_info {
     uint8_t blah[560];
     uint32_t error_code;
@@ -208,8 +201,8 @@ X(int32_t,    property_read_query_memsize, avs_reader_t reader, AVS_FILE f, int*
 X(property_t, property_create, int flags, void *buffer, uint32_t buffer_size) \
 X(int,        property_insert_read, property_t prop, node_t node, avs_reader_t reader, AVS_FILE f) \
 X(int,        property_mem_write, property_t prop, char* output, int output_size) \
+X(int,        property_query_size, property_t prop) \
 X(void,       property_destroy, property_t prop) \
-X(int,        property_node_query_stat, property_t prop, node_t node, struct node_size *size) \
 /* md5sum *sha1 if needed) */ \
 X(mdigest_p,  mdigest_create, mdigest_algs_t algorithm) \
 X(void,       mdigest_update, mdigest_p digest, const char* data, int size) \

src/hook.cpp

@@ -310,7 +310,7 @@ uint32_t handle_file_open(HookFile &file) {
 
     auto ret = file.call_real();
     if(file.ramfs_demangle()) {
-        ramfs_demangler_on_fs_open(file.get_path_to_open(), ret);
+        ramfs_demangler_on_fs_open(file.path, ret);
     }
     // log_verbose("(returned %d)", ret);
     return ret;
@@ -460,6 +460,7 @@ extern "C" {
         log_info(".pak dumper mode enabled");
 #endif
 
+        init_modpath_handler();
         cache_mods();
 
         // hook pkfs, not big enough to be its own file

src/imagefs.cpp

@@ -187,7 +187,7 @@ void parse_texturelist(HookFile &file) {
     auto ifs_path = file.norm_path;
     // truncate
     ifs_path.resize(ifs_path.size() - strlen("/tex/texturelist.xml"));
-    //log_misc("Reading ifs %s", ifs_path.c_str());
+    // log_misc("Reading ifs %s", ifs_path.c_str());
     auto ifs_mod_path = ifs_path;
     string_replace(ifs_mod_path, ".ifs", "_ifs");
 
@@ -267,7 +267,7 @@ void parse_texturelist(HookFile &file) {
             // it's a 4u16
             sscanf(imgrect->value(), "%" SCNu16 " %" SCNu16 " %" SCNu16 " %" SCNu16, &dimensions[0], &dimensions[1], &dimensions[2], &dimensions[3]);
 
-            //log_misc("Image '%s' compress %d format %d", tmp, compress, format_type);
+            // log_misc("Image '%s' compress %d format %d", name->value(), compress, format_type);
             image_t image_info;
             image_info.name = name->value();
             image_info.name_md5 = md5_sum(name->value());

src/modpath_handler.cpp

@@ -79,28 +79,19 @@ void cache_mods(void) {
 // data, data2, data_op2 etc
 // data is "flat", all others must have their own special subfolders
 static vector<string> game_folders;
-static CriticalSectionLock game_folders_mtx;
 
-optional<string> normalise_path(const string &_path) {
-    // one-off init
-    if (game_folders.empty()) {
-        game_folders_mtx.lock();
-
-        // check again in case init was raced
-        if (game_folders.empty()) {
-            for (auto folder : folders_in_folder(".")) {
-                // data is the normal case we transparently handle
-                if (!strcasecmp(folder.c_str(), "data")) {
-                    continue;
-                }
-
-                game_folders.push_back(folder + "/");
-            }
+void init_modpath_handler(void) {
+    for (auto folder : folders_in_folder(".")) {
+        // data is the normal case we transparently handle
+        if (!strcasecmp(folder.c_str(), "data")) {
+            continue;
         }
-        game_folders_mtx.unlock();
-        // all access past here is read-only, don't use the mutex any more
+
+        game_folders.push_back(folder + "/");
     }
+}
 
+optional<string> normalise_path(const string &_path) {
     auto path = _path;
     ramfs_demangler_demangle_if_possible(path);
 

src/modpath_handler.h

@@ -17,6 +17,7 @@ using std::vector;
 #define MOD_FOLDER "./data_mods"
 #define CACHE_FOLDER MOD_FOLDER "/_cache"
 
+void init_modpath_handler(void);
 void cache_mods(void);
 vector<string> available_mods();
 // mutates source string to be all lowercase

src/playpen.cpp

@@ -19,6 +19,8 @@ typedef void (*avs_log_writer_t)(const char *chars, uint32_t nchars, void *ctx);
 X("XCgsqzn0000129", void,   avs_boot, node_t config, void *com_heap, size_t sz_com_heap, void *reserved, avs_log_writer_t log_writer, void *log_context) \
 X("XCgsqzn000012a", void,   avs_shutdown) \
 X("XCgsqzn00000a1", node_t, property_search, property_t prop, node_t node, const char *path) \
+X("XCgsqzn0000048", int, avs_fs_addfs, void* filesys) \
+X("XCgsqzn0000159", void*, avs_filesys_ramfs) \
 
 #define AVS_FUNC_PTR(obfus_name, ret_type, name, ...) ret_type (* name )( __VA_ARGS__ );
 FOREACH_EXTRA_FUNC(AVS_FUNC_PTR)
@@ -29,6 +31,8 @@ static bool print_logs = true;
 
 #include "texbin.hpp"
 
+#define log_assert(cond) if(!(cond)) {log_fatal("Assertion failed:" #cond);}
+
 // decompressed_length MUST be set and will be updated on finish
 unsigned char* lz_decompress(unsigned char* input, size_t length, size_t *decompressed_length) {
     auto compressor = cstream_create(AVS_DECOMPRESS_AVSLZ);
@@ -106,6 +110,26 @@ optional<std::vector<uint8_t>> readFile(const char* filename)
 }
 
 void avs_playpen() {
+    avs_fs_addfs(avs_filesys_ramfs());
+
+    auto f = hook_avs_fs_open("/data/graphic/gmframe29.ifs", avs_open_mode_read(), 420);
+    log_assert(f >= 0);
+
+    // avs_file_to_vec but using the hook_read so ramfs demangler catches it
+    avs_stat stat = {0};
+    avs_fs_fstat(f, &stat);
+    std::vector<uint8_t> contents;
+    contents.resize(stat.filesize);
+    hook_avs_fs_read(f, &contents[0], stat.filesize);
+    avs_fs_close(f);
+
+    char args[] = "base=0x0000000000000000,size=0x00000000,mode=ro";
+    snprintf(args, sizeof(args), "base=0x%p,size=0x%llx,mode=ro", &contents[0], contents.size());
+    log_assert(hook_avs_fs_mount("/afpr2318908", "image.bin", "ramfs", args) >= 0);
+    log_assert(hook_avs_fs_mount("/afp23/data/graphic/gmframe29.ifs", "/afpr2318908/image.bin", "imagefs", NULL) >= 0);
+    hook_avs_fs_open("/afp23/data/graphic/gmframe29.ifs/tex/texturelist.xml", avs_open_mode_read(), 420);
+    hook_avs_fs_open("/afp23/data/graphic/gmframe29.ifs/tex/643b3d20d1b19dfe98e9e23a59a72bba", avs_open_mode_read(), 420);
+
     // log_info("loading file");
     // auto _debug = readFile("debug.bin");
     // if(!_debug) {
@@ -132,11 +156,11 @@ void avs_playpen() {
 
     // auto _tex = Texbin::from_path("tex_l44qb_smc_sm.bin");
     // auto tex = Texbin::from_path("tex_custom.bin");
-    auto tex = Texbin::from_path("tex_l44_paseli_info.bin");
-    if(!tex) {
-        return;
-    }
-    tex->debug();
+    // auto tex = Texbin::from_path("tex_l44_paseli_info.bin");
+    // if(!tex) {
+    //     return;
+    // }
+    // tex->debug();
 
 // #ifdef TEXBIN_VERBOSE
 //     tex->debug();
@@ -306,6 +330,11 @@ void log_writer(const char *chars, uint32_t nchars, void *ctx) {
 static const char *boot_cfg = R"(<?xml version="1.0" encoding="SHIFT_JIS"?>
 <config>
   <fs>
+    <nr_filesys __type="u16">16</nr_filesys>
+    <nr_mountpoint __type="u16">1024</nr_mountpoint>
+    <nr_mounttable __type="u16">32</nr_mounttable>
+    <nr_filedesc __type="u16">4096</nr_filedesc>
+    <link_limit __type="u16">8</link_limit>
     <root><device>.</device></root>
     <mounttable>
       <vfs name="boot" fstype="fs" src="dev/raw" dst="/dev/raw" opt="vf=1,posix=1"/>
@@ -320,8 +349,10 @@ static const char *boot_cfg = R"(<?xml version="1.0" encoding="SHIFT_JIS"?>
 </config>
 )";
 
+static size_t off;
 static size_t read_str(int32_t context, void *dst_buf, size_t count) {
-    memcpy(dst_buf, boot_cfg, count);
+    memcpy(dst_buf, &boot_cfg[off], count);
+    off += count;
     return count;
 }
 
@@ -346,6 +377,7 @@ bool load_dll(void) {
 void boot_avs(void) {
     auto avs_heap = malloc(DEFAULT_HEAP_SIZE);
 
+    off = 0;
     int prop_len = property_read_query_memsize(read_str, 0, 0, 0);
     if (prop_len <= 0) {
         log_fatal("error reading config (size <= 0)");
@@ -357,6 +389,7 @@ void boot_avs(void) {
         log_fatal("cannot create property");
         return;
     }
+    off = 0;
     if (!property_insert_read(avs_config, 0, read_str, 0)) {
         log_fatal("avs-core", "cannot read property");
         return;

src/ramfs_demangler.cpp

@@ -63,14 +63,14 @@ static CriticalSectionLock mangling_mtx;
 // since we call this from a function that is already taking the lock
 static void ramfs_demangler_demangle_if_possible_nolock(std::string& raw_path);
 
-void ramfs_demangler_on_fs_open(const std::string& norm_path, AVS_FILE open_result) {
-	if (open_result < 0 || !string_ends_with(norm_path.c_str(), ".ifs")) {
+void ramfs_demangler_on_fs_open(const std::string& path, AVS_FILE open_result) {
+	if (open_result < 0 || !string_ends_with(path.c_str(), ".ifs")) {
 		return;
 	}
 
 	mangling_mtx.lock();
 
-	auto existing_info = cleanup_map.find(norm_path);
+	auto existing_info = cleanup_map.find(path);
 	if (existing_info != cleanup_map.end()) {
 		file_cleanup_info_t cleanup = existing_info->second;
 
@@ -92,8 +92,8 @@ void ramfs_demangler_on_fs_open(const std::string& norm_path, AVS_FILE open_resu
 		nullopt,
 		nullopt
 	};
-	cleanup_map[norm_path] = cleanup;
-	open_file_map[open_result] = norm_path;
+	cleanup_map[path] = cleanup;
+	open_file_map[open_result] = path;
 
 	mangling_mtx.unlock();
 }
@@ -179,7 +179,7 @@ void ramfs_demangler_demangle_if_possible(std::string& raw_path) {
 
 	auto search = mangling_map.longest_prefix(raw_path);
 	if (search != mangling_map.end()) {
-		//log_verbose("can demangle %s to %s", search.key().c_str(), search->c_str());
+		// log_verbose("can demangle %s to %s", search.key().c_str(), search->c_str());
 		string_replace(raw_path, search.key().c_str(), search->c_str());
 	}