From aa6f1491d5a15e2da44a9516a9e847c1482625db Mon Sep 17 00:00:00 2001 From: Will Toohey Date: Wed, 13 Sep 2023 23:43:01 +1000 Subject: [PATCH] Support hooking pkfs file opens without unpacking --- src/hook.cpp | 269 ++++++++++++++++++++++++++++------------ src/hook.h | 56 +++++++++ src/imagefs.cpp | 27 ++-- src/imagefs.hpp | 19 +-- src/modpath_handler.cpp | 3 +- src/modpath_handler.h | 2 +- src/utils.cpp | 6 +- src/utils.hpp | 3 +- 8 files changed, 276 insertions(+), 109 deletions(-) diff --git a/src/hook.cpp b/src/hook.cpp index 0c3a055..bf443e4 100644 --- a/src/hook.cpp +++ b/src/hook.cpp @@ -1,10 +1,4 @@ -#include -#include #include -#include -#include -#include -#include #include "hook.h" @@ -42,7 +36,113 @@ using std::string; // debugging //#define ALWAYS_CACHE +// used by pkfs hooks - we don't want to hook avs_fs_open if we just hooked pkfs +thread_local static bool inside_pkfs_hook; + +unsigned int (*pkfs_fs_open)(const char *path); +unsigned int (*pkfs_fs_fstat)(unsigned int f, struct avs_stat *stat); +unsigned int (*pkfs_fs_read)(unsigned int f, void *buf, int sz); +unsigned int (*pkfs_fs_close)(unsigned int f); + +class AvsHookFile : public HookFile { + using HookFile::HookFile; + + std::optional> load_to_vec() override { + AVS_FILE f = avs_fs_open(get_path_to_open().c_str(), avs_open_mode_read(), 420); + if (f >= 0) { + auto ret = avs_file_to_vec(f); + avs_fs_close(f); + return ret; + } else { + return nullopt; + } + } +}; +class AvsOpenHookFile : public AvsHookFile { + private: + uint16_t mode; + int flags; + + public: + AvsOpenHookFile(const std::string path, const std::string norm_path, uint16_t mode, int flags) + : AvsHookFile(path, norm_path) + , mode(mode) + , flags(flags) + {} + + bool ramfs_demangle() override {return true;}; + + uint32_t call_real() override { + log_if_modfile(); + return (uint32_t)avs_fs_open(get_path_to_open().c_str(), mode, flags); + } +}; +class AvsLstatHookFile : public AvsHookFile { + private: + struct avs_stat *st; + + public: + AvsLstatHookFile(const std::string path, const std::string norm_path, struct avs_stat *st) + : AvsHookFile(path, norm_path) + , st(st) + {} + + uint32_t call_real() override { + log_if_modfile(); + return (uint32_t)avs_fs_lstat(get_path_to_open().c_str(), st); + } +}; + +class AvsConvertPathHookFile : public AvsHookFile { + private: + char *dest_name; + + public: + AvsConvertPathHookFile(const std::string path, const std::string norm_path, char *dest_name) + : AvsHookFile(path, norm_path) + , dest_name(dest_name) + {} + + uint32_t call_real() override { + log_if_modfile(); + return (uint32_t)avs_fs_convert_path(dest_name, get_path_to_open().c_str()); + } +}; + +class PkfsHookFile : public HookFile { + public: + PkfsHookFile(const std::string path, const std::string norm_path) + : HookFile(path, norm_path) + {} + + bool ramfs_demangle() {return false;}; + + uint32_t call_real() override { + log_if_modfile(); + // note that this also hides the avs_fs_open of the pakfile holding a + // particular file - acceptable compromise IMO + inside_pkfs_hook = true; + auto ret = pkfs_fs_open(get_path_to_open().c_str()); + inside_pkfs_hook = false; + return ret; + } + + std::optional> load_to_vec() override { + AVS_FILE f = pkfs_fs_open(get_path_to_open().c_str()); + if (f != 0) { + avs_stat stat = {0}; // file type is shared! + pkfs_fs_fstat(f, &stat); + std::vector ret; + ret.resize(stat.filesize); + pkfs_fs_read(f, &ret[0], stat.filesize); + pkfs_fs_close(f); + return ret; + } else { + return nullopt; + } + } +}; // this should probably be part of the modpath h/cpp void list_pngs_onefolder(string_set &names, string const& folder) { @@ -75,11 +175,10 @@ string_set list_pngs(string const&folder) { return ret; } -void handle_texbin(string const& path, string const&norm_path, optional &mod_path) { +void handle_texbin(HookFile &file) { auto start = time(); - auto bin_orig_path = mod_path ? *mod_path : path; // may have layered pre-built mod .bin with extra PNGs - auto bin_mod_path = norm_path; + auto bin_mod_path = file.norm_path; // mod texbins strip the .bin off the end. This isn't consistent with the _ifs // used for ifs files, but it's consistent with gitadora-texbintool, the *only* // tool to extract .bin files currently. @@ -116,7 +215,7 @@ void handle_texbin(string const& path, string const&norm_path, optional return; } - string out = CACHE_FOLDER "/" + norm_path; + string out = CACHE_FOLDER "/" + file.norm_path; auto out_hashed = out + ".hashed"; uint8_t hash[MD5_LEN]; @@ -131,12 +230,12 @@ void handle_texbin(string const& path, string const&norm_path, optional } auto time_out = file_time(out.c_str()); - auto newest = file_time(bin_orig_path.c_str()); + auto newest = file_time(file.get_path_to_open().c_str()); for (auto &path : pngs_list) newest = std::max(newest, file_time(path.c_str())); // no need to merge - timestamps all up to date, dll not newer, files haven't been deleted if(time_out >= newest && time_out >= dll_time && memcmp(hash, cache_hash, sizeof(hash)) == 0) { - mod_path = out; + file.mod_path = out; log_misc("texbin cache up to date, skip"); return; } @@ -146,11 +245,9 @@ void handle_texbin(string const& path, string const&norm_path, optional // log_verbose(" memcmp(hash, cache_hash, sizeof(hash)) == 0 == %d", memcmp(hash, cache_hash, sizeof(hash)) == 0); Texbin texbin; - AVS_FILE f = avs_fs_open(bin_orig_path.c_str(), avs_open_mode_read(), 420); - if (f >= 0) { - auto orig_data = avs_file_to_vec(f); - avs_fs_close(f); - + auto _orig_data = file.load_to_vec(); + if (_orig_data) { + auto orig_data = *_orig_data; // one extra copy which *sucks* but whatever std::istringstream stream(string((char*)&orig_data[0], orig_data.size())); auto _texbin = Texbin::from_stream(stream); @@ -160,7 +257,7 @@ void handle_texbin(string const& path, string const&norm_path, optional } texbin = *_texbin; } else { - log_info("Found texbin mods but no original file, creating from scratch: \"%s\"", norm_path.c_str()); + log_info("Found texbin mods but no original file, creating from scratch: \"%s\"", file.norm_path.c_str()); } auto folder_terminator = out.rfind("/"); @@ -187,11 +284,42 @@ void handle_texbin(string const& path, string const&norm_path, optional fwrite(hash, 1, sizeof(hash), cache_hashfile); fclose(cache_hashfile); } - mod_path = out; + file.mod_path = out; log_misc("Texbin generation took %d ms", time() - start); } +uint32_t handle_file_open(HookFile &file) { + auto norm_copy = file.norm_path; + file.mod_path = find_first_modfile(norm_copy); + // mod ifs paths use _ifs, go one at a time for ifs-inside-ifs + while (!file.mod_path && string_replace_first(norm_copy, ".ifs", "_ifs")) { + file.mod_path = find_first_modfile(norm_copy); + } + + if(string_ends_with(file.path, ".xml")) { + merge_xmls(file); + } + + if(string_ends_with(file.path, ".bin")) { + handle_texbin(file); + } + + if (string_ends_with(file.path, "texturelist.xml")) { + parse_texturelist(file); + } + else { + handle_texture(file); + } + + auto ret = file.call_real(); + if(file.ramfs_demangle()) { + ramfs_demangler_on_fs_open(file.get_path_to_open(), ret); + } + // log_verbose("(returned %d)", ret); + return ret; +} + int hook_avs_fs_lstat(const char* name, struct avs_stat *st) { if (name == NULL) return avs_fs_lstat(name, st); @@ -199,26 +327,14 @@ int hook_avs_fs_lstat(const char* name, struct avs_stat *st) { log_verbose("statting %s", name); string path = name; - // can it be modded? + // can it be modded ie is it under /data ? auto norm_path = normalise_path(path); - if(!norm_path) + if (!norm_path) return avs_fs_lstat(name, st); + // unpack success + AvsLstatHookFile file(path, *norm_path, st); - auto mod_path = find_first_modfile(*norm_path); - - if (mod_path) { - log_verbose("Overwriting lstat"); - return avs_fs_lstat(mod_path->c_str(), st); - } - // called prior to avs_fs_open - if(string_ends_with(path.c_str(), ".bin")) { - handle_texbin(name, *norm_path, mod_path); - if(mod_path) { - log_verbose("Overwriting texbin lstat"); - return avs_fs_lstat(mod_path->c_str(), st); - } - } - return avs_fs_lstat(name, st); + return handle_file_open(file); } int hook_avs_fs_convert_path(char dest_name[256], const char *name) { @@ -228,18 +344,14 @@ int hook_avs_fs_convert_path(char dest_name[256], const char *name) { log_verbose("convert_path %s", name); string path = name; - // can it be modded? + // can it be modded ie is it under /data ? auto norm_path = normalise_path(path); if (!norm_path) return avs_fs_convert_path(dest_name, name); + // unpack success + AvsConvertPathHookFile file(path, *norm_path, dest_name); - auto mod_path = find_first_modfile(*norm_path); - - if (mod_path) { - log_verbose("Overwriting convert_path"); - return avs_fs_convert_path(dest_name, mod_path->c_str()); - } - return avs_fs_convert_path(dest_name, name); + return handle_file_open(file); } int hook_avs_fs_mount(const char* mountpoint, const char* fsroot, const char* fstype, const char* args) { @@ -255,7 +367,7 @@ size_t hook_avs_fs_read(AVS_FILE context, void* bytes, size_t nbytes) { } AVS_FILE hook_avs_fs_open(const char* name, uint16_t mode, int flags) { - if(name == NULL) + if(name == NULL || inside_pkfs_hook) return avs_fs_open(name, mode, flags); log_verbose("opening %s mode %d flags %d", name, mode, flags); // only touch reads - new AVS has bitflags (R=1,W=2), old AVS has enum (R=0,W=1,RW=2) @@ -263,45 +375,30 @@ AVS_FILE hook_avs_fs_open(const char* name, uint16_t mode, int flags) { return avs_fs_open(name, mode, flags); } string path = name; - string orig_path = name; // can it be modded ie is it under /data ? - auto _norm_path = normalise_path(path); - if (!_norm_path) + auto norm_path = normalise_path(path); + if (!norm_path) return avs_fs_open(name, mode, flags); // unpack success - auto norm_path = *_norm_path; - - auto mod_path = find_first_modfile(norm_path); - // mod ifs paths use _ifs, go one at a time for ifs-inside-ifs - while (!mod_path && string_replace_first(norm_path, ".ifs", "_ifs")) { - mod_path = find_first_modfile(norm_path); - } + AvsOpenHookFile file(path, *norm_path, mode, flags); - if(string_ends_with(path.c_str(), ".xml")) { - merge_xmls(orig_path, norm_path, mod_path); - } + return handle_file_open(file); +} - if(string_ends_with(path.c_str(), ".bin")) { - handle_texbin(orig_path, norm_path, mod_path); - } +unsigned int hook_pkfs_open(const char *name) { + log_verbose("pkfs_open %s", name); - if (string_ends_with(path.c_str(), "texturelist.xml")) { - parse_texturelist(orig_path, norm_path, mod_path); - } - else { - handle_texture(norm_path, mod_path); - } + string path = name; - if (mod_path) { - log_info("Using %s", mod_path->c_str()); - } + // can it be modded ie is it under /data ? + auto norm_path = normalise_path(path); + if (!norm_path) + return pkfs_fs_open(name); + // unpack success + PkfsHookFile file(path, *norm_path); - auto to_open = mod_path ? *mod_path : orig_path; - auto ret = avs_fs_open(to_open.c_str(), mode, flags); - ramfs_demangler_on_fs_open(to_open, ret); - // log_verbose("(returned %d)", ret); - return ret; + return handle_file_open(file); } extern "C" { @@ -336,7 +433,27 @@ extern "C" { cache_mods(); - //jb_texhook_init(); + // hook pkfs, not big enough to be its own file + if(MH_CreateHookApi(L"libpackfs.dll", "?pkfs_fs_open@@YAIPBD@Z", (LPVOID)&hook_pkfs_open, (LPVOID*)&pkfs_fs_open) == MH_OK) { + auto mod = GetModuleHandleA("libpackfs.dll"); + pkfs_fs_fstat = (decltype(pkfs_fs_fstat))GetProcAddress(mod, "?pkfs_fs_fstat@@YAEIPAUT_AVS_FS_STAT@@@Z"); + pkfs_fs_read = (decltype(pkfs_fs_read))GetProcAddress(mod, "?pkfs_fs_read@@YAHIPAXH@Z"); + pkfs_fs_close = (decltype(pkfs_fs_close))GetProcAddress(mod, "?pkfs_fs_close@@YAHI@Z"); + } else if(MH_CreateHookApi(L"pkfs.dll", "pkfs_fs_open", (LPVOID)&hook_pkfs_open, (LPVOID*)&pkfs_fs_open) == MH_OK) { + // jubeat DLL has no mangling - only one of these will succeed (if at all) + auto mod = GetModuleHandleA("pkfs.dll"); + pkfs_fs_fstat = (decltype(pkfs_fs_fstat))GetProcAddress(mod, "pkfs_fs_fstat"); + pkfs_fs_read = (decltype(pkfs_fs_read))GetProcAddress(mod, "pkfs_fs_read"); + pkfs_fs_close = (decltype(pkfs_fs_close))GetProcAddress(mod, "pkfs_fs_close"); + } + + if(pkfs_fs_open) { + if(pkfs_fs_fstat && pkfs_fs_read && pkfs_fs_close) { + log_info("pkfs hooks activated"); + } else { + log_fatal("Couldn't fully init pkfs hook - open an issue!"); + } + } if (MH_EnableHook(MH_ALL_HOOKS) != MH_OK) { log_warning("Couldn't enable hooks"); diff --git a/src/hook.h b/src/hook.h index 4ab2806..0229908 100644 --- a/src/hook.h +++ b/src/hook.h @@ -1,7 +1,10 @@ #pragma once #include +#include +#include #include "avs.h" +#include "log.hpp" #include "utils.hpp" extern uint64_t dll_time; @@ -18,3 +21,56 @@ extern "C" { __declspec(dllexport) int init(void); extern HMODULE my_module; } + +// Used to simplify file opens - pkfs/avs_fs have different signatures, but +// otherwise have almost the exact same semantics and mod behaviour. It lets you +// replace a file's path, or read the original file (using the correct API) into +// a vector. +// NOTE: the APIs to read XML files (specifically prop_from_file_path and +// rapidxml_from_avs_filepath) still use avs_fs_open, because I can't find any +// evidence the games are using XMLs in a way that anybody would want to mod. +// May this decision not bite me later... +class HookFile { + public: + // The original path requested by the game + const std::string path; + // Regardless of how many prefixes, extraneous slashes, back/forward + // slashes, the normalised path is the canonical game-folder-relative path + // used to search for mods eg: + // graphics/ver03/cmn_sys.ifs + // data2/graphics/whatever.ifs + const std::string norm_path; + // If a mod has been found, this is its path. This can be used to overwrite + // an entire ifs, but also have a subsequent mod overwrite an individual + // file inside that ifs + std::optional mod_path; + + // Using the mod_path (if available) or path, call the original open func. + // The return type is conveniently the same width for both pkfs and avs_fs. + // Will need rework if another opening method is found... + virtual uint32_t call_real() = 0; + + // Load the mod_path (if available) or path into a vector + virtual std::optional> load_to_vec() = 0; + + const std::string& get_path_to_open() { + return mod_path ? *mod_path : path; + } + + void log_if_modfile() { + if (mod_path) { + log_info("Using %s", mod_path->c_str()); + } + } + + // Whether this should be included in the ramfs demangler machinery (yes for + // avs/pkfs_open, no for lstat/convert_path) + virtual bool ramfs_demangle() {return false;}; + + HookFile(const std::string path, const std::string norm_path) + : path(path) + , norm_path(norm_path) + , mod_path(std::nullopt) + {} + virtual ~HookFile() {} +}; diff --git a/src/imagefs.cpp b/src/imagefs.cpp index 2e85151..fdb39a6 100644 --- a/src/imagefs.cpp +++ b/src/imagefs.cpp @@ -10,7 +10,6 @@ #include "3rd_party/rapidxml_print.hpp" #include "avs.h" -#include "hook.h" #include "log.hpp" #include "modpath_handler.h" #include "texture_packer.h" @@ -181,11 +180,11 @@ bool add_images_to_list(string_set &extra_pngs, rapidxml::xml_node<> *texturelis return true; } -void parse_texturelist(string const&path, string const&norm_path, optional &mod_path) { +void parse_texturelist(HookFile &file) { bool prop_was_rewritten = false; // get a reasonable base path - auto ifs_path = norm_path; + auto ifs_path = file.norm_path; // truncate ifs_path.resize(ifs_path.size() - strlen("/tex/texturelist.xml")); //log_misc("Reading ifs %s", ifs_path.c_str()); @@ -198,7 +197,7 @@ void parse_texturelist(string const&path, string const&norm_path, optional texturelist; auto success = rapidxml_from_avs_filepath(path_to_open, texturelist, texturelist); if (!success) @@ -300,7 +299,7 @@ void parse_texturelist(string const&path, string const&norm_path, optional &mod_path) { +void handle_texture(HookFile &file) { ifs_textures_mtx.lock(); - auto tex_search = ifs_textures.find(norm_path); + auto tex_search = ifs_textures.find(file.norm_path); if (tex_search == ifs_textures.end()) { ifs_textures_mtx.unlock(); return; @@ -437,27 +436,27 @@ void handle_texture(string const&norm_path, optional &mod_path) { log_verbose("Mapped file %s found!", png_path->c_str()); if (cache_texture(*png_path, tex)) { - mod_path = tex.cache_file(); + file.mod_path = tex.cache_file(); } return; } -void merge_xmls(string const& path, string const&norm_path, optional &mod_path) { +void merge_xmls(HookFile &file) { auto start = time(); // initialize since we're GOTO-ing like naughty people string out; string out_folder; rapidxml::xml_document<> merged_xml; - auto merge_path = norm_path; + auto merge_path = file.norm_path; string_replace(merge_path, ".xml", ".merged.xml"); auto to_merge = find_all_modfile(merge_path); // nothing to do... if (to_merge.size() == 0) return; - auto starting = mod_path ? *mod_path : path; - out = CACHE_FOLDER "/" + norm_path; + auto starting = file.get_path_to_open(); + out = CACHE_FOLDER "/" + file.norm_path; auto out_hashed = out + ".hashed"; uint8_t hash[MD5_LEN]; @@ -477,7 +476,7 @@ void merge_xmls(string const& path, string const&norm_path, optional &mo newest = std::max(newest, file_time(path.c_str())); // no need to merge - timestamps all up to date, dll not newer, files haven't been deleted if(time_out >= newest && time_out >= dll_time && memcmp(hash, cache_hash, sizeof(hash)) == 0) { - mod_path = out; + file.mod_path = out; return; } @@ -517,7 +516,7 @@ void merge_xmls(string const& path, string const&norm_path, optional &mo fwrite(hash, 1, sizeof(hash), cache_hashfile); fclose(cache_hashfile); } - mod_path = out; + file.mod_path = out; log_misc("Merge took %d ms", time() - start); } diff --git a/src/imagefs.hpp b/src/imagefs.hpp index a2ed0dd..03b6b49 100644 --- a/src/imagefs.hpp +++ b/src/imagefs.hpp @@ -1,16 +1,5 @@ -#include -#include +#include "hook.h" -void handle_texture( - std::string const&norm_path, - std::optional &mod_path -); -void parse_texturelist( - std::string const&path, - std::string const&norm_path, - std::optional &mod_path -); -void merge_xmls( - std::string const& path, - std::string const&norm_path, - std::optional &mod_path); +void handle_texture(HookFile &file); +void parse_texturelist(HookFile &file); +void merge_xmls(HookFile &file); diff --git a/src/modpath_handler.cpp b/src/modpath_handler.cpp index 8f98f5f..1eab060 100644 --- a/src/modpath_handler.cpp +++ b/src/modpath_handler.cpp @@ -81,7 +81,7 @@ void cache_mods(void) { static vector game_folders; static CriticalSectionLock game_folders_mtx; -optional normalise_path(string &path) { +optional normalise_path(const string &_path) { // one-off init if (game_folders.empty()) { game_folders_mtx.lock(); @@ -101,6 +101,7 @@ optional normalise_path(string &path) { // all access past here is read-only, don't use the mutex any more } + auto path = _path; ramfs_demangler_demangle_if_possible(path); auto data_pos = string_find_icase(path, "data/"); diff --git a/src/modpath_handler.h b/src/modpath_handler.h index 04ebac8..6bb5d8d 100644 --- a/src/modpath_handler.h +++ b/src/modpath_handler.h @@ -20,7 +20,7 @@ using std::vector; void cache_mods(void); vector available_mods(); // mutates source string to be all lowercase -optional normalise_path(string &path); +optional normalise_path(const string &path); optional find_first_modfile(const string &norm_path); optional find_first_modfolder(const string &norm_path); vector find_all_modfile(const string &norm_path); diff --git a/src/utils.cpp b/src/utils.cpp index f9d4e84..cc6c1ba 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -18,7 +18,7 @@ char* snprintf_auto(const char* fmt, ...) { return s; } -int string_ends_with(const char * str, const char * suffix) { +bool string_ends_with(const char * str, const char * suffix) { size_t str_len = strlen(str); size_t suffix_len = strlen(suffix); @@ -27,6 +27,10 @@ int string_ends_with(const char * str, const char * suffix) { (0 == _stricmp(str + (str_len - suffix_len), suffix)); } +bool string_ends_with(const std::string &str, const char * suffix) { + return string_ends_with(str.c_str(), suffix); +} + void string_replace(std::string &str, const char* from, const char* to) { auto to_len = strlen(to); auto from_len = strlen(from); diff --git a/src/utils.hpp b/src/utils.hpp index 1c05c52..14aa9fa 100644 --- a/src/utils.hpp +++ b/src/utils.hpp @@ -12,7 +12,8 @@ #define lenof(x) (sizeof(x) / sizeof(*x)) char* snprintf_auto(const char* fmt, ...); -int string_ends_with(const char * str, const char * suffix); +bool string_ends_with(const char * str, const char * suffix); +bool string_ends_with(const std::string &str, const char * suffix); // case insensitive void string_replace(std::string &str, const char* from, const char* to); // // case insensitive