From c1eb71fa3342ab44f7bf91648afc1e3a126c4485 Mon Sep 17 00:00:00 2001 From: Nicholas Estelami Date: Sun, 3 Apr 2022 15:48:44 -0400 Subject: [PATCH] Updated python script to use new "extract directory" mode. Additionally fixed oversight with audio files and bug in OTRGui. --- OTRExporter/OTRExporter/Main.cpp | 4 + OTRExporter/extract_assets.py | 99 ++----------------- OTRExporter/extract_assets_old.py | 125 ++++++++++++++++++++++++ OTRGui/src/game/game.cpp | 9 +- OTRGui/src/impl/extractor/extractor.cpp | 1 - ZAPDTR/ZAPD/Main.cpp | 12 --- ZAPDTR/ZAPD/ZFile.cpp | 26 +++++ 7 files changed, 167 insertions(+), 109 deletions(-) create mode 100644 OTRExporter/extract_assets_old.py diff --git a/OTRExporter/OTRExporter/Main.cpp b/OTRExporter/OTRExporter/Main.cpp index e62ed5ef4..a58f94ae3 100644 --- a/OTRExporter/OTRExporter/Main.cpp +++ b/OTRExporter/OTRExporter/Main.cpp @@ -78,6 +78,10 @@ static void ExporterProgramEnd() auto fileData = File::ReadAllBytes(item); otrArchive->AddFile(StringHelper::Split(item, "Extract\\")[1], (uintptr_t)fileData.data(), fileData.size()); } + + otrArchive->AddFile("Audiobank", (uintptr_t)Globals::Instance->GetBaseromFile("Audiobank").data(), Globals::Instance->GetBaseromFile("Audiobank").size()); + otrArchive->AddFile("Audioseq", (uintptr_t)Globals::Instance->GetBaseromFile("Audioseq").data(), Globals::Instance->GetBaseromFile("Audioseq").size()); + otrArchive->AddFile("Audiotable", (uintptr_t)Globals::Instance->GetBaseromFile("Audiotable").data(), Globals::Instance->GetBaseromFile("Audiotable").size()); } } diff --git a/OTRExporter/extract_assets.py b/OTRExporter/extract_assets.py index 2922bbf06..bb9ed177c 100755 --- a/OTRExporter/extract_assets.py +++ b/OTRExporter/extract_assets.py @@ -4,21 +4,11 @@ import argparse, json, os, signal, time, sys, shutil from multiprocessing import Pool, cpu_count, Event, Manager, ProcessError import shutil -def SignalHandler(sig, frame): - print(f'Signal {sig} received. Aborting...') - mainAbort.set() - # Don't exit immediately to update the extracted assets file. - -def BuildOTR(): - shutil.copyfile("baserom/Audiobank", "Extract/Audiobank") - shutil.copyfile("baserom/Audioseq", "Extract/Audioseq") - shutil.copyfile("baserom/Audiotable", "Extract/Audiotable") - +def BuildOTR(xmlPath): shutil.copytree("assets", "Extract/assets") execStr = "x64\\Release\\ZAPD.exe" if sys.platform == "win32" else "../ZAPD/ZAPD.out" - - execStr += " botr -se OTR" + execStr += " ed -i %s -b baserom.z64 -fl CFG\\filelists -o placeholder -osf placeholder -gsf 1 -rconf CFG/Config.xml -se OTR" % (xmlPath) print(execStr) exitValue = os.system(execStr) @@ -28,52 +18,12 @@ def BuildOTR(): print("Aborting...", file=os.sys.stderr) print("\n") -def ExtractFile(xmlPath, outputPath, outputSourcePath): - execStr = "x64\\Release\\ZAPD.exe" if sys.platform == "win32" else "../ZAPD/ZAPD.out" - execStr += " e -eh -i %s -b baserom/ -o %s -osf %s -gsf 1 -rconf CFG/Config.xml -se OTR" % (xmlPath, outputPath, outputSourcePath) - - if "overlays" in xmlPath: - execStr += " --static" - - print(execStr) - exitValue = os.system(execStr) - #exitValue = 0 - if exitValue != 0: - print("\n") - print("Error when extracting from file " + xmlPath, file=os.sys.stderr) - print("Aborting...", file=os.sys.stderr) - print("\n") - -def ExtractFunc(fullPath): - *pathList, xmlName = fullPath.split(os.sep) - objectName = os.path.splitext(xmlName)[0] - - outPath = os.path.join("..\\soh\\assets\\", *pathList[5:], objectName) - os.makedirs(outPath, exist_ok=True) - outSourcePath = outPath - - ExtractFile(fullPath, outPath, outSourcePath) - -def initializeWorker(abort, test): - global globalAbort - globalAbort = abort - - def main(): parser = argparse.ArgumentParser(description="baserom asset extractor") - parser.add_argument("-s", "--single", help="asset path relative to assets/, e.g. objects/gameplay_keep") - parser.add_argument("-f", "--force", help="Force the extraction of every xml instead of checking the touched ones.", action="store_true") - parser.add_argument("-u", "--unaccounted", help="Enables ZAPD unaccounted detector warning system.", action="store_true") parser.add_argument("-v", "--version", help="Sets game version.") args = parser.parse_args() - - global mainAbort - mainAbort = Event() - manager = Manager() - signal.signal(signal.SIGINT, SignalHandler) - - extractedAssetsTracker = manager.dict() - + + # TODO: Read from makerom file to automatically determine game version xmlVer = "GC_NMQ_D" if (args.version == "gc_pal_nmpq"): @@ -81,45 +31,10 @@ def main(): elif (args.version == "dbg_mq"): xmlVer = "GC_MQ_D" - asset_path = args.single - if asset_path is not None: - fullPath = os.path.join("..\\soh\\assets", "xml", asset_path + ".xml") - if not os.path.exists(fullPath): - print(f"Error. File {fullPath} doesn't exists.", file=os.sys.stderr) - exit(1) - - ExtractFunc(fullPath) - else: - extract_text_path = "assets/text/message_data.h" - if os.path.isfile(extract_text_path): - extract_text_path = None - extract_staff_text_path = "assets/text/message_data_staff.h" - if os.path.isfile(extract_staff_text_path): - extract_staff_text_path = None - - xmlFiles = [] - for currentPath, _, files in os.walk(os.path.join("..\\soh\\assets\\xml\\", xmlVer)): - for file in files: - fullPath = os.path.join(currentPath, file) - if file.endswith(".xml"): - xmlFiles.append(fullPath) - - try: - numCores = 2 - print("Extracting assets with " + str(numCores) + " CPU cores.") - with Pool(numCores, initializer=initializeWorker, initargs=(mainAbort, 0)) as p: - p.map(ExtractFunc, xmlFiles) - except Exception as e: - print("Warning: Multiprocessing exception ocurred.", file=os.sys.stderr) - print("Disabling mutliprocessing.", file=os.sys.stderr) - - initializeWorker(mainAbort, 0) - for singlePath in xmlFiles: - ExtractFunc(singlePath) - - - BuildOTR() + if (os.path.exists("Extract")): shutil.rmtree("Extract") + + BuildOTR("..\\soh\\assets\\xml\\" + xmlVer + "\\") if __name__ == "__main__": main() \ No newline at end of file diff --git a/OTRExporter/extract_assets_old.py b/OTRExporter/extract_assets_old.py new file mode 100644 index 000000000..2922bbf06 --- /dev/null +++ b/OTRExporter/extract_assets_old.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 + +import argparse, json, os, signal, time, sys, shutil +from multiprocessing import Pool, cpu_count, Event, Manager, ProcessError +import shutil + +def SignalHandler(sig, frame): + print(f'Signal {sig} received. Aborting...') + mainAbort.set() + # Don't exit immediately to update the extracted assets file. + +def BuildOTR(): + shutil.copyfile("baserom/Audiobank", "Extract/Audiobank") + shutil.copyfile("baserom/Audioseq", "Extract/Audioseq") + shutil.copyfile("baserom/Audiotable", "Extract/Audiotable") + + shutil.copytree("assets", "Extract/assets") + + execStr = "x64\\Release\\ZAPD.exe" if sys.platform == "win32" else "../ZAPD/ZAPD.out" + + execStr += " botr -se OTR" + + print(execStr) + exitValue = os.system(execStr) + if exitValue != 0: + print("\n") + print("Error when building the OTR file...", file=os.sys.stderr) + print("Aborting...", file=os.sys.stderr) + print("\n") + +def ExtractFile(xmlPath, outputPath, outputSourcePath): + execStr = "x64\\Release\\ZAPD.exe" if sys.platform == "win32" else "../ZAPD/ZAPD.out" + execStr += " e -eh -i %s -b baserom/ -o %s -osf %s -gsf 1 -rconf CFG/Config.xml -se OTR" % (xmlPath, outputPath, outputSourcePath) + + if "overlays" in xmlPath: + execStr += " --static" + + print(execStr) + exitValue = os.system(execStr) + #exitValue = 0 + if exitValue != 0: + print("\n") + print("Error when extracting from file " + xmlPath, file=os.sys.stderr) + print("Aborting...", file=os.sys.stderr) + print("\n") + +def ExtractFunc(fullPath): + *pathList, xmlName = fullPath.split(os.sep) + objectName = os.path.splitext(xmlName)[0] + + outPath = os.path.join("..\\soh\\assets\\", *pathList[5:], objectName) + os.makedirs(outPath, exist_ok=True) + outSourcePath = outPath + + ExtractFile(fullPath, outPath, outSourcePath) + +def initializeWorker(abort, test): + global globalAbort + globalAbort = abort + + +def main(): + parser = argparse.ArgumentParser(description="baserom asset extractor") + parser.add_argument("-s", "--single", help="asset path relative to assets/, e.g. objects/gameplay_keep") + parser.add_argument("-f", "--force", help="Force the extraction of every xml instead of checking the touched ones.", action="store_true") + parser.add_argument("-u", "--unaccounted", help="Enables ZAPD unaccounted detector warning system.", action="store_true") + parser.add_argument("-v", "--version", help="Sets game version.") + args = parser.parse_args() + + global mainAbort + mainAbort = Event() + manager = Manager() + signal.signal(signal.SIGINT, SignalHandler) + + extractedAssetsTracker = manager.dict() + + xmlVer = "GC_NMQ_D" + + if (args.version == "gc_pal_nmpq"): + xmlVer = "GC_NMQ_PAL_F" + elif (args.version == "dbg_mq"): + xmlVer = "GC_MQ_D" + + asset_path = args.single + if asset_path is not None: + fullPath = os.path.join("..\\soh\\assets", "xml", asset_path + ".xml") + if not os.path.exists(fullPath): + print(f"Error. File {fullPath} doesn't exists.", file=os.sys.stderr) + exit(1) + + ExtractFunc(fullPath) + else: + extract_text_path = "assets/text/message_data.h" + if os.path.isfile(extract_text_path): + extract_text_path = None + extract_staff_text_path = "assets/text/message_data_staff.h" + if os.path.isfile(extract_staff_text_path): + extract_staff_text_path = None + + xmlFiles = [] + for currentPath, _, files in os.walk(os.path.join("..\\soh\\assets\\xml\\", xmlVer)): + for file in files: + fullPath = os.path.join(currentPath, file) + if file.endswith(".xml"): + xmlFiles.append(fullPath) + + try: + numCores = 2 + print("Extracting assets with " + str(numCores) + " CPU cores.") + with Pool(numCores, initializer=initializeWorker, initargs=(mainAbort, 0)) as p: + p.map(ExtractFunc, xmlFiles) + except Exception as e: + print("Warning: Multiprocessing exception ocurred.", file=os.sys.stderr) + print("Disabling mutliprocessing.", file=os.sys.stderr) + + initializeWorker(mainAbort, 0) + for singlePath in xmlFiles: + ExtractFunc(singlePath) + + + BuildOTR() + shutil.rmtree("Extract") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/OTRGui/src/game/game.cpp b/OTRGui/src/game/game.cpp index 2f5512385..88bd7201f 100644 --- a/OTRGui/src/game/game.cpp +++ b/OTRGui/src/game/game.cpp @@ -87,10 +87,11 @@ void ExtractRom() if (MoonUtils::exists("Extract")) MoonUtils::rm("Extract"); MoonUtils::mkdir("Extract"); - MoonUtils::copy("tmp/baserom/Audiobank", "Extract/Audiobank"); - MoonUtils::copy("tmp/baserom/Audioseq", "Extract/Audioseq"); - MoonUtils::copy("tmp/baserom/Audiotable", "Extract/Audiotable"); - MoonUtils::copy("tmp/baserom/version", "Extract/version"); + //MoonUtils::copy("tmp/baserom/Audiobank", "Extract/Audiobank"); + //MoonUtils::copy("tmp/baserom/Audioseq", "Extract/Audioseq"); + //MoonUtils::copy("tmp/baserom/Audiotable", "Extract/Audiotable"); + //MoonUtils::copy("tmp/baserom/version", "Extract/version"); + MoonUtils::write("Extract/version", (char*)&version.crc, sizeof(version.crc)); MoonUtils::copy("assets/game/", "Extract/assets/"); diff --git a/OTRGui/src/impl/extractor/extractor.cpp b/OTRGui/src/impl/extractor/extractor.cpp index e28fa5513..a2eab1bb6 100644 --- a/OTRGui/src/impl/extractor/extractor.cpp +++ b/OTRGui/src/impl/extractor/extractor.cpp @@ -83,7 +83,6 @@ void startWorker(RomVersion version) { Util::write("tmp/baserom/version", (char*)&version.crc, sizeof(version.crc)); - if (oldExtractMode) { std::vector files; diff --git a/ZAPDTR/ZAPD/Main.cpp b/ZAPDTR/ZAPD/Main.cpp index 1a99d346a..dd53b9c67 100644 --- a/ZAPDTR/ZAPD/Main.cpp +++ b/ZAPDTR/ZAPD/Main.cpp @@ -387,18 +387,6 @@ int main(int argc, char* argv[]) { BuildAssetBlob(Globals::Instance->inputPath, Globals::Instance->outputPath); } - /* - else if (fileMode == ZFileMode::BuildOverlay) - { - ZOverlay* overlay = - ZOverlay::FromBuild(Path::GetDirectoryName(Globals::Instance->inputPath), - Path::GetDirectoryName(Globals::Instance->cfgPath)); - - if (overlay != nullptr) - File::WriteAllText(Globals::Instance->outputPath.string(), - overlay->GetSourceOutputCode("")); - } - */ if (exporterSet != nullptr && exporterSet->endProgramFunc != nullptr) exporterSet->endProgramFunc(); diff --git a/ZAPDTR/ZAPD/ZFile.cpp b/ZAPDTR/ZAPD/ZFile.cpp index b706c1914..9ff7a6823 100644 --- a/ZAPDTR/ZAPD/ZFile.cpp +++ b/ZAPDTR/ZAPD/ZFile.cpp @@ -823,6 +823,32 @@ void ZFile::GenerateSourceHeaderFiles() if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory) File::WriteAllText(headerFilename, formatter.GetOutput()); + else if (Globals::Instance->sourceOutputPath != "") + { + std::string xmlPath = xmlFilePath.string(); + xmlPath = StringHelper::Replace(xmlPath, "\\", "/"); + auto pathList = StringHelper::Split(xmlPath, "/"); + std::string outPath = ""; + + for (int i = 0; i < 3; i++) + outPath += pathList[i] + "/"; + + for (int i = 5; i < pathList.size(); i++) + { + if (i == pathList.size() - 1) + { + outPath += Path::GetFileNameWithoutExtension(pathList[i]) + "/"; + outPath += outName.string() + ".h"; + } + else + outPath += pathList[i]; + + if (i < pathList.size() - 1) + outPath += "/"; + } + + File::WriteAllText(outPath, formatter.GetOutput()); + } } std::string ZFile::GetHeaderInclude() const