Updated python script to use new "extract directory" mode.

Additionally fixed oversight with audio files and bug in OTRGui.

OTRExporter/OTRExporter/Main.cpp

@@ -78,6 +78,10 @@ static void ExporterProgramEnd()
 			auto fileData = File::ReadAllBytes(item);
 			otrArchive->AddFile(StringHelper::Split(item, "Extract\\")[1], (uintptr_t)fileData.data(), fileData.size());
 		}
+
+		otrArchive->AddFile("Audiobank", (uintptr_t)Globals::Instance->GetBaseromFile("Audiobank").data(), Globals::Instance->GetBaseromFile("Audiobank").size());
+		otrArchive->AddFile("Audioseq", (uintptr_t)Globals::Instance->GetBaseromFile("Audioseq").data(), Globals::Instance->GetBaseromFile("Audioseq").size());
+		otrArchive->AddFile("Audiotable", (uintptr_t)Globals::Instance->GetBaseromFile("Audiotable").data(), Globals::Instance->GetBaseromFile("Audiotable").size());
 	}
 }
 

OTRExporter/extract_assets.py

@@ -4,21 +4,11 @@ import argparse, json, os, signal, time, sys, shutil
 from multiprocessing import Pool, cpu_count, Event, Manager, ProcessError
 import shutil
 
-def SignalHandler(sig, frame):
+def BuildOTR(xmlPath):
-    print(f'Signal {sig} received. Aborting...')
-    mainAbort.set()
-    # Don't exit immediately to update the extracted assets file.
-
-def BuildOTR():
-    shutil.copyfile("baserom/Audiobank", "Extract/Audiobank")
-    shutil.copyfile("baserom/Audioseq", "Extract/Audioseq")
-    shutil.copyfile("baserom/Audiotable", "Extract/Audiotable")
-
     shutil.copytree("assets", "Extract/assets")
 
     execStr = "x64\\Release\\ZAPD.exe" if sys.platform == "win32" else "../ZAPD/ZAPD.out"
-
+    execStr += " ed -i %s -b baserom.z64 -fl CFG\\filelists -o placeholder -osf placeholder -gsf 1 -rconf CFG/Config.xml -se OTR" % (xmlPath)
-    execStr += " botr -se OTR"
 
     print(execStr)
     exitValue = os.system(execStr)
@@ -28,52 +18,12 @@ def BuildOTR():
         print("Aborting...", file=os.sys.stderr)
         print("\n")
 
-def ExtractFile(xmlPath, outputPath, outputSourcePath):
-    execStr = "x64\\Release\\ZAPD.exe" if sys.platform == "win32" else "../ZAPD/ZAPD.out"
-    execStr += " e -eh -i %s -b baserom/ -o %s -osf %s -gsf 1 -rconf CFG/Config.xml -se OTR" % (xmlPath, outputPath, outputSourcePath)
-
-    if "overlays" in xmlPath:
-        execStr += " --static"
-
-    print(execStr)
-    exitValue = os.system(execStr)
-    #exitValue = 0
-    if exitValue != 0:
-        print("\n")
-        print("Error when extracting from file " + xmlPath, file=os.sys.stderr)
-        print("Aborting...", file=os.sys.stderr)
-        print("\n")
-
-def ExtractFunc(fullPath):
-    *pathList, xmlName = fullPath.split(os.sep)
-    objectName = os.path.splitext(xmlName)[0]
-
-    outPath = os.path.join("..\\soh\\assets\\", *pathList[5:], objectName)
-    os.makedirs(outPath, exist_ok=True)
-    outSourcePath = outPath
-
-    ExtractFile(fullPath, outPath, outSourcePath)
-
-def initializeWorker(abort, test):
-    global globalAbort
-    globalAbort = abort
-
-
 def main():
     parser = argparse.ArgumentParser(description="baserom asset extractor")
-    parser.add_argument("-s", "--single", help="asset path relative to assets/, e.g. objects/gameplay_keep")
-    parser.add_argument("-f", "--force", help="Force the extraction of every xml instead of checking the touched ones.", action="store_true")
-    parser.add_argument("-u", "--unaccounted", help="Enables ZAPD unaccounted detector warning system.", action="store_true")
     parser.add_argument("-v", "--version", help="Sets game version.")
     args = parser.parse_args()
     
-    global mainAbort
+    # TODO: Read from makerom file to automatically determine game version
-    mainAbort = Event()
-    manager = Manager()
-    signal.signal(signal.SIGINT, SignalHandler)
-
-    extractedAssetsTracker = manager.dict()
-
     xmlVer = "GC_NMQ_D"
 
     if (args.version == "gc_pal_nmpq"):
@@ -81,45 +31,10 @@ def main():
     elif (args.version == "dbg_mq"):
         xmlVer = "GC_MQ_D"
 
-    asset_path = args.single
+    if (os.path.exists("Extract")):
-    if asset_path is not None:
-        fullPath = os.path.join("..\\soh\\assets", "xml", asset_path + ".xml")
-        if not os.path.exists(fullPath):
-            print(f"Error. File {fullPath} doesn't exists.", file=os.sys.stderr)
-            exit(1)
-
-        ExtractFunc(fullPath)
-    else:
-        extract_text_path = "assets/text/message_data.h"
-        if os.path.isfile(extract_text_path):
-            extract_text_path = None
-        extract_staff_text_path = "assets/text/message_data_staff.h"
-        if os.path.isfile(extract_staff_text_path):
-            extract_staff_text_path = None
-
-        xmlFiles = []
-        for currentPath, _, files in os.walk(os.path.join("..\\soh\\assets\\xml\\", xmlVer)):
-            for file in files:
-                fullPath = os.path.join(currentPath, file)
-                if file.endswith(".xml"):
-                    xmlFiles.append(fullPath)
-
-        try:
-            numCores = 2
-            print("Extracting assets with " + str(numCores) + " CPU cores.")
-            with Pool(numCores, initializer=initializeWorker, initargs=(mainAbort, 0)) as p:
-                p.map(ExtractFunc, xmlFiles)
-        except Exception as e:
-            print("Warning: Multiprocessing exception ocurred.", file=os.sys.stderr)
-            print("Disabling mutliprocessing.", file=os.sys.stderr)
-
-            initializeWorker(mainAbort, 0)
-            for singlePath in xmlFiles:
-                ExtractFunc(singlePath)
-
-
-        BuildOTR()
         shutil.rmtree("Extract")
     
+    BuildOTR("..\\soh\\assets\\xml\\" + xmlVer + "\\")
+
 if __name__ == "__main__":
     main()

OTRExporter/extract_assets_old.py

@@ -0,0 +1,125 @@
+#!/usr/bin/env python3
+
+import argparse, json, os, signal, time, sys, shutil
+from multiprocessing import Pool, cpu_count, Event, Manager, ProcessError
+import shutil
+
+def SignalHandler(sig, frame):
+    print(f'Signal {sig} received. Aborting...')
+    mainAbort.set()
+    # Don't exit immediately to update the extracted assets file.
+
+def BuildOTR():
+    shutil.copyfile("baserom/Audiobank", "Extract/Audiobank")
+    shutil.copyfile("baserom/Audioseq", "Extract/Audioseq")
+    shutil.copyfile("baserom/Audiotable", "Extract/Audiotable")
+
+    shutil.copytree("assets", "Extract/assets")
+
+    execStr = "x64\\Release\\ZAPD.exe" if sys.platform == "win32" else "../ZAPD/ZAPD.out"
+
+    execStr += " botr -se OTR"
+
+    print(execStr)
+    exitValue = os.system(execStr)
+    if exitValue != 0:
+        print("\n")
+        print("Error when building the OTR file...", file=os.sys.stderr)
+        print("Aborting...", file=os.sys.stderr)
+        print("\n")
+
+def ExtractFile(xmlPath, outputPath, outputSourcePath):
+    execStr = "x64\\Release\\ZAPD.exe" if sys.platform == "win32" else "../ZAPD/ZAPD.out"
+    execStr += " e -eh -i %s -b baserom/ -o %s -osf %s -gsf 1 -rconf CFG/Config.xml -se OTR" % (xmlPath, outputPath, outputSourcePath)
+
+    if "overlays" in xmlPath:
+        execStr += " --static"
+
+    print(execStr)
+    exitValue = os.system(execStr)
+    #exitValue = 0
+    if exitValue != 0:
+        print("\n")
+        print("Error when extracting from file " + xmlPath, file=os.sys.stderr)
+        print("Aborting...", file=os.sys.stderr)
+        print("\n")
+
+def ExtractFunc(fullPath):
+    *pathList, xmlName = fullPath.split(os.sep)
+    objectName = os.path.splitext(xmlName)[0]
+
+    outPath = os.path.join("..\\soh\\assets\\", *pathList[5:], objectName)
+    os.makedirs(outPath, exist_ok=True)
+    outSourcePath = outPath
+
+    ExtractFile(fullPath, outPath, outSourcePath)
+
+def initializeWorker(abort, test):
+    global globalAbort
+    globalAbort = abort
+
+
+def main():
+    parser = argparse.ArgumentParser(description="baserom asset extractor")
+    parser.add_argument("-s", "--single", help="asset path relative to assets/, e.g. objects/gameplay_keep")
+    parser.add_argument("-f", "--force", help="Force the extraction of every xml instead of checking the touched ones.", action="store_true")
+    parser.add_argument("-u", "--unaccounted", help="Enables ZAPD unaccounted detector warning system.", action="store_true")
+    parser.add_argument("-v", "--version", help="Sets game version.")
+    args = parser.parse_args()
+
+    global mainAbort
+    mainAbort = Event()
+    manager = Manager()
+    signal.signal(signal.SIGINT, SignalHandler)
+
+    extractedAssetsTracker = manager.dict()
+
+    xmlVer = "GC_NMQ_D"
+
+    if (args.version == "gc_pal_nmpq"):
+        xmlVer = "GC_NMQ_PAL_F"
+    elif (args.version == "dbg_mq"):
+        xmlVer = "GC_MQ_D"
+
+    asset_path = args.single
+    if asset_path is not None:
+        fullPath = os.path.join("..\\soh\\assets", "xml", asset_path + ".xml")
+        if not os.path.exists(fullPath):
+            print(f"Error. File {fullPath} doesn't exists.", file=os.sys.stderr)
+            exit(1)
+
+        ExtractFunc(fullPath)
+    else:
+        extract_text_path = "assets/text/message_data.h"
+        if os.path.isfile(extract_text_path):
+            extract_text_path = None
+        extract_staff_text_path = "assets/text/message_data_staff.h"
+        if os.path.isfile(extract_staff_text_path):
+            extract_staff_text_path = None
+
+        xmlFiles = []
+        for currentPath, _, files in os.walk(os.path.join("..\\soh\\assets\\xml\\", xmlVer)):
+            for file in files:
+                fullPath = os.path.join(currentPath, file)
+                if file.endswith(".xml"):
+                    xmlFiles.append(fullPath)
+
+        try:
+            numCores = 2
+            print("Extracting assets with " + str(numCores) + " CPU cores.")
+            with Pool(numCores, initializer=initializeWorker, initargs=(mainAbort, 0)) as p:
+                p.map(ExtractFunc, xmlFiles)
+        except Exception as e:
+            print("Warning: Multiprocessing exception ocurred.", file=os.sys.stderr)
+            print("Disabling mutliprocessing.", file=os.sys.stderr)
+
+            initializeWorker(mainAbort, 0)
+            for singlePath in xmlFiles:
+                ExtractFunc(singlePath)
+
+
+        BuildOTR()
+        shutil.rmtree("Extract")
+
+if __name__ == "__main__":
+    main()

OTRGui/src/game/game.cpp

@@ -87,10 +87,11 @@ void ExtractRom()
 		if (MoonUtils::exists("Extract")) MoonUtils::rm("Extract");
 
 		MoonUtils::mkdir("Extract");
-		MoonUtils::copy("tmp/baserom/Audiobank", "Extract/Audiobank");
+		//MoonUtils::copy("tmp/baserom/Audiobank", "Extract/Audiobank");
-		MoonUtils::copy("tmp/baserom/Audioseq", "Extract/Audioseq");
+		//MoonUtils::copy("tmp/baserom/Audioseq", "Extract/Audioseq");
-		MoonUtils::copy("tmp/baserom/Audiotable", "Extract/Audiotable");
+		//MoonUtils::copy("tmp/baserom/Audiotable", "Extract/Audiotable");
-		MoonUtils::copy("tmp/baserom/version", "Extract/version");
+		//MoonUtils::copy("tmp/baserom/version", "Extract/version");
+		MoonUtils::write("Extract/version", (char*)&version.crc, sizeof(version.crc));
 
 		MoonUtils::copy("assets/game/", "Extract/assets/");
 

OTRGui/src/impl/extractor/extractor.cpp

@@ -83,7 +83,6 @@ void startWorker(RomVersion version) {
 
 	Util::write("tmp/baserom/version", (char*)&version.crc, sizeof(version.crc));
 
-
 	if (oldExtractMode)
 	{
 		std::vector<std::string> files;

ZAPDTR/ZAPD/Main.cpp

@@ -387,18 +387,6 @@ int main(int argc, char* argv[])
 	{
 		BuildAssetBlob(Globals::Instance->inputPath, Globals::Instance->outputPath);
 	}
-	/*
-	else if (fileMode == ZFileMode::BuildOverlay)
-	{
-		ZOverlay* overlay =
-			ZOverlay::FromBuild(Path::GetDirectoryName(Globals::Instance->inputPath),
-		                        Path::GetDirectoryName(Globals::Instance->cfgPath));
-
-		if (overlay != nullptr)
-			File::WriteAllText(Globals::Instance->outputPath.string(),
-			                   overlay->GetSourceOutputCode(""));
-	}
-	*/
 
 	if (exporterSet != nullptr && exporterSet->endProgramFunc != nullptr)
 		exporterSet->endProgramFunc();

ZAPDTR/ZAPD/ZFile.cpp

@@ -823,6 +823,32 @@ void ZFile::GenerateSourceHeaderFiles()
 
 	if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
 		File::WriteAllText(headerFilename, formatter.GetOutput());
+	else if (Globals::Instance->sourceOutputPath != "")
+	{
+		std::string xmlPath = xmlFilePath.string();
+		xmlPath = StringHelper::Replace(xmlPath, "\\", "/");
+		auto pathList = StringHelper::Split(xmlPath, "/");
+		std::string outPath = "";
+		
+		for (int i = 0; i < 3; i++)
+			outPath += pathList[i] + "/";
+
+		for (int i = 5; i < pathList.size(); i++)
+		{
+			if (i == pathList.size() - 1)
+			{
+				outPath += Path::GetFileNameWithoutExtension(pathList[i]) + "/";
+				outPath += outName.string() + ".h";
+			}
+			else
+				outPath += pathList[i];
+
+			if (i < pathList.size() - 1)
+				outPath += "/";
+		}
+
+		File::WriteAllText(outPath, formatter.GetOutput());
+	}
 }
 
 std::string ZFile::GetHeaderInclude() const