build/package/ship soh.otr (#2699)

* update

* Update generate-builds.yml

* Update generate-builds.yml

* sohotr in CI?

* paths

* gitignore soh.otr

* Fix

* no COMMAND maybe?

* try generating otr as it's own step

* deps

* f

* g

* h

* a

* cc

* ccc

* m

* blarg

* try uploading

* try

* s

* copy things

* wii u?

* wiiu

* don't fail

* windows without cpack?

* wut

* whoops

* see all the things plz

* b

* v

* zip working maybe

* ok but this tho

* pdb and switch

* mac lus

* soh.otr in appimage

* down first

* mac

* bundle soh otr into mac app and read it from the bundle

* appimage fix

* try just downloading for mac?

* i like to mv it mv it

* mv to the right place

* no more otrgui

* bring back otrgui and assets

* echo blarg

* echo

* zapd.exe

* clean and remove

* a

* space

---------

Co-authored-by: louis <louist103@gmail.com>
Co-authored-by: louist103 <35883445+louist103@users.noreply.github.com>
Co-authored-by: briaguya <briaguya@alice>
Co-authored-by: Adam Bird <archez39@me.com>

.github/workflows/generate-builds.yml

@@ -6,7 +6,55 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 jobs:
+  generate-soh-otr:
+    runs-on: ${{ (vars.LINUX_RUNNER && fromJSON(vars.LINUX_RUNNER)) || 'ubuntu-latest' }}
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+    - name: ccache
+      uses: hendrikmuhs/ccache-action@v1.2
+      with:
+        key: ${{ runner.os }}-soh-otr-ccache
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y $(cat .github/workflows/apt-deps.txt)
+    - name: Install latest SDL
+      run: |
+        export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
+        wget https://www.libsdl.org/release/SDL2-2.26.1.tar.gz
+        tar -xzf SDL2-2.26.1.tar.gz
+        cd SDL2-2.26.1
+        ./configure
+        make -j 10
+        sudo make install
+        sudo cp -av /usr/local/lib/libSDL* /lib/x86_64-linux-gnu/
+    - name: Install latest SDL_net
+      run: |
+        export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
+        wget https://www.libsdl.org/projects/SDL_net/release/SDL2_net-2.2.0.tar.gz
+        tar -xzf SDL2_net-2.2.0.tar.gz
+        cd SDL2_net-2.2.0
+        ./configure
+        make -j 10
+        sudo make install
+        sudo cp -av /usr/local/lib/libSDL* /lib/x86_64-linux-gnu/
+    - name: Generate soh.otr
+      run: |
+        export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
+        cmake --no-warn-unused-cli -H. -Bbuild-cmake -GNinja -DCMAKE_BUILD_TYPE:STRING=Release
+        cmake --build build-cmake --config Release --target ZAPD
+        mkdir Extract
+        cp -r OTRExporter/assets/ ./Extract/assets
+        ./build-cmake/ZAPD/ZAPD.out botr -se OTR --norom
+    - uses: actions/upload-artifact@v3
+      with:
+        name: soh.otr
+        path: soh.otr
+        retention-days: 1
   build-macos:
+    needs: generate-soh-otr
     runs-on: ${{ (vars.MAC_RUNNER && fromJSON(vars.MAC_RUNNER)) || 'macos-12' }}
     steps:
     - uses: actions/checkout@v3
@@ -47,11 +95,16 @@ jobs:
         brew uninstall --ignore-dependencies libpng
         sudo port install $(cat .github/workflows/macports-deps.txt)
         brew install ninja
+    - name: Download soh.otr
+      uses: actions/download-artifact@v3
+      with:
+        name: soh.otr
     - name: Build SoH
       run: |
         export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
         cmake --no-warn-unused-cli -H. -Bbuild-cmake -GNinja -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64"
         cmake --build build-cmake --config Release --parallel 10
+        mv soh.otr build-cmake/soh
         (cd build-cmake && cpack)
 
         mv _packages/*.dmg SoH.dmg
@@ -64,6 +117,7 @@ jobs:
           SoH.dmg
           readme.txt
   build-linux:
+    needs: generate-soh-otr
     strategy:
       fail-fast: true
       matrix:
@@ -107,11 +161,14 @@ jobs:
         make -j 10
         sudo make install
         sudo cp -av /usr/local/lib/libSDL* /lib/x86_64-linux-gnu/
+    - name: Download soh.otr
+      uses: actions/download-artifact@v3
+      with:
+        name: soh.otr
     - name: Build SoH
       run: |
         export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
         cmake --no-warn-unused-cli -H. -Bbuild-cmake -GNinja -DCMAKE_BUILD_TYPE:STRING=Release
-        cmake --build build-cmake --target OTRGui -j3
         cmake --build build-cmake --config Release -j3
         (cd build-cmake && cpack -G External)
 
@@ -128,6 +185,7 @@ jobs:
           soh.appimage
           readme.txt
   build-switch:
+    needs: generate-soh-otr
     runs-on: ${{ (vars.LINUX_RUNNER && fromJSON(vars.LINUX_RUNNER)) || 'ubuntu-latest' }}
     container:
       image: devkitpro/devkita64:latest
@@ -150,14 +208,20 @@ jobs:
 
         mv build-switch/soh/*.nro soh.nro
         mv README.md readme.txt
+    - name: Download soh.otr
+      uses: actions/download-artifact@v3
+      with:
+        name: soh.otr
     - name: Upload build
       uses: actions/upload-artifact@v3
       with:
         name: soh-switch
         path: |
           soh.nro
+          soh.otr
           readme.txt
   build-wiiu:
+    needs: generate-soh-otr
     runs-on: ${{ (vars.LINUX_RUNNER && fromJSON(vars.LINUX_RUNNER)) || 'ubuntu-latest' }}
     container:
       image: devkitpro/devkitppc:latest
@@ -184,6 +248,10 @@ jobs:
       env:
         DEVKITPRO: /opt/devkitpro
         DEVKITPPC: /opt/devkitpro/devkitPPC
+    - name: Download soh.otr
+      uses: actions/download-artifact@v3
+      with:
+        name: soh.otr
     - name: Upload build
       uses: actions/upload-artifact@v3
       with:
@@ -191,8 +259,10 @@ jobs:
         path: |
           soh.rpx
           soh.wuhb
+          soh.otr
           readme.txt
   build-windows:
+    needs: generate-soh-otr
     runs-on: ${{ (vars.WINDOWS_RUNNER && fromJSON(vars.WINDOWS_RUNNER)) || 'windows-latest' }}
     steps:
     - name: Install dependencies
@@ -208,19 +278,29 @@ jobs:
       with:
         key: ${{ runner.os }}-ccache
     - uses: ilammy/msvc-dev-cmd@v1
-    - name: Setup 7-Zip
-      run: |
-        "C:\Program Files\7-Zip" >> $env:GITHUB_PATH
     - name: Build SoH
       run: |
         set $env:PATH="$env:USERPROFILE/.cargo/bin;$env:PATH"
         cmake -S . -B build-windows -G Ninja -DCMAKE_MAKE_PROGRAM=ninja -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
         cmake --build build-windows --target OTRGui --config Release --parallel 10
         cmake --build build-windows --config Release --parallel 10
-        cd build-windows
+
-        cpack -G ZIP
+        mkdir soh-windows
+        mv ./x64/Release/soh.exe ./soh-windows/soh.exe
+        mkdir soh-windows/debug
+        mv ./x64/Release/soh.pdb ./soh-windows/debug/soh.pdb
+        mv ./x64/Release/OTRGui.exe ./soh-windows/OTRGui.exe
+        mv ./README.md ./soh-windows/readme.txt
+        mv ./build-windows/gamecontrollerdb.txt ./soh-windows/gamecontrollerdb.txt
+        mv ./build-windows/OTRGui/assets ./soh-windows
+        mv ./build-windows/ZAPD/ZAPD.exe ./soh-windows/assets/extractor/ZAPD.exe
+    - name: Download soh.otr
+      uses: actions/download-artifact@v3
+      with:
+        name: soh.otr
+        path: soh-windows
     - name: Upload build
       uses: actions/upload-artifact@v3
       with:
         name: soh-windows
-        path: _packages/*.zip
+        path: soh-windows

.gitignore

@@ -409,8 +409,7 @@ DebugObj/*
 ReleaseObj/*
 .tags
 tags
-oot.otr
+*.otr
-oot-mq.otr
 *.sav
 shipofharkinian.ini
 shipofharkinian.json

CMakeLists.txt

@@ -100,6 +100,7 @@ set_property(TARGET soh PROPERTY APPIMAGE_ICON_FILE "${CMAKE_BINARY_DIR}/sohIcon
 
 if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux")
 install(PROGRAMS "${CMAKE_SOURCE_DIR}/scripts/linux/appimage/soh.sh" DESTINATION . COMPONENT appimage)
+install(FILES "${CMAKE_SOURCE_DIR}/soh.otr" DESTINATION . COMPONENT appimage)
 endif()
 
 find_package(Python3 COMPONENTS Interpreter)
@@ -113,7 +114,7 @@ add_custom_target(
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/OTRExporter
     COMMENT "Running asset extraction..."
     DEPENDS ZAPD
-    BYPRODUCTS oot.otr ${CMAKE_SOURCE_DIR}/oot.otr oot-mq.otr ${CMAKE_SOURCE_DIR}/oot-mq.otr
+    BYPRODUCTS oot.otr ${CMAKE_SOURCE_DIR}/oot.otr oot-mq.otr ${CMAKE_SOURCE_DIR}/oot-mq.otr ${CMAKE_SOURCE_DIR}/soh.otr
 )
 
 if(CMAKE_SYSTEM_NAME MATCHES "Linux")

OTRExporter/OTRExporter/Main.cpp

@@ -64,27 +64,23 @@ static void ExporterParseFileMode(const std::string& buildMode, ZFileMode& fileM
 
 static void ExporterProgramEnd()
 {
+	uint32_t crc = 0xFFFFFFFF;
+	const uint8_t endianness = (uint8_t)Endianness::Big;
+	
 	if (Globals::Instance->fileMode == ZFileMode::ExtractDirectory)
 	{
+		std::string romPath = Globals::Instance->baseRomPath.string();
+		const std::vector<uint8_t>& romData = File::ReadAllBytes(romPath);
+		crc = BitConverter::ToUInt32BE(romData, 0x10);
 		printf("Creating version file...\n");
 
 		// Get crc from rom
-		std::string romPath = Globals::Instance->baseRomPath.string();
-		std::vector<uint8_t> romData = File::ReadAllBytes(romPath);
-		uint32_t crc = BitConverter::ToUInt32BE(romData, 0x10);
-        uint8_t endianness = (uint8_t)Endianness::Big;
 
-		// Write crc to version file
+		MemoryStream *versionStream = new MemoryStream();
-		fs::path versionPath("Extract/version");
-        MemoryStream* versionStream = new MemoryStream();
 		BinaryWriter writer(versionStream);
 		writer.SetEndianness(Endianness::Big);
 		writer.Write(endianness);
 		writer.Write(crc);
-		std::ofstream versionFile(versionPath.c_str(), std::ios::out | std::ios::binary);
-        versionFile.write(versionStream->ToVector().data(), versionStream->GetLength());
-		versionFile.flush();
-		versionFile.close();
 		writer.Close();
 
 		printf("Created version file.\n");
@@ -92,7 +88,10 @@ static void ExporterProgramEnd()
 		printf("Generating OTR Archive...\n");
 		otrArchive = Ship::Archive::CreateArchive(otrFileName, 40000);
 
-		for (auto item : files) {
+		otrArchive->AddFile("version", (uintptr_t)versionStream->ToVector().data(), versionStream->GetLength());
+
+		for (const auto& item : files)
+		{
 			std::string fName = item.first;
 			if (fName.find("gTitleZeldaShieldLogoMQTex") != std::string::npos && !ZRom(romPath).IsMQ())
 			{
@@ -102,64 +101,72 @@ static void ExporterProgramEnd()
 					fName.replace(pos, 27, "gTitleZeldaShieldLogoTex");
 				}
 			}
-			auto fileData = item.second;
+			const auto& fileData = item.second;
 			otrArchive->AddFile(fName, (uintptr_t)fileData.data(),
 								fileData.size());
 		}
+	}
+	otrArchive = nullptr;
 
 	// Add any additional files that need to be manually copied...
-		auto lst = Directory::ListFiles("Extract");
+	const auto& lst = Directory::ListFiles("Extract");
+	std::shared_ptr<Ship::Archive> sohOtr = Ship::Archive::CreateArchive("soh.otr", 4096);
+	//sohOtr->AddFile("version", (uintptr_t)versionStream->ToVector().data(), versionStream->GetLength());
 
-		for (auto item : lst)
+	for (const auto& item : lst)
 	{
 		std::vector<std::string> splitPath = StringHelper::Split(item, ".");
 
-			if (splitPath.size() >= 3) {
+		if (splitPath.size() >= 3)
-				std::string extension = splitPath.at(splitPath.size() - 1);
+		{
-				std::string format = splitPath.at(splitPath.size() - 2);
+			const std::string& extension = splitPath.at(splitPath.size() - 1);
+			const std::string& format = splitPath.at(splitPath.size() - 2);
 			splitPath.pop_back();
 			splitPath.pop_back();
 			std::string afterPath = std::accumulate(splitPath.begin(), splitPath.end(), std::string(""));
-				if (extension == "png" && (format == "rgba32" || format == "rgb5a1" || format == "i4" || format == "i8" || format == "ia4" || format == "ia8" || format == "ia16" || format == "ci4" || format == "ci8")) {
+			if (extension == "png" && (format == "rgba32" || format == "rgb5a1" || format == "i4" || format == "i8" || format == "ia4" || format == "ia8" || format == "ia16" || format == "ci4" || format == "ci8"))
+			{
 				ZTexture tex(nullptr);
 				Globals::Instance->buildRawTexture = true;
 				tex.FromPNG(item, ZTexture::GetTextureTypeFromString(format));
-					printf("otrArchive->AddFile(%s)\n", StringHelper::Split(afterPath, "Extract/")[1].c_str());
+				printf("sohOtr->AddFile(%s)\n", StringHelper::Split(afterPath, "Extract/")[1].c_str());
+
+				OTRExporter_Texture exporter;
 
-					auto exporter = new OTRExporter_Texture();
 				MemoryStream* stream = new MemoryStream();
 				BinaryWriter writer(stream);
 
- 					exporter->Save(&tex, "", &writer);
+				exporter.Save(&tex, "", &writer);
 
 				std::string src = tex.GetBodySourceCode();
- 					writer.Write((char*) src.c_str(), src.size());
+				writer.Write((char *)src.c_str(), src.size());
 
 				std::vector<char> fileData = stream->ToVector();
- 					otrArchive->AddFile(StringHelper::Split(afterPath, "Extract/assets/")[1], (uintptr_t)fileData.data(), fileData.size());
+				sohOtr->AddFile(StringHelper::Split(afterPath, "Extract/assets/")[1], (uintptr_t)fileData.data(), fileData.size());
 				continue;
 			}
 		}
 
-			if(item.find("accessibility") != std::string::npos) {
+		if (item.find("accessibility") != std::string::npos)
+		{
 			std::string extension = splitPath.at(splitPath.size() - 1);
 			splitPath.pop_back();
-				if(extension == "json"){
+			if (extension == "json")
-					auto fileData = File::ReadAllBytes(item);
+			{
+				const auto &fileData = File::ReadAllBytes(item);
 				printf("Adding accessibility texts %s\n", StringHelper::Split(item, "texts/")[1].c_str());
-					otrArchive->AddFile(StringHelper::Split(item, "Extract/assets/")[1], (uintptr_t)fileData.data(), fileData.size());
+				sohOtr->AddFile(StringHelper::Split(item, "Extract/assets/")[1], (uintptr_t)fileData.data(), fileData.size());
 			}
 			continue;
 		}
 
-			auto fileData = File::ReadAllBytes(item);
+		const auto& fileData = File::ReadAllBytes(item);
-			printf("otrArchive->AddFile(%s)\n", StringHelper::Split(item, "Extract/")[1].c_str());
+		printf("sohOtr->AddFile(%s)\n", StringHelper::Split(item, "Extract/")[1].c_str());
-			otrArchive->AddFile(StringHelper::Split(item, item.find("Extract/assets/") != std::string::npos ? "Extract/assets/" : "Extract/")[1], (uintptr_t)fileData.data(), fileData.size());
+		sohOtr->AddFile(StringHelper::Split(item, item.find("Extract/assets/") != std::string::npos ? "Extract/assets/" : "Extract/")[1], (uintptr_t)fileData.data(), fileData.size());
-		}
 	}
+	sohOtr = nullptr;
 }
 
-
 static void ExporterParseArgs(int argc, char* argv[], int& i)
 {
 	std::string arg = argv[i];

ZAPDTR/ZAPD/Globals.h

@@ -54,7 +54,7 @@ public:
 	bool useLegacyZDList;
 	bool singleThreaded;
 	VerbosityLevel verbosity;  // ZAPD outputs additional information
-	ZFileMode fileMode;
+	ZFileMode fileMode = ZFileMode::Invalid;
 	fs::path baseRomPath, inputPath, outputPath, sourceOutputPath, cfgPath, fileListPath;
 	TextureType texType;
 	ZGame game;
@@ -65,6 +65,7 @@ public:
 	bool forceUnaccountedStatic = false;
 	bool otrMode = true;
 	bool buildRawTexture = false;
+	bool onlyGenSohOtr = false;
 
 	ZRom* rom;
 	std::vector<ZFile*> files;

ZAPDTR/ZAPD/Main.cpp

@@ -260,10 +260,23 @@ int main(int argc, char* argv[])
 		{
 			Globals::Instance->buildRawTexture = true;
 		}
+		else if (arg == "--norom") 
+		{
+			Globals::Instance->onlyGenSohOtr = true;
 		}
+	}
+
 
 	// Parse File Mode
 	ExporterSet* exporterSet = Globals::Instance->GetExporterSet();
+	
+	if(Globals::Instance->onlyGenSohOtr) {
+		exporterSet->endProgramFunc();
+
+		delete g;
+		return 0;
+	}
+	
 	std::string buildMode = argv[1];
 	ZFileMode fileMode = ZFileMode::Invalid;
 
@@ -420,6 +433,8 @@ int main(int argc, char* argv[])
 	if (exporterSet != nullptr && exporterSet->endProgramFunc != nullptr)
 		exporterSet->endProgramFunc();
 
+end:
+
 	delete g;
 	return 0;
 }

copy-existing-otrs.cmake

@@ -6,6 +6,10 @@ if(EXISTS ${SOURCE_DIR}/OTRExporter/oot-mq.otr)
     execute_process(COMMAND ${CMAKE_COMMAND} -E copy oot-mq.otr ${SOURCE_DIR})
     execute_process(COMMAND ${CMAKE_COMMAND} -E copy oot-mq.otr ${BINARY_DIR}/soh/)
 endif()
+if(EXISTS ${SOURCE_DIR}/OTRExporter/soh.otr)
+    execute_process(COMMAND ${CMAKE_COMMAND} -E copy soh.otr ${SOURCE_DIR})
+    execute_process(COMMAND ${CMAKE_COMMAND} -E copy soh.otr ${BINARY_DIR}/soh/)
+endif()
 if(NOT EXISTS ${SOURCE_DIR}/oot.otr AND NOT EXISTS ${SOURCE_DIR}/oot-mq.otr)
     message(FATAL_ERROR, "No OTR files found.")
 endif()

scripts/linux/appimage/soh.sh

@@ -9,6 +9,10 @@ if [ -z ${SHIP_HOME+x} ]; then
 export SHIP_HOME=$PWD
 fi
 
+if [ -z ${SHIP_BIN_DIR+x} ]; then
+export SHIP_BIN_DIR="$HERE/usr/bin"
+fi
+
 while [[ (! -e "$SHIP_HOME"/oot.otr) || (! -e "$SHIP_HOME"/oot-mq.otr) ]]; do
         for romfile in "$SHIP_HOME"/*.*64
         do
@@ -95,10 +99,10 @@ while [[ (! -e "$SHIP_HOME"/oot.otr) || (! -e "$SHIP_HOME"/oot-mq.otr) ]]; do
             rm -r "$ASSETDIR"
             exit
         else
-            (cd "$HERE/usr/bin"; ./soh.elf)
+            (cd "$SHIP_BIN_DIR"; ./soh.elf)
             exit
         fi
         rm -r "$ASSETDIR"
 done	
-	(cd "$HERE/usr/bin"; ./soh.elf)
+	(cd "$SHIP_BIN_DIR"; ./soh.elf)
 exit

soh/CMakeLists.txt

@@ -687,6 +687,7 @@ execute_process(COMMAND ${CURL} -sSfL https://raw.githubusercontent.com/gabomdq/
 if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin")
 configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/macosx/Info.plist.in ${CMAKE_BINARY_DIR}/macosx/Info.plist @ONLY)
 INSTALL(FILES ${CMAKE_BINARY_DIR}/gamecontrollerdb.txt DESTINATION ../MacOS COMPONENT ship)
+INSTALL(FILES ${CMAKE_BINARY_DIR}/soh/soh.otr DESTINATION ../Resources COMPONENT ship)
 elseif(NOT "${CMAKE_SYSTEM_NAME}" MATCHES "NintendoSwitch|CafeOS")
 INSTALL(FILES ${CMAKE_BINARY_DIR}/gamecontrollerdb.txt DESTINATION . COMPONENT ship)
 endif()

soh/soh/OTRGlobals.cpp

@@ -205,6 +205,10 @@ OTRGlobals::OTRGlobals() {
     if (std::filesystem::exists(ootPath)) {
         OTRFiles.push_back(ootPath);
     }
+    std::string sohOtrPath = Ship::Window::GetPathRelativeToAppBundle("soh.otr");
+    if (std::filesystem::exists(sohOtrPath)) {
+        OTRFiles.push_back(sohOtrPath);
+    }
     std::string patchesPath = Ship::Window::GetPathRelativeToAppDirectory("mods");
     if (patchesPath.length() > 0 && std::filesystem::exists(patchesPath)) {
         if (std::filesystem::is_directory(patchesPath)) {