From c4ce2f10cd1076136f4a16c677738309f16bb301 Mon Sep 17 00:00:00 2001 From: briaguya <70942617+briaguya-ai@users.noreply.github.com> Date: Wed, 12 Apr 2023 21:14:33 -0400 Subject: [PATCH] build/package/ship `soh.otr` (#2699) * update * Update generate-builds.yml * Update generate-builds.yml * sohotr in CI? * paths * gitignore soh.otr * Fix * no COMMAND maybe? * try generating otr as it's own step * deps * f * g * h * a * cc * ccc * m * blarg * try uploading * try * s * copy things * wii u? * wiiu * don't fail * windows without cpack? * wut * whoops * see all the things plz * b * v * zip working maybe * ok but this tho * pdb and switch * mac lus * soh.otr in appimage * down first * mac * bundle soh otr into mac app and read it from the bundle * appimage fix * try just downloading for mac? * i like to mv it mv it * mv to the right place * no more otrgui * bring back otrgui and assets * echo blarg * echo * zapd.exe * clean and remove * a * space --------- Co-authored-by: louis Co-authored-by: louist103 <35883445+louist103@users.noreply.github.com> Co-authored-by: briaguya Co-authored-by: Adam Bird --- .github/workflows/generate-builds.yml | 94 ++++++++++++++-- .gitignore | 3 +- CMakeLists.txt | 3 +- OTRExporter/OTRExporter/Main.cpp | 147 ++++++++++++++------------ ZAPDTR/ZAPD/Globals.h | 3 +- ZAPDTR/ZAPD/Main.cpp | 15 +++ copy-existing-otrs.cmake | 4 + scripts/linux/appimage/soh.sh | 8 +- soh/CMakeLists.txt | 1 + soh/soh/OTRGlobals.cpp | 4 + 10 files changed, 199 insertions(+), 83 deletions(-) diff --git a/.github/workflows/generate-builds.yml b/.github/workflows/generate-builds.yml index 01a2f9ec1..a406fb6f8 100644 --- a/.github/workflows/generate-builds.yml +++ b/.github/workflows/generate-builds.yml @@ -6,7 +6,55 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: + generate-soh-otr: + runs-on: ${{ (vars.LINUX_RUNNER && fromJSON(vars.LINUX_RUNNER)) || 'ubuntu-latest' }} + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: ccache + uses: hendrikmuhs/ccache-action@v1.2 + with: + key: ${{ runner.os }}-soh-otr-ccache + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y $(cat .github/workflows/apt-deps.txt) + - name: Install latest SDL + run: | + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + wget https://www.libsdl.org/release/SDL2-2.26.1.tar.gz + tar -xzf SDL2-2.26.1.tar.gz + cd SDL2-2.26.1 + ./configure + make -j 10 + sudo make install + sudo cp -av /usr/local/lib/libSDL* /lib/x86_64-linux-gnu/ + - name: Install latest SDL_net + run: | + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + wget https://www.libsdl.org/projects/SDL_net/release/SDL2_net-2.2.0.tar.gz + tar -xzf SDL2_net-2.2.0.tar.gz + cd SDL2_net-2.2.0 + ./configure + make -j 10 + sudo make install + sudo cp -av /usr/local/lib/libSDL* /lib/x86_64-linux-gnu/ + - name: Generate soh.otr + run: | + export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" + cmake --no-warn-unused-cli -H. -Bbuild-cmake -GNinja -DCMAKE_BUILD_TYPE:STRING=Release + cmake --build build-cmake --config Release --target ZAPD + mkdir Extract + cp -r OTRExporter/assets/ ./Extract/assets + ./build-cmake/ZAPD/ZAPD.out botr -se OTR --norom + - uses: actions/upload-artifact@v3 + with: + name: soh.otr + path: soh.otr + retention-days: 1 build-macos: + needs: generate-soh-otr runs-on: ${{ (vars.MAC_RUNNER && fromJSON(vars.MAC_RUNNER)) || 'macos-12' }} steps: - uses: actions/checkout@v3 @@ -47,11 +95,16 @@ jobs: brew uninstall --ignore-dependencies libpng sudo port install $(cat .github/workflows/macports-deps.txt) brew install ninja + - name: Download soh.otr + uses: actions/download-artifact@v3 + with: + name: soh.otr - name: Build SoH run: | export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" cmake --no-warn-unused-cli -H. -Bbuild-cmake -GNinja -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_OSX_ARCHITECTURES="x86_64;arm64" cmake --build build-cmake --config Release --parallel 10 + mv soh.otr build-cmake/soh (cd build-cmake && cpack) mv _packages/*.dmg SoH.dmg @@ -64,6 +117,7 @@ jobs: SoH.dmg readme.txt build-linux: + needs: generate-soh-otr strategy: fail-fast: true matrix: @@ -107,11 +161,14 @@ jobs: make -j 10 sudo make install sudo cp -av /usr/local/lib/libSDL* /lib/x86_64-linux-gnu/ + - name: Download soh.otr + uses: actions/download-artifact@v3 + with: + name: soh.otr - name: Build SoH run: | export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH" cmake --no-warn-unused-cli -H. -Bbuild-cmake -GNinja -DCMAKE_BUILD_TYPE:STRING=Release - cmake --build build-cmake --target OTRGui -j3 cmake --build build-cmake --config Release -j3 (cd build-cmake && cpack -G External) @@ -128,6 +185,7 @@ jobs: soh.appimage readme.txt build-switch: + needs: generate-soh-otr runs-on: ${{ (vars.LINUX_RUNNER && fromJSON(vars.LINUX_RUNNER)) || 'ubuntu-latest' }} container: image: devkitpro/devkita64:latest @@ -150,14 +208,20 @@ jobs: mv build-switch/soh/*.nro soh.nro mv README.md readme.txt + - name: Download soh.otr + uses: actions/download-artifact@v3 + with: + name: soh.otr - name: Upload build uses: actions/upload-artifact@v3 with: name: soh-switch path: | soh.nro + soh.otr readme.txt build-wiiu: + needs: generate-soh-otr runs-on: ${{ (vars.LINUX_RUNNER && fromJSON(vars.LINUX_RUNNER)) || 'ubuntu-latest' }} container: image: devkitpro/devkitppc:latest @@ -184,6 +248,10 @@ jobs: env: DEVKITPRO: /opt/devkitpro DEVKITPPC: /opt/devkitpro/devkitPPC + - name: Download soh.otr + uses: actions/download-artifact@v3 + with: + name: soh.otr - name: Upload build uses: actions/upload-artifact@v3 with: @@ -191,8 +259,10 @@ jobs: path: | soh.rpx soh.wuhb + soh.otr readme.txt build-windows: + needs: generate-soh-otr runs-on: ${{ (vars.WINDOWS_RUNNER && fromJSON(vars.WINDOWS_RUNNER)) || 'windows-latest' }} steps: - name: Install dependencies @@ -208,19 +278,29 @@ jobs: with: key: ${{ runner.os }}-ccache - uses: ilammy/msvc-dev-cmd@v1 - - name: Setup 7-Zip - run: | - "C:\Program Files\7-Zip" >> $env:GITHUB_PATH - name: Build SoH run: | set $env:PATH="$env:USERPROFILE/.cargo/bin;$env:PATH" cmake -S . -B build-windows -G Ninja -DCMAKE_MAKE_PROGRAM=ninja -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache cmake --build build-windows --target OTRGui --config Release --parallel 10 cmake --build build-windows --config Release --parallel 10 - cd build-windows - cpack -G ZIP + + mkdir soh-windows + mv ./x64/Release/soh.exe ./soh-windows/soh.exe + mkdir soh-windows/debug + mv ./x64/Release/soh.pdb ./soh-windows/debug/soh.pdb + mv ./x64/Release/OTRGui.exe ./soh-windows/OTRGui.exe + mv ./README.md ./soh-windows/readme.txt + mv ./build-windows/gamecontrollerdb.txt ./soh-windows/gamecontrollerdb.txt + mv ./build-windows/OTRGui/assets ./soh-windows + mv ./build-windows/ZAPD/ZAPD.exe ./soh-windows/assets/extractor/ZAPD.exe + - name: Download soh.otr + uses: actions/download-artifact@v3 + with: + name: soh.otr + path: soh-windows - name: Upload build uses: actions/upload-artifact@v3 with: name: soh-windows - path: _packages/*.zip + path: soh-windows diff --git a/.gitignore b/.gitignore index ff0a8688e..58d6bb578 100644 --- a/.gitignore +++ b/.gitignore @@ -409,8 +409,7 @@ DebugObj/* ReleaseObj/* .tags tags -oot.otr -oot-mq.otr +*.otr *.sav shipofharkinian.ini shipofharkinian.json diff --git a/CMakeLists.txt b/CMakeLists.txt index e6d37fd45..7f3c32572 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,6 +100,7 @@ set_property(TARGET soh PROPERTY APPIMAGE_ICON_FILE "${CMAKE_BINARY_DIR}/sohIcon if("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") install(PROGRAMS "${CMAKE_SOURCE_DIR}/scripts/linux/appimage/soh.sh" DESTINATION . COMPONENT appimage) +install(FILES "${CMAKE_SOURCE_DIR}/soh.otr" DESTINATION . COMPONENT appimage) endif() find_package(Python3 COMPONENTS Interpreter) @@ -113,7 +114,7 @@ add_custom_target( WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/OTRExporter COMMENT "Running asset extraction..." DEPENDS ZAPD - BYPRODUCTS oot.otr ${CMAKE_SOURCE_DIR}/oot.otr oot-mq.otr ${CMAKE_SOURCE_DIR}/oot-mq.otr + BYPRODUCTS oot.otr ${CMAKE_SOURCE_DIR}/oot.otr oot-mq.otr ${CMAKE_SOURCE_DIR}/oot-mq.otr ${CMAKE_SOURCE_DIR}/soh.otr ) if(CMAKE_SYSTEM_NAME MATCHES "Linux") diff --git a/OTRExporter/OTRExporter/Main.cpp b/OTRExporter/OTRExporter/Main.cpp index 3bc9da046..fb49210b9 100644 --- a/OTRExporter/OTRExporter/Main.cpp +++ b/OTRExporter/OTRExporter/Main.cpp @@ -64,35 +64,34 @@ static void ExporterParseFileMode(const std::string& buildMode, ZFileMode& fileM static void ExporterProgramEnd() { + uint32_t crc = 0xFFFFFFFF; + const uint8_t endianness = (uint8_t)Endianness::Big; + if (Globals::Instance->fileMode == ZFileMode::ExtractDirectory) { + std::string romPath = Globals::Instance->baseRomPath.string(); + const std::vector& romData = File::ReadAllBytes(romPath); + crc = BitConverter::ToUInt32BE(romData, 0x10); printf("Creating version file...\n"); // Get crc from rom - std::string romPath = Globals::Instance->baseRomPath.string(); - std::vector romData = File::ReadAllBytes(romPath); - uint32_t crc = BitConverter::ToUInt32BE(romData, 0x10); - uint8_t endianness = (uint8_t)Endianness::Big; - // Write crc to version file - fs::path versionPath("Extract/version"); - MemoryStream* versionStream = new MemoryStream(); - BinaryWriter writer(versionStream); - writer.SetEndianness(Endianness::Big); - writer.Write(endianness); - writer.Write(crc); - std::ofstream versionFile(versionPath.c_str(), std::ios::out | std::ios::binary); - versionFile.write(versionStream->ToVector().data(), versionStream->GetLength()); - versionFile.flush(); - versionFile.close(); - writer.Close(); + MemoryStream *versionStream = new MemoryStream(); + BinaryWriter writer(versionStream); + writer.SetEndianness(Endianness::Big); + writer.Write(endianness); + writer.Write(crc); + writer.Close(); printf("Created version file.\n"); printf("Generating OTR Archive...\n"); otrArchive = Ship::Archive::CreateArchive(otrFileName, 40000); - for (auto item : files) { + otrArchive->AddFile("version", (uintptr_t)versionStream->ToVector().data(), versionStream->GetLength()); + + for (const auto& item : files) + { std::string fName = item.first; if (fName.find("gTitleZeldaShieldLogoMQTex") != std::string::npos && !ZRom(romPath).IsMQ()) { @@ -102,63 +101,71 @@ static void ExporterProgramEnd() fName.replace(pos, 27, "gTitleZeldaShieldLogoTex"); } } - auto fileData = item.second; + const auto& fileData = item.second; otrArchive->AddFile(fName, (uintptr_t)fileData.data(), - fileData.size()); - } - - // Add any additional files that need to be manually copied... - auto lst = Directory::ListFiles("Extract"); - - for (auto item : lst) - { - std::vector splitPath = StringHelper::Split(item, "."); - - if (splitPath.size() >= 3) { - std::string extension = splitPath.at(splitPath.size() - 1); - std::string format = splitPath.at(splitPath.size() - 2); - splitPath.pop_back(); - splitPath.pop_back(); - std::string afterPath = std::accumulate(splitPath.begin(), splitPath.end(), std::string("")); - if (extension == "png" && (format == "rgba32" || format == "rgb5a1" || format == "i4" || format == "i8" || format == "ia4" || format == "ia8" || format == "ia16" || format == "ci4" || format == "ci8")) { - ZTexture tex(nullptr); - Globals::Instance->buildRawTexture = true; - tex.FromPNG(item, ZTexture::GetTextureTypeFromString(format)); - printf("otrArchive->AddFile(%s)\n", StringHelper::Split(afterPath, "Extract/")[1].c_str()); - - auto exporter = new OTRExporter_Texture(); - MemoryStream* stream = new MemoryStream(); - BinaryWriter writer(stream); - - exporter->Save(&tex, "", &writer); - - std::string src = tex.GetBodySourceCode(); - writer.Write((char*) src.c_str(), src.size()); - - std::vector fileData = stream->ToVector(); - otrArchive->AddFile(StringHelper::Split(afterPath, "Extract/assets/")[1], (uintptr_t)fileData.data(), fileData.size()); - continue; - } - } - - if(item.find("accessibility") != std::string::npos) { - std::string extension = splitPath.at(splitPath.size() - 1); - splitPath.pop_back(); - if(extension == "json"){ - auto fileData = File::ReadAllBytes(item); - printf("Adding accessibility texts %s\n", StringHelper::Split(item, "texts/")[1].c_str()); - otrArchive->AddFile(StringHelper::Split(item, "Extract/assets/")[1], (uintptr_t)fileData.data(), fileData.size()); - } - continue; - } - - auto fileData = File::ReadAllBytes(item); - printf("otrArchive->AddFile(%s)\n", StringHelper::Split(item, "Extract/")[1].c_str()); - otrArchive->AddFile(StringHelper::Split(item, item.find("Extract/assets/") != std::string::npos ? "Extract/assets/" : "Extract/")[1], (uintptr_t)fileData.data(), fileData.size()); + fileData.size()); } } -} + otrArchive = nullptr; + // Add any additional files that need to be manually copied... + const auto& lst = Directory::ListFiles("Extract"); + std::shared_ptr sohOtr = Ship::Archive::CreateArchive("soh.otr", 4096); + //sohOtr->AddFile("version", (uintptr_t)versionStream->ToVector().data(), versionStream->GetLength()); + + for (const auto& item : lst) + { + std::vector splitPath = StringHelper::Split(item, "."); + + if (splitPath.size() >= 3) + { + const std::string& extension = splitPath.at(splitPath.size() - 1); + const std::string& format = splitPath.at(splitPath.size() - 2); + splitPath.pop_back(); + splitPath.pop_back(); + std::string afterPath = std::accumulate(splitPath.begin(), splitPath.end(), std::string("")); + if (extension == "png" && (format == "rgba32" || format == "rgb5a1" || format == "i4" || format == "i8" || format == "ia4" || format == "ia8" || format == "ia16" || format == "ci4" || format == "ci8")) + { + ZTexture tex(nullptr); + Globals::Instance->buildRawTexture = true; + tex.FromPNG(item, ZTexture::GetTextureTypeFromString(format)); + printf("sohOtr->AddFile(%s)\n", StringHelper::Split(afterPath, "Extract/")[1].c_str()); + + OTRExporter_Texture exporter; + + MemoryStream* stream = new MemoryStream(); + BinaryWriter writer(stream); + + exporter.Save(&tex, "", &writer); + + std::string src = tex.GetBodySourceCode(); + writer.Write((char *)src.c_str(), src.size()); + + std::vector fileData = stream->ToVector(); + sohOtr->AddFile(StringHelper::Split(afterPath, "Extract/assets/")[1], (uintptr_t)fileData.data(), fileData.size()); + continue; + } + } + + if (item.find("accessibility") != std::string::npos) + { + std::string extension = splitPath.at(splitPath.size() - 1); + splitPath.pop_back(); + if (extension == "json") + { + const auto &fileData = File::ReadAllBytes(item); + printf("Adding accessibility texts %s\n", StringHelper::Split(item, "texts/")[1].c_str()); + sohOtr->AddFile(StringHelper::Split(item, "Extract/assets/")[1], (uintptr_t)fileData.data(), fileData.size()); + } + continue; + } + + const auto& fileData = File::ReadAllBytes(item); + printf("sohOtr->AddFile(%s)\n", StringHelper::Split(item, "Extract/")[1].c_str()); + sohOtr->AddFile(StringHelper::Split(item, item.find("Extract/assets/") != std::string::npos ? "Extract/assets/" : "Extract/")[1], (uintptr_t)fileData.data(), fileData.size()); + } + sohOtr = nullptr; +} static void ExporterParseArgs(int argc, char* argv[], int& i) { diff --git a/ZAPDTR/ZAPD/Globals.h b/ZAPDTR/ZAPD/Globals.h index e0cf1d1bc..629a5d353 100644 --- a/ZAPDTR/ZAPD/Globals.h +++ b/ZAPDTR/ZAPD/Globals.h @@ -54,7 +54,7 @@ public: bool useLegacyZDList; bool singleThreaded; VerbosityLevel verbosity; // ZAPD outputs additional information - ZFileMode fileMode; + ZFileMode fileMode = ZFileMode::Invalid; fs::path baseRomPath, inputPath, outputPath, sourceOutputPath, cfgPath, fileListPath; TextureType texType; ZGame game; @@ -65,6 +65,7 @@ public: bool forceUnaccountedStatic = false; bool otrMode = true; bool buildRawTexture = false; + bool onlyGenSohOtr = false; ZRom* rom; std::vector files; diff --git a/ZAPDTR/ZAPD/Main.cpp b/ZAPDTR/ZAPD/Main.cpp index d596d0545..38b0f1597 100644 --- a/ZAPDTR/ZAPD/Main.cpp +++ b/ZAPDTR/ZAPD/Main.cpp @@ -260,10 +260,23 @@ int main(int argc, char* argv[]) { Globals::Instance->buildRawTexture = true; } + else if (arg == "--norom") + { + Globals::Instance->onlyGenSohOtr = true; + } } + // Parse File Mode ExporterSet* exporterSet = Globals::Instance->GetExporterSet(); + + if(Globals::Instance->onlyGenSohOtr) { + exporterSet->endProgramFunc(); + + delete g; + return 0; + } + std::string buildMode = argv[1]; ZFileMode fileMode = ZFileMode::Invalid; @@ -420,6 +433,8 @@ int main(int argc, char* argv[]) if (exporterSet != nullptr && exporterSet->endProgramFunc != nullptr) exporterSet->endProgramFunc(); +end: + delete g; return 0; } diff --git a/copy-existing-otrs.cmake b/copy-existing-otrs.cmake index b3510830d..007376d7b 100644 --- a/copy-existing-otrs.cmake +++ b/copy-existing-otrs.cmake @@ -6,6 +6,10 @@ if(EXISTS ${SOURCE_DIR}/OTRExporter/oot-mq.otr) execute_process(COMMAND ${CMAKE_COMMAND} -E copy oot-mq.otr ${SOURCE_DIR}) execute_process(COMMAND ${CMAKE_COMMAND} -E copy oot-mq.otr ${BINARY_DIR}/soh/) endif() +if(EXISTS ${SOURCE_DIR}/OTRExporter/soh.otr) + execute_process(COMMAND ${CMAKE_COMMAND} -E copy soh.otr ${SOURCE_DIR}) + execute_process(COMMAND ${CMAKE_COMMAND} -E copy soh.otr ${BINARY_DIR}/soh/) +endif() if(NOT EXISTS ${SOURCE_DIR}/oot.otr AND NOT EXISTS ${SOURCE_DIR}/oot-mq.otr) message(FATAL_ERROR, "No OTR files found.") endif() \ No newline at end of file diff --git a/scripts/linux/appimage/soh.sh b/scripts/linux/appimage/soh.sh index 384311d4c..b8de73641 100644 --- a/scripts/linux/appimage/soh.sh +++ b/scripts/linux/appimage/soh.sh @@ -9,6 +9,10 @@ if [ -z ${SHIP_HOME+x} ]; then export SHIP_HOME=$PWD fi +if [ -z ${SHIP_BIN_DIR+x} ]; then +export SHIP_BIN_DIR="$HERE/usr/bin" +fi + while [[ (! -e "$SHIP_HOME"/oot.otr) || (! -e "$SHIP_HOME"/oot-mq.otr) ]]; do for romfile in "$SHIP_HOME"/*.*64 do @@ -95,10 +99,10 @@ while [[ (! -e "$SHIP_HOME"/oot.otr) || (! -e "$SHIP_HOME"/oot-mq.otr) ]]; do rm -r "$ASSETDIR" exit else - (cd "$HERE/usr/bin"; ./soh.elf) + (cd "$SHIP_BIN_DIR"; ./soh.elf) exit fi rm -r "$ASSETDIR" done - (cd "$HERE/usr/bin"; ./soh.elf) + (cd "$SHIP_BIN_DIR"; ./soh.elf) exit diff --git a/soh/CMakeLists.txt b/soh/CMakeLists.txt index 577c0c33c..1a873fbf8 100644 --- a/soh/CMakeLists.txt +++ b/soh/CMakeLists.txt @@ -687,6 +687,7 @@ execute_process(COMMAND ${CURL} -sSfL https://raw.githubusercontent.com/gabomdq/ if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin") configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/macosx/Info.plist.in ${CMAKE_BINARY_DIR}/macosx/Info.plist @ONLY) INSTALL(FILES ${CMAKE_BINARY_DIR}/gamecontrollerdb.txt DESTINATION ../MacOS COMPONENT ship) +INSTALL(FILES ${CMAKE_BINARY_DIR}/soh/soh.otr DESTINATION ../Resources COMPONENT ship) elseif(NOT "${CMAKE_SYSTEM_NAME}" MATCHES "NintendoSwitch|CafeOS") INSTALL(FILES ${CMAKE_BINARY_DIR}/gamecontrollerdb.txt DESTINATION . COMPONENT ship) endif() diff --git a/soh/soh/OTRGlobals.cpp b/soh/soh/OTRGlobals.cpp index d3e038a63..638cf162b 100644 --- a/soh/soh/OTRGlobals.cpp +++ b/soh/soh/OTRGlobals.cpp @@ -205,6 +205,10 @@ OTRGlobals::OTRGlobals() { if (std::filesystem::exists(ootPath)) { OTRFiles.push_back(ootPath); } + std::string sohOtrPath = Ship::Window::GetPathRelativeToAppBundle("soh.otr"); + if (std::filesystem::exists(sohOtrPath)) { + OTRFiles.push_back(sohOtrPath); + } std::string patchesPath = Ship::Window::GetPathRelativeToAppDirectory("mods"); if (patchesPath.length() > 0 && std::filesystem::exists(patchesPath)) { if (std::filesystem::is_directory(patchesPath)) {