Further optimized OTRExporter, saving around ~20 seconds.

This commit is contained in:
Jack Walker 2022-03-31 00:01:26 -04:00
parent 869e00d5bd
commit dba4cf6c14
31 changed files with 5284 additions and 164 deletions

View File

@ -345,6 +345,8 @@ baserom/
*.otr
*.swp
*.a
*.z64
*.n64
Extract/
tmp.txt

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -756,42 +756,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
word1 = value.words.w1 | 0xF0000000;
}
else
//if (dList->vertices.size() > 0)
{
// Connect neighboring vertex arrays
std::vector<std::pair<uint32_t, std::vector<ZVtx>>> vertsKeys(dList->vertices.begin(),
dList->vertices.end());
if (vertsKeys.size() > 0)
{
auto lastItem = vertsKeys[0];
for (size_t i = 1; i < vertsKeys.size(); i++)
{
auto curItem = vertsKeys[i];
int32_t sizeDiff = curItem.first - (lastItem.first + (lastItem.second.size() * 16));
// Make sure there isn't an unaccounted inbetween these two
if (sizeDiff == 0)
{
for (auto v : curItem.second)
{
dList->vertices[lastItem.first].push_back(v);
lastItem.second.push_back(v);
}
dList->vertices.erase(curItem.first);
vertsKeys.erase(vertsKeys.begin() + i);
i--;
continue;
}
lastItem = curItem;
}
}
// Write CRC64 of vtx file name
uint32_t addr = data & 0xFFFFFFFF;
@ -829,7 +794,6 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
if (files.find(fName) == files.end() && !File::Exists("Extract\\" + fName))
{
//printf("Exporting VTX Data %s\n", fName.c_str());
// Write vertices to file
MemoryStream* vtxStream = new MemoryStream();
BinaryWriter vtxWriter = BinaryWriter(vtxStream);
@ -852,47 +816,40 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
vtxWriter.Write((uint32_t)ZResourceType::Vertex);
vtxWriter.Write((uint32_t)arrCnt);
size_t sz = dList->vertices[vtxDecl->address].size();
auto start = std::chrono::steady_clock::now();
//if (sz > 0)
// God dammit this is so dumb
for (size_t i = 0; i < split.size(); i++)
{
auto start = std::chrono::steady_clock::now();
std::string line = split[i];
// God dammit this is so dumb
for (size_t i = 0; i < split.size(); i++)
if (StringHelper::Contains(line, "VTX("))
{
std::string line = split[i];
auto split2 = StringHelper::Split(StringHelper::Split(StringHelper::Split(line, "VTX(")[1], ")")[0], ",");
if (StringHelper::Contains(line, "VTX("))
{
auto split2 = StringHelper::Split(StringHelper::Split(StringHelper::Split(line, "VTX(")[1], ")")[0], ",");
vtxWriter.Write((int16_t)std::stoi(split2[0], nullptr, 10)); // v.x
vtxWriter.Write((int16_t)std::stoi(split2[1], nullptr, 10)); // v.y
vtxWriter.Write((int16_t)std::stoi(split2[2], nullptr, 10)); // v.z
vtxWriter.Write((int16_t)std::stoi(split2[0], nullptr, 10)); // v.x
vtxWriter.Write((int16_t)std::stoi(split2[1], nullptr, 10)); // v.y
vtxWriter.Write((int16_t)std::stoi(split2[2], nullptr, 10)); // v.z
vtxWriter.Write((int16_t)0); // v.flag
vtxWriter.Write((int16_t)0); // v.flag
vtxWriter.Write((int16_t)std::stoi(split2[3], nullptr, 10)); // v.s
vtxWriter.Write((int16_t)std::stoi(split2[4], nullptr, 10)); // v.t
vtxWriter.Write((uint8_t)std::stoi(split2[5], nullptr, 10)); // v.r
vtxWriter.Write((uint8_t)std::stoi(split2[6], nullptr, 10)); // v.g
vtxWriter.Write((uint8_t)std::stoi(split2[7], nullptr, 10)); // v.b
vtxWriter.Write((uint8_t)std::stoi(split2[8], nullptr, 10)); // v.a
}
vtxWriter.Write((int16_t)std::stoi(split2[3], nullptr, 10)); // v.s
vtxWriter.Write((int16_t)std::stoi(split2[4], nullptr, 10)); // v.t
vtxWriter.Write((uint8_t)std::stoi(split2[5], nullptr, 10)); // v.r
vtxWriter.Write((uint8_t)std::stoi(split2[6], nullptr, 10)); // v.g
vtxWriter.Write((uint8_t)std::stoi(split2[7], nullptr, 10)); // v.b
vtxWriter.Write((uint8_t)std::stoi(split2[8], nullptr, 10)); // v.a
}
if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
File::WriteAllBytes("Extract\\" + fName, vtxStream->ToVector());
else
files[fName] = vtxStream->ToVector();
auto end = std::chrono::steady_clock::now();
size_t diff = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
//printf("Exported VTX Array %s in %zums\n", fName.c_str(), diff);
}
if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
File::WriteAllBytes("Extract\\" + fName, vtxStream->ToVector());
else
files[fName] = vtxStream->ToVector();
auto end = std::chrono::steady_clock::now();
size_t diff = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
}
}
else
@ -900,15 +857,6 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
spdlog::error("vtxDecl == nullptr!");
}
}
/*else
{
writer->Write(word0);
writer->Write(word1);
word0 = 0;
word1 = 0;
spdlog::error("dList->vertices.size() <= 0!");
}*/
}
break;
}

View File

@ -25,6 +25,7 @@ bool single_thread = false;
bool hide_second_btn = false;
RomVersion version;
const char* patched_rom = "tmp/rom.z64";
extern bool oldExtractMode;
static std::string currentStep = "None";
@ -137,7 +138,9 @@ void OTRGame::draw() {
UIUtils::GuiShadowText(("Rom Type: " + version.version).c_str(), 32, text_y, 10, WHITE, BLACK);
UIUtils::GuiShadowText("Tool Version: 1.0", 32, text_y + 15, 10, WHITE, BLACK);
UIUtils::GuiShadowText("OTR Version: 1.0", 32, text_y + 30, 10, WHITE, BLACK);
UIUtils::GuiToggle(&single_thread, "Single Thread", 32, text_y + 40, currentStep != NULLSTR);
if (oldExtractMode)
UIUtils::GuiToggle(&single_thread, "Single Thread", 32, text_y + 40, currentStep != NULLSTR);
if(!hide_second_btn && UIUtils::GuiIconButton("Folder", "Open\nShip Folder", 109, 50, currentStep != NULLSTR, "Select your Ship of Harkinian Folder\n\nYou could use another folder\nfor development purposes")) {
const std::string path = NativeFS->LaunchFileExplorer(LaunchType::FOLDER);

View File

@ -14,7 +14,7 @@
#endif
namespace Util = MoonUtils;
static bool oldExtractMode = false;
bool oldExtractMode = false;
static int maxResources = 0;
static int extractedResources = 0;
bool buildingOtr = false;

View File

@ -1,6 +1,6 @@
#pragma once
static uint32_t CRC32B(unsigned char* message, int32_t size)
static uint32_t CRC32B(const unsigned char* message, int32_t size)
{
int32_t byte, crc;
int32_t mask;

View File

@ -1,6 +1,7 @@
#include "Declaration.h"
#include "Globals.h"
#include "ZVtx.h"
#include "Utils/StringHelper.h"
Declaration::Declaration(offset_t nAddress, DeclarationAlignment nAlignment, size_t nSize,
@ -61,6 +62,12 @@ Declaration::Declaration(offset_t nAddress, const std::string& nIncludePath, siz
varName = nVarName;
}
Declaration::~Declaration()
{
//for (auto item : vertexHack)
//delete item;
}
bool Declaration::IsStatic() const
{
switch (staticConf)

View File

@ -22,6 +22,8 @@ enum class StaticConfig
On
};
class ZVtx;
class Declaration
{
public:
@ -38,6 +40,8 @@ public:
std::string varName;
std::string includePath;
std::vector<ZVtx*> vertexHack;
bool isExternal = false;
bool isArray = false;
bool forceArrayCnt = false;
@ -65,6 +69,8 @@ public:
Declaration(offset_t nAddress, const std::string& nIncludePath, size_t nSize,
const std::string& nVarType, const std::string& nVarName);
~Declaration();
bool IsStatic() const;
std::string GetNormalDeclarationStr() const;

View File

@ -93,6 +93,14 @@ ExporterSet* Globals::GetExporterSet()
return nullptr;
}
std::vector<uint8_t> Globals::GetBaseromFile(std::string fileName)
{
if (fileMode == ZFileMode::ExtractDirectory)
return rom->GetFile(StringHelper::Split(fileName, "baserom/")[1]);
else
return File::ReadAllBytes(fileName);
}
bool Globals::GetSegmentedPtrName(segptr_t segAddress, ZFile* currentFile,
const std::string& expectedType, std::string& declName)
{

View File

@ -5,6 +5,7 @@
#include <vector>
#include "GameConfig.h"
#include "ZFile.h"
#include <ZRom.h>
class ZRoom;
@ -62,6 +63,7 @@ public:
bool forceUnaccountedStatic = false;
bool otrMode = true;
ZRom* rom;
std::vector<ZFile*> files;
std::vector<ZFile*> externalFiles;
std::vector<int32_t> segments;
@ -80,6 +82,8 @@ public:
ZResourceExporter* GetExporter(ZResourceType resType);
ExporterSet* GetExporterSet();
std::vector<uint8_t> GetBaseromFile(std::string fileName);
/**
* Search in every file (and the symbol map) for the `segAddress` passed as parameter.
* If the segment of `currentFile` is the same segment of `segAddress`, then that file will be

View File

@ -253,6 +253,9 @@ int main(int argc, char* argv[])
Globals::Instance->fileMode = fileMode;
if (fileMode == ZFileMode::ExtractDirectory)
Globals::Instance->rom = new ZRom("baserom.z64");
// We've parsed through our commands once. If an exporter exists, it's been set by now.
// Now we'll parse through them again but pass them on to our exporter if one is available.
@ -319,20 +322,13 @@ int main(int argc, char* argv[])
if (!parseSuccessful)
return 1;
/*for (auto file : Globals::Instance->files)
delete file;*/
for (int i = 0; i < Globals::Instance->files.size(); i++)
{
//if (!Globals::Instance->files[i]->isExternalFile)
{
delete Globals::Instance->files[i];
Globals::Instance->files.erase(Globals::Instance->files.begin() + i);
i--;
}
delete Globals::Instance->files[i];
Globals::Instance->files.erase(Globals::Instance->files.begin() + i);
i--;
}
//Globals::Instance->files.clear();
Globals::Instance->externalFiles.clear();
Globals::Instance->segments.clear();
Globals::Instance->cfg.segmentRefFiles.clear();
@ -387,6 +383,7 @@ int main(int argc, char* argv[])
{
BuildAssetBlob(Globals::Instance->inputPath, Globals::Instance->outputPath);
}
/*
else if (fileMode == ZFileMode::BuildOverlay)
{
ZOverlay* overlay =
@ -397,6 +394,7 @@ int main(int argc, char* argv[])
File::WriteAllText(Globals::Instance->outputPath.string(),
overlay->GetSourceOutputCode(""));
}
*/
if (exporterSet != nullptr && exporterSet->endProgramFunc != nullptr)
exporterSet->endProgramFunc();

View File

@ -1,3 +1,4 @@
#if 0
#include "ZOverlay.h"
#include <cassert>
@ -350,3 +351,4 @@ ELFIO::Elf_Half ZOverlay::FindSymbolInSection(const std::string& curSymName,
}
return SHN_UNDEF;
}
#endif

View File

@ -1,5 +1,7 @@
#pragma once
#if 0
#include "Utils/Directory.h"
#include "ZResource.h"
#include "elfio/elfio.hpp"
@ -73,3 +75,4 @@ private:
ELFIO::Elf_Half FindSymbolInSection(const std::string& curSymName, ELFIO::section* sectionData,
ELFIO::elfio& reader, size_t readerId);
};
#endif

View File

@ -192,6 +192,7 @@
<ClCompile Include="OutputFormatter.cpp" />
<ClCompile Include="Overlays\ZOverlay.cpp" />
<ClCompile Include="WarningHandler.cpp" />
<ClCompile Include="yaz0\yaz0.cpp" />
<ClCompile Include="ZArray.cpp" />
<ClCompile Include="ZBackground.cpp" />
<ClCompile Include="ZCutsceneMM.cpp" />
@ -199,6 +200,7 @@
<ClCompile Include="ZMtx.cpp" />
<ClCompile Include="ZPath.cpp" />
<ClCompile Include="ZPlayerAnimationData.cpp" />
<ClCompile Include="ZRom.cpp" />
<ClCompile Include="ZRoom\Commands\SetActorCutsceneList.cpp" />
<ClCompile Include="ZRoom\Commands\SetAnimatedMaterialList.cpp" />
<ClCompile Include="ZRoom\Commands\SetCsCamera.cpp" />
@ -279,6 +281,8 @@
<ClInclude Include="OutputFormatter.h" />
<ClInclude Include="Overlays\ZOverlay.h" />
<ClInclude Include="WarningHandler.h" />
<ClInclude Include="yaz0\readwrite.h" />
<ClInclude Include="yaz0\yaz0.h" />
<ClInclude Include="ZAnimation.h" />
<ClInclude Include="ZArray.h" />
<ClInclude Include="ZBackground.h" />
@ -292,6 +296,7 @@
<ClInclude Include="ZMtx.h" />
<ClInclude Include="ZPath.h" />
<ClInclude Include="ZPlayerAnimationData.h" />
<ClInclude Include="ZRom.h" />
<ClInclude Include="ZRoom\Commands\SetActorCutsceneList.h" />
<ClInclude Include="ZRoom\Commands\SetAnimatedMaterialList.h" />
<ClInclude Include="ZRoom\Commands\SetCsCamera.h" />

View File

@ -58,6 +58,12 @@
<Filter Include="NuGet">
<UniqueIdentifier>{730beb67-6d59-4849-9d9b-702c4a565fc0}</UniqueIdentifier>
</Filter>
<Filter Include="Source Files\Yaz0">
<UniqueIdentifier>{b26457d2-cdb8-4c92-9ed7-a55bf6d3619e}</UniqueIdentifier>
</Filter>
<Filter Include="Header Files\Yaz0">
<UniqueIdentifier>{9651a041-1019-4486-9e90-1dccfa9471e9}</UniqueIdentifier>
</Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="Main.cpp">
@ -282,6 +288,12 @@
<ClCompile Include="ZText.cpp">
<Filter>Source Files\Z64</Filter>
</ClCompile>
<ClCompile Include="ZRom.cpp">
<Filter>Source Files\Z64</Filter>
</ClCompile>
<ClCompile Include="yaz0\yaz0.cpp">
<Filter>Source Files\Yaz0</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="ZRoom\ZRoom.h">
@ -539,6 +551,15 @@
<ClInclude Include="ZText.h">
<Filter>Header Files\Z64</Filter>
</ClInclude>
<ClInclude Include="ZRom.h">
<Filter>Header Files\Z64</Filter>
</ClInclude>
<ClInclude Include="yaz0\readwrite.h">
<Filter>Header Files\Yaz0</Filter>
</ClInclude>
<ClInclude Include="yaz0\yaz0.h">
<Filter>Header Files\Yaz0</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<Text Include="..\SymbolMap_OoTMqDbg.txt">

View File

@ -112,12 +112,15 @@ void ZNormalAnimation::DeclareReferences(const std::string& prefix)
const uint8_t lineLength = 14;
const uint8_t offset = 0;
for (size_t i = 0; i < rotationValues.size(); i++)
if (!Globals::Instance->otrMode)
{
valuesStr += StringHelper::Sprintf("0x%04X, ", rotationValues[i]);
for (size_t i = 0; i < rotationValues.size(); i++)
{
valuesStr += StringHelper::Sprintf("0x%04X, ", rotationValues[i]);
if ((i - offset + 1) % lineLength == 0)
valuesStr += "\n ";
if ((i - offset + 1) % lineLength == 0)
valuesStr += "\n ";
}
}
parent->AddDeclarationArray(rotationValuesOffset, DeclarationAlignment::Align4,
@ -125,13 +128,17 @@ void ZNormalAnimation::DeclareReferences(const std::string& prefix)
StringHelper::Sprintf("%sFrameData", defaultPrefix.c_str()),
rotationValues.size(), valuesStr);
for (size_t i = 0; i < rotationIndices.size(); i++)
if (!Globals::Instance->otrMode)
{
indicesStr += StringHelper::Sprintf(" { 0x%04X, 0x%04X, 0x%04X },", rotationIndices[i].x,
rotationIndices[i].y, rotationIndices[i].z);
for (size_t i = 0; i < rotationIndices.size(); i++)
{
indicesStr +=
StringHelper::Sprintf(" { 0x%04X, 0x%04X, 0x%04X },", rotationIndices[i].x,
rotationIndices[i].y, rotationIndices[i].z);
if (i != (rotationIndices.size() - 1))
indicesStr += "\n";
if (i != (rotationIndices.size() - 1))
indicesStr += "\n";
}
}
parent->AddDeclarationArray(rotationIndicesOffset, DeclarationAlignment::Align4,

View File

@ -102,8 +102,7 @@ std::string ZArray::GetBodySourceCode() const
const auto& res = resList[i];
output += "\t";
if (res->GetResourceType() == ZResourceType::Scalar ||
res->GetResourceType() == ZResourceType::Vertex)
if (res->GetResourceType() == ZResourceType::Scalar || res->GetResourceType() == ZResourceType::Vertex)
output += resList.at(i)->GetBodySourceCode();
else
output += StringHelper::Sprintf("{ %s }", resList.at(i)->GetBodySourceCode().c_str());

View File

@ -97,12 +97,15 @@ void ZCollisionHeader::DeclareReferences(const std::string& prefix)
if (waterBoxes.size() > 0)
{
for (size_t i = 0; i < waterBoxes.size(); i++)
if (!Globals::Instance->otrMode)
{
declaration +=
StringHelper::Sprintf("\t{ %s },", waterBoxes[i].GetBodySourceCode().c_str());
if (i + 1 < waterBoxes.size())
declaration += "\n";
for (size_t i = 0; i < waterBoxes.size(); i++)
{
declaration +=
StringHelper::Sprintf("\t{ %s },", waterBoxes[i].GetBodySourceCode().c_str());
if (i + 1 < waterBoxes.size())
declaration += "\n";
}
}
parent->AddDeclarationArray(
@ -115,14 +118,17 @@ void ZCollisionHeader::DeclareReferences(const std::string& prefix)
{
declaration.clear();
for (size_t i = 0; i < polygons.size(); i++)
if (!Globals::Instance->otrMode)
{
declaration += StringHelper::Sprintf(
"\t{ 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X },",
polygons[i].type, polygons[i].vtxA, polygons[i].vtxB, polygons[i].vtxC,
polygons[i].a, polygons[i].b, polygons[i].c, polygons[i].d);
if (i + 1 < polygons.size())
declaration += "\n";
for (size_t i = 0; i < polygons.size(); i++)
{
declaration += StringHelper::Sprintf(
"\t{ 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X },",
polygons[i].type, polygons[i].vtxA, polygons[i].vtxB, polygons[i].vtxC,
polygons[i].a, polygons[i].b, polygons[i].c, polygons[i].d);
if (i + 1 < polygons.size())
declaration += "\n";
}
}
parent->AddDeclarationArray(
@ -132,13 +138,16 @@ void ZCollisionHeader::DeclareReferences(const std::string& prefix)
}
declaration.clear();
for (size_t i = 0; i < polygonTypes.size(); i++)
if (!Globals::Instance->otrMode)
{
declaration += StringHelper::Sprintf("\t{ 0x%08lX, 0x%08lX },", polygonTypes[i] >> 32,
polygonTypes[i] & 0xFFFFFFFF);
for (size_t i = 0; i < polygonTypes.size(); i++)
{
declaration += StringHelper::Sprintf("\t{ 0x%08lX, 0x%08lX },", polygonTypes[i] >> 32,
polygonTypes[i] & 0xFFFFFFFF);
if (i < polygonTypes.size() - 1)
declaration += "\n";
if (i < polygonTypes.size() - 1)
declaration += "\n";
}
}
if (polyTypeDefAddress != 0)
@ -154,13 +163,16 @@ void ZCollisionHeader::DeclareReferences(const std::string& prefix)
{
declaration.clear();
for (size_t i = 0; i < vertices.size(); i++)
if (!Globals::Instance->otrMode)
{
declaration +=
StringHelper::Sprintf("\t{ %s },", vertices[i].GetBodySourceCode().c_str());
for (size_t i = 0; i < vertices.size(); i++)
{
declaration +=
StringHelper::Sprintf("\t{ %s },", vertices[i].GetBodySourceCode().c_str());
if (i < vertices.size() - 1)
declaration += "\n";
if (i < vertices.size() - 1)
declaration += "\n";
}
}
const auto& first = vertices.front();
@ -177,6 +189,9 @@ std::string ZCollisionHeader::GetBodySourceCode() const
{
std::string declaration = "";
if (Globals::Instance->otrMode)
return declaration;
declaration += "\n";
declaration += StringHelper::Sprintf("\t{ %i, %i, %i },\n", absMinX, absMinY, absMinZ);

View File

@ -1647,7 +1647,9 @@ static int32_t GfxdCallback_Vtx(uint32_t seg, int32_t count)
}
self->references.push_back(seg);
gfxd_puts("@r");
if (!Globals::Instance->otrMode)
gfxd_puts("@r");
return 1;
}
@ -1805,6 +1807,23 @@ void ZDisplayList::DeclareReferences(const std::string& prefix)
curAddr, firstVtx.GetDeclarationAlignment(),
item.second.size() * firstVtx.GetRawDataSize(), firstVtx.GetSourceTypeName(),
firstVtx.GetDefaultName(name), item.second.size(), declaration);
/*for (auto vtx : item.second)
{
ZVtx* nVtx = new ZVtx(vtx.parent);
nVtx->x = vtx.x;
nVtx->y = vtx.y;
nVtx->z = vtx.z;
nVtx->flag = vtx.flag;
nVtx->s = vtx.s;
nVtx->t = vtx.t;
nVtx->r = vtx.r;
nVtx->g = vtx.g;
nVtx->b = vtx.b;
nVtx->a = vtx.a;
decl->vertexHack.push_back(nVtx);
}*/
decl->isExternal = true;
}
}
@ -1850,15 +1869,15 @@ void ZDisplayList::DeclareReferences(const std::string& prefix)
{
auto& item = vertices[vtxKeys[i]];
std::string declaration;
//std::string declaration;
for (auto& vtx : item)
declaration += StringHelper::Sprintf("\t%s,\n", vtx.GetBodySourceCode().c_str());
//for (auto& vtx : item)
//declaration += StringHelper::Sprintf("\t%s,\n", vtx.GetBodySourceCode().c_str());
// Ensure there's always a trailing line feed to prevent dumb warnings.
// Please don't remove this line, unless you somehow made a way to prevent
// that warning when building the OoT repo.
declaration += "\n";
//declaration += "\n";
if (parent != nullptr)
{
@ -1870,12 +1889,6 @@ void ZDisplayList::DeclareReferences(const std::string& prefix)
else
vtxName = StringHelper::Sprintf("%sVtx_%06X", prefix.c_str(), vtxKeys[i]);
if (StringHelper::Contains(vtxName, "4B18"))
{
int bp = 0;
}
auto filepath = Globals::Instance->outputPath / vtxName;
std::string incStr = StringHelper::Sprintf("%s.%s.inc", filepath.string().c_str(), "vtx");

View File

@ -190,7 +190,8 @@ void ZFile::ParseXML(tinyxml2::XMLElement* reader, const std::string& filename)
HANDLE_ERROR_PROCESS(WarningType::Always, errorHeader, "");
}
rawData = File::ReadAllBytes((basePath / name).string());
//rawData = File::ReadAllBytes((basePath / name).string());
rawData = Globals::Instance->GetBaseromFile((basePath / name).string());
if (reader->Attribute("RangeEnd") == nullptr)
rangeEnd = rawData.size();
@ -1000,6 +1001,10 @@ std::string ZFile::ProcessDeclarations()
lastItem.second->size += curItem.second->size;
lastItem.second->arrayItemCnt += curItem.second->arrayItemCnt;
lastItem.second->text += "\n" + curItem.second->text;
for (auto vtx : curItem.second->vertexHack)
lastItem.second->vertexHack.push_back(vtx);
declarations.erase(curItem.first);
declarationKeys.erase(declarationKeys.begin() + i);
delete curItem.second;

View File

@ -108,12 +108,12 @@ public:
static void RegisterNode(std::string nodeName, ZResourceFactoryFunc* nodeFunc);
protected:
std::vector<uint8_t> rawData;
std::string name;
fs::path outName = "";
fs::path basePath;
fs::path outputPath;
fs::path xmlFilePath;
std::vector<uint8_t> rawData;
// Keep track of every texture of this ZFile.
// The pointers declared here are "borrowed" (somebody else is the owner),

View File

@ -8,6 +8,7 @@
#include "ZFile.h"
#include <Globals.h>
#include <ZDisplayList.h>
#include <ZArray.h>
ZResource::ZResource(ZFile* nParent)
{
@ -18,6 +19,7 @@ ZResource::ZResource(ZFile* nParent)
sourceOutput = "";
rawDataIndex = 0;
outputDeclaration = true;
hash = 0;
RegisterRequiredAttribute("Name");
RegisterOptionalAttribute("OutName");
@ -119,14 +121,21 @@ void ZResource::ParseXML(tinyxml2::XMLElement* reader)
name = registeredAttributes.at("Name").value;
static std::regex r("[a-zA-Z_]+[a-zA-Z0-9_]*", std::regex::icase | std::regex::optimize);
if (!isInner || (isInner && name != ""))
// Disable this check for OTR file generation for now since it takes up a considerable amount of CPU time
if (!Globals::Instance->otrMode)
{
if (!std::regex_match(name, r))
static std::regex r("[a-zA-Z_]+[a-zA-Z0-9_]*",
std::regex::icase | std::regex::optimize);
if (!isInner || (isInner && name != ""))
{
HANDLE_ERROR_RESOURCE(WarningType::InvalidAttributeValue, parent, this,
rawDataIndex, "invalid value found for 'Name' attribute", "");
if (!std::regex_match(name, r))
{
HANDLE_ERROR_RESOURCE(WarningType::InvalidAttributeValue, parent, this,
rawDataIndex, "invalid value found for 'Name' attribute",
"");
}
}
}
@ -273,6 +282,21 @@ void ZResource::GetSourceOutputCode([[maybe_unused]] const std::string& prefix)
else
decl->text = bodyStr;
// OTRTODO: This is a hack and we need something more elegant in the future...
if (GetResourceType() == ZResourceType::Array)
{
ZArray* arr = (ZArray*)this;
if (arr->resList[0]->GetResourceType() == ZResourceType::Vertex)
{
for (int i = 0; i < arr->resList.size(); i++)
{
ZVtx* vtx = (ZVtx*)arr->resList[i];
decl->vertexHack.push_back(vtx);
}
}
}
if (decl != nullptr)
decl->staticConf = staticConf;
}

201
ZAPDTR/ZAPD/ZRom.cpp Normal file
View File

@ -0,0 +1,201 @@
#include "ZRom.h"
#include "Utils/BitConverter.h"
#include "Utils/File.h"
#include "Utils/Directory.h"
#include "yaz0/yaz0.h"
#ifndef _MSC_VER
#include <byteswap.h>
#endif
namespace fs = std::filesystem;
#define DMA_ENTRY_SIZE 16
#if defined(_MSC_VER)
#define __bswap_32 _byteswap_ulong
#define bswap_32 _byteswap_ulong
#endif
// ROM DMA Table Start
#define OOT_OFF_NTSC_10_RC 0x7430
#define OOT_OFF_NTSC_10 0x7430
#define OOT_OFF_NTSC_11 0x7430
#define OOT_OFF_PAL_10 0x7950
#define OOT_OFF_NTSC_12 0x7960
#define OOT_OFF_PAL_11 0x7950
#define OOT_OFF_JP_GC 0x7170
#define OOT_OFF_JP_MQ 0x7170
#define OOT_OFF_US_GC 0x7170
#define OOT_OFF_US_MQ 0x7170
#define OOT_OFF_PAL_GC_DBG1 0x12F70
#define OOT_OFF_PAL_MQ_DBG 0x12F70
#define OOT_OFF_PAL_GC_DBG2 0x12F70
#define OOT_OFF_PAL_GC 0x7170
#define OOT_OFF_PAL_MQ 0x7170
#define OOT_OFF_JP_GC_CE 007170
#define OOT_OFF_CN_IQUE 0xB7A0
#define OOT_OFF_TW_IQUE 0xB240
#define MM_OFF_US_10 0x1A500
#define MM_OFF_JP_10 0x1C110
#define MM_OFF_JP_11 0x1C050
#define MM_OFF_DBG 0x24F60
#define OOT_NTSC_10 0xEC7011B7
#define OOT_NTSC_11 0xD43DA81F
#define OOT_NTSC_12 0x693BA2AE
#define OOT_PAL_10 0xB044B569
#define OOT_PAL_11 0xB2055FBD
#define OOT_NTSC_JP_GC_CE 0xF7F52DB8
#define OOT_NTSC_JP_GC 0xF611F4BA
#define OOT_NTSC_US_GC 0xF3DD35BA
#define OOT_PAL_GC 0x09465AC3
#define OOT_NTSC_JP_MQ 0xF43B45BA
#define OOT_NTSC_US_MQ 0xF034001A
#define OOT_PAL_MQ 0x1D4136F3
#define OOT_PAL_GC_DBG1 0x871E1C92 // 03-21-2002 build
#define OOT_PAL_GC_DBG2 0x87121EFE // 03-13-2002 build
#define OOT_PAL_GC_MQ_DBG 0x917D18F6
#define OOT_IQUE_TW 0x3D81FB3E
#define OOT_IQUE_CN 0xB1E1E07B
#define OOT_UNKNOWN 0xFFFFFFFF
ZRom::ZRom(std::string romPath)
{
RomVersion version;
romData = File::ReadAllBytes(romPath);
version.crc = BitConverter::ToInt32BE(romData, 0x10);
switch (version.crc)
{
case OOT_NTSC_10:
version.version = "N64 NTSC 1.0";
version.listPath = "ntsc_oot.txt";
version.offset = OOT_OFF_NTSC_10;
break;
case OOT_NTSC_11:
version.version = "N64 NTSC 1.1";
version.listPath = "ntsc_oot.txt";
version.offset = OOT_OFF_NTSC_11;
break;
case OOT_NTSC_12:
version.version = "N64 NTSC 1.2";
version.listPath = "ntsc_oot.txt";
version.offset = OOT_OFF_NTSC_12;
break;
case OOT_PAL_10:
version.version = "N64 PAL 1.0";
version.listPath = "pal_oot.txt";
version.offset = OOT_OFF_PAL_10;
break;
case OOT_PAL_11:
version.version = "N64 PAL 1.1";
version.listPath = "pal_oot.txt";
version.offset = OOT_OFF_PAL_11;
break;
case OOT_NTSC_JP_GC:
version.version = "JP GameCube (MQ Disk)";
version.listPath = "gamecube.txt";
version.offset = OOT_OFF_JP_GC;
break;
case OOT_NTSC_JP_GC_CE:
version.version = "GameCube (Collectors Edition Disk)";
version.listPath = "gamecube.txt";
version.offset = OOT_OFF_JP_GC_CE;
break;
case OOT_NTSC_JP_MQ:
version.version = "JP Master Quest";
version.listPath = "gamecube.txt";
version.offset = OOT_OFF_JP_MQ;
break;
case OOT_NTSC_US_MQ:
version.version = "NTSC Master Quest";
version.listPath = "gamecube.txt";
version.offset = OOT_OFF_JP_MQ;
break;
case OOT_NTSC_US_GC:
version.version = "NTSC GameCube";
version.listPath = "gamecube.txt";
version.offset = OOT_OFF_US_MQ;
break;
case OOT_PAL_GC:
version.version = "PAL GameCube";
version.listPath = "gamecube_pal.txt";
version.offset = OOT_OFF_PAL_GC;
break;
case OOT_PAL_MQ:
version.version = "PAL Master Quest";
version.listPath = "gamecube_pal.txt";
version.offset = OOT_OFF_PAL_MQ;
break;
case OOT_PAL_GC_DBG1:
version.version = "GameCube Debug 1.0";
version.listPath = "dbg.txt";
version.offset = OOT_OFF_PAL_GC_DBG1;
break;
case OOT_PAL_GC_DBG2:
version.version = "GameCube Debug 2.0";
version.listPath = "dbg.txt";
version.offset = OOT_OFF_PAL_GC_DBG2;
break;
case OOT_PAL_GC_MQ_DBG:
version.version = "GameCube MQ-Debug";
version.listPath = "dbg.txt";
version.offset = OOT_OFF_PAL_MQ_DBG;
break;
case OOT_IQUE_CN:
version.version = "OoT IQue";
version.listPath = "ique.txt";
version.offset = OOT_OFF_CN_IQUE;
break;
case OOT_IQUE_TW:
version.version = "TW IQue";
version.listPath = "ique.txt";
version.offset = OOT_OFF_TW_IQUE;
break;
}
auto path = StringHelper::Sprintf("CFG/filelists/%s", version.listPath.c_str());
auto txt = File::ReadAllText(path);
std::vector<std::string> lines = StringHelper::Split(txt, "\n");
std::vector<uint8_t> decompressedData(1);
for (int i = 0; i < lines.size(); i++)
{
lines[i] = StringHelper::Strip(lines[i], "\r");
const int romOffset = version.offset + (DMA_ENTRY_SIZE * i);
const int virtStart = BitConverter::ToInt32BE(romData, romOffset + 0);
const int virtEnd = BitConverter::ToInt32BE(romData, romOffset + 4);
const int physStart = BitConverter::ToInt32BE(romData, romOffset + 8);
const int physEnd = BitConverter::ToInt32BE(romData, romOffset + 12);
const bool compressed = physEnd != 0;
int size = compressed ? physEnd - physStart : virtEnd - virtStart;
auto outData = std::vector<uint8_t>();
outData.resize(size);
memcpy(outData.data(), romData.data() + physStart, size);
if (compressed)
{
int decSize = virtEnd - virtStart;
decompressedData = std::vector<uint8_t>();
decompressedData.resize(decSize);
yaz0_decode(outData.data(), decompressedData.data(), decSize);
files[lines[i]] = decompressedData;
}
else
files[lines[i]] = outData;
}
int bp = 0;
}
std::vector<uint8_t> ZRom::GetFile(std::string fileName)
{
return files[fileName];
}

27
ZAPDTR/ZAPD/ZRom.h Normal file
View File

@ -0,0 +1,27 @@
#pragma once
#include <stdint.h>
#include <vector>
#include <map>
#include <string>
class ZRom
{
public:
ZRom(std::string romPath);
std::vector<uint8_t> GetFile(std::string fileName);
protected:
std::vector<uint8_t> romData;
std::map<std::string, std::vector<uint8_t>> files;
};
struct RomVersion
{
std::string version = "None";
std::string error = "None";
std::string listPath = "None";
int offset;
uint32_t crc;
};

View File

@ -21,11 +21,12 @@ void ZText::ParseRawData()
const auto& rawData = parent->GetRawData();
uint32_t currentPtr = StringHelper::StrToL(registeredAttributes.at("CodeOffset").value, 16);
std::vector<uint8_t> codeData = File::ReadAllBytes(Globals::Instance->baseRomPath.string() + "\\code");
std::vector<uint8_t> codeData = Globals::Instance->GetBaseromFile(Globals::Instance->baseRomPath.string() + "code");
//std::vector<uint8_t> codeData = File::ReadAllBytes(Globals::Instance->baseRomPath.string() + "\\code");
// In some cases with the multi-process extractor it seems that it fails to read the code file if something else is reading from it at the same time.
while (codeData.size() == 0)
codeData = File::ReadAllBytes(Globals::Instance->baseRomPath.string() + "\\code");
//while (codeData.size() == 0)
//codeData = File::ReadAllBytes(Globals::Instance->baseRomPath.string() + "\\code");
while (true)
{

View File

@ -807,25 +807,30 @@ Declaration* ZTexture::DeclareVar(const std::string& prefix,
std::string ZTexture::GetBodySourceCode() const
{
std::string sourceOutput;
size_t texSizeInc = (dWordAligned) ? 8 : 4;
for (size_t i = 0; i < textureDataRaw.size(); i += texSizeInc)
{
if (i % 32 == 0)
sourceOutput += " ";
if (dWordAligned)
sourceOutput +=
StringHelper::Sprintf("0x%016llX, ", BitConverter::ToUInt64BE(textureDataRaw, i));
else
sourceOutput +=
StringHelper::Sprintf("0x%08llX, ", BitConverter::ToUInt32BE(textureDataRaw, i));
if (i % 32 == 24)
sourceOutput += StringHelper::Sprintf(" // 0x%06X \n", rawDataIndex + ((i / 32) * 32));
}
// Ensure there's always a trailing line feed to prevent dumb warnings.
// Please don't remove this line, unless you somehow made a way to prevent
// that warning when building the OoT repo.
sourceOutput += "\n";
if (!Globals::Instance->otrMode)
{
size_t texSizeInc = (dWordAligned) ? 8 : 4;
for (size_t i = 0; i < textureDataRaw.size(); i += texSizeInc)
{
if (i % 32 == 0)
sourceOutput += " ";
if (dWordAligned)
sourceOutput += StringHelper::Sprintf("0x%016llX, ",
BitConverter::ToUInt64BE(textureDataRaw, i));
else
sourceOutput += StringHelper::Sprintf("0x%08llX, ",
BitConverter::ToUInt32BE(textureDataRaw, i));
if (i % 32 == 24)
sourceOutput +=
StringHelper::Sprintf(" // 0x%06X \n", rawDataIndex + ((i / 32) * 32));
}
// Ensure there's always a trailing line feed to prevent dumb warnings.
// Please don't remove this line, unless you somehow made a way to prevent
// that warning when building the OoT repo.
sourceOutput += "\n";
}
return sourceOutput;
}
@ -847,8 +852,11 @@ std::string ZTexture::GetSourceTypeName() const
void ZTexture::CalcHash()
{
auto parentRawData = parent->GetRawData();
hash = CRC32B(parentRawData.data() + rawDataIndex, GetRawDataSize());
//if (hash == 0)
{
const auto& parentRawData = parent->GetRawData();
hash = CRC32B(parentRawData.data() + rawDataIndex, GetRawDataSize());
}
}
std::string ZTexture::GetExternalExtension() const

View File

@ -0,0 +1,29 @@
#ifndef __READWRITE_H__
#define __READWRITE_H__
#include <stdint.h>
/* variables */
union {
uint32_t u;
float f;
} __u32_f32_union__;
#define U32(x) \
((uint32_t)((((uint8_t*)(x))[0] << 24) | (((uint8_t*)(x))[1] << 16) | \
(((uint8_t*)(x))[2] << 8) | ((uint8_t*)(x))[3]))
#define U16(x) ((uint16_t)(((*((uint8_t*)(x))) << 8) | ((uint8_t*)(x))[1]))
#define U8(x) ((uint8_t)((uint8_t*)(x))[0])
#define S32(x) ((int32_t)(U32(x)))
#define S16(x) ((int16_t)(U16(x)))
#define F32(x) (((__u32_f32_union__.u = U32(x)) & 0) + __u32_f32_union__.f)
#define W32(x, v) \
{ \
*((uint8_t*)x + 3) = ((v)&0xFF); \
*((uint8_t*)x + 2) = (((v) >> 8) & 0xFF); \
*((uint8_t*)x + 1) = (((v) >> 16) & 0xFF); \
*((uint8_t*)x + 0) = (((v) >> 24) & 0xFF); \
}
#endif

227
ZAPDTR/ZAPD/yaz0/yaz0.cpp Normal file
View File

@ -0,0 +1,227 @@
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <list>
#include "readwrite.h"
#include "yaz0.h"
typedef uint8_t u8;
typedef uint16_t u16;
typedef uint32_t u32;
/* internal declarations */
int yaz0_encode_internal(const u8* src, int srcSize, u8* Data);
int yaz0_get_size(u8* src) { return U32(src + 0x4); }
u32 toDWORD(u32 d) {
u8 w1 = d & 0xFF;
u8 w2 = (d >> 8) & 0xFF;
u8 w3 = (d >> 16) & 0xFF;
u8 w4 = d >> 24;
return (w1 << 24) | (w2 << 16) | (w3 << 8) | w4;
}
// simple and straight encoding scheme for Yaz0
u32 longest_match_brute(const u8* src, int size, int pos, u32* pMatchPos) {
int startPos = pos - 0x1000;
int max_match_size = size - pos;
u32 best_match_size = 0;
u32 best_match_pos = 0;
if (max_match_size < 3) return 0;
if (startPos < 0) startPos = 0;
if (max_match_size > 0x111) max_match_size = 0x111;
for (int i = startPos; i < pos; i++) {
int current_size;
for (current_size = 0; current_size < max_match_size; current_size++) {
if (src[i + current_size] != src[pos + current_size]) {
break;
}
}
if (current_size > best_match_size) {
best_match_size = current_size;
best_match_pos = i;
if (best_match_size == 0x111) break;
}
}
*pMatchPos = best_match_pos;
return best_match_size;
}
u32 longest_match_rabinkarp(const u8* src, int size, int pos, u32* match_pos) {
int startPos = pos - 0x1000;
int max_match_size = size - pos;
u32 best_match_size = 0;
u32 best_match_pos = 0;
if (max_match_size < 3) return 0;
if (startPos < 0) startPos = 0;
if (max_match_size > 0x111) max_match_size = 0x111;
int find_hash = src[pos] << 16 | src[pos + 1] << 8 | src[pos + 2];
int current_hash = src[startPos] << 16 | src[startPos + 1] << 8 | src[startPos + 2];
for (int i = startPos; i < pos; i++) {
if(current_hash == find_hash) {
int current_size;
for (current_size = 3; current_size < max_match_size; current_size++) {
if (src[i + current_size] != src[pos + current_size]) {
break;
}
}
if (current_size > best_match_size) {
best_match_size = current_size;
best_match_pos = i;
if (best_match_size == 0x111) break;
}
}
current_hash = (current_hash << 8 | src[i + 3]) & 0xFFFFFF;
}
*match_pos = best_match_pos;
return best_match_size;
}
int yaz0_encode_internal(const u8* src, int srcSize, u8* Data) {
int srcPos = 0;
int bitmask = 0x80;
u8 currCodeByte = 0;
int currCodeBytePos = 0;
int pos = currCodeBytePos + 1;
while (srcPos < srcSize) {
u32 numBytes;
u32 matchPos;
numBytes = longest_match_rabinkarp(src, srcSize, srcPos, &matchPos);
//fprintf(stderr, "pos %x len %x pos %x\n", srcPos, (int)numBytes, (int)matchPos);
if (numBytes < 3) {
//fprintf(stderr, "single byte %02x\n", src[srcPos]);
Data[pos++] = src[srcPos++];
currCodeByte |= bitmask;
} else {
// RLE part
u32 dist = srcPos - matchPos - 1;
if (numBytes >= 0x12) // 3 byte encoding
{
Data[pos++] = dist >> 8; // 0R
Data[pos++] = dist & 0xFF; // FF
if (numBytes > 0xFF + 0x12) numBytes = 0xFF + 0x12;
Data[pos++] = numBytes - 0x12;
} else // 2 byte encoding
{
Data[pos++] = ((numBytes - 2) << 4) | (dist >> 8);
Data[pos++] = dist & 0xFF;
}
srcPos += numBytes;
}
bitmask >>= 1;
// write eight codes
if (!bitmask) {
Data[currCodeBytePos] = currCodeByte;
currCodeBytePos = pos++;
currCodeByte = 0;
bitmask = 0x80;
}
}
if (bitmask) {
Data[currCodeBytePos] = currCodeByte;
}
return pos;
}
std::vector<uint8_t> yaz0_encode_fast(const u8* src, int src_size) {
std::vector<uint8_t> buffer;
std::vector<std::list<uint32_t>> lut;
lut.resize(0x1000000);
for (int i = 0; i < src_size - 3; ++i) {
lut[src[i + 0] << 16 | src[i + 1] << 8 | src[i + 2]].push_back(i);
}
return buffer;
}
std::vector<uint8_t> yaz0_encode(const u8* src, int src_size) {
std::vector<uint8_t> buffer(src_size * 10 / 8 + 16);
u8* dst = buffer.data();
// write 4 bytes yaz0 header
memcpy(dst, "Yaz0", 4);
// write 4 bytes uncompressed size
W32(dst + 4, src_size);
// encode
int dst_size = yaz0_encode_internal(src, src_size, dst + 16);
int aligned_size = (dst_size + 31) & -16;
buffer.resize(aligned_size);
#if 0
std::vector<uint8_t> decompressed(src_size);
yaz0_decode(buffer.data(), decompressed.data(), src_size);
if(memcmp(src, decompressed.data(), src_size)) {
fprintf(stderr, "Decompressed buffer is different from original\n");
}
#endif
return buffer;
}
void yaz0_decode(const uint8_t* source, uint8_t* decomp, int32_t decompSize) {
uint32_t srcPlace = 0, dstPlace = 0;
uint32_t i, dist, copyPlace, numBytes;
uint8_t codeByte, byte1, byte2;
uint8_t bitCount = 0;
source += 0x10;
while (dstPlace < decompSize) {
/* If there are no more bits to test, get a new byte */
if (!bitCount) {
codeByte = source[srcPlace++];
bitCount = 8;
}
/* If bit 7 is a 1, just copy 1 byte from source to destination */
/* Else do some decoding */
if (codeByte & 0x80) {
decomp[dstPlace++] = source[srcPlace++];
} else {
/* Get 2 bytes from source */
byte1 = source[srcPlace++];
byte2 = source[srcPlace++];
/* Calculate distance to move in destination */
/* And the number of bytes to copy */
dist = ((byte1 & 0xF) << 8) | byte2;
copyPlace = dstPlace - (dist + 1);
numBytes = byte1 >> 4;
/* Do more calculations on the number of bytes to copy */
if (!numBytes)
numBytes = source[srcPlace++] + 0x12;
else
numBytes += 2;
/* Copy data from a previous point in destination */
/* to current point in destination */
for (i = 0; i < numBytes; i++) decomp[dstPlace++] = decomp[copyPlace++];
}
/* Set up for the next read cycle */
codeByte = codeByte << 1;
bitCount--;
}
}

6
ZAPDTR/ZAPD/yaz0/yaz0.h Normal file
View File

@ -0,0 +1,6 @@
#pragma once
#include <vector>
void yaz0_decode(const uint8_t* src, uint8_t* dest, int32_t destsize);
std::vector<uint8_t> yaz0_encode(const uint8_t* src, int src_size);