Added support for multiple game versions (#107)

* WIP Multiversion support

* GC PAL Non-MQ support complete

* Updated OtrGui to handle different game versions

* Added version file

* Added new extract mode to ZAPD and optimized OTR gen time

* Fixed bug causing crash

* Further optimized OTRExporter, saving around ~20 seconds.

* ZAPD is now multi-threaded.

* Fixed merge issue

* Fixed memory leak and fog issue on pause screen.

* Additional fog fixes.

Co-authored-by: Jack Walker <7463599+Jack-Walker@users.noreply.github.com>
This commit is contained in:
Nicholas Estelami 2022-03-31 19:42:44 -04:00 committed by GitHub
parent 572e9fb9d0
commit c80f9fbd57
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1203 changed files with 30620 additions and 501 deletions

View File

@ -345,6 +345,8 @@ baserom/
*.otr
*.swp
*.a
*.z64
*.n64
Extract/
tmp.txt

View File

@ -2,7 +2,7 @@
<SymbolMap File="SymbolMap_OoTMqDbg.txt"/>
<ActorList File="ActorList_OoTMqDbg.txt"/>
<ObjectList File="ObjectList_OoTMqDbg.txt"/>
<ExternalXMLFolder Path="../soh/assets/xml/"/>
<ExternalXMLFolder Path="../soh/assets/xml/GC_NMQ_PAL_F/"/>
<TexturePool File="TexturePool.xml"/>
<ExternalFile XmlPath="objects/gameplay_keep.xml" OutPath="objects/gameplay_keep/"/>
</Root>

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -209,7 +209,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
break;
case G_MTX:
{
if ((!Globals::Instance->HasSegment(GETSEGNUM(data))) || ((data & 0xFFFFFFFF) == 0x07000000)) // En_Zf and En_Ny place a DL in segment 7
if ((!Globals::Instance->HasSegment(GETSEGNUM(data), res->parent->workerID)) || ((data & 0xFFFFFFFF) == 0x07000000)) // En_Zf and En_Ny place a DL in segment 7
{
uint32_t pp = (data & 0x000000FF00000000) >> 32;
uint32_t mm = (data & 0x00000000FFFFFFFF);
@ -370,7 +370,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
//std::string fName = StringHelper::Sprintf("%s\\%s", GetParentFolderName(res).c_str(), dListDecl2->varName.c_str());
std::string fName = OTRExporter_DisplayList::GetPathToRes(res, dListDecl2->varName.c_str());
if (!File::Exists("Extract\\" + fName))
if (files.find(fName) == files.end() && !File::Exists("Extract\\" + fName))
{
MemoryStream* dlStream = new MemoryStream();
BinaryWriter dlWriter = BinaryWriter(dlStream);
@ -382,7 +382,10 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
//otrArchive->RemoveFile(fName);
#endif
File::WriteAllBytes("Extract\\" + fName, dlStream->ToVector());
if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
File::WriteAllBytes("Extract\\" + fName, dlStream->ToVector());
else
files[fName] = dlStream->ToVector();
//otrArchive->AddFile(fName, (uintptr_t)dlStream->ToVector().data(), dlWriter.GetBaseAddress());
}
@ -401,7 +404,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
//case G_BRANCH_Z:
case G_DL:
{
if ((!Globals::Instance->HasSegment(GETSEGNUM(data)) && (int)opF3D != G_BRANCH_Z)
if ((!Globals::Instance->HasSegment(GETSEGNUM(data), res->parent->workerID) && (int)opF3D != G_BRANCH_Z)
|| ((data & 0xFFFFFFFF) == 0x07000000)) // En_Zf and En_Ny place a DL in segment 7
{
int32_t pp = (data & 0x00FF000000000000) >> 56;
@ -464,14 +467,17 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
//std::string fName = StringHelper::Sprintf("%s\\%s", GetParentFolderName(res).c_str(), dListDecl2->varName.c_str());
std::string fName = OTRExporter_DisplayList::GetPathToRes(res, dListDecl2->varName.c_str());
if (!File::Exists("Extract\\" + fName))
if (files.find(fName) == files.end() && !File::Exists("Extract\\" + fName))
{
MemoryStream* dlStream = new MemoryStream();
BinaryWriter dlWriter = BinaryWriter(dlStream);
Save(dList->otherDLists[i], outPath, &dlWriter);
File::WriteAllBytes("Extract\\" + fName, dlStream->ToVector());
if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
File::WriteAllBytes("Extract\\" + fName, dlStream->ToVector());
else
files[fName] = dlStream->ToVector();
}
}
else
@ -675,7 +681,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
uint32_t seg = data & 0xFFFFFFFF;
int32_t texAddress = Seg2Filespace(data, dList->parent->baseAddress);
if (!Globals::Instance->HasSegment(GETSEGNUM(seg)))
if (!Globals::Instance->HasSegment(GETSEGNUM(seg), res->parent->workerID))
{
int32_t __ = (data & 0x00FF000000000000) >> 48;
int32_t www = (data & 0x00000FFF00000000) >> 32;
@ -693,7 +699,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
else
{
std::string texName = "";
bool foundDecl = Globals::Instance->GetSegmentedPtrName(seg, dList->parent, "", texName);
bool foundDecl = Globals::Instance->GetSegmentedPtrName(seg, dList->parent, "", texName, res->parent->workerID);
int32_t __ = (data & 0x00FF000000000000) >> 48;
int32_t www = (data & 0x00000FFF00000000) >> 32;
@ -712,7 +718,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
if (foundDecl)
{
ZFile* assocFile = Globals::Instance->GetSegment(GETSEGNUM(seg));
ZFile* assocFile = Globals::Instance->GetSegment(GETSEGNUM(seg), res->parent->workerID);
std::string assocFileName = assocFile->GetName();
std::string fName = "";
@ -750,42 +756,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
word1 = value.words.w1 | 0xF0000000;
}
else
//if (dList->vertices.size() > 0)
{
// Connect neighboring vertex arrays
std::vector<std::pair<uint32_t, std::vector<ZVtx>>> vertsKeys(dList->vertices.begin(),
dList->vertices.end());
if (vertsKeys.size() > 0)
{
auto lastItem = vertsKeys[0];
for (size_t i = 1; i < vertsKeys.size(); i++)
{
auto curItem = vertsKeys[i];
int32_t sizeDiff = curItem.first - (lastItem.first + (lastItem.second.size() * 16));
// Make sure there isn't an unaccounted inbetween these two
if (sizeDiff == 0)
{
for (auto v : curItem.second)
{
dList->vertices[lastItem.first].push_back(v);
lastItem.second.push_back(v);
}
dList->vertices.erase(curItem.first);
vertsKeys.erase(vertsKeys.begin() + i);
i--;
continue;
}
lastItem = curItem;
}
}
// Write CRC64 of vtx file name
uint32_t addr = data & 0xFFFFFFFF;
@ -793,10 +764,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
addr -= dList->parent->baseAddress;
auto segOffset = GETSEGOFFSET(addr);
//uint32_t seg = data & 0xFFFFFFFF;
Declaration* vtxDecl = dList->parent->GetDeclarationRanged(segOffset);
//std::string vtxName = "";
//bool foundDecl = Globals::Instance->GetSegmentedPtrName(seg, dList->parent, "", vtxName);
int32_t aa = (data & 0x000000FF00000000ULL) >> 32;
int32_t nn = (data & 0x000FF00000000000ULL) >> 44;
@ -822,9 +790,8 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
word0 = hash >> 32;
word1 = hash & 0xFFFFFFFF;
if (!File::Exists("Extract\\" + fName))
if (files.find(fName) == files.end() && !File::Exists("Extract\\" + fName))
{
//printf("Exporting VTX Data %s\n", fName.c_str());
// Write vertices to file
MemoryStream* vtxStream = new MemoryStream();
BinaryWriter vtxWriter = BinaryWriter(vtxStream);
@ -847,44 +814,40 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
vtxWriter.Write((uint32_t)ZResourceType::Vertex);
vtxWriter.Write((uint32_t)arrCnt);
size_t sz = dList->vertices[vtxDecl->address].size();
auto start = std::chrono::steady_clock::now();
//if (sz > 0)
// God dammit this is so dumb
for (size_t i = 0; i < split.size(); i++)
{
auto start = std::chrono::steady_clock::now();
std::string line = split[i];
// God dammit this is so dumb
for (size_t i = 0; i < split.size(); i++)
if (StringHelper::Contains(line, "VTX("))
{
std::string line = split[i];
auto split2 = StringHelper::Split(StringHelper::Split(StringHelper::Split(line, "VTX(")[1], ")")[0], ",");
if (StringHelper::Contains(line, "VTX("))
{
auto split2 = StringHelper::Split(StringHelper::Split(StringHelper::Split(line, "VTX(")[1], ")")[0], ",");
vtxWriter.Write((int16_t)std::stoi(split2[0], nullptr, 10)); // v.x
vtxWriter.Write((int16_t)std::stoi(split2[1], nullptr, 10)); // v.y
vtxWriter.Write((int16_t)std::stoi(split2[2], nullptr, 10)); // v.z
vtxWriter.Write((int16_t)std::stoi(split2[0], nullptr, 10)); // v.x
vtxWriter.Write((int16_t)std::stoi(split2[1], nullptr, 10)); // v.y
vtxWriter.Write((int16_t)std::stoi(split2[2], nullptr, 10)); // v.z
vtxWriter.Write((int16_t)0); // v.flag
vtxWriter.Write((int16_t)0); // v.flag
vtxWriter.Write((int16_t)std::stoi(split2[3], nullptr, 10)); // v.s
vtxWriter.Write((int16_t)std::stoi(split2[4], nullptr, 10)); // v.t
vtxWriter.Write((uint8_t)std::stoi(split2[5], nullptr, 10)); // v.r
vtxWriter.Write((uint8_t)std::stoi(split2[6], nullptr, 10)); // v.g
vtxWriter.Write((uint8_t)std::stoi(split2[7], nullptr, 10)); // v.b
vtxWriter.Write((uint8_t)std::stoi(split2[8], nullptr, 10)); // v.a
}
vtxWriter.Write((int16_t)std::stoi(split2[3], nullptr, 10)); // v.s
vtxWriter.Write((int16_t)std::stoi(split2[4], nullptr, 10)); // v.t
vtxWriter.Write((uint8_t)std::stoi(split2[5], nullptr, 10)); // v.r
vtxWriter.Write((uint8_t)std::stoi(split2[6], nullptr, 10)); // v.g
vtxWriter.Write((uint8_t)std::stoi(split2[7], nullptr, 10)); // v.b
vtxWriter.Write((uint8_t)std::stoi(split2[8], nullptr, 10)); // v.a
}
File::WriteAllBytes("Extract\\" + fName, vtxStream->ToVector());
auto end = std::chrono::steady_clock::now();
size_t diff = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
//printf("Exported VTX Array %s in %zums\n", fName.c_str(), diff);
}
if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
File::WriteAllBytes("Extract\\" + fName, vtxStream->ToVector());
else
files[fName] = vtxStream->ToVector();
auto end = std::chrono::steady_clock::now();
size_t diff = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
}
}
else
@ -892,15 +855,6 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
spdlog::error("vtxDecl == nullptr!");
}
}
/*else
{
writer->Write(word0);
writer->Write(word1);
word0 = 0;
word1 = 0;
spdlog::error("dList->vertices.size() <= 0!");
}*/
}
break;
}

View File

@ -25,6 +25,7 @@ std::string otrFileName = "oot.otr";
std::shared_ptr<Ship::Archive> otrArchive;
BinaryWriter* fileWriter;
std::chrono::steady_clock::time_point fileStart, resStart;
std::map<std::string, std::vector<char>> files;
void InitVersionInfo();
@ -39,6 +40,8 @@ static void ExporterParseFileMode(const std::string& buildMode, ZFileMode& fileM
{
fileMode = (ZFileMode)ExporterFileMode::BuildOTR;
printf("BOTR: Generating OTR Archive...\n");
if (File::Exists(otrFileName))
otrArchive = std::shared_ptr<Ship::Archive>(new Ship::Archive(otrFileName, true));
else
@ -54,6 +57,31 @@ static void ExporterParseFileMode(const std::string& buildMode, ZFileMode& fileM
}
}
static void ExporterProgramEnd()
{
if (Globals::Instance->fileMode == ZFileMode::ExtractDirectory)
{
printf("Generating OTR Archive...\n");
otrArchive = Ship::Archive::CreateArchive(otrFileName, 65536 / 2);
for (auto item : files)
{
auto fileData = item.second;
otrArchive->AddFile(item.first, (uintptr_t)fileData.data(), fileData.size());
}
// Add any additional files that need to be manually copied...
auto lst = Directory::ListFiles("Extract");
for (auto item : lst)
{
auto fileData = File::ReadAllBytes(item);
otrArchive->AddFile(StringHelper::Split(item, "Extract\\")[1], (uintptr_t)fileData.data(), fileData.size());
}
}
}
static void ExporterParseArgs(int argc, char* argv[], int& i)
{
std::string arg = argv[i];
@ -85,6 +113,7 @@ static void ExporterFileBegin(ZFile* file)
static void ExporterFileEnd(ZFile* file)
{
int bp = 0;
}
static void ExporterResourceEnd(ZResource* res, BinaryWriter& writer)
@ -124,7 +153,10 @@ static void ExporterResourceEnd(ZResource* res, BinaryWriter& writer)
else
fName = StringHelper::Sprintf("%s\\%s", oName.c_str(), rName.c_str());
File::WriteAllBytes("Extract\\" + fName, strem->ToVector());
if (Globals::Instance->fileMode == ZFileMode::ExtractDirectory)
files[fName] = strem->ToVector();
else
File::WriteAllBytes("Extract\\" + fName, strem->ToVector());
}
auto end = std::chrono::steady_clock::now();
@ -155,6 +187,8 @@ static void ImportExporters()
exporterSet->beginXMLFunc = ExporterXMLBegin;
exporterSet->endXMLFunc = ExporterXMLEnd;
exporterSet->resSaveFunc = ExporterResourceEnd;
exporterSet->endProgramFunc = ExporterProgramEnd;
exporterSet->exporters[ZResourceType::Background] = new OTRExporter_Background();
exporterSet->exporters[ZResourceType::Texture] = new OTRExporter_Texture();
exporterSet->exporters[ZResourceType::Room] = new OTRExporter_Room();

View File

@ -2,4 +2,5 @@
#include <Archive.h>
extern std::shared_ptr<Ship::Archive> otrArchive;
extern std::shared_ptr<Ship::Archive> otrArchive;
extern std::map<std::string, std::vector<char>> files;

View File

@ -63,6 +63,12 @@
<ClCompile Include="VersionInfo.cpp" />
<ClCompile Include="VtxExporter.cpp" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\OTRGui\build\ZERO_CHECK.vcxproj">
<Project>{02d10590-9542-3f55-aaf8-6055677e2a2a}</Project>
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
</ProjectReference>
</ItemGroup>
<PropertyGroup Label="Globals">
<VCProjectVersion>16.0</VCProjectVersion>
<Keyword>Win32Proj</Keyword>
@ -118,19 +124,31 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(SolutionDir)otrlib;$(SolutionDir)\ZAPD\ZAPD\;$(SolutionDir)\ZAPD\lib\tinyxml2;$(SolutionDir)\ZAPD\lib\libgfxd;$(SolutionDir)\ZAPD\lib\elfio;$(SolutionDir)\ZAPD\lib\assimp\include;$(SolutionDir)\ZAPD\lib\stb;$(ProjectDir);$(IncludePath)</IncludePath>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(ProjectDir)..\..\ZAPDTR\ZAPD;$(ProjectDir)..\..\ZAPDTR\lib\tinyxml2;$(ProjectDir)..\..\ZAPDTR\lib\libgfxd;$(ProjectDir)..\..\ZAPDTR\ZAPDUtils;$(ProjectDir)..\..\libultraship\libultraship;$(ProjectDir)..\..\libultraship\libultraship\lib\spdlog\include;$(ProjectDir)..\..\libultraship\libultraship\Lib\Fast3D\U64;$(IncludePath)</IncludePath>
<LibraryPath>$(ProjectDir)..\..\libultraship\libultraship;$(LibraryPath)</LibraryPath>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(ProjectDir)..\..\ZAPDTR\ZAPD;$(ProjectDir)..\..\ZAPDTR\lib\tinyxml2;$(ProjectDir)..\..\ZAPDTR\lib\libgfxd;$(ProjectDir)..\..\ZAPDTR\ZAPDUtils;$(ProjectDir)..\..\libultraship\libultraship;$(ProjectDir)..\..\libultraship\libultraship\lib\spdlog\include;$(ProjectDir)..\..\libultraship\libultraship\Lib\Fast3D\U64;$(IncludePath)</IncludePath>
<LibraryPath>$(ProjectDir)..\..\libultraship\libultraship;$(LibraryPath)</LibraryPath>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>

View File

@ -407,7 +407,7 @@ void OTRExporter_Room::Save(ZResource* res, const fs::path& outPath, BinaryWrite
{
uint32_t seg = cmdHeaders->headers[i] & 0xFFFFFFFF;
std::string headerName = "";
bool foundDecl = Globals::Instance->GetSegmentedPtrName(seg, room->parent, "", headerName);
bool foundDecl = Globals::Instance->GetSegmentedPtrName(seg, room->parent, "", headerName, res->parent->workerID);
if (headerName == "NULL")
writer->Write("");
else
@ -443,7 +443,7 @@ void OTRExporter_Room::Save(ZResource* res, const fs::path& outPath, BinaryWrite
SetCutscenes* cmdSetCutscenes = (SetCutscenes*)cmd;
std::string listName;
Globals::Instance->GetSegmentedPtrName(cmdSetCutscenes->cmdArg2, room->parent, "CutsceneData", listName);
Globals::Instance->GetSegmentedPtrName(cmdSetCutscenes->cmdArg2, room->parent, "CutsceneData", listName, res->parent->workerID);
std::string fName = OTRExporter_DisplayList::GetPathToRes(room, listName);
//std::string fName = StringHelper::Sprintf("%s\\%s", OTRExporter_DisplayList::GetParentFolderName(room).c_str(), listName.c_str());
writer->Write(fName);
@ -452,8 +452,11 @@ void OTRExporter_Room::Save(ZResource* res, const fs::path& outPath, BinaryWrite
BinaryWriter csWriter = BinaryWriter(csStream);
OTRExporter_Cutscene cs;
cs.Save(cmdSetCutscenes->cutscenes[0], "", &csWriter);
File::WriteAllBytes("Extract\\" + fName, csStream->ToVector());
if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
File::WriteAllBytes("Extract\\" + fName, csStream->ToVector());
else
files[fName] = csStream->ToVector();
//std::string fName = OTRExporter_DisplayList::GetPathToRes(res, vtxDecl->varName);
//otrArchive->AddFile(fName, (uintptr_t)csStream->ToVector().data(), csWriter.GetBaseAddress());
@ -477,7 +480,10 @@ void OTRExporter_Room::Save(ZResource* res, const fs::path& outPath, BinaryWrite
OTRExporter_Path pathExp;
pathExp.Save(&cmdSetPathways->pathwayList, outPath, &pathWriter);
File::WriteAllBytes("Extract\\" + path, pathStream->ToVector());
if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
File::WriteAllBytes("Extract\\" + path, pathStream->ToVector());
else
files[path] = pathStream->ToVector();
//otrArchive->AddFile(path, (uintptr_t)pathStream->ToVector().data(), pathWriter.GetBaseAddress());

View File

@ -23,7 +23,7 @@ void OTRExporter_Skeleton::Save(ZResource* res, const fs::path& outPath, BinaryW
Declaration* skelDecl = skel->parent->GetDeclarationRanged(GETSEGOFFSET(skel->limbsTable.limbsAddresses[i]));
std::string name;
bool foundDecl = Globals::Instance->GetSegmentedPtrName(skel->limbsTable.limbsAddresses[i], skel->parent, "", name);
bool foundDecl = Globals::Instance->GetSegmentedPtrName(skel->limbsTable.limbsAddresses[i], skel->parent, "", name, res->parent->workerID);
if (foundDecl)
{
if (name.at(0) == '&')

View File

@ -86,7 +86,7 @@ void OTRExporter_SkeletonLimb::Save(ZResource* res, const fs::path& outPath, Bin
if (limb->childPtr != 0)
{
std::string name;
bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->childPtr, limb->parent, "", name);
bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->childPtr, limb->parent, "", name, res->parent->workerID);
if (foundDecl)
{
if (name.at(0) == '&')
@ -107,7 +107,7 @@ void OTRExporter_SkeletonLimb::Save(ZResource* res, const fs::path& outPath, Bin
if (limb->siblingPtr != 0)
{
std::string name;
bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->siblingPtr, limb->parent, "", name);
bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->siblingPtr, limb->parent, "", name, res->parent->workerID);
if (foundDecl)
{
if (name.at(0) == '&')
@ -128,7 +128,7 @@ void OTRExporter_SkeletonLimb::Save(ZResource* res, const fs::path& outPath, Bin
if (limb->dListPtr != 0)
{
std::string name;
bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->dListPtr, limb->parent, "", name);
bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->dListPtr, limb->parent, "", name, res->parent->workerID);
if (foundDecl)
{
if (name.at(0) == '&')
@ -149,7 +149,7 @@ void OTRExporter_SkeletonLimb::Save(ZResource* res, const fs::path& outPath, Bin
if (limb->dList2Ptr != 0)
{
std::string name;
bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->dList2Ptr, limb->parent, "", name);
bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->dList2Ptr, limb->parent, "", name, res->parent->workerID);
if (foundDecl)
{
if (name.at(0) == '&')

View File

@ -48,7 +48,7 @@ def ExtractFunc(fullPath):
*pathList, xmlName = fullPath.split(os.sep)
objectName = os.path.splitext(xmlName)[0]
outPath = os.path.join("..\\soh\\assets\\", *pathList[4:], objectName)
outPath = os.path.join("..\\soh\\assets\\", *pathList[5:], objectName)
os.makedirs(outPath, exist_ok=True)
outSourcePath = outPath
@ -64,6 +64,7 @@ def main():
parser.add_argument("-s", "--single", help="asset path relative to assets/, e.g. objects/gameplay_keep")
parser.add_argument("-f", "--force", help="Force the extraction of every xml instead of checking the touched ones.", action="store_true")
parser.add_argument("-u", "--unaccounted", help="Enables ZAPD unaccounted detector warning system.", action="store_true")
parser.add_argument("-v", "--version", help="Sets game version.")
args = parser.parse_args()
global mainAbort
@ -73,6 +74,13 @@ def main():
extractedAssetsTracker = manager.dict()
xmlVer = "GC_NMQ_D"
if (args.version == "gc_pal_nmpq"):
xmlVer = "GC_NMQ_PAL_F"
elif (args.version == "dbg_mq"):
xmlVer = "GC_MQ_D"
asset_path = args.single
if asset_path is not None:
fullPath = os.path.join("..\\soh\\assets", "xml", asset_path + ".xml")
@ -90,7 +98,7 @@ def main():
extract_staff_text_path = None
xmlFiles = []
for currentPath, _, files in os.walk(os.path.join("..\\soh\\assets", "xml")):
for currentPath, _, files in os.walk(os.path.join("..\\soh\\assets\\xml\\", xmlVer)):
for file in files:
fullPath = os.path.join(currentPath, file)
if file.endswith(".xml"):

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -2,7 +2,7 @@
<SymbolMap File="symbols/SymbolMap_OoTMqDbg.txt"/>
<ActorList File="symbols/ActorList_OoTMqDbg.txt"/>
<ObjectList File="symbols/ObjectList_OoTMqDbg.txt"/>
<ExternalXMLFolder Path="assets/extractor/xmls/"/>
<ExternalXMLFolder Path="assets/extractor/xmls/GC_MQ_D/"/>
<TexturePool File="TexturePool.xml"/>
<ExternalFile XmlPath="objects/gameplay_keep.xml" OutPath="objects/gameplay_keep/"/>
</Root>

View File

@ -0,0 +1,8 @@
<Root>
<SymbolMap File="symbols/SymbolMap_OoTMqDbg.txt"/>
<ActorList File="symbols/ActorList_OoTMqDbg.txt"/>
<ObjectList File="symbols/ObjectList_OoTMqDbg.txt"/>
<ExternalXMLFolder Path="assets/extractor/xmls/GC_NMQ_D/"/>
<TexturePool File="TexturePool.xml"/>
<ExternalFile XmlPath="objects/gameplay_keep.xml" OutPath="objects/gameplay_keep/"/>
</Root>

View File

@ -0,0 +1,8 @@
<Root>
<SymbolMap File="symbols/SymbolMap_OoTMqDbg.txt"/>
<ActorList File="symbols/ActorList_OoTMqDbg.txt"/>
<ObjectList File="symbols/ObjectList_OoTMqDbg.txt"/>
<ExternalXMLFolder Path="assets/extractor/xmls/GC_NMQ_PAL_F/"/>
<TexturePool File="TexturePool.xml"/>
<ExternalFile XmlPath="objects/gameplay_keep.xml" OutPath="objects/gameplay_keep/"/>
</Root>

File diff suppressed because it is too large Load Diff

View File

@ -26,6 +26,7 @@ bool single_thread = false;
bool hide_second_btn = false;
RomVersion version;
const char* patched_rom = "tmp/rom.z64";
extern bool oldExtractMode;
static std::string currentStep = "None";
@ -72,11 +73,28 @@ void OTRGame::init(){
}
}
void ExtractRom() {
const WriteResult result = ExtractBaserom(patched_rom);
void ExtractRom()
{
WriteResult result;
if (oldExtractMode)
ExtractBaserom(patched_rom);
else
result.error = NULLSTR;
if (result.error == NULLSTR) {
if (MoonUtils::exists("oot.otr")) MoonUtils::rm("oot.otr");
startWorker();
if (MoonUtils::exists("Extract")) MoonUtils::rm("Extract");
MoonUtils::mkdir("Extract");
MoonUtils::copy("tmp/baserom/Audiobank", "Extract/Audiobank");
MoonUtils::copy("tmp/baserom/Audioseq", "Extract/Audioseq");
MoonUtils::copy("tmp/baserom/Audiotable", "Extract/Audiotable");
MoonUtils::copy("tmp/baserom/version", "Extract/version");
MoonUtils::copy("assets/game/", "Extract/assets/");
startWorker(version);
extracting = true;
}
}
@ -131,7 +149,7 @@ void OTRGame::draw() {
}
// Clamp the window to the borders of the monitors
if (wndPos.x < vsX1) wndPos.x = vsX1;
if (wndPos.x < vsX1) wndPos.x = vsX1;
if (wndPos.y < vsY1) wndPos.y = vsY1;
if (wndPos.x + windowSize.x > vsX2) wndPos.x = vsX2 - windowSize.x;
@ -160,7 +178,9 @@ void OTRGame::draw() {
UIUtils::GuiShadowText(("Rom Type: " + version.version).c_str(), 32, text_y, 10, WHITE, BLACK);
UIUtils::GuiShadowText("Tool Version: 1.0", 32, text_y + 15, 10, WHITE, BLACK);
UIUtils::GuiShadowText("OTR Version: 1.0", 32, text_y + 30, 10, WHITE, BLACK);
UIUtils::GuiToggle(&single_thread, "Single Thread", 32, text_y + 40, currentStep != NULLSTR);
if (oldExtractMode)
UIUtils::GuiToggle(&single_thread, "Single Thread", 32, text_y + 40, currentStep != NULLSTR);
if (!hide_second_btn && UIUtils::GuiIconButton("Folder", "Open\nShip Folder", 109, 50, currentStep != NULLSTR, "Select your Ship of Harkinian Folder\n\nYou could use another folder\nfor development purposes")) {
const std::string path = NativeFS->LaunchFileExplorer(LaunchType::FOLDER);

View File

@ -83,37 +83,37 @@ RomVersion GetVersion(FILE* rom) {
break;
case OOT_NTSC_JP_GC:
version.version = "JP GameCube (MQ Disk)";
version.listPath = "gamecube_mq.txt";
version.listPath = "gamecube.txt";
version.offset = OOT_OFF_JP_GC;
break;
case OOT_NTSC_JP_GC_CE:
version.version = "GameCube (Collectors Edition Disk)";
version.listPath = "gamecube_mq.txt";
version.listPath = "gamecube.txt";
version.offset = OOT_OFF_JP_GC_CE;
break;
case OOT_NTSC_JP_MQ:
version.version = "JP Master Quest";
version.listPath = "gamecube_mq.txt";
version.listPath = "gamecube.txt";
version.offset = OOT_OFF_JP_MQ;
break;
case OOT_NTSC_US_MQ:
version.version = "NTSC Master Quest";
version.listPath = "gamecube_mq.txt";
version.listPath = "gamecube.txt";
version.offset = OOT_OFF_JP_MQ;
break;
case OOT_NTSC_US_GC:
version.version = "NTSC GameCube";
version.listPath = "gamecube_mq.txt";
version.listPath = "gamecube.txt";
version.offset = OOT_OFF_US_MQ;
break;
case OOT_PAL_GC:
version.version = "PAL GameCube";
version.listPath = "gamecube_mq.txt";
version.listPath = "gamecube_pal.txt";
version.offset = OOT_OFF_PAL_GC;
break;
case OOT_PAL_MQ:
version.version = "PAL Master Quest";
version.listPath = "pal_mq.txt";
version.listPath = "gamecube_pal.txt";
version.offset = OOT_OFF_PAL_MQ;
break;
case OOT_PAL_GC_DBG1:
@ -179,6 +179,8 @@ WriteResult ExtractBaserom(const char* romPath) {
const std::vector<std::string> lines = MoonUtils::split(read(MoonUtils::join("assets/extractor/filelists", version.listPath)), '\n');
std::vector<uint8_t> decompressedData(1);
for (int i = 0; i < lines.size(); i++) {
FILE* outFile = fopen(MoonUtils::join("tmp/baserom", lines[i]).c_str(), "wb");
const int romOffset = version.offset + (DMA_ENTRY_SIZE * i);
@ -196,10 +198,13 @@ WriteResult ExtractBaserom(const char* romPath) {
auto outData = new uint8_t[size];
memcpy(outData, romData + physStart, size);
if (compressed) {
std::vector<uint8_t> compressedData = yaz0_encode(outData, size);
outData = compressedData.data();
size = compressedData.size();
int decSize = virtEnd - virtStart;
decompressedData = std::vector<uint8_t>(decSize);
yaz0_decode(outData, decompressedData.data(), decSize);
outData = decompressedData.data();
size = decSize;
}
fwrite(outData, sizeof(char), size, outFile);

View File

@ -1,23 +1,7 @@
#ifndef EXTRACT_BASEROM_H_
#define EXTRACT_BASEROM_H_
#define OOT_NTSC_10 0xEC7011B7
#define OOT_NTSC_11 0xD43DA81F
#define OOT_NTSC_12 0x693BA2AE
#define OOT_PAL_10 0xB044B569
#define OOT_PAL_11 0xB2055FBD
#define OOT_NTSC_JP_GC_CE 0xF7F52DB8
#define OOT_NTSC_JP_GC 0xF611F4BA
#define OOT_NTSC_US_GC 0xF3DD35BA
#define OOT_PAL_GC 0x09465AC3
#define OOT_NTSC_JP_MQ 0xF43B45BA
#define OOT_NTSC_US_MQ 0xF034001A
#define OOT_PAL_MQ 0x1D4136F3
#define OOT_PAL_GC_DBG1 0x871E1C92 // 03-21-2002 build
#define OOT_PAL_GC_DBG2 0x87121EFE // 03-13-2002 build
#define OOT_PAL_GC_MQ_DBG 0x917D18F6
#define OOT_IQUE_TW 0x3D81FB3E
#define OOT_IQUE_CN 0xB1E1E07B
#include "../../libultraship/libultraship/GameVersions.h"
#include <cstdio>
#include <string>

View File

@ -5,6 +5,7 @@
#include "utils/mutils.h"
#include "ctpl/ctpl_stl.h"
#include <thread>
#include <impl/baserom_extractor/baserom_extractor.h>
#ifdef _WIN32
#define PLATFORM Platforms::WINDOWS
@ -13,6 +14,7 @@
#endif
namespace Util = MoonUtils;
bool oldExtractMode = false;
static int maxResources = 0;
static int extractedResources = 0;
bool buildingOtr = false;
@ -22,19 +24,29 @@ bool isWindows() {
return (PLATFORM == Platforms::WINDOWS);
}
void BuildOTR(const std::string output) {
Util::copy("tmp/baserom/Audiobank", "Extract/Audiobank");
Util::copy("tmp/baserom/Audioseq", "Extract/Audioseq");
Util::copy("tmp/baserom/Audiotable", "Extract/Audiotable");
Util::copy("assets/game/", "Extract/assets/");
std::string execStr = Util::format("assets/extractor/%s", isWindows() ? "ZAPD.exe" : "ZAPD.out") + " botr -se OTR";
ProcessResult result = NativeFS->LaunchProcess(execStr);
if(result.exitCode != 0) {
std::cout << "\nError when building the OTR file with error code: " << result.exitCode << " !" << std::endl;
std::cout << "Aborting...\n" << std::endl;
std::string GetXMLVersion(RomVersion version)
{
switch (version.crc)
{
case OOT_PAL_GC_DBG1: return "GC_NMQ_D";
case OOT_PAL_GC_DBG2: return "GC_MQ_D";
case OOT_PAL_GC: return "GC_NMQ_PAL_F";
}
return "ERROR";
}
void BuildOTR(const std::string output) {
if (oldExtractMode)
{
std::string execStr = Util::format("assets/extractor/%s", isWindows() ? "ZAPD.exe" : "ZAPD.out") + " botr -se OTR";
ProcessResult result = NativeFS->LaunchProcess(execStr);
if (result.exitCode != 0) {
std::cout << "\nError when building the OTR file with error code: " << result.exitCode << " !" << std::endl;
std::cout << "Aborting...\n" << std::endl;
}
}
setCurrentStep("Done!");
if (output == ".") return;
@ -44,9 +56,9 @@ void BuildOTR(const std::string output) {
MoonUtils::copy("oot.otr", outputPath);
}
void ExtractFile(std::string xmlPath, std::string outPath, std::string outSrcPath) {
void ExtractFile(std::string xmlPath, std::string outPath, std::string outSrcPath, RomVersion version) {
std::string execStr = Util::format("assets/extractor/%s", isWindows() ? "ZAPD.exe" : "ZAPD.out");
std::string args = Util::format(" e -eh -i %s -b tmp/baserom/ -o %s -osf %s -gsf 1 -rconf assets/extractor/Config.xml -se OTR %s", xmlPath.c_str(), outPath.c_str(), outSrcPath.c_str(), xmlPath.find("overlays") != std::string::npos ? "--static" : "");
std::string args = Util::format(" e -eh -i %s -b tmp/baserom/ -o %s -osf %s -gsf 1 -rconf assets/extractor/Config_%s.xml -se OTR %s", xmlPath.c_str(), outPath.c_str(), outSrcPath.c_str(), GetXMLVersion(version).c_str(), xmlPath.find("overlays") != std::string::npos ? "--static" : "");
ProcessResult result = NativeFS->LaunchProcess(execStr + args);
if (result.exitCode != 0) {
@ -55,49 +67,78 @@ void ExtractFile(std::string xmlPath, std::string outPath, std::string outSrcPat
}
}
void ExtractFunc(std::string fullPath) {
void ExtractFunc(std::string fullPath, RomVersion version) {
std::vector<std::string> path = Util::split(fullPath, Util::pathSeparator());
std::string outPath = Util::join(Util::join("assets/extractor/xmls/output", path[4]), Util::basename(fullPath));
Util::mkdir(outPath);
ExtractFile(fullPath, outPath, outPath);
ExtractFile(fullPath, outPath, outPath, version);
setCurrentStep("Extracting: " + Util::basename(fullPath));
extractedResources++;
}
void startWorker() {
std::string path = "assets/extractor/xmls";
std::vector<std::string> files;
Util::dirscan(path, files);
std::vector<std::string> xmlFiles;
void startWorker(RomVersion version) {
std::string path = "assets/extractor/xmls/";
const int num_threads = std::thread::hardware_concurrency();
ctpl::thread_pool pool(num_threads / 2);
for(auto &file : files) {
if (file.find(".xml") != std::string::npos) xmlFiles.push_back(file);
}
path += GetXMLVersion(version);
for (auto& file : xmlFiles) {
if(single_thread) {
ExtractFunc(file);
} else {
pool.push([file](int) {
ExtractFunc(file);
});
Util::write("tmp/baserom/version", (char*)&version.crc, sizeof(version.crc));
if (oldExtractMode)
{
std::vector<std::string> files;
Util::dirscan(path, files);
std::vector<std::string> xmlFiles;
const int num_threads = std::thread::hardware_concurrency();
ctpl::thread_pool pool(num_threads / 2);
for (auto& file : files) {
if (file.find(".xml") != std::string::npos) xmlFiles.push_back(file);
}
}
maxResources = xmlFiles.size();
for (auto& file : xmlFiles) {
if (single_thread) {
ExtractFunc(file, version);
}
else {
pool.push([file, version](int) {
ExtractFunc(file, version);
});
}
}
maxResources = xmlFiles.size();
}
else
{
std::string execStr = Util::format("assets/extractor/%s", isWindows() ? "ZAPD.exe" : "ZAPD.out");
std::string args = Util::format(" ed -eh -i %s -b tmp/rom.z64 -fl assets/extractor/filelists -o %s -osf %s -gsf 1 -rconf assets/extractor/Config_%s.xml -se OTR %s", path.c_str(), path + "../", path + "../", GetXMLVersion(version).c_str(), "");
ProcessResult result = NativeFS->LaunchProcess(execStr + args);
if (result.exitCode != 0) {
std::cout << "\nError when extracting the ROM with error code: " << result.exitCode << " !" << std::endl;
std::cout << "Aborting...\n" << std::endl;
}
else
{
printf("All done?\n");
}
maxResources = 1;
}
}
void updateWorker(const std::string& output) {
if (maxResources > 0 && !buildingOtr && extractedResources >= maxResources) {
if (maxResources > 0 && !buildingOtr && (extractedResources >= maxResources || !oldExtractMode))
{
setCurrentStep("Building OTR...");
if (skipFrames < 3) {
skipFrames++;
return;
}
buildingOtr = true;
if (single_thread){
if (single_thread || !oldExtractMode){
BuildOTR(output);
return;
}

View File

@ -5,5 +5,7 @@ enum Platforms {
WINDOWS, LINUX
};
void startWorker();
struct RomVersion;
void startWorker(RomVersion version);
void updateWorker(const std::string& output);

View File

@ -72,7 +72,11 @@ namespace MoonUtils {
vector<string> result;
stringstream ss (s);
string item;
while (getline(ss, item, delim)) {
while (getline(ss, item, delim))
{
if (item.at(item.size() - 1) == '\r')
item = item.substr(0, item.size() - 1);
result.push_back (item);
}
return result;

View File

@ -1,6 +1,6 @@
#pragma once
static uint32_t CRC32B(unsigned char* message, int32_t size)
static uint32_t CRC32B(const unsigned char* message, int32_t size)
{
int32_t byte, crc;
int32_t mask;

View File

@ -1,6 +1,7 @@
#include "Declaration.h"
#include "Globals.h"
#include "ZVtx.h"
#include "Utils/StringHelper.h"
Declaration::Declaration(offset_t nAddress, DeclarationAlignment nAlignment, size_t nSize,
@ -61,6 +62,12 @@ Declaration::Declaration(offset_t nAddress, const std::string& nIncludePath, siz
varName = nVarName;
}
Declaration::~Declaration()
{
//for (auto item : vertexHack)
//delete item;
}
bool Declaration::IsStatic() const
{
switch (staticConf)

View File

@ -22,6 +22,8 @@ enum class StaticConfig
On
};
class ZVtx;
class Declaration
{
public:
@ -38,6 +40,8 @@ public:
std::string varName;
std::string includePath;
std::vector<ZVtx*> vertexHack;
bool isExternal = false;
bool isArray = false;
bool forceArrayCnt = false;
@ -65,6 +69,8 @@ public:
Declaration(offset_t nAddress, const std::string& nIncludePath, size_t nSize,
const std::string& nVarType, const std::string& nVarName);
~Declaration();
bool IsStatic() const;
std::string GetNormalDeclarationStr() const;

View File

15
ZAPDTR/ZAPD/FileWorker.h Normal file
View File

@ -0,0 +1,15 @@
#pragma once
#include <map>
#include <string>
#include <vector>
#include "ZFile.h"
class FileWorker
{
public:
std::vector<ZFile*> files;
std::vector<ZFile*> externalFiles;
std::vector<int32_t> segments;
std::map<int32_t, std::vector<ZFile*>> segmentRefFiles;
};

View File

@ -34,30 +34,88 @@ Globals::~Globals()
}
}
void Globals::AddSegment(int32_t segment, ZFile* file)
void Globals::AddSegment(int32_t segment, ZFile* file, int workerID)
{
if (std::find(segments.begin(), segments.end(), segment) == segments.end())
segments.push_back(segment);
if (cfg.segmentRefFiles.find(segment) == cfg.segmentRefFiles.end())
cfg.segmentRefFiles[segment] = std::vector<ZFile*>();
cfg.segmentRefFiles[segment].push_back(file);
}
bool Globals::HasSegment(int32_t segment)
{
return std::find(segments.begin(), segments.end(), segment) != segments.end();
}
ZFile* Globals::GetSegment(int32_t segment)
{
if (HasSegment(segment))
if (!Globals::Instance->singleThreaded)
{
int idx = std::find(segments.begin(), segments.end(), segment) - segments.begin();
return files[idx];
auto worker = workerData[workerID];
if (std::find(worker->segments.begin(), worker->segments.end(), segment) ==
worker->segments.end())
worker->segments.push_back(segment);
if (worker->segmentRefFiles.find(segment) == worker->segmentRefFiles.end())
worker->segmentRefFiles[segment] = std::vector<ZFile*>();
worker->segmentRefFiles[segment].push_back(file);
}
else
return nullptr;
{
if (std::find(segments.begin(), segments.end(), segment) == segments.end())
segments.push_back(segment);
if (cfg.segmentRefFiles.find(segment) == cfg.segmentRefFiles.end())
cfg.segmentRefFiles[segment] = std::vector<ZFile*>();
cfg.segmentRefFiles[segment].push_back(file);
}
}
bool Globals::HasSegment(int32_t segment, int workerID)
{
if (!Globals::Instance->singleThreaded)
return std::find(workerData[workerID]->segments.begin(),
workerData[workerID]->segments.end(), segment) != workerData[workerID]->segments.end();
else
return std::find(segments.begin(), segments.end(), segment) != segments.end();
}
ZFile* Globals::GetSegment(int32_t segment, int workerID)
{
if (!Globals::Instance->singleThreaded)
{
if (HasSegment(segment, workerID))
{
int idx = std::find(workerData[workerID]->segments.begin(),
workerData[workerID]->segments.end(), segment) -
workerData[workerID]->segments.begin();
return workerData[workerID]->files[idx];
}
else
return nullptr;
}
else
{
if (HasSegment(segment, workerID))
{
int idx = std::find(segments.begin(), segments.end(), segment) - segments.begin();
return files[idx];
}
else
return nullptr;
}
}
std::map<int32_t, std::vector<ZFile*>> Globals::GetSegmentRefFiles(int workerID)
{
if (!Globals::Instance->singleThreaded)
return workerData[workerID]->segmentRefFiles;
else
return cfg.segmentRefFiles;
}
void Globals::AddFile(ZFile* file, int workerID)
{
if (singleThreaded)
files.push_back(file);
else
workerData[workerID]->files.push_back(file);
}
void Globals::AddExternalFile(ZFile* file, int workerID)
{
if (singleThreaded)
externalFiles.push_back(file);
else
workerData[workerID]->externalFiles.push_back(file);
}
std::map<std::string, ExporterSet*>& Globals::GetExporterMap()
@ -93,8 +151,22 @@ ExporterSet* Globals::GetExporterSet()
return nullptr;
}
std::vector<uint8_t> Globals::GetBaseromFile(std::string fileName)
{
if (fileMode == ZFileMode::ExtractDirectory)
{
if (StringHelper::Contains(fileName, "baserom/"))
fileName = StringHelper::Split(fileName, "baserom/")[1];
return rom->GetFile(fileName);
}
else
return File::ReadAllBytes(fileName);
}
bool Globals::GetSegmentedPtrName(segptr_t segAddress, ZFile* currentFile,
const std::string& expectedType, std::string& declName)
const std::string& expectedType, std::string& declName, int workerID)
{
if (segAddress == 0)
{
@ -130,9 +202,11 @@ bool Globals::GetSegmentedPtrName(segptr_t segAddress, ZFile* currentFile,
if (currentFile->GetDeclarationPtrName(segAddress, expectedType, declName))
return true;
}
else if (HasSegment(segment))
else if (HasSegment(segment, workerID))
{
for (auto file : cfg.segmentRefFiles[segment])
// OTRTODO: Multithreading
auto segs = GetSegmentRefFiles(workerID);
for (auto file : segs[segment])
{
offset = Seg2Filespace(segAddress, file->baseAddress);
@ -176,7 +250,7 @@ bool Globals::GetSegmentedPtrName(segptr_t segAddress, ZFile* currentFile,
bool Globals::GetSegmentedArrayIndexedName(segptr_t segAddress, size_t elementSize,
ZFile* currentFile, const std::string& expectedType,
std::string& declName)
std::string& declName, int workerID)
{
if (segAddress == 0)
{
@ -193,9 +267,11 @@ bool Globals::GetSegmentedArrayIndexedName(segptr_t segAddress, size_t elementSi
if (addressFound)
return true;
}
else if (HasSegment(segment))
else if (HasSegment(segment, workerID))
{
for (auto file : cfg.segmentRefFiles[segment])
// OTRTODO: Multithreading
auto segs = GetSegmentRefFiles(workerID);
for (auto file : segs[segment])
{
if (file->IsSegmentedInFilespaceRange(segAddress))
{

View File

@ -5,6 +5,8 @@
#include <vector>
#include "GameConfig.h"
#include "ZFile.h"
#include <ZRom.h>
#include <FileWorker.h>
class ZRoom;
@ -36,6 +38,7 @@ public:
ExporterSetFuncVoid3 beginXMLFunc = nullptr;
ExporterSetFuncVoid3 endXMLFunc = nullptr;
ExporterSetResSave resSaveFunc = nullptr;
ExporterSetFuncVoid3 endProgramFunc = nullptr;
};
class Globals
@ -49,9 +52,10 @@ public:
bool outputCrc = false;
bool profile; // Measure performance of certain operations
bool useLegacyZDList;
bool singleThreaded;
VerbosityLevel verbosity; // ZAPD outputs additional information
ZFileMode fileMode;
fs::path baseRomPath, inputPath, outputPath, sourceOutputPath, cfgPath;
fs::path baseRomPath, inputPath, outputPath, sourceOutputPath, cfgPath, fileListPath;
TextureType texType;
ZGame game;
GameConfig cfg;
@ -61,10 +65,13 @@ public:
bool forceUnaccountedStatic = false;
bool otrMode = true;
ZRom* rom;
std::vector<ZFile*> files;
std::vector<ZFile*> externalFiles;
std::vector<int32_t> segments;
std::map<int, FileWorker*> workerData;
std::string currentExporter;
static std::map<std::string, ExporterSet*>& GetExporterMap();
static void AddExporter(std::string exporterName, ExporterSet* exporterSet);
@ -72,13 +79,18 @@ public:
Globals();
~Globals();
void AddSegment(int32_t segment, ZFile* file);
bool HasSegment(int32_t segment);
ZFile* GetSegment(int32_t segment);
void AddSegment(int32_t segment, ZFile* file, int workerID);
bool HasSegment(int32_t segment, int workerID);
ZFile* GetSegment(int32_t segment, int workerID);
std::map<int32_t, std::vector<ZFile*>> GetSegmentRefFiles(int workerID);
void AddFile(ZFile* file, int workerID);
void AddExternalFile(ZFile* file, int workerID);
ZResourceExporter* GetExporter(ZResourceType resType);
ExporterSet* GetExporterSet();
std::vector<uint8_t> GetBaseromFile(std::string fileName);
/**
* Search in every file (and the symbol map) for the `segAddress` passed as parameter.
* If the segment of `currentFile` is the same segment of `segAddress`, then that file will be
@ -88,8 +100,8 @@ public:
* in which case `declName` will be set to the address formatted as a pointer.
*/
bool GetSegmentedPtrName(segptr_t segAddress, ZFile* currentFile,
const std::string& expectedType, std::string& declName);
const std::string& expectedType, std::string& declName, int workerID);
bool GetSegmentedArrayIndexedName(segptr_t segAddress, size_t elementSize, ZFile* currentFile,
const std::string& expectedType, std::string& declName);
const std::string& expectedType, std::string& declName, int workerID);
};

View File

@ -23,16 +23,20 @@
#include <string>
#include <string_view>
#include "tinyxml2.h"
#include <ctpl_stl.h>
//extern const char gBuildHash[];
const char gBuildHash[] = "";
bool Parse(const fs::path& xmlFilePath, const fs::path& basePath, const fs::path& outPath,
ZFileMode fileMode);
ZFileMode fileMode, int workerID);
void BuildAssetTexture(const fs::path& pngFilePath, TextureType texType, const fs::path& outPath);
void BuildAssetBackground(const fs::path& imageFilePath, const fs::path& outPath);
void BuildAssetBlob(const fs::path& blobFilePath, const fs::path& outPath);
int ExtractFunc(int workerID, int fileListSize, std::string fileListItem, ZFileMode fileMode);
volatile int numWorkersLeft = 0;
#if !defined(_MSC_VER) && !defined(__CYGWIN__)
#define ARRAY_COUNT(arr) (sizeof(arr) / sizeof(arr[0]))
@ -182,6 +186,10 @@ int main(int argc, char* argv[])
{
Globals::Instance->cfgPath = argv[++i];
}
else if (arg == "-fl") // Set baserom filelist path
{
Globals::Instance->fileListPath = argv[++i];
}
else if (arg == "-rconf") // Read Config File
{
Globals::Instance->cfg.ReadConfigFile(argv[++i]);
@ -242,6 +250,8 @@ int main(int argc, char* argv[])
fileMode = ZFileMode::BuildBlob;
else if (buildMode == "e")
fileMode = ZFileMode::Extract;
else if (buildMode == "ed")
fileMode = ZFileMode::ExtractDirectory;
else if (exporterSet != nullptr && exporterSet->parseFileModeFunc != nullptr)
exporterSet->parseFileModeFunc(buildMode, fileMode);
@ -251,6 +261,11 @@ int main(int argc, char* argv[])
return 1;
}
Globals::Instance->fileMode = fileMode;
if (fileMode == ZFileMode::ExtractDirectory)
Globals::Instance->rom = new ZRom(Globals::Instance->baseRomPath.string());
// We've parsed through our commands once. If an exporter exists, it's been set by now.
// Now we'll parse through them again but pass them on to our exporter if one is available.
@ -269,7 +284,7 @@ int main(int argc, char* argv[])
}
// TODO: switch
if (fileMode == ZFileMode::Extract || fileMode == ZFileMode::BuildSourceFile)
if (fileMode == ZFileMode::Extract || fileMode == ZFileMode::BuildSourceFile || fileMode == ZFileMode::ExtractDirectory)
{
bool procFileModeSuccess = false;
@ -278,30 +293,85 @@ int main(int argc, char* argv[])
if (!procFileModeSuccess)
{
bool parseSuccessful;
for (auto& extFile : Globals::Instance->cfg.externalFiles)
if (fileMode == ZFileMode::ExtractDirectory)
{
fs::path externalXmlFilePath =
Globals::Instance->cfg.externalXmlFolder / extFile.xmlPath;
std::vector<std::string> fileList =
Directory::ListFiles(Globals::Instance->inputPath.string());
if (Globals::Instance->verbosity >= VerbosityLevel::VERBOSITY_INFO)
const int num_threads = std::thread::hardware_concurrency();
ctpl::thread_pool pool(num_threads / 2);
bool parseSuccessful;
auto start = std::chrono::steady_clock::now();
int fileListSize = fileList.size();
Globals::Instance->singleThreaded = false;
for (int i = 0; i < fileListSize; i++)
Globals::Instance->workerData[i] = new FileWorker();
numWorkersLeft = fileListSize;
for (int i = 0; i < fileListSize; i++)
{
printf("Parsing external file from config: '%s'\n",
externalXmlFilePath.c_str());
if (Globals::Instance->singleThreaded)
{
ExtractFunc(i, fileList.size(), fileList[i], fileMode);
}
else
{
std::string fileListItem = fileList[i];
pool.push([i, fileListSize, fileListItem, fileMode](int) {
ExtractFunc(i, fileListSize, fileListItem, fileMode);
});
}
}
parseSuccessful = Parse(externalXmlFilePath, Globals::Instance->baseRomPath,
extFile.outPath, ZFileMode::ExternalFile);
if (!Globals::Instance->singleThreaded)
{
while (true)
{
if (numWorkersLeft <= 0)
break;
std::this_thread::sleep_for(std::chrono::milliseconds(250));
}
}
auto end = std::chrono::steady_clock::now();
auto diff =
std::chrono::duration_cast<std::chrono::seconds>(end - start).count();
printf("Generated OTR File Data in %i seconds\n", diff);
}
else
{
bool parseSuccessful;
for (auto& extFile : Globals::Instance->cfg.externalFiles)
{
fs::path externalXmlFilePath =
Globals::Instance->cfg.externalXmlFolder / extFile.xmlPath;
if (Globals::Instance->verbosity >= VerbosityLevel::VERBOSITY_INFO)
{
printf("Parsing external file from config: '%s'\n",
externalXmlFilePath.c_str());
}
parseSuccessful = Parse(externalXmlFilePath, Globals::Instance->baseRomPath,
extFile.outPath, ZFileMode::ExternalFile, 0);
if (!parseSuccessful)
return 1;
}
parseSuccessful =
Parse(Globals::Instance->inputPath, Globals::Instance->baseRomPath,
Globals::Instance->outputPath, fileMode, 0);
if (!parseSuccessful)
return 1;
}
parseSuccessful = Parse(Globals::Instance->inputPath, Globals::Instance->baseRomPath,
Globals::Instance->outputPath, fileMode);
if (!parseSuccessful)
return 1;
}
}
else if (fileMode == ZFileMode::BuildTexture)
@ -317,6 +387,7 @@ int main(int argc, char* argv[])
{
BuildAssetBlob(Globals::Instance->inputPath, Globals::Instance->outputPath);
}
/*
else if (fileMode == ZFileMode::BuildOverlay)
{
ZOverlay* overlay =
@ -327,13 +398,77 @@ int main(int argc, char* argv[])
File::WriteAllText(Globals::Instance->outputPath.string(),
overlay->GetSourceOutputCode(""));
}
*/
if (exporterSet != nullptr && exporterSet->endProgramFunc != nullptr)
exporterSet->endProgramFunc();
delete g;
return 0;
}
int ExtractFunc(int workerID, int fileListSize, std::string fileListItem, ZFileMode fileMode)
{
bool parseSuccessful;
printf("(%i / %i): %s\n", (workerID + 1), fileListSize, fileListItem.c_str());
for (auto& extFile : Globals::Instance->cfg.externalFiles)
{
fs::path externalXmlFilePath = Globals::Instance->cfg.externalXmlFolder / extFile.xmlPath;
if (Globals::Instance->verbosity >= VerbosityLevel::VERBOSITY_INFO)
{
printf("Parsing external file from config: '%s'\n", externalXmlFilePath.c_str());
}
parseSuccessful = Parse(externalXmlFilePath, Globals::Instance->baseRomPath,
extFile.outPath, ZFileMode::ExternalFile, workerID);
if (!parseSuccessful)
return 1;
}
parseSuccessful = Parse(fileListItem, Globals::Instance->baseRomPath,
Globals::Instance->outputPath, fileMode, workerID);
if (!parseSuccessful)
return 1;
if (Globals::Instance->singleThreaded)
{
for (int i = 0; i < Globals::Instance->files.size(); i++)
{
delete Globals::Instance->files[i];
Globals::Instance->files.erase(Globals::Instance->files.begin() + i);
i--;
}
Globals::Instance->externalFiles.clear();
Globals::Instance->segments.clear();
Globals::Instance->cfg.segmentRefFiles.clear();
}
else
{
for (int i = 0; i < Globals::Instance->workerData[workerID]->files.size(); i++)
{
delete Globals::Instance->workerData[workerID]->files[i];
Globals::Instance->workerData[workerID]->files.erase(
Globals::Instance->workerData[workerID]->files.begin() +
i);
i--;
}
Globals::Instance->workerData[workerID]->externalFiles.clear();
Globals::Instance->workerData[workerID]->segments.clear();
Globals::Instance->workerData[workerID]->segmentRefFiles.clear();
numWorkersLeft--;
}
}
bool Parse(const fs::path& xmlFilePath, const fs::path& basePath, const fs::path& outPath,
ZFileMode fileMode)
ZFileMode fileMode, int workerID)
{
tinyxml2::XMLDocument doc;
tinyxml2::XMLError eResult = doc.LoadFile(xmlFilePath.string().c_str());
@ -361,11 +496,11 @@ bool Parse(const fs::path& xmlFilePath, const fs::path& basePath, const fs::path
{
if (std::string_view(child->Name()) == "File")
{
ZFile* file = new ZFile(fileMode, child, basePath, outPath, "", xmlFilePath);
Globals::Instance->files.push_back(file);
ZFile* file = new ZFile(fileMode, child, basePath, outPath, "", xmlFilePath, workerID);
Globals::Instance->AddFile(file, workerID);
if (fileMode == ZFileMode::ExternalFile)
{
Globals::Instance->externalFiles.push_back(file);
Globals::Instance->AddExternalFile(file, workerID);
file->isExternalFile = true;
}
}
@ -398,7 +533,7 @@ bool Parse(const fs::path& xmlFilePath, const fs::path& basePath, const fs::path
}
// Recursion. What can go wrong?
Parse(externalXmlFilePath, basePath, externalOutFilePath, ZFileMode::ExternalFile);
Parse(externalXmlFilePath, basePath, externalOutFilePath, ZFileMode::ExternalFile, workerID);
}
else
{
@ -417,7 +552,14 @@ bool Parse(const fs::path& xmlFilePath, const fs::path& basePath, const fs::path
if (exporterSet != nullptr && exporterSet->beginXMLFunc != nullptr)
exporterSet->beginXMLFunc();
for (ZFile* file : Globals::Instance->files)
std::vector<ZFile*> files;
if (Globals::Instance->singleThreaded)
files = Globals::Instance->files;
else
files = Globals::Instance->workerData[workerID]->files;
for (ZFile* file : files)
{
if (fileMode == ZFileMode::BuildSourceFile)
file->BuildSourceFile();

View File

@ -199,8 +199,10 @@ std::string Struct_800A598C::GetBodySourceCode() const
{
std::string unk_8_Str;
std::string unk_C_Str;
Globals::Instance->GetSegmentedPtrName(unk_8, parent, "Struct_800A57C0", unk_8_Str);
Globals::Instance->GetSegmentedPtrName(unk_C, parent, "Struct_800A598C_2", unk_C_Str);
Globals::Instance->GetSegmentedPtrName(unk_8, parent, "Struct_800A57C0", unk_8_Str,
parent->workerID);
Globals::Instance->GetSegmentedPtrName(unk_C, parent, "Struct_800A598C_2", unk_C_Str,
parent->workerID);
std::string entryStr = StringHelper::Sprintf("\n\t\tARRAY_COUNTU(%s), ARRAY_COUNTU(%s),\n",
unk_8_Str.c_str(), unk_C_Str.c_str());
@ -316,8 +318,9 @@ std::string Struct_800A5E28::GetBodySourceCode() const
{
std::string unk_4_Str;
std::string unk_8_Str;
Globals::Instance->GetSegmentedPtrName(unk_4, parent, "Struct_800A598C", unk_4_Str);
Globals::Instance->GetSegmentedPtrName(unk_8, parent, "Gfx", unk_8_Str);
Globals::Instance->GetSegmentedPtrName(unk_4, parent, "Struct_800A598C", unk_4_Str,
parent->workerID);
Globals::Instance->GetSegmentedPtrName(unk_8, parent, "Gfx", unk_8_Str, parent->workerID);
std::string entryStr = "\n";
entryStr += StringHelper::Sprintf("\t%i, ARRAY_COUNTU(%s),\n", unk_0, unk_4_Str.c_str());

View File

@ -3,7 +3,7 @@
void OutputFormatter::Flush()
{
//if (!Globals::Instance->otrMode)
//if (!Globals::Instance->otrMode) // OTRTODO: MULTITHREADING
{
if (col > lineLimit && !Globals::Instance->otrMode)
{
@ -31,6 +31,10 @@ void OutputFormatter::Flush()
int OutputFormatter::Write(const char* buf, int count)
{
// OTRTODO
//if (!Globals::Instance->singleThreaded)
//return 0;
for (int i = 0; i < count; i++)
{
char c = buf[i];
@ -92,7 +96,7 @@ int OutputFormatter::Write(const std::string& buf)
return Write(buf.data(), buf.size());
}
OutputFormatter* OutputFormatter::Instance;
__declspec(thread) OutputFormatter* OutputFormatter::Instance;
int OutputFormatter::WriteStatic(const char* buf, int count)
{

View File

@ -25,7 +25,7 @@ private:
void Flush();
static OutputFormatter* Instance;
static __declspec(thread) OutputFormatter* Instance;
static int WriteStatic(const char* buf, int count);
public:

View File

@ -1,3 +1,4 @@
#if 0
#include "ZOverlay.h"
#include <cassert>
@ -350,3 +351,4 @@ ELFIO::Elf_Half ZOverlay::FindSymbolInSection(const std::string& curSymName,
}
return SHN_UNDEF;
}
#endif

View File

@ -1,5 +1,7 @@
#pragma once
#if 0
#include "Utils/Directory.h"
#include "ZResource.h"
#include "elfio/elfio.hpp"
@ -73,3 +75,4 @@ private:
ELFIO::Elf_Half FindSymbolInSection(const std::string& curSymName, ELFIO::section* sectionData,
ELFIO::elfio& reader, size_t readerId);
};
#endif

View File

@ -74,15 +74,29 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LibraryPath>$(OutDir);$(ProjectDir)..\lib\libgfxd;$(ProjectDir)..\..\OTRExporter\packages\libpng-v142.1.6.37.2\build\native\lib\x64\v142\Debug\;$(ProjectDir)..\..\libultraship\libultraship\;$(LibraryPath)</LibraryPath>
<IncludePath>$(ProjectDir)..\ZAPDUtils;$(ProjectDir)..\lib\tinyxml2;$(ProjectDir)..\lib\libgfxd;$(ProjectDir)..\lib\elfio;$(ProjectDir)..\lib\stb;$(ProjectDir);$(IncludePath)</IncludePath>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<IncludePath>$(ProjectDir)..\ZAPDUtils;$(ProjectDir)..\lib\tinyxml2;$(ProjectDir)..\lib\libgfxd;$(ProjectDir)..\lib\elfio;$(ProjectDir)..\lib\stb;$(ProjectDir);$(IncludePath)</IncludePath>
<LibraryPath>$(OutDir);$(ProjectDir)..\lib\libgfxd;$(ProjectDir)..\..\OTRExporter\packages\libpng-v142.1.6.37.2\build\native\lib\x64\v142\Debug\;$(ProjectDir)..\..\libultraship\libultraship\;$(LibraryPath)</LibraryPath>
<PreBuildEventUseInBuild>false</PreBuildEventUseInBuild>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<IncludePath>$(SolutionDir)ZAPD\lib\tinyxml2;$(SolutionDir)ZAPD\lib\libgfxd;$(SolutionDir)ZAPD\lib\elfio;$(SolutionDir)ZAPD\lib\stb;$(ProjectDir);$(IncludePath)</IncludePath>
<LibraryPath>$(SolutionDir)ZAPD\lib\libgfxd;$(SolutionDir)x64\Debug;$(SolutionDir)packages\libpng.1.6.28.1\build\native\lib\x64\v140\dynamic\Debug;$(LibraryPath)</LibraryPath>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
@ -170,6 +184,7 @@
<ClCompile Include="..\lib\libgfxd\uc_f3dex2.c" />
<ClCompile Include="..\lib\libgfxd\uc_f3dexb.c" />
<ClCompile Include="Declaration.cpp" />
<ClCompile Include="FileWorker.cpp" />
<ClCompile Include="GameConfig.cpp" />
<ClCompile Include="Globals.cpp" />
<ClCompile Include="ImageBackend.cpp" />
@ -178,6 +193,7 @@
<ClCompile Include="OutputFormatter.cpp" />
<ClCompile Include="Overlays\ZOverlay.cpp" />
<ClCompile Include="WarningHandler.cpp" />
<ClCompile Include="yaz0\yaz0.cpp" />
<ClCompile Include="ZArray.cpp" />
<ClCompile Include="ZBackground.cpp" />
<ClCompile Include="ZCutsceneMM.cpp" />
@ -185,6 +201,7 @@
<ClCompile Include="ZMtx.cpp" />
<ClCompile Include="ZPath.cpp" />
<ClCompile Include="ZPlayerAnimationData.cpp" />
<ClCompile Include="ZRom.cpp" />
<ClCompile Include="ZRoom\Commands\SetActorCutsceneList.cpp" />
<ClCompile Include="ZRoom\Commands\SetAnimatedMaterialList.cpp" />
<ClCompile Include="ZRoom\Commands\SetCsCamera.cpp" />
@ -257,7 +274,9 @@
<ClInclude Include="..\lib\stb\stb_image_write.h" />
<ClInclude Include="..\lib\stb\tinyxml2.h" />
<ClInclude Include="CRC32.h" />
<ClInclude Include="ctpl_stl.h" />
<ClInclude Include="Declaration.h" />
<ClInclude Include="FileWorker.h" />
<ClInclude Include="GameConfig.h" />
<ClInclude Include="Globals.h" />
<ClInclude Include="ImageBackend.h" />
@ -265,6 +284,8 @@
<ClInclude Include="OutputFormatter.h" />
<ClInclude Include="Overlays\ZOverlay.h" />
<ClInclude Include="WarningHandler.h" />
<ClInclude Include="yaz0\readwrite.h" />
<ClInclude Include="yaz0\yaz0.h" />
<ClInclude Include="ZAnimation.h" />
<ClInclude Include="ZArray.h" />
<ClInclude Include="ZBackground.h" />
@ -278,6 +299,7 @@
<ClInclude Include="ZMtx.h" />
<ClInclude Include="ZPath.h" />
<ClInclude Include="ZPlayerAnimationData.h" />
<ClInclude Include="ZRom.h" />
<ClInclude Include="ZRoom\Commands\SetActorCutsceneList.h" />
<ClInclude Include="ZRoom\Commands\SetAnimatedMaterialList.h" />
<ClInclude Include="ZRoom\Commands\SetCsCamera.h" />
@ -334,6 +356,12 @@
<ItemGroup>
<None Include="packages.config" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\OTRGui\build\ZERO_CHECK.vcxproj">
<Project>{02d10590-9542-3f55-aaf8-6055677e2a2a}</Project>
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
</ProjectReference>
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
<Import Project="..\..\OTRExporter\packages\zlib.v120.windesktop.msvcstl.dyn.rt-dyn.1.2.8.8\build\native\zlib.v120.windesktop.msvcstl.dyn.rt-dyn.targets" Condition="Exists('..\..\OTRExporter\packages\zlib.v120.windesktop.msvcstl.dyn.rt-dyn.1.2.8.8\build\native\zlib.v120.windesktop.msvcstl.dyn.rt-dyn.targets')" />

View File

@ -58,6 +58,12 @@
<Filter Include="NuGet">
<UniqueIdentifier>{730beb67-6d59-4849-9d9b-702c4a565fc0}</UniqueIdentifier>
</Filter>
<Filter Include="Source Files\Yaz0">
<UniqueIdentifier>{b26457d2-cdb8-4c92-9ed7-a55bf6d3619e}</UniqueIdentifier>
</Filter>
<Filter Include="Header Files\Yaz0">
<UniqueIdentifier>{9651a041-1019-4486-9e90-1dccfa9471e9}</UniqueIdentifier>
</Filter>
</ItemGroup>
<ItemGroup>
<ClCompile Include="Main.cpp">
@ -282,6 +288,15 @@
<ClCompile Include="ZText.cpp">
<Filter>Source Files\Z64</Filter>
</ClCompile>
<ClCompile Include="ZRom.cpp">
<Filter>Source Files\Z64</Filter>
</ClCompile>
<ClCompile Include="yaz0\yaz0.cpp">
<Filter>Source Files\Yaz0</Filter>
</ClCompile>
<ClCompile Include="FileWorker.cpp">
<Filter>Source Files</Filter>
</ClCompile>
</ItemGroup>
<ItemGroup>
<ClInclude Include="ZRoom\ZRoom.h">
@ -539,6 +554,21 @@
<ClInclude Include="ZText.h">
<Filter>Header Files\Z64</Filter>
</ClInclude>
<ClInclude Include="ZRom.h">
<Filter>Header Files\Z64</Filter>
</ClInclude>
<ClInclude Include="yaz0\readwrite.h">
<Filter>Header Files\Yaz0</Filter>
</ClInclude>
<ClInclude Include="yaz0\yaz0.h">
<Filter>Header Files\Yaz0</Filter>
</ClInclude>
<ClInclude Include="FileWorker.h">
<Filter>Header Files</Filter>
</ClInclude>
<ClInclude Include="ctpl_stl.h">
<Filter>Header Files\Libraries</Filter>
</ClInclude>
</ItemGroup>
<ItemGroup>
<Text Include="..\SymbolMap_OoTMqDbg.txt">

View File

@ -112,12 +112,15 @@ void ZNormalAnimation::DeclareReferences(const std::string& prefix)
const uint8_t lineLength = 14;
const uint8_t offset = 0;
for (size_t i = 0; i < rotationValues.size(); i++)
if (!Globals::Instance->otrMode)
{
valuesStr += StringHelper::Sprintf("0x%04X, ", rotationValues[i]);
for (size_t i = 0; i < rotationValues.size(); i++)
{
valuesStr += StringHelper::Sprintf("0x%04X, ", rotationValues[i]);
if ((i - offset + 1) % lineLength == 0)
valuesStr += "\n ";
if ((i - offset + 1) % lineLength == 0)
valuesStr += "\n ";
}
}
parent->AddDeclarationArray(rotationValuesOffset, DeclarationAlignment::Align4,
@ -125,13 +128,17 @@ void ZNormalAnimation::DeclareReferences(const std::string& prefix)
StringHelper::Sprintf("%sFrameData", defaultPrefix.c_str()),
rotationValues.size(), valuesStr);
for (size_t i = 0; i < rotationIndices.size(); i++)
if (!Globals::Instance->otrMode)
{
indicesStr += StringHelper::Sprintf(" { 0x%04X, 0x%04X, 0x%04X },", rotationIndices[i].x,
rotationIndices[i].y, rotationIndices[i].z);
for (size_t i = 0; i < rotationIndices.size(); i++)
{
indicesStr +=
StringHelper::Sprintf(" { 0x%04X, 0x%04X, 0x%04X },", rotationIndices[i].x,
rotationIndices[i].y, rotationIndices[i].z);
if (i != (rotationIndices.size() - 1))
indicesStr += "\n";
if (i != (rotationIndices.size() - 1))
indicesStr += "\n";
}
}
parent->AddDeclarationArray(rotationIndicesOffset, DeclarationAlignment::Align4,
@ -143,10 +150,11 @@ void ZNormalAnimation::DeclareReferences(const std::string& prefix)
std::string ZNormalAnimation::GetBodySourceCode() const
{
std::string frameDataName;
Globals::Instance->GetSegmentedPtrName(rotationValuesSeg, parent, "s16", frameDataName);
Globals::Instance->GetSegmentedPtrName(rotationValuesSeg, parent, "s16", frameDataName,
parent->workerID);
std::string jointIndicesName;
Globals::Instance->GetSegmentedPtrName(rotationIndicesSeg, parent, "JointIndex",
jointIndicesName);
jointIndicesName, parent->workerID);
std::string headerStr =
StringHelper::Sprintf("\n\t{ %i }, %s,\n", frameCount, frameDataName.c_str());
@ -183,7 +191,7 @@ void ZLinkAnimation::ParseRawData()
std::string ZLinkAnimation::GetBodySourceCode() const
{
std::string segSymbol;
Globals::Instance->GetSegmentedPtrName(segmentAddress, parent, "", segSymbol);
Globals::Instance->GetSegmentedPtrName(segmentAddress, parent, "", segSymbol, parent->workerID);
return StringHelper::Sprintf("\n\t{ %i }, %s\n", frameCount, segSymbol.c_str());
}
@ -383,12 +391,13 @@ void ZCurveAnimation::DeclareReferences(const std::string& prefix)
std::string ZCurveAnimation::GetBodySourceCode() const
{
std::string refIndexStr;
Globals::Instance->GetSegmentedPtrName(refIndex, parent, "u8", refIndexStr);
Globals::Instance->GetSegmentedPtrName(refIndex, parent, "u8", refIndexStr, parent->workerID);
std::string transformDataStr;
Globals::Instance->GetSegmentedPtrName(transformData, parent, "TransformData",
transformDataStr);
transformDataStr, parent->workerID);
std::string copyValuesStr;
Globals::Instance->GetSegmentedPtrName(copyValues, parent, "s16", copyValuesStr);
Globals::Instance->GetSegmentedPtrName(copyValues, parent, "s16", copyValuesStr,
parent->workerID);
return StringHelper::Sprintf("\n\t%s,\n\t%s,\n\t%s,\n\t%i, %i\n", refIndexStr.c_str(),
transformDataStr.c_str(), copyValuesStr.c_str(), unk_0C, unk_10);
@ -510,8 +519,10 @@ std::string ZLegacyAnimation::GetBodySourceCode() const
std::string frameDataName;
std::string jointKeyName;
Globals::Instance->GetSegmentedPtrName(frameData, parent, "s16", frameDataName);
Globals::Instance->GetSegmentedPtrName(jointKey, parent, "JointKey", jointKeyName);
Globals::Instance->GetSegmentedPtrName(frameData, parent, "s16", frameDataName,
parent->workerID);
Globals::Instance->GetSegmentedPtrName(jointKey, parent, "JointKey", jointKeyName,
parent->workerID);
body += StringHelper::Sprintf("\t%i, %i,\n", frameCount, limbCount);
body += StringHelper::Sprintf("\t%s,\n", frameDataName.c_str());

View File

@ -102,8 +102,7 @@ std::string ZArray::GetBodySourceCode() const
const auto& res = resList[i];
output += "\t";
if (res->GetResourceType() == ZResourceType::Scalar ||
res->GetResourceType() == ZResourceType::Vertex)
if (res->GetResourceType() == ZResourceType::Scalar || res->GetResourceType() == ZResourceType::Vertex)
output += resList.at(i)->GetBodySourceCode();
else
output += StringHelper::Sprintf("{ %s }", resList.at(i)->GetBodySourceCode().c_str());

View File

@ -150,8 +150,11 @@ std::string ZBackground::GetExternalExtension() const
void ZBackground::Save(const fs::path& outFolder)
{
fs::path filepath = outFolder / (outName + "." + GetExternalExtension());
File::WriteAllBytes(filepath.string(), data);
if (!Globals::Instance->otrMode)
{
fs::path filepath = outFolder / (outName + "." + GetExternalExtension());
File::WriteAllBytes(filepath.string(), data);
}
}
std::string ZBackground::GetBodySourceCode() const

View File

@ -97,12 +97,15 @@ void ZCollisionHeader::DeclareReferences(const std::string& prefix)
if (waterBoxes.size() > 0)
{
for (size_t i = 0; i < waterBoxes.size(); i++)
if (!Globals::Instance->otrMode)
{
declaration +=
StringHelper::Sprintf("\t{ %s },", waterBoxes[i].GetBodySourceCode().c_str());
if (i + 1 < waterBoxes.size())
declaration += "\n";
for (size_t i = 0; i < waterBoxes.size(); i++)
{
declaration +=
StringHelper::Sprintf("\t{ %s },", waterBoxes[i].GetBodySourceCode().c_str());
if (i + 1 < waterBoxes.size())
declaration += "\n";
}
}
parent->AddDeclarationArray(
@ -115,14 +118,17 @@ void ZCollisionHeader::DeclareReferences(const std::string& prefix)
{
declaration.clear();
for (size_t i = 0; i < polygons.size(); i++)
if (!Globals::Instance->otrMode)
{
declaration += StringHelper::Sprintf(
"\t{ 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X },",
polygons[i].type, polygons[i].vtxA, polygons[i].vtxB, polygons[i].vtxC,
polygons[i].a, polygons[i].b, polygons[i].c, polygons[i].d);
if (i + 1 < polygons.size())
declaration += "\n";
for (size_t i = 0; i < polygons.size(); i++)
{
declaration += StringHelper::Sprintf(
"\t{ 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X, 0x%04X },",
polygons[i].type, polygons[i].vtxA, polygons[i].vtxB, polygons[i].vtxC,
polygons[i].a, polygons[i].b, polygons[i].c, polygons[i].d);
if (i + 1 < polygons.size())
declaration += "\n";
}
}
parent->AddDeclarationArray(
@ -132,13 +138,16 @@ void ZCollisionHeader::DeclareReferences(const std::string& prefix)
}
declaration.clear();
for (size_t i = 0; i < polygonTypes.size(); i++)
if (!Globals::Instance->otrMode)
{
declaration += StringHelper::Sprintf("\t{ 0x%08lX, 0x%08lX },", polygonTypes[i] >> 32,
polygonTypes[i] & 0xFFFFFFFF);
for (size_t i = 0; i < polygonTypes.size(); i++)
{
declaration += StringHelper::Sprintf("\t{ 0x%08lX, 0x%08lX },", polygonTypes[i] >> 32,
polygonTypes[i] & 0xFFFFFFFF);
if (i < polygonTypes.size() - 1)
declaration += "\n";
if (i < polygonTypes.size() - 1)
declaration += "\n";
}
}
if (polyTypeDefAddress != 0)
@ -154,13 +163,16 @@ void ZCollisionHeader::DeclareReferences(const std::string& prefix)
{
declaration.clear();
for (size_t i = 0; i < vertices.size(); i++)
if (!Globals::Instance->otrMode)
{
declaration +=
StringHelper::Sprintf("\t{ %s },", vertices[i].GetBodySourceCode().c_str());
for (size_t i = 0; i < vertices.size(); i++)
{
declaration +=
StringHelper::Sprintf("\t{ %s },", vertices[i].GetBodySourceCode().c_str());
if (i < vertices.size() - 1)
declaration += "\n";
if (i < vertices.size() - 1)
declaration += "\n";
}
}
const auto& first = vertices.front();
@ -177,29 +189,36 @@ std::string ZCollisionHeader::GetBodySourceCode() const
{
std::string declaration = "";
if (Globals::Instance->otrMode)
return declaration;
declaration += "\n";
declaration += StringHelper::Sprintf("\t{ %i, %i, %i },\n", absMinX, absMinY, absMinZ);
declaration += StringHelper::Sprintf("\t{ %i, %i, %i },\n", absMaxX, absMaxY, absMaxZ);
std::string vtxName;
Globals::Instance->GetSegmentedPtrName(vtxAddress, parent, "Vec3s", vtxName);
Globals::Instance->GetSegmentedPtrName(vtxAddress, parent, "Vec3s", vtxName, parent->workerID);
declaration += StringHelper::Sprintf("\t%i,\n\t%s,\n", numVerts, vtxName.c_str());
std::string polyName;
Globals::Instance->GetSegmentedPtrName(polyAddress, parent, "CollisionPoly", polyName);
Globals::Instance->GetSegmentedPtrName(polyAddress, parent, "CollisionPoly", polyName,
parent->workerID);
declaration += StringHelper::Sprintf("\t%i,\n\t%s,\n", numPolygons, polyName.c_str());
std::string surfaceName;
Globals::Instance->GetSegmentedPtrName(polyTypeDefAddress, parent, "SurfaceType", surfaceName);
Globals::Instance->GetSegmentedPtrName(polyTypeDefAddress, parent, "SurfaceType", surfaceName,
parent->workerID);
declaration += StringHelper::Sprintf("\t%s,\n", surfaceName.c_str());
std::string camName;
Globals::Instance->GetSegmentedPtrName(camDataAddress, parent, "CamData", camName);
Globals::Instance->GetSegmentedPtrName(camDataAddress, parent, "CamData", camName,
parent->workerID);
declaration += StringHelper::Sprintf("\t%s,\n", camName.c_str());
std::string waterBoxName;
Globals::Instance->GetSegmentedPtrName(waterBoxAddress, parent, "WaterBox", waterBoxName);
Globals::Instance->GetSegmentedPtrName(waterBoxAddress, parent, "WaterBox", waterBoxName,
parent->workerID);
declaration += StringHelper::Sprintf("\t%i,\n\t%s\n", numWaterBoxes, waterBoxName.c_str());
return declaration;

View File

@ -553,7 +553,8 @@ int32_t ZDisplayList::OptimizationCheck_LoadTextureBlock(int32_t startIndex, std
lastTexSeg = segmentNumber;
Globals::Instance->GetSegmentedPtrName(data & 0xFFFFFFFF, parent, "", texStr);
Globals::Instance->GetSegmentedPtrName(data & 0xFFFFFFFF, parent, "", texStr,
parent->workerID);
}
// gsDPSetTile
@ -705,7 +706,7 @@ void ZDisplayList::Opcode_G_DL(uint64_t data, const std::string& prefix, char* l
if (pp != 0)
{
if (!Globals::Instance->HasSegment(segNum))
if (!Globals::Instance->HasSegment(segNum, parent->workerID))
sprintf(line, "gsSPBranchList(0x%08" PRIX64 "),", data & 0xFFFFFFFF);
else if (dListDecl != nullptr)
sprintf(line, "gsSPBranchList(%s),", dListDecl->varName.c_str());
@ -715,7 +716,7 @@ void ZDisplayList::Opcode_G_DL(uint64_t data, const std::string& prefix, char* l
}
else
{
if (!Globals::Instance->HasSegment(segNum))
if (!Globals::Instance->HasSegment(segNum, parent->workerID))
sprintf(line, "gsSPDisplayList(0x%08" PRIX64 "),", data & 0xFFFFFFFF);
else if (dListDecl != nullptr)
sprintf(line, "gsSPDisplayList(%s),", dListDecl->varName.c_str());
@ -726,7 +727,7 @@ void ZDisplayList::Opcode_G_DL(uint64_t data, const std::string& prefix, char* l
// if (segNum == 8 || segNum == 9 || segNum == 10 || segNum == 11 || segNum == 12 || segNum ==
// 13) // Used for runtime-generated display lists
if (!Globals::Instance->HasSegment(segNum))
if (!Globals::Instance->HasSegment(segNum, parent->workerID))
{
if (pp != 0)
sprintf(line, "gsSPBranchList(0x%08" PRIX64 "),", data & 0xFFFFFFFF);
@ -847,7 +848,7 @@ void ZDisplayList::Opcode_G_VTX(uint64_t data, char* line)
}
// Hack: Don't extract vertices from a unknown segment.
if (!Globals::Instance->HasSegment(GETSEGNUM(data)))
if (!Globals::Instance->HasSegment(GETSEGNUM(data), parent->workerID))
{
segptr_t segmented = data & 0xFFFFFFFF;
references.push_back(segmented);
@ -951,7 +952,7 @@ void ZDisplayList::Opcode_G_SETTIMG(uint64_t data, const std::string& prefix, ch
if (parent != nullptr)
{
if (Globals::Instance->HasSegment(segmentNumber))
if (Globals::Instance->HasSegment(segmentNumber, parent->workerID))
texDecl = parent->GetDeclaration(texAddress);
else
texDecl = parent->GetDeclaration(data);
@ -959,7 +960,7 @@ void ZDisplayList::Opcode_G_SETTIMG(uint64_t data, const std::string& prefix, ch
if (texDecl != nullptr)
sprintf(texStr, "%s", texDecl->varName.c_str());
else if (data != 0 && Globals::Instance->HasSegment(segmentNumber))
else if (data != 0 && Globals::Instance->HasSegment(segmentNumber, parent->workerID))
sprintf(texStr, "%sTex_%06X", prefix.c_str(), texAddress);
else
{
@ -972,7 +973,7 @@ void ZDisplayList::Opcode_G_SETTIMG(uint64_t data, const std::string& prefix, ch
else
{
std::string texName;
Globals::Instance->GetSegmentedPtrName(data, parent, "", texName);
Globals::Instance->GetSegmentedPtrName(data, parent, "", texName, parent->workerID);
sprintf(line, "gsDPSetTextureImage(%s, %s, %i, %s),", fmtTbl[fmt], sizTbl[siz], www + 1,
texName.c_str());
}
@ -1647,7 +1648,9 @@ static int32_t GfxdCallback_Vtx(uint32_t seg, int32_t count)
}
self->references.push_back(seg);
gfxd_puts("@r");
if (!Globals::Instance->otrMode)
gfxd_puts("@r");
return 1;
}
@ -1670,7 +1673,7 @@ static int32_t GfxdCallback_Texture(segptr_t seg, int32_t fmt, int32_t siz, int3
self->TextureGenCheck();
std::string texName;
Globals::Instance->GetSegmentedPtrName(seg, self->parent, "", texName);
Globals::Instance->GetSegmentedPtrName(seg, self->parent, "", texName, self->parent->workerID);
gfxd_puts(texName.c_str());
@ -1694,7 +1697,7 @@ static int32_t GfxdCallback_Palette(uint32_t seg, [[maybe_unused]] int32_t idx,
self->TextureGenCheck();
std::string palName;
Globals::Instance->GetSegmentedPtrName(seg, self->parent, "", palName);
Globals::Instance->GetSegmentedPtrName(seg, self->parent, "", palName, self->parent->workerID);
gfxd_puts(palName.c_str());
@ -1708,7 +1711,8 @@ static int32_t GfxdCallback_DisplayList(uint32_t seg)
uint32_t dListSegNum = GETSEGNUM(seg);
std::string dListName = "";
bool addressFound = Globals::Instance->GetSegmentedPtrName(seg, self->parent, "Gfx", dListName);
bool addressFound = Globals::Instance->GetSegmentedPtrName(seg, self->parent, "Gfx", dListName,
self->parent->workerID);
if (!addressFound && self->parent->segment == dListSegNum)
{
@ -1731,7 +1735,8 @@ static int32_t GfxdCallback_Matrix(uint32_t seg)
std::string mtxName;
ZDisplayList* self = static_cast<ZDisplayList*>(gfxd_udata_get());
bool addressFound = Globals::Instance->GetSegmentedPtrName(seg, self->parent, "Mtx", mtxName);
bool addressFound = Globals::Instance->GetSegmentedPtrName(seg, self->parent, "Mtx", mtxName,
self->parent->workerID);
if (!addressFound && GETSEGNUM(seg) == self->parent->segment)
{
Declaration* decl =
@ -1805,6 +1810,23 @@ void ZDisplayList::DeclareReferences(const std::string& prefix)
curAddr, firstVtx.GetDeclarationAlignment(),
item.second.size() * firstVtx.GetRawDataSize(), firstVtx.GetSourceTypeName(),
firstVtx.GetDefaultName(name), item.second.size(), declaration);
/*for (auto vtx : item.second)
{
ZVtx* nVtx = new ZVtx(vtx.parent);
nVtx->x = vtx.x;
nVtx->y = vtx.y;
nVtx->z = vtx.z;
nVtx->flag = vtx.flag;
nVtx->s = vtx.s;
nVtx->t = vtx.t;
nVtx->r = vtx.r;
nVtx->g = vtx.g;
nVtx->b = vtx.b;
nVtx->a = vtx.a;
decl->vertexHack.push_back(nVtx);
}*/
decl->isExternal = true;
}
}
@ -1850,15 +1872,15 @@ void ZDisplayList::DeclareReferences(const std::string& prefix)
{
auto& item = vertices[vtxKeys[i]];
std::string declaration;
//std::string declaration;
for (auto& vtx : item)
declaration += StringHelper::Sprintf("\t%s,\n", vtx.GetBodySourceCode().c_str());
//for (auto& vtx : item)
//declaration += StringHelper::Sprintf("\t%s,\n", vtx.GetBodySourceCode().c_str());
// Ensure there's always a trailing line feed to prevent dumb warnings.
// Please don't remove this line, unless you somehow made a way to prevent
// that warning when building the OoT repo.
declaration += "\n";
//declaration += "\n";
if (parent != nullptr)
{
@ -1870,12 +1892,6 @@ void ZDisplayList::DeclareReferences(const std::string& prefix)
else
vtxName = StringHelper::Sprintf("%sVtx_%06X", prefix.c_str(), vtxKeys[i]);
if (StringHelper::Contains(vtxName, "4B18"))
{
int bp = 0;
}
auto filepath = Globals::Instance->outputPath / vtxName;
std::string incStr = StringHelper::Sprintf("%s.%s.inc", filepath.string().c_str(), "vtx");
@ -1991,7 +2007,7 @@ bool ZDisplayList::TextureGenCheck(int32_t texWidth, int32_t texHeight, uint32_t
texWidth, texHeight, texIsPalette, texAddr);
if ((texSeg != 0 || texAddr != 0) && texWidth > 0 && texHeight > 0 && texLoaded &&
Globals::Instance->HasSegment(segmentNumber))
Globals::Instance->HasSegment(segmentNumber, self->parent->workerID))
{
ZFile* auxParent = nullptr;
if (segmentNumber == self->parent->segment)
@ -2002,7 +2018,8 @@ bool ZDisplayList::TextureGenCheck(int32_t texWidth, int32_t texHeight, uint32_t
{
// Try to find a non-external file (i.e., one we are actually extracting)
// which has the same segment number we are looking for.
for (auto& otherFile : Globals::Instance->cfg.segmentRefFiles[segmentNumber])
auto segs = Globals::Instance->GetSegmentRefFiles(self->parent->workerID);
for (auto& otherFile : segs[segmentNumber])
{
if (!otherFile->isExternalFile)
{

View File

@ -41,6 +41,7 @@ ZFile::ZFile()
baseAddress = 0;
rangeStart = 0x000000000;
rangeEnd = 0xFFFFFFFF;
workerID = 0;
}
ZFile::ZFile(const fs::path& nOutPath, const std::string& nName) : ZFile()
@ -51,7 +52,7 @@ ZFile::ZFile(const fs::path& nOutPath, const std::string& nName) : ZFile()
}
ZFile::ZFile(ZFileMode nMode, tinyxml2::XMLElement* reader, const fs::path& nBasePath,
const fs::path& nOutPath, const std::string& filename, const fs::path& nXmlFilePath)
const fs::path& nOutPath, const std::string& filename, const fs::path& nXmlFilePath, int nWorkerID)
: ZFile()
{
xmlFilePath = nXmlFilePath;
@ -66,6 +67,7 @@ ZFile::ZFile(ZFileMode nMode, tinyxml2::XMLElement* reader, const fs::path& nBas
outputPath = nOutPath;
mode = nMode;
workerID = nWorkerID;
ParseXML(reader, filename);
if (mode != ZFileMode::ExternalFile)
@ -167,7 +169,7 @@ void ZFile::ParseXML(tinyxml2::XMLElement* reader, const std::string& filename)
}
}
}
Globals::Instance->AddSegment(segment, this);
Globals::Instance->AddSegment(segment, this, workerID);
if (Globals::Instance->verbosity >= VerbosityLevel::VERBOSITY_INFO)
{
@ -181,16 +183,22 @@ void ZFile::ParseXML(tinyxml2::XMLElement* reader, const std::string& filename)
}
}
if (mode == ZFileMode::Extract || mode == ZFileMode::ExternalFile)
if (mode == ZFileMode::Extract || mode == ZFileMode::ExternalFile || mode == ZFileMode::ExtractDirectory)
{
if (!File::Exists((basePath / name).string()))
if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
{
std::string errorHeader = StringHelper::Sprintf("binary file '%s' does not exist.",
(basePath / name).c_str());
HANDLE_ERROR_PROCESS(WarningType::Always, errorHeader, "");
if (!File::Exists((basePath / name).string()))
{
std::string errorHeader = StringHelper::Sprintf("binary file '%s' does not exist.",
(basePath / name).c_str());
HANDLE_ERROR_PROCESS(WarningType::Always, errorHeader, "");
}
}
rawData = File::ReadAllBytes((basePath / name).string());
if (Globals::Instance->fileMode == ZFileMode::ExtractDirectory)
rawData = Globals::Instance->GetBaseromFile(name);
else
rawData = Globals::Instance->GetBaseromFile((basePath / name).string());
if (reader->Attribute("RangeEnd") == nullptr)
rangeEnd = rawData.size();
@ -260,7 +268,7 @@ void ZFile::ParseXML(tinyxml2::XMLElement* reader, const std::string& filename)
{
ZResource* nRes = nodeMap[nodeName](this);
if (mode == ZFileMode::Extract || mode == ZFileMode::ExternalFile)
if (mode == ZFileMode::Extract || mode == ZFileMode::ExternalFile || mode == ZFileMode::ExtractDirectory)
nRes->ExtractFromXML(child, rawDataIndex);
switch (nRes->GetResourceType())
@ -813,7 +821,8 @@ void ZFile::GenerateSourceHeaderFiles()
if (Globals::Instance->verbosity >= VerbosityLevel::VERBOSITY_INFO)
printf("Writing H file: %s\n", headerFilename.c_str());
File::WriteAllText(headerFilename, formatter.GetOutput());
if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
File::WriteAllText(headerFilename, formatter.GetOutput());
}
std::string ZFile::GetHeaderInclude() const
@ -999,6 +1008,10 @@ std::string ZFile::ProcessDeclarations()
lastItem.second->size += curItem.second->size;
lastItem.second->arrayItemCnt += curItem.second->arrayItemCnt;
lastItem.second->text += "\n" + curItem.second->text;
for (auto vtx : curItem.second->vertexHack)
lastItem.second->vertexHack.push_back(vtx);
declarations.erase(curItem.first);
declarationKeys.erase(declarationKeys.begin() + i);
delete curItem.second;
@ -1087,7 +1100,7 @@ void ZFile::ProcessDeclarationText(Declaration* decl)
{
std::string vtxName;
Globals::Instance->GetSegmentedArrayIndexedName(decl->references[refIndex], 0x10, this,
"Vtx", vtxName);
"Vtx", vtxName, workerID);
decl->text.replace(i, 2, vtxName);
refIndex++;
@ -1190,6 +1203,9 @@ void ZFile::HandleUnaccountedData()
uint32_t lastSize = 0;
std::vector<offset_t> declsAddresses;
if (Globals::Instance->otrMode)
return;
for (const auto& item : declarations)
{
declsAddresses.push_back(item.first);

View File

@ -16,6 +16,7 @@ enum class ZFileMode
BuildBackground,
Extract,
ExternalFile,
ExtractDirectory,
Invalid,
Custom = 1000, // Used for exporter file modes
};
@ -34,6 +35,8 @@ public:
std::string defines;
std::vector<ZResource*> resources;
int workerID;
// Default to using virtual addresses
uint32_t segment = 0x80;
uint32_t baseAddress, rangeStart, rangeEnd;
@ -41,7 +44,7 @@ public:
ZFile(const fs::path& nOutPath, const std::string& nName);
ZFile(ZFileMode nMode, tinyxml2::XMLElement* reader, const fs::path& nBasePath,
const fs::path& nOutPath, const std::string& filename, const fs::path& nXmlFilePath);
const fs::path& nOutPath, const std::string& filename, const fs::path& nXmlFilePath, int nWorkerID);
~ZFile();
std::string GetName() const;
@ -107,12 +110,12 @@ public:
static void RegisterNode(std::string nodeName, ZResourceFactoryFunc* nodeFunc);
protected:
std::vector<uint8_t> rawData;
std::string name;
fs::path outName = "";
fs::path basePath;
fs::path outputPath;
fs::path xmlFilePath;
std::vector<uint8_t> rawData;
// Keep track of every texture of this ZFile.
// The pointers declared here are "borrowed" (somebody else is the owner),

View File

@ -218,18 +218,25 @@ size_t ZLimb::GetRawDataSize() const
std::string ZLimb::GetBodySourceCode() const
{
if (Globals::Instance->otrMode)
return "";
std::string dListStr;
std::string dListStr2;
Globals::Instance->GetSegmentedArrayIndexedName(dListPtr, 8, parent, "Gfx", dListStr);
Globals::Instance->GetSegmentedArrayIndexedName(dList2Ptr, 8, parent, "Gfx", dListStr2);
Globals::Instance->GetSegmentedArrayIndexedName(dListPtr, 8, parent, "Gfx", dListStr,
parent->workerID);
Globals::Instance->GetSegmentedArrayIndexedName(dList2Ptr, 8, parent, "Gfx", dListStr2,
parent->workerID);
std::string entryStr = "\n\t";
if (type == ZLimbType::Legacy)
{
std::string childName;
std::string siblingName;
Globals::Instance->GetSegmentedPtrName(childPtr, parent, "LegacyLimb", childName);
Globals::Instance->GetSegmentedPtrName(siblingPtr, parent, "LegacyLimb", siblingName);
Globals::Instance->GetSegmentedPtrName(childPtr, parent, "LegacyLimb", childName,
parent->workerID);
Globals::Instance->GetSegmentedPtrName(siblingPtr, parent, "LegacyLimb", siblingName,
parent->workerID);
entryStr += StringHelper::Sprintf("%s,\n", dListStr.c_str());
entryStr +=
@ -261,7 +268,8 @@ std::string ZLimb::GetBodySourceCode() const
case ZLimbType::Skin:
{
std::string skinSegmentStr;
Globals::Instance->GetSegmentedPtrName(skinSegment, parent, "", skinSegmentStr);
Globals::Instance->GetSegmentedPtrName(skinSegment, parent, "", skinSegmentStr,
parent->workerID);
entryStr +=
StringHelper::Sprintf("\t0x%02X, %s\n", skinSegmentType, skinSegmentStr.c_str());
}
@ -367,7 +375,7 @@ void ZLimb::DeclareDList(segptr_t dListSegmentedPtr, const std::string& prefix,
std::string dlistName;
bool declFound = Globals::Instance->GetSegmentedArrayIndexedName(dListSegmentedPtr, 8, parent,
"Gfx", dlistName);
"Gfx", dlistName, parent->workerID);
if (declFound)
return;

View File

@ -142,8 +142,8 @@ void PathwayEntry::DeclareReferences(const std::string& prefix)
return;
std::string pointsName;
bool addressFound =
Globals::Instance->GetSegmentedPtrName(listSegmentAddress, parent, "Vec3s", pointsName);
bool addressFound = Globals::Instance->GetSegmentedPtrName(listSegmentAddress, parent, "Vec3s",
pointsName, parent->workerID);
if (addressFound)
return;
@ -177,7 +177,8 @@ std::string PathwayEntry::GetBodySourceCode() const
{
std::string declaration;
std::string listName;
Globals::Instance->GetSegmentedPtrName(listSegmentAddress, parent, "Vec3s", listName);
Globals::Instance->GetSegmentedPtrName(listSegmentAddress, parent, "Vec3s", listName,
parent->workerID);
if (Globals::Instance->game == ZGame::MM_RETAIL)
declaration +=

View File

@ -3,6 +3,7 @@
#include "Utils/BitConverter.h"
#include "Utils/StringHelper.h"
#include "ZFile.h"
#include <Globals.h>
REGISTER_ZFILENODE(PlayerAnimationData, ZPlayerAnimationData);
@ -54,6 +55,9 @@ std::string ZPlayerAnimationData::GetBodySourceCode() const
{
std::string declaration = "";
if (Globals::Instance->otrMode)
return "";
size_t index = 0;
for (const auto& entry : limbRotData)
{

View File

@ -8,6 +8,7 @@
#include "ZFile.h"
#include <Globals.h>
#include <ZDisplayList.h>
#include <ZArray.h>
ZResource::ZResource(ZFile* nParent)
{
@ -18,6 +19,7 @@ ZResource::ZResource(ZFile* nParent)
sourceOutput = "";
rawDataIndex = 0;
outputDeclaration = true;
hash = 0;
RegisterRequiredAttribute("Name");
RegisterOptionalAttribute("OutName");
@ -119,14 +121,21 @@ void ZResource::ParseXML(tinyxml2::XMLElement* reader)
name = registeredAttributes.at("Name").value;
static std::regex r("[a-zA-Z_]+[a-zA-Z0-9_]*", std::regex::icase | std::regex::optimize);
if (!isInner || (isInner && name != ""))
// Disable this check for OTR file generation for now since it takes up a considerable amount of CPU time
if (!Globals::Instance->otrMode)
{
if (!std::regex_match(name, r))
static std::regex r("[a-zA-Z_]+[a-zA-Z0-9_]*",
std::regex::icase | std::regex::optimize);
if (!isInner || (isInner && name != ""))
{
HANDLE_ERROR_RESOURCE(WarningType::InvalidAttributeValue, parent, this,
rawDataIndex, "invalid value found for 'Name' attribute", "");
if (!std::regex_match(name, r))
{
HANDLE_ERROR_RESOURCE(WarningType::InvalidAttributeValue, parent, this,
rawDataIndex, "invalid value found for 'Name' attribute",
"");
}
}
}
@ -273,6 +282,21 @@ void ZResource::GetSourceOutputCode([[maybe_unused]] const std::string& prefix)
else
decl->text = bodyStr;
// OTRTODO: This is a hack and we need something more elegant in the future...
if (GetResourceType() == ZResourceType::Array)
{
ZArray* arr = (ZArray*)this;
if (arr->resList[0]->GetResourceType() == ZResourceType::Vertex)
{
for (int i = 0; i < arr->resList.size(); i++)
{
ZVtx* vtx = (ZVtx*)arr->resList[i];
decl->vertexHack.push_back(vtx);
}
}
}
if (decl != nullptr)
decl->staticConf = staticConf;
}

202
ZAPDTR/ZAPD/ZRom.cpp Normal file
View File

@ -0,0 +1,202 @@
#include "ZRom.h"
#include "Utils/BitConverter.h"
#include "Utils/File.h"
#include "Utils/Directory.h"
#include "yaz0/yaz0.h"
#ifndef _MSC_VER
#include <byteswap.h>
#endif
#include <Globals.h>
namespace fs = std::filesystem;
#define DMA_ENTRY_SIZE 16
#if defined(_MSC_VER)
#define __bswap_32 _byteswap_ulong
#define bswap_32 _byteswap_ulong
#endif
// ROM DMA Table Start
#define OOT_OFF_NTSC_10_RC 0x7430
#define OOT_OFF_NTSC_10 0x7430
#define OOT_OFF_NTSC_11 0x7430
#define OOT_OFF_PAL_10 0x7950
#define OOT_OFF_NTSC_12 0x7960
#define OOT_OFF_PAL_11 0x7950
#define OOT_OFF_JP_GC 0x7170
#define OOT_OFF_JP_MQ 0x7170
#define OOT_OFF_US_GC 0x7170
#define OOT_OFF_US_MQ 0x7170
#define OOT_OFF_PAL_GC_DBG1 0x12F70
#define OOT_OFF_PAL_MQ_DBG 0x12F70
#define OOT_OFF_PAL_GC_DBG2 0x12F70
#define OOT_OFF_PAL_GC 0x7170
#define OOT_OFF_PAL_MQ 0x7170
#define OOT_OFF_JP_GC_CE 007170
#define OOT_OFF_CN_IQUE 0xB7A0
#define OOT_OFF_TW_IQUE 0xB240
#define MM_OFF_US_10 0x1A500
#define MM_OFF_JP_10 0x1C110
#define MM_OFF_JP_11 0x1C050
#define MM_OFF_DBG 0x24F60
#define OOT_NTSC_10 0xEC7011B7
#define OOT_NTSC_11 0xD43DA81F
#define OOT_NTSC_12 0x693BA2AE
#define OOT_PAL_10 0xB044B569
#define OOT_PAL_11 0xB2055FBD
#define OOT_NTSC_JP_GC_CE 0xF7F52DB8
#define OOT_NTSC_JP_GC 0xF611F4BA
#define OOT_NTSC_US_GC 0xF3DD35BA
#define OOT_PAL_GC 0x09465AC3
#define OOT_NTSC_JP_MQ 0xF43B45BA
#define OOT_NTSC_US_MQ 0xF034001A
#define OOT_PAL_MQ 0x1D4136F3
#define OOT_PAL_GC_DBG1 0x871E1C92 // 03-21-2002 build
#define OOT_PAL_GC_DBG2 0x87121EFE // 03-13-2002 build
#define OOT_PAL_GC_MQ_DBG 0x917D18F6
#define OOT_IQUE_TW 0x3D81FB3E
#define OOT_IQUE_CN 0xB1E1E07B
#define OOT_UNKNOWN 0xFFFFFFFF
ZRom::ZRom(std::string romPath)
{
RomVersion version;
romData = File::ReadAllBytes(romPath);
version.crc = BitConverter::ToInt32BE(romData, 0x10);
switch (version.crc)
{
case OOT_NTSC_10:
version.version = "N64 NTSC 1.0";
version.listPath = "ntsc_oot.txt";
version.offset = OOT_OFF_NTSC_10;
break;
case OOT_NTSC_11:
version.version = "N64 NTSC 1.1";
version.listPath = "ntsc_oot.txt";
version.offset = OOT_OFF_NTSC_11;
break;
case OOT_NTSC_12:
version.version = "N64 NTSC 1.2";
version.listPath = "ntsc_oot.txt";
version.offset = OOT_OFF_NTSC_12;
break;
case OOT_PAL_10:
version.version = "N64 PAL 1.0";
version.listPath = "pal_oot.txt";
version.offset = OOT_OFF_PAL_10;
break;
case OOT_PAL_11:
version.version = "N64 PAL 1.1";
version.listPath = "pal_oot.txt";
version.offset = OOT_OFF_PAL_11;
break;
case OOT_NTSC_JP_GC:
version.version = "JP GameCube (MQ Disk)";
version.listPath = "gamecube.txt";
version.offset = OOT_OFF_JP_GC;
break;
case OOT_NTSC_JP_GC_CE:
version.version = "GameCube (Collectors Edition Disk)";
version.listPath = "gamecube.txt";
version.offset = OOT_OFF_JP_GC_CE;
break;
case OOT_NTSC_JP_MQ:
version.version = "JP Master Quest";
version.listPath = "gamecube.txt";
version.offset = OOT_OFF_JP_MQ;
break;
case OOT_NTSC_US_MQ:
version.version = "NTSC Master Quest";
version.listPath = "gamecube.txt";
version.offset = OOT_OFF_JP_MQ;
break;
case OOT_NTSC_US_GC:
version.version = "NTSC GameCube";
version.listPath = "gamecube.txt";
version.offset = OOT_OFF_US_MQ;
break;
case OOT_PAL_GC:
version.version = "PAL GameCube";
version.listPath = "gamecube_pal.txt";
version.offset = OOT_OFF_PAL_GC;
break;
case OOT_PAL_MQ:
version.version = "PAL Master Quest";
version.listPath = "gamecube_pal.txt";
version.offset = OOT_OFF_PAL_MQ;
break;
case OOT_PAL_GC_DBG1:
version.version = "GameCube Debug 1.0";
version.listPath = "dbg.txt";
version.offset = OOT_OFF_PAL_GC_DBG1;
break;
case OOT_PAL_GC_DBG2:
version.version = "GameCube Debug 2.0";
version.listPath = "dbg.txt";
version.offset = OOT_OFF_PAL_GC_DBG2;
break;
case OOT_PAL_GC_MQ_DBG:
version.version = "GameCube MQ-Debug";
version.listPath = "dbg.txt";
version.offset = OOT_OFF_PAL_MQ_DBG;
break;
case OOT_IQUE_CN:
version.version = "OoT IQue";
version.listPath = "ique.txt";
version.offset = OOT_OFF_CN_IQUE;
break;
case OOT_IQUE_TW:
version.version = "TW IQue";
version.listPath = "ique.txt";
version.offset = OOT_OFF_TW_IQUE;
break;
}
auto path = StringHelper::Sprintf("%s/%s", Globals::Instance->fileListPath.string().c_str(), version.listPath.c_str());
auto txt = File::ReadAllText(path);
std::vector<std::string> lines = StringHelper::Split(txt, "\n");
std::vector<uint8_t> decompressedData(1);
for (int i = 0; i < lines.size(); i++)
{
lines[i] = StringHelper::Strip(lines[i], "\r");
const int romOffset = version.offset + (DMA_ENTRY_SIZE * i);
const int virtStart = BitConverter::ToInt32BE(romData, romOffset + 0);
const int virtEnd = BitConverter::ToInt32BE(romData, romOffset + 4);
const int physStart = BitConverter::ToInt32BE(romData, romOffset + 8);
const int physEnd = BitConverter::ToInt32BE(romData, romOffset + 12);
const bool compressed = physEnd != 0;
int size = compressed ? physEnd - physStart : virtEnd - virtStart;
auto outData = std::vector<uint8_t>();
outData.resize(size);
memcpy(outData.data(), romData.data() + physStart, size);
if (compressed)
{
int decSize = virtEnd - virtStart;
decompressedData = std::vector<uint8_t>();
decompressedData.resize(decSize);
yaz0_decode(outData.data(), decompressedData.data(), decSize);
files[lines[i]] = decompressedData;
}
else
files[lines[i]] = outData;
}
int bp = 0;
}
std::vector<uint8_t> ZRom::GetFile(std::string fileName)
{
return files[fileName];
}

27
ZAPDTR/ZAPD/ZRom.h Normal file
View File

@ -0,0 +1,27 @@
#pragma once
#include <stdint.h>
#include <vector>
#include <map>
#include <string>
class ZRom
{
public:
ZRom(std::string romPath);
std::vector<uint8_t> GetFile(std::string fileName);
protected:
std::vector<uint8_t> romData;
std::map<std::string, std::vector<uint8_t>> files;
};
struct RomVersion
{
std::string version = "None";
std::string error = "None";
std::string listPath = "None";
int offset;
uint32_t crc;
};

View File

@ -54,7 +54,8 @@ void SetActorCutsceneList::DeclareReferences(const std::string& prefix)
std::string SetActorCutsceneList::GetBodySourceCode() const
{
std::string listName;
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "ActorCutscene", listName);
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "ActorCutscene", listName,
parent->workerID);
return StringHelper::Sprintf("SCENE_CMD_ACTOR_CUTSCENE_LIST(%i, %s)", cutscenes.size(),
listName.c_str());
}

View File

@ -81,7 +81,8 @@ void SetActorList::DeclareReferencesLate(const std::string& prefix)
std::string SetActorList::GetBodySourceCode() const
{
std::string listName;
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "ActorEntry", listName);
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "ActorEntry", listName,
parent->workerID);
if (numActors != actors.size())
{
printf("%s: numActors(%i) ~ actors(%li)\n", parent->GetName().c_str(), numActors,

View File

@ -48,7 +48,8 @@ void SetAlternateHeaders::DeclareReferencesLate(const std::string& prefix)
for (size_t i = 0; i < headers.size(); i++)
{
std::string altHeaderName;
Globals::Instance->GetSegmentedPtrName(headers.at(i), parent, "", altHeaderName);
Globals::Instance->GetSegmentedPtrName(headers.at(i), parent, "", altHeaderName,
parent->workerID);
declaration += StringHelper::Sprintf("\t%s,", altHeaderName.c_str());
@ -66,7 +67,8 @@ void SetAlternateHeaders::DeclareReferencesLate(const std::string& prefix)
std::string SetAlternateHeaders::GetBodySourceCode() const
{
std::string listName;
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "SceneCmd*", listName);
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "SceneCmd*", listName,
parent->workerID);
return StringHelper::Sprintf("SCENE_CMD_ALTERNATE_HEADER_LIST(%s)", listName.c_str());
}

View File

@ -33,7 +33,8 @@ void SetAnimatedMaterialList::DeclareReferences(const std::string& prefix)
std::string SetAnimatedMaterialList::GetBodySourceCode() const
{
std::string listName;
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "AnimatedMaterial", listName);
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "AnimatedMaterial", listName,
parent->workerID);
return StringHelper::Sprintf("SCENE_CMD_ANIMATED_MATERIAL_LIST(%s)", listName.c_str());
}

View File

@ -29,7 +29,8 @@ void SetCollisionHeader::DeclareReferences(const std::string& prefix)
std::string SetCollisionHeader::GetBodySourceCode() const
{
std::string listName;
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "CollisionHeader", listName);
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "CollisionHeader", listName,
parent->workerID);
return StringHelper::Sprintf("SCENE_CMD_COL_HEADER(%s)", listName.c_str());
}

View File

@ -71,7 +71,7 @@ void SetCsCamera::DeclareReferences(const std::string& prefix)
{
std::string camPointsName;
Globals::Instance->GetSegmentedPtrName(cameras.at(0).GetCamAddress(), parent, "Vec3s",
camPointsName);
camPointsName, parent->workerID);
std::string declaration;
size_t index = 0;
@ -103,7 +103,8 @@ void SetCsCamera::DeclareReferences(const std::string& prefix)
std::string SetCsCamera::GetBodySourceCode() const
{
std::string listName;
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "CsCameraEntry", listName);
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "CsCameraEntry", listName,
parent->workerID);
return StringHelper::Sprintf("SCENE_CMD_ACTOR_CUTSCENE_CAM_LIST(%i, %s)", cameras.size(),
listName.c_str());
}

View File

@ -86,7 +86,8 @@ void SetCutscenes::ParseRawData()
std::string SetCutscenes::GetBodySourceCode() const
{
std::string listName;
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "CutsceneData", listName);
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "CutsceneData", listName,
parent->workerID);
if (Globals::Instance->game == ZGame::MM_RETAIL)
return StringHelper::Sprintf("SCENE_CMD_CUTSCENE_LIST(%i, %s)", numCutscenes,

View File

@ -63,7 +63,8 @@ void SetEntranceList::DeclareReferencesLate([[maybe_unused]] const std::string&
std::string SetEntranceList::GetBodySourceCode() const
{
std::string listName;
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "EntranceEntry", listName);
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "EntranceEntry", listName,
parent->workerID);
return StringHelper::Sprintf("SCENE_CMD_ENTRANCE_LIST(%s)", listName.c_str());
}

View File

@ -58,7 +58,7 @@ void SetExitList::DeclareReferencesLate([[maybe_unused]] const std::string& pref
std::string SetExitList::GetBodySourceCode() const
{
std::string listName;
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "u16", listName);
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "u16", listName, parent->workerID);
return StringHelper::Sprintf("SCENE_CMD_EXIT_LIST(%s)", listName.c_str());
}

View File

@ -52,7 +52,8 @@ void SetLightList::DeclareReferences(const std::string& prefix)
std::string SetLightList::GetBodySourceCode() const
{
std::string listName;
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "LightInfo", listName);
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "LightInfo", listName,
parent->workerID);
return StringHelper::Sprintf("SCENE_CMD_LIGHT_LIST(%i, %s)", numLights, listName.c_str());
}

View File

@ -44,7 +44,8 @@ void SetLightingSettings::DeclareReferences(const std::string& prefix)
std::string SetLightingSettings::GetBodySourceCode() const
{
std::string listName;
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "LightSettings", listName);
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "LightSettings", listName,
parent->workerID);
return StringHelper::Sprintf("SCENE_CMD_ENV_LIGHT_SETTINGS(%i, %s)", settings.size(),
listName.c_str());
}

View File

@ -78,7 +78,7 @@ std::string SetMesh::GenDListExterns(ZDisplayList* dList)
std::string SetMesh::GetBodySourceCode() const
{
std::string list;
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "", list);
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "", list, parent->workerID);
return StringHelper::Sprintf("SCENE_CMD_MESH(%s)", list.c_str());
}
@ -129,8 +129,8 @@ std::string PolygonDlist::GetBodySourceCode() const
std::string bodyStr;
std::string opaStr;
std::string xluStr;
Globals::Instance->GetSegmentedPtrName(opa, parent, "Gfx", opaStr);
Globals::Instance->GetSegmentedPtrName(xlu, parent, "Gfx", xluStr);
Globals::Instance->GetSegmentedPtrName(opa, parent, "Gfx", opaStr, parent->workerID);
Globals::Instance->GetSegmentedPtrName(xlu, parent, "Gfx", xluStr, parent->workerID);
if (polyType == 2)
{
@ -294,7 +294,7 @@ std::string BgImage::GetBodySourceCode() const
}
std::string backgroundName;
Globals::Instance->GetSegmentedPtrName(source, parent, "", backgroundName);
Globals::Instance->GetSegmentedPtrName(source, parent, "", backgroundName, parent->workerID);
bodyStr += StringHelper::Sprintf("%s, ", backgroundName.c_str());
bodyStr += "\n ";
if (!isSubStruct)
@ -493,7 +493,7 @@ std::string PolygonType1::GetBodySourceCode() const
bodyStr += StringHelper::Sprintf("%i, %i, ", type, format);
std::string dlistStr;
Globals::Instance->GetSegmentedPtrName(dlist, parent, "", dlistStr);
Globals::Instance->GetSegmentedPtrName(dlist, parent, "", dlistStr, parent->workerID);
bodyStr += StringHelper::Sprintf("%s, ", dlistStr.c_str());
bodyStr += "}, \n";
@ -505,7 +505,7 @@ std::string PolygonType1::GetBodySourceCode() const
bodyStr += single.GetBodySourceCode();
break;
case 2:
Globals::Instance->GetSegmentedPtrName(list, parent, "BgImage", listStr);
Globals::Instance->GetSegmentedPtrName(list, parent, "BgImage", listStr, parent->workerID);
bodyStr += StringHelper::Sprintf(" %i, %s, \n", count, listStr.c_str());
break;
@ -592,7 +592,7 @@ void PolygonType2::DeclareReferences(const std::string& prefix)
std::string PolygonType2::GetBodySourceCode() const
{
std::string listName;
Globals::Instance->GetSegmentedPtrName(start, parent, "", listName);
Globals::Instance->GetSegmentedPtrName(start, parent, "", listName, parent->workerID);
std::string body = StringHelper::Sprintf("\n %i, %i,\n", type, polyDLists.size());
body += StringHelper::Sprintf(" %s,\n", listName.c_str());

View File

@ -50,7 +50,8 @@ void SetMinimapChests::DeclareReferences(const std::string& prefix)
std::string SetMinimapChests::GetBodySourceCode() const
{
std::string listName;
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "MinimapChest", listName);
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "MinimapChest", listName,
parent->workerID);
return StringHelper::Sprintf("SCENE_CMD_MINIMAP_COMPASS_ICON_INFO(0x%02X, %s)", chests.size(),
listName.c_str());
}

View File

@ -52,7 +52,8 @@ void SetMinimapList::DeclareReferences(const std::string& prefix)
{
std::string listName;
Globals::Instance->GetSegmentedPtrName(listSegmentAddr, parent, "MinimapEntry", listName);
Globals::Instance->GetSegmentedPtrName(listSegmentAddr, parent, "MinimapEntry", listName,
parent->workerID);
std::string declaration = StringHelper::Sprintf("\n\t%s, 0x%08X\n", listName.c_str(), unk4);
parent->AddDeclaration(
@ -65,7 +66,8 @@ void SetMinimapList::DeclareReferences(const std::string& prefix)
std::string SetMinimapList::GetBodySourceCode() const
{
std::string listName;
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "MinimapList", listName);
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "MinimapList", listName,
parent->workerID);
return StringHelper::Sprintf("SCENE_CMD_MINIMAP_INFO(%s)", listName.c_str());
}

View File

@ -51,7 +51,7 @@ void SetObjectList::DeclareReferences(const std::string& prefix)
std::string SetObjectList::GetBodySourceCode() const
{
std::string listName;
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "s16", listName);
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "s16", listName, parent->workerID);
return StringHelper::Sprintf("SCENE_CMD_OBJECT_LIST(%i, %s)", objects.size(), listName.c_str());
}

View File

@ -50,7 +50,7 @@ void SetPathways::DeclareReferencesLate(const std::string& prefix)
std::string SetPathways::GetBodySourceCode() const
{
std::string listName;
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "Path", listName);
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "Path", listName, parent->workerID);
return StringHelper::Sprintf("SCENE_CMD_PATH_LIST(%s)", listName.c_str());
}

View File

@ -34,7 +34,7 @@ void SetRoomList::DeclareReferences(const std::string& prefix)
std::string SetRoomList::GetBodySourceCode() const
{
std::string listName;
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "RomFile", listName);
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "RomFile", listName, parent->workerID);
return StringHelper::Sprintf("SCENE_CMD_ROOM_LIST(%i, %s)", romfile->rooms.size(),
listName.c_str());
}

View File

@ -51,7 +51,8 @@ void SetStartPositionList::DeclareReferences(const std::string& prefix)
std::string SetStartPositionList::GetBodySourceCode() const
{
std::string listName;
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "ActorEntry", listName);
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "ActorEntry", listName,
parent->workerID);
return StringHelper::Sprintf("SCENE_CMD_SPAWN_LIST(%i, %s)", actors.size(), listName.c_str());
}

View File

@ -52,7 +52,8 @@ void SetTransitionActorList::DeclareReferences(const std::string& prefix)
std::string SetTransitionActorList::GetBodySourceCode() const
{
std::string listName;
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "TransitionActorEntry", listName);
Globals::Instance->GetSegmentedPtrName(cmdArg2, parent, "TransitionActorEntry", listName,
parent->workerID);
return StringHelper::Sprintf("SCENE_CMD_TRANSITION_ACTOR_LIST(%i, %s)", transitionActors.size(),
listName.c_str());
}

View File

@ -89,7 +89,8 @@ void ZSkeleton::DeclareReferences(const std::string& prefix)
std::string ZSkeleton::GetBodySourceCode() const
{
std::string limbArrayName;
Globals::Instance->GetSegmentedPtrName(limbsArrayAddress, parent, "", limbArrayName);
Globals::Instance->GetSegmentedPtrName(limbsArrayAddress, parent, "", limbArrayName,
parent->workerID);
switch (type)
{
@ -245,7 +246,8 @@ std::string ZLimbTable::GetBodySourceCode() const
for (size_t i = 0; i < count; i++)
{
std::string limbName;
Globals::Instance->GetSegmentedPtrName(limbsAddresses[i], parent, "", limbName);
Globals::Instance->GetSegmentedPtrName(limbsAddresses[i], parent, "", limbName,
parent->workerID);
body += StringHelper::Sprintf("\t%s,", limbName.c_str());
if (i + 1 < count)

View File

@ -21,11 +21,12 @@ void ZText::ParseRawData()
const auto& rawData = parent->GetRawData();
uint32_t currentPtr = StringHelper::StrToL(registeredAttributes.at("CodeOffset").value, 16);
std::vector<uint8_t> codeData = File::ReadAllBytes(Globals::Instance->baseRomPath.string() + "\\code");
std::vector<uint8_t> codeData;
// In some cases with the multi-process extractor it seems that it fails to read the code file if something else is reading from it at the same time.
while (codeData.size() == 0)
codeData = File::ReadAllBytes(Globals::Instance->baseRomPath.string() + "\\code");
if (Globals::Instance->fileMode == ZFileMode::ExtractDirectory)
codeData = Globals::Instance->GetBaseromFile("code");
else
codeData = Globals::Instance->GetBaseromFile(Globals::Instance->baseRomPath.string() + "code");
while (true)
{

View File

@ -807,25 +807,30 @@ Declaration* ZTexture::DeclareVar(const std::string& prefix,
std::string ZTexture::GetBodySourceCode() const
{
std::string sourceOutput;
size_t texSizeInc = (dWordAligned) ? 8 : 4;
for (size_t i = 0; i < textureDataRaw.size(); i += texSizeInc)
{
if (i % 32 == 0)
sourceOutput += " ";
if (dWordAligned)
sourceOutput +=
StringHelper::Sprintf("0x%016llX, ", BitConverter::ToUInt64BE(textureDataRaw, i));
else
sourceOutput +=
StringHelper::Sprintf("0x%08llX, ", BitConverter::ToUInt32BE(textureDataRaw, i));
if (i % 32 == 24)
sourceOutput += StringHelper::Sprintf(" // 0x%06X \n", rawDataIndex + ((i / 32) * 32));
}
// Ensure there's always a trailing line feed to prevent dumb warnings.
// Please don't remove this line, unless you somehow made a way to prevent
// that warning when building the OoT repo.
sourceOutput += "\n";
if (!Globals::Instance->otrMode)
{
size_t texSizeInc = (dWordAligned) ? 8 : 4;
for (size_t i = 0; i < textureDataRaw.size(); i += texSizeInc)
{
if (i % 32 == 0)
sourceOutput += " ";
if (dWordAligned)
sourceOutput += StringHelper::Sprintf("0x%016llX, ",
BitConverter::ToUInt64BE(textureDataRaw, i));
else
sourceOutput += StringHelper::Sprintf("0x%08llX, ",
BitConverter::ToUInt32BE(textureDataRaw, i));
if (i % 32 == 24)
sourceOutput +=
StringHelper::Sprintf(" // 0x%06X \n", rawDataIndex + ((i / 32) * 32));
}
// Ensure there's always a trailing line feed to prevent dumb warnings.
// Please don't remove this line, unless you somehow made a way to prevent
// that warning when building the OoT repo.
sourceOutput += "\n";
}
return sourceOutput;
}
@ -847,8 +852,11 @@ std::string ZTexture::GetSourceTypeName() const
void ZTexture::CalcHash()
{
auto parentRawData = parent->GetRawData();
hash = CRC32B(parentRawData.data() + rawDataIndex, GetRawDataSize());
//if (hash == 0)
{
const auto& parentRawData = parent->GetRawData();
hash = CRC32B(parentRawData.data() + rawDataIndex, GetRawDataSize());
}
}
std::string ZTexture::GetExternalExtension() const

View File

@ -349,9 +349,12 @@ std::string TextureColorChangingParams::GetBodySourceCode() const
std::string envColorListName;
std::string frameDataListName;
Globals::Instance->GetSegmentedPtrName(primColorListAddress, parent, "", primColorListName);
Globals::Instance->GetSegmentedPtrName(envColorListAddress, parent, "", envColorListName);
Globals::Instance->GetSegmentedPtrName(frameDataListAddress, parent, "", frameDataListName);
Globals::Instance->GetSegmentedPtrName(primColorListAddress, parent, "", primColorListName,
parent->workerID);
Globals::Instance->GetSegmentedPtrName(envColorListAddress, parent, "", envColorListName,
parent->workerID);
Globals::Instance->GetSegmentedPtrName(frameDataListAddress, parent, "", frameDataListName,
parent->workerID);
std::string bodyStr = StringHelper::Sprintf(
"\n %d, %d, %s, %s, %s,\n", animLength, colorListCount, primColorListName.c_str(),
@ -423,7 +426,8 @@ void TextureCyclingParams::DeclareReferences([[maybe_unused]] const std::string&
for (const auto& tex : textureList)
{
bool texFound = Globals::Instance->GetSegmentedPtrName(tex, parent, "", texName);
bool texFound =
Globals::Instance->GetSegmentedPtrName(tex, parent, "", texName, parent->workerID);
// texName is a raw segmented pointer. This occurs if the texture is not declared
// separately since we cannot read the format. In theory we could scan DLists for the
@ -477,9 +481,10 @@ std::string TextureCyclingParams::GetBodySourceCode() const
std::string textureListName;
std::string textureIndexListName;
Globals::Instance->GetSegmentedPtrName(textureListAddress, parent, "", textureListName);
Globals::Instance->GetSegmentedPtrName(textureListAddress, parent, "", textureListName,
parent->workerID);
Globals::Instance->GetSegmentedPtrName(textureIndexListAddress, parent, "",
textureIndexListName);
textureIndexListName, parent->workerID);
std::string bodyStr = StringHelper::Sprintf(
"\n %d, %s, %s,\n", cycleLength, textureListName.c_str(), textureIndexListName.c_str());
@ -652,7 +657,8 @@ std::string ZTextureAnimation::GetBodySourceCode() const
for (const auto& entry : entries)
{
std::string paramName;
Globals::Instance->GetSegmentedPtrName(entry.paramsPtr, parent, "", paramName);
Globals::Instance->GetSegmentedPtrName(entry.paramsPtr, parent, "", paramName,
parent->workerID);
bodyStr += StringHelper::Sprintf("\t{ %d, %d, %s },\n", entry.segment, entry.type,
paramName.c_str());

251
ZAPDTR/ZAPD/ctpl_stl.h Normal file
View File

@ -0,0 +1,251 @@
/*********************************************************
*
* Copyright (C) 2014 by Vitaliy Vitsentiy
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*********************************************************/
#ifndef __ctpl_stl_thread_pool_H__
#define __ctpl_stl_thread_pool_H__
#include <functional>
#include <thread>
#include <atomic>
#include <vector>
#include <memory>
#include <exception>
#include <future>
#include <mutex>
#include <queue>
// thread pool to run user's functors with signature
// ret func(int id, other_params)
// where id is the index of the thread that runs the functor
// ret is some return type
namespace ctpl {
namespace detail {
template <typename T>
class Queue {
public:
bool push(T const & value) {
std::unique_lock<std::mutex> lock(this->mutex);
this->q.push(value);
return true;
}
// deletes the retrieved element, do not use for non integral types
bool pop(T & v) {
std::unique_lock<std::mutex> lock(this->mutex);
if (this->q.empty())
return false;
v = this->q.front();
this->q.pop();
return true;
}
bool empty() {
std::unique_lock<std::mutex> lock(this->mutex);
return this->q.empty();
}
private:
std::queue<T> q;
std::mutex mutex;
};
}
class thread_pool {
public:
thread_pool() { this->init(); }
thread_pool(int nThreads) { this->init(); this->resize(nThreads); }
// the destructor waits for all the functions in the queue to be finished
~thread_pool() {
this->stop(true);
}
// get the number of running threads in the pool
int size() { return static_cast<int>(this->threads.size()); }
// number of idle threads
int n_idle() { return this->nWaiting; }
std::thread & get_thread(int i) { return *this->threads[i]; }
// change the number of threads in the pool
// should be called from one thread, otherwise be careful to not interleave, also with this->stop()
// nThreads must be >= 0
void resize(int nThreads) {
if (!this->isStop && !this->isDone) {
int oldNThreads = static_cast<int>(this->threads.size());
if (oldNThreads <= nThreads) { // if the number of threads is increased
this->threads.resize(nThreads);
this->flags.resize(nThreads);
for (int i = oldNThreads; i < nThreads; ++i) {
this->flags[i] = std::make_shared<std::atomic<bool>>(false);
this->set_thread(i);
}
}
else { // the number of threads is decreased
for (int i = oldNThreads - 1; i >= nThreads; --i) {
*this->flags[i] = true; // this thread will finish
this->threads[i]->detach();
}
{
// stop the detached threads that were waiting
std::unique_lock<std::mutex> lock(this->mutex);
this->cv.notify_all();
}
this->threads.resize(nThreads); // safe to delete because the threads are detached
this->flags.resize(nThreads); // safe to delete because the threads have copies of shared_ptr of the flags, not originals
}
}
}
// empty the queue
void clear_queue() {
std::function<void(int id)> * _f;
while (this->q.pop(_f))
delete _f; // empty the queue
}
// pops a functional wrapper to the original function
std::function<void(int)> pop() {
std::function<void(int id)> * _f = nullptr;
this->q.pop(_f);
std::unique_ptr<std::function<void(int id)>> func(_f); // at return, delete the function even if an exception occurred
std::function<void(int)> f;
if (_f)
f = *_f;
return f;
}
// wait for all computing threads to finish and stop all threads
// may be called asynchronously to not pause the calling thread while waiting
// if isWait == true, all the functions in the queue are run, otherwise the queue is cleared without running the functions
void stop(bool isWait = false) {
if (!isWait) {
if (this->isStop)
return;
this->isStop = true;
for (int i = 0, n = this->size(); i < n; ++i) {
*this->flags[i] = true; // command the threads to stop
}
this->clear_queue(); // empty the queue
}
else {
if (this->isDone || this->isStop)
return;
this->isDone = true; // give the waiting threads a command to finish
}
{
std::unique_lock<std::mutex> lock(this->mutex);
this->cv.notify_all(); // stop all waiting threads
}
for (int i = 0; i < static_cast<int>(this->threads.size()); ++i) { // wait for the computing threads to finish
if (this->threads[i]->joinable())
this->threads[i]->join();
}
// if there were no threads in the pool but some functors in the queue, the functors are not deleted by the threads
// therefore delete them here
this->clear_queue();
this->threads.clear();
this->flags.clear();
}
template<typename F, typename... Rest>
auto push(F && f, Rest&&... rest) ->std::future<decltype(f(0, rest...))> {
auto pck = std::make_shared<std::packaged_task<decltype(f(0, rest...))(int)>>(
std::bind(std::forward<F>(f), std::placeholders::_1, std::forward<Rest>(rest)...)
);
auto _f = new std::function<void(int id)>([pck](int id) {
(*pck)(id);
});
this->q.push(_f);
std::unique_lock<std::mutex> lock(this->mutex);
this->cv.notify_one();
return pck->get_future();
}
// run the user's function that excepts argument int - id of the running thread. returned value is templatized
// operator returns std::future, where the user can get the result and rethrow the catched exceptins
template<typename F>
auto push(F && f) ->std::future<decltype(f(0))> {
auto pck = std::make_shared<std::packaged_task<decltype(f(0))(int)>>(std::forward<F>(f));
auto _f = new std::function<void(int id)>([pck](int id) {
(*pck)(id);
});
this->q.push(_f);
std::unique_lock<std::mutex> lock(this->mutex);
this->cv.notify_one();
return pck->get_future();
}
private:
// deleted
thread_pool(const thread_pool &);// = delete;
thread_pool(thread_pool &&);// = delete;
thread_pool & operator=(const thread_pool &);// = delete;
thread_pool & operator=(thread_pool &&);// = delete;
void set_thread(int i) {
std::shared_ptr<std::atomic<bool>> flag(this->flags[i]); // a copy of the shared ptr to the flag
auto f = [this, i, flag/* a copy of the shared ptr to the flag */]() {
std::atomic<bool> & _flag = *flag;
std::function<void(int id)> * _f;
bool isPop = this->q.pop(_f);
while (true) {
while (isPop) { // if there is anything in the queue
std::unique_ptr<std::function<void(int id)>> func(_f); // at return, delete the function even if an exception occurred
(*_f)(i);
if (_flag)
return; // the thread is wanted to stop, return even if the queue is not empty yet
else
isPop = this->q.pop(_f);
}
// the queue is empty here, wait for the next command
std::unique_lock<std::mutex> lock(this->mutex);
++this->nWaiting;
this->cv.wait(lock, [this, &_f, &isPop, &_flag](){ isPop = this->q.pop(_f); return isPop || this->isDone || _flag; });
--this->nWaiting;
if (!isPop)
return; // if the queue is empty and this->isDone == true or *flag then return
}
};
this->threads[i].reset(new std::thread(f)); // compiler may not support std::make_unique()
}
void init() { this->nWaiting = 0; this->isStop = false; this->isDone = false; }
std::vector<std::unique_ptr<std::thread>> threads;
std::vector<std::shared_ptr<std::atomic<bool>>> flags;
detail::Queue<std::function<void(int id)> *> q;
std::atomic<bool> isDone;
std::atomic<bool> isStop;
std::atomic<int> nWaiting; // how many threads are waiting
std::mutex mutex;
std::condition_variable cv;
};
}
#endif // __ctpl_stl_thread_pool_H__

View File

@ -0,0 +1,29 @@
#ifndef __READWRITE_H__
#define __READWRITE_H__
#include <stdint.h>
/* variables */
union {
uint32_t u;
float f;
} __u32_f32_union__;
#define U32(x) \
((uint32_t)((((uint8_t*)(x))[0] << 24) | (((uint8_t*)(x))[1] << 16) | \
(((uint8_t*)(x))[2] << 8) | ((uint8_t*)(x))[3]))
#define U16(x) ((uint16_t)(((*((uint8_t*)(x))) << 8) | ((uint8_t*)(x))[1]))
#define U8(x) ((uint8_t)((uint8_t*)(x))[0])
#define S32(x) ((int32_t)(U32(x)))
#define S16(x) ((int16_t)(U16(x)))
#define F32(x) (((__u32_f32_union__.u = U32(x)) & 0) + __u32_f32_union__.f)
#define W32(x, v) \
{ \
*((uint8_t*)x + 3) = ((v)&0xFF); \
*((uint8_t*)x + 2) = (((v) >> 8) & 0xFF); \
*((uint8_t*)x + 1) = (((v) >> 16) & 0xFF); \
*((uint8_t*)x + 0) = (((v) >> 24) & 0xFF); \
}
#endif

227
ZAPDTR/ZAPD/yaz0/yaz0.cpp Normal file
View File

@ -0,0 +1,227 @@
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <list>
#include "readwrite.h"
#include "yaz0.h"
typedef uint8_t u8;
typedef uint16_t u16;
typedef uint32_t u32;
/* internal declarations */
int yaz0_encode_internal(const u8* src, int srcSize, u8* Data);
int yaz0_get_size(u8* src) { return U32(src + 0x4); }
u32 toDWORD(u32 d) {
u8 w1 = d & 0xFF;
u8 w2 = (d >> 8) & 0xFF;
u8 w3 = (d >> 16) & 0xFF;
u8 w4 = d >> 24;
return (w1 << 24) | (w2 << 16) | (w3 << 8) | w4;
}
// simple and straight encoding scheme for Yaz0
u32 longest_match_brute(const u8* src, int size, int pos, u32* pMatchPos) {
int startPos = pos - 0x1000;
int max_match_size = size - pos;
u32 best_match_size = 0;
u32 best_match_pos = 0;
if (max_match_size < 3) return 0;
if (startPos < 0) startPos = 0;
if (max_match_size > 0x111) max_match_size = 0x111;
for (int i = startPos; i < pos; i++) {
int current_size;
for (current_size = 0; current_size < max_match_size; current_size++) {
if (src[i + current_size] != src[pos + current_size]) {
break;
}
}
if (current_size > best_match_size) {
best_match_size = current_size;
best_match_pos = i;
if (best_match_size == 0x111) break;
}
}
*pMatchPos = best_match_pos;
return best_match_size;
}
u32 longest_match_rabinkarp(const u8* src, int size, int pos, u32* match_pos) {
int startPos = pos - 0x1000;
int max_match_size = size - pos;
u32 best_match_size = 0;
u32 best_match_pos = 0;
if (max_match_size < 3) return 0;
if (startPos < 0) startPos = 0;
if (max_match_size > 0x111) max_match_size = 0x111;
int find_hash = src[pos] << 16 | src[pos + 1] << 8 | src[pos + 2];
int current_hash = src[startPos] << 16 | src[startPos + 1] << 8 | src[startPos + 2];
for (int i = startPos; i < pos; i++) {
if(current_hash == find_hash) {
int current_size;
for (current_size = 3; current_size < max_match_size; current_size++) {
if (src[i + current_size] != src[pos + current_size]) {
break;
}
}
if (current_size > best_match_size) {
best_match_size = current_size;
best_match_pos = i;
if (best_match_size == 0x111) break;
}
}
current_hash = (current_hash << 8 | src[i + 3]) & 0xFFFFFF;
}
*match_pos = best_match_pos;
return best_match_size;
}
int yaz0_encode_internal(const u8* src, int srcSize, u8* Data) {
int srcPos = 0;
int bitmask = 0x80;
u8 currCodeByte = 0;
int currCodeBytePos = 0;
int pos = currCodeBytePos + 1;
while (srcPos < srcSize) {
u32 numBytes;
u32 matchPos;
numBytes = longest_match_rabinkarp(src, srcSize, srcPos, &matchPos);
//fprintf(stderr, "pos %x len %x pos %x\n", srcPos, (int)numBytes, (int)matchPos);
if (numBytes < 3) {
//fprintf(stderr, "single byte %02x\n", src[srcPos]);
Data[pos++] = src[srcPos++];
currCodeByte |= bitmask;
} else {
// RLE part
u32 dist = srcPos - matchPos - 1;
if (numBytes >= 0x12) // 3 byte encoding
{
Data[pos++] = dist >> 8; // 0R
Data[pos++] = dist & 0xFF; // FF
if (numBytes > 0xFF + 0x12) numBytes = 0xFF + 0x12;
Data[pos++] = numBytes - 0x12;
} else // 2 byte encoding
{
Data[pos++] = ((numBytes - 2) << 4) | (dist >> 8);
Data[pos++] = dist & 0xFF;
}
srcPos += numBytes;
}
bitmask >>= 1;
// write eight codes
if (!bitmask) {
Data[currCodeBytePos] = currCodeByte;
currCodeBytePos = pos++;
currCodeByte = 0;
bitmask = 0x80;
}
}
if (bitmask) {
Data[currCodeBytePos] = currCodeByte;
}
return pos;
}
std::vector<uint8_t> yaz0_encode_fast(const u8* src, int src_size) {
std::vector<uint8_t> buffer;
std::vector<std::list<uint32_t>> lut;
lut.resize(0x1000000);
for (int i = 0; i < src_size - 3; ++i) {
lut[src[i + 0] << 16 | src[i + 1] << 8 | src[i + 2]].push_back(i);
}
return buffer;
}
std::vector<uint8_t> yaz0_encode(const u8* src, int src_size) {
std::vector<uint8_t> buffer(src_size * 10 / 8 + 16);
u8* dst = buffer.data();
// write 4 bytes yaz0 header
memcpy(dst, "Yaz0", 4);
// write 4 bytes uncompressed size
W32(dst + 4, src_size);
// encode
int dst_size = yaz0_encode_internal(src, src_size, dst + 16);
int aligned_size = (dst_size + 31) & -16;
buffer.resize(aligned_size);
#if 0
std::vector<uint8_t> decompressed(src_size);
yaz0_decode(buffer.data(), decompressed.data(), src_size);
if(memcmp(src, decompressed.data(), src_size)) {
fprintf(stderr, "Decompressed buffer is different from original\n");
}
#endif
return buffer;
}
void yaz0_decode(const uint8_t* source, uint8_t* decomp, int32_t decompSize) {
uint32_t srcPlace = 0, dstPlace = 0;
uint32_t i, dist, copyPlace, numBytes;
uint8_t codeByte, byte1, byte2;
uint8_t bitCount = 0;
source += 0x10;
while (dstPlace < decompSize) {
/* If there are no more bits to test, get a new byte */
if (!bitCount) {
codeByte = source[srcPlace++];
bitCount = 8;
}
/* If bit 7 is a 1, just copy 1 byte from source to destination */
/* Else do some decoding */
if (codeByte & 0x80) {
decomp[dstPlace++] = source[srcPlace++];
} else {
/* Get 2 bytes from source */
byte1 = source[srcPlace++];
byte2 = source[srcPlace++];
/* Calculate distance to move in destination */
/* And the number of bytes to copy */
dist = ((byte1 & 0xF) << 8) | byte2;
copyPlace = dstPlace - (dist + 1);
numBytes = byte1 >> 4;
/* Do more calculations on the number of bytes to copy */
if (!numBytes)
numBytes = source[srcPlace++] + 0x12;
else
numBytes += 2;
/* Copy data from a previous point in destination */
/* to current point in destination */
for (i = 0; i < numBytes; i++) decomp[dstPlace++] = decomp[copyPlace++];
}
/* Set up for the next read cycle */
codeByte = codeByte << 1;
bitCount--;
}
}

6
ZAPDTR/ZAPD/yaz0/yaz0.h Normal file
View File

@ -0,0 +1,6 @@
#pragma once
#include <vector>
void yaz0_decode(const uint8_t* src, uint8_t* dest, int32_t destsize);
std::vector<uint8_t> yaz0_encode(const uint8_t* src, int src_size);

View File

@ -72,15 +72,27 @@
<PropertyGroup Label="UserMacros" />
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
@ -174,6 +186,12 @@
<ClCompile Include="Utils\MemoryStream.cpp" />
<ClCompile Include="Utils\StringHelper.cpp" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\OTRGui\build\ZERO_CHECK.vcxproj">
<Project>{02d10590-9542-3f55-aaf8-6055677e2a2a}</Project>
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
</ProjectReference>
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>

View File

@ -12,14 +12,14 @@
<Filter Include="Header Files\Utils">
<UniqueIdentifier>{d8c2c1e7-b065-4b0f-86a2-46ab46eedc0b}</UniqueIdentifier>
</Filter>
<Filter Include="Header Files\Source Files">
<Filter Include="Source Files">
<UniqueIdentifier>{4FC737F1-C7A5-4376-A066-2A32D752A2FF}</UniqueIdentifier>
<Extensions>cpp;c;cc;cxx;c++;cppm;ixx;def;odl;idl;hpj;bat;asm;asmx</Extensions>
</Filter>
<Filter Include="Header Files\Source Files\Utils">
<Filter Include="Source Files\Utils">
<UniqueIdentifier>{e047919d-7186-49ca-b115-e48fbb5c8743}</UniqueIdentifier>
</Filter>
<Filter Include="Header Files\Source Files\Libraries">
<Filter Include="Source Files\Libraries">
<UniqueIdentifier>{3de9dd46-0dfd-4d48-9f20-9f24e5b80fe0}</UniqueIdentifier>
</Filter>
</ItemGroup>
@ -69,19 +69,19 @@
</ItemGroup>
<ItemGroup>
<ClCompile Include="Utils\BinaryWriter.cpp">
<Filter>Header Files\Source Files\Utils</Filter>
<Filter>Source Files\Utils</Filter>
</ClCompile>
<ClCompile Include="Utils\MemoryStream.cpp">
<Filter>Header Files\Source Files\Utils</Filter>
<Filter>Source Files\Utils</Filter>
</ClCompile>
<ClCompile Include="Utils\BinaryReader.cpp">
<Filter>Header Files\Source Files\Utils</Filter>
<Filter>Source Files\Utils</Filter>
</ClCompile>
<ClCompile Include="..\lib\tinyxml2\tinyxml2.cpp">
<Filter>Header Files\Source Files\Libraries</Filter>
<Filter>Source Files\Libraries</Filter>
</ClCompile>
<ClCompile Include="Utils\StringHelper.cpp">
<Filter>Header Files\Source Files\Utils</Filter>
<Filter>Source Files\Utils</Filter>
</ClCompile>
</ItemGroup>
</Project>

View File

@ -2,6 +2,8 @@
#define GFXD_PRIV_H
#include "gfxd.h"
#define CONFIG_MT
#ifdef CONFIG_MT
# ifdef _MSC_VER
# define TLOCAL __declspec(thread)
@ -9,7 +11,7 @@
# define TLOCAL _Thread_local
# endif
#else
# define TLOCAL
#define TLOCAL
#endif
#define UCFUNC static inline

View File

@ -0,0 +1,20 @@
#pragma once
#define OOT_NTSC_10 0xEC7011B7
#define OOT_NTSC_11 0xD43DA81F
#define OOT_NTSC_12 0x693BA2AE
#define OOT_PAL_10 0xB044B569
#define OOT_PAL_11 0xB2055FBD
#define OOT_NTSC_JP_GC_CE 0xF7F52DB8
#define OOT_NTSC_JP_GC 0xF611F4BA
#define OOT_NTSC_US_GC 0xF3DD35BA
#define OOT_PAL_GC 0x09465AC3
#define OOT_NTSC_JP_MQ 0xF43B45BA
#define OOT_NTSC_US_MQ 0xF034001A
#define OOT_PAL_MQ 0x1D4136F3
#define OOT_PAL_GC_DBG1 0x871E1C92 // 03-21-2002 build
#define OOT_PAL_GC_DBG2 0x87121EFE // 03-13-2002 build
#define OOT_PAL_GC_MQ_DBG 0x917D18F6
#define OOT_IQUE_TW 0x3D81FB3E
#define OOT_IQUE_CN 0xB1E1E07B
#define OOT_UNKNOWN 0xFFFFFFFF

View File

@ -2381,12 +2381,14 @@ static void gfx_run_dl(Gfx* cmd) {
cmd++;
uint64_t hash = ((uint64_t)cmd->words.w0 << 32) + (uint64_t)cmd->words.w1;
ResourceMgr_GetNameByCRC(hash, fileName);
#if _DEBUG && 0
char* tex = ResourceMgr_LoadTexByCRC(hash);
#if _DEBUG
//ResourceMgr_GetNameByCRC(hash, fileName);
//printf("G_SETTIMG_OTR: %s, %08X\n", fileName, hash);
ResourceMgr_GetNameByCRC(hash, fileName);
printf("G_SETTIMG_OTR: %s, %08X\n", fileName, hash);
#else
char* tex = NULL;
#endif
if (addr != NULL)

View File

@ -21,7 +21,7 @@ struct TextureCacheKey {
uint8_t palette_index;
bool operator==(const TextureCacheKey&) const noexcept = default;
struct Hasher {
size_t operator()(const TextureCacheKey& key) const noexcept {
uintptr_t addr = (uintptr_t)key.texture_addr;

View File

@ -3,6 +3,7 @@
#include "spdlog/spdlog.h"
#include "File.h"
#include "Archive.h"
#include "GameVersions.h"
#include <Utils/StringHelper.h>
#include "Lib/StormLib/StormLib.h"
@ -11,6 +12,8 @@ namespace Ship {
ResourceMgr::ResourceMgr(std::shared_ptr<GlobalCtx2> Context, std::string MainPath, std::string PatchesPath) : Context(Context), bIsRunning(false), FileLoadThread(nullptr) {
OTR = std::make_shared<Archive>(MainPath, PatchesPath, false);
gameVersion = OOT_UNKNOWN;
if (OTR->IsMainMPQValid())
Start();
}
@ -86,7 +89,10 @@ namespace Ship {
OTR->LoadFile(ToLoad->path, true, ToLoad);
//Lock.lock();
FileCache[ToLoad->path] = ToLoad->bIsLoaded && !ToLoad->bHasLoadError ? ToLoad : nullptr;
if (!ToLoad->bHasLoadError)
FileCache[ToLoad->path] = ToLoad->bIsLoaded && !ToLoad->bHasLoadError ? ToLoad : nullptr;
//Lock.unlock();
SPDLOG_DEBUG("Loaded File {} on ResourceMgr thread", ToLoad->path);
@ -124,44 +130,62 @@ namespace Ship {
}
}
auto UnmanagedRes = ResourceLoader::LoadResource(ToLoad->File);
if (UnmanagedRes != nullptr)
if (!ToLoad->File->bHasLoadError)
{
UnmanagedRes->resMgr = this;
auto Res = std::shared_ptr<Resource>(UnmanagedRes);
auto UnmanagedRes = ResourceLoader::LoadResource(ToLoad->File);
if (Res != nullptr) {
std::unique_lock<std::mutex> Lock(ToLoad->ResourceLoadMutex);
if (UnmanagedRes != nullptr)
{
UnmanagedRes->resMgr = this;
auto Res = std::shared_ptr<Resource>(UnmanagedRes);
ToLoad->bHasResourceLoaded = true;
ToLoad->Resource = Res;
ResourceCache[Res->file->path] = Res;
if (Res != nullptr) {
std::unique_lock<std::mutex> Lock(ToLoad->ResourceLoadMutex);
SPDLOG_DEBUG("Loaded Resource {} on ResourceMgr thread", ToLoad->File->path);
ToLoad->bHasResourceLoaded = true;
ToLoad->Resource = Res;
ResourceCache[Res->file->path] = Res;
// Disabled for now because it can cause random crashes
//FileCache[Res->File->path] = nullptr;
//FileCache.erase(FileCache.find(Res->File->path));
Res->file = nullptr;
SPDLOG_DEBUG("Loaded Resource {} on ResourceMgr thread", ToLoad->File->path);
// Disabled for now because it can cause random crashes
//FileCache[Res->File->path] = nullptr;
//FileCache.erase(FileCache.find(Res->File->path));
Res->file = nullptr;
}
else {
ToLoad->bHasResourceLoaded = false;
ToLoad->Resource = nullptr;
SPDLOG_ERROR("Resource load FAILED {} on ResourceMgr thread", ToLoad->File->path);
}
//ResLock.lock();
//ResLock.unlock();
}
else {
ToLoad->bHasResourceLoaded = false;
ToLoad->Resource = nullptr;
SPDLOG_ERROR("Resource load FAILED {} on ResourceMgr thread", ToLoad->File->path);
}
//ResLock.lock();
//ResLock.unlock();
ToLoad->ResourceLoadNotifier.notify_all();
}
else
{
ToLoad->bHasResourceLoaded = false;
ToLoad->Resource = nullptr;
}
ToLoad->ResourceLoadNotifier.notify_all();
}
SPDLOG_INFO("Resource Manager LoadResourceThread ended");
}
uint32_t ResourceMgr::GetGameVersion()
{
return gameVersion;
}
void ResourceMgr::SetGameVersion(uint32_t newGameVersion)
{
gameVersion = newGameVersion;
}
std::shared_ptr<File> ResourceMgr::LoadFileAsync(std::string FilePath) {
const std::lock_guard<std::mutex> Lock(FileLoadMutex);
// File NOT already loaded...?
@ -232,9 +256,16 @@ namespace Ship {
std::shared_ptr<File> FileData = LoadFile(FilePath);
Promise->File = FileData;
Promise->bHasResourceLoaded = false;
ResourceLoadQueue.push(Promise);
ResourceLoadNotifier.notify_all();
if (Promise->File->bHasLoadError)
{
Promise->bHasResourceLoaded = true;
}
else
{
Promise->bHasResourceLoaded = false;
ResourceLoadQueue.push(Promise);
ResourceLoadNotifier.notify_all();
}
} else {
Promise->bHasResourceLoaded = true;
Promise->Resource = resCacheFind->second;

View File

@ -29,6 +29,8 @@ namespace Ship
void InvalidateResourceCache();
uint32_t GetGameVersion();
void SetGameVersion(uint32_t newGameVersion);
std::shared_ptr<File> LoadFileAsync(std::string FilePath);
std::shared_ptr<File> LoadFile(std::string FilePath);
std::shared_ptr<Ship::Resource> GetCachedFile(std::string FilePath);
@ -58,5 +60,6 @@ namespace Ship
std::condition_variable FileLoadNotifier;
std::condition_variable ResourceLoadNotifier;
volatile bool bIsRunning;
uint32_t gameVersion;
};
}

View File

@ -101,31 +101,49 @@
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(ProjectDir)..\..\ZAPDTR\ZAPDUtils;$(ProjectDir)Lib\Fast3D\U64;$(ProjectDir)Lib\libjpeg\include;$(ProjectDir)Lib\spdlog\include;$(ProjectDir)Lib\SDL;$(ProjectDir)Lib\GLEW;$(IncludePath)</IncludePath>
<LibraryPath>$(ProjectDir)Lib\SDL\lib\x86;$(LibraryPath)</LibraryPath>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Testing|Win32'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(ProjectDir)..\..\ZAPDTR\ZAPDUtils;$(ProjectDir)Lib\Fast3D\U64;$(ProjectDir)Lib\libjpeg\include;$(ProjectDir)Lib\spdlog\include;$(ProjectDir)Lib\SDL;$(ProjectDir)Lib\GLEW;$(IncludePath)</IncludePath>
<LibraryPath>$(ProjectDir)Lib\SDL\lib\x86;$(LibraryPath)</LibraryPath>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(ProjectDir)..\..\ZAPDTR\ZAPDUtils;$(ProjectDir)Lib\Fast3D\U64;$(ProjectDir)Lib\spdlog\include;$(ProjectDir)Lib\SDL;$(ProjectDir)Lib\GLEW;$(IncludePath)</IncludePath>
<LibraryPath>$(ProjectDir)Lib\SDL\lib\x86;$(LibraryPath)</LibraryPath>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(ProjectDir)..\..\ZAPDTR\ZAPDUtils;$(ProjectDir)Lib\Fast3D\U64;$(ProjectDir)Lib\spdlog\include;$(ProjectDir)Lib\SDL;$(ProjectDir)Lib\GLEW;$(IncludePath)</IncludePath>
<LibraryPath>$(ProjectDir)Lib\SDL\lib\x64;$(LibraryPath)</LibraryPath>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Testing|x64'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(ProjectDir)..\..\ZAPDTR\ZAPDUtils;$(ProjectDir)Lib\Fast3D\U64;$(ProjectDir)Lib\spdlog\include;$(ProjectDir)Lib\SDL;$(ProjectDir)Lib\GLEW;$(IncludePath)</IncludePath>
<LibraryPath>$(ProjectDir)Lib\SDL\lib\x64;$(LibraryPath)</LibraryPath>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(ProjectDir)..\..\ZAPDTR\ZAPDUtils;$(ProjectDir)Lib\Fast3D\U64;$(ProjectDir)Lib\spdlog\include;$(ProjectDir)Lib\SDL;$(ProjectDir)Lib\GLEW;$(IncludePath)</IncludePath>
<LibraryPath>$(ProjectDir)Lib\SDL\lib\x64;$(LibraryPath)</LibraryPath>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>
@ -326,6 +344,7 @@
<ClInclude Include="Cvar.h" />
<ClInclude Include="Environment.h" />
<ClInclude Include="GameSettings.h" />
<ClInclude Include="GameVersions.h" />
<ClInclude Include="Lib\ImGui\backends\imgui_impl_dx11.h" />
<ClInclude Include="Lib\ImGui\backends\imgui_impl_win32.h" />
<ClInclude Include="Lib\stb\stb_image_write.h" />
@ -416,6 +435,12 @@
<ClInclude Include="SDLController.h" />
<ClInclude Include="WindowShim.h" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\OTRGui\build\ZERO_CHECK.vcxproj">
<Project>{02d10590-9542-3f55-aaf8-6055677e2a2a}</Project>
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
</ProjectReference>
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>

View File

@ -626,5 +626,8 @@
<ClInclude Include="GameSettings.h">
<Filter>Source Files\CustomImpl</Filter>
</ClInclude>
<ClInclude Include="GameVersions.h">
<Filter>Source Files\Resources</Filter>
</ClInclude>
</ItemGroup>
</Project>

Some files were not shown because too many files have changed in this diff Show More