Added support for multiple game versions (#107)

* WIP Multiversion support

* GC PAL Non-MQ support complete

* Updated OtrGui to handle different game versions

* Added version file

* Added new extract mode to ZAPD and optimized OTR gen time

* Fixed bug causing crash

* Further optimized OTRExporter, saving around ~20 seconds.

* ZAPD is now multi-threaded.

* Fixed merge issue

* Fixed memory leak and fog issue on pause screen.

* Additional fog fixes.

Co-authored-by: Jack Walker <7463599+Jack-Walker@users.noreply.github.com>
This commit is contained in:
Nicholas Estelami 2022-03-31 19:42:44 -04:00 committed by GitHub
parent 572e9fb9d0
commit c80f9fbd57
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1203 changed files with 30620 additions and 501 deletions

View File

@ -345,6 +345,8 @@ baserom/
*.otr
*.swp
*.a
*.z64
*.n64
Extract/
tmp.txt

View File

@ -2,7 +2,7 @@
<SymbolMap File="SymbolMap_OoTMqDbg.txt"/>
<ActorList File="ActorList_OoTMqDbg.txt"/>
<ObjectList File="ObjectList_OoTMqDbg.txt"/>
<ExternalXMLFolder Path="../soh/assets/xml/"/>
<ExternalXMLFolder Path="../soh/assets/xml/GC_NMQ_PAL_F/"/>
<TexturePool File="TexturePool.xml"/>
<ExternalFile XmlPath="objects/gameplay_keep.xml" OutPath="objects/gameplay_keep/"/>
</Root>

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -209,7 +209,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
break;
case G_MTX:
{
if ((!Globals::Instance->HasSegment(GETSEGNUM(data))) || ((data & 0xFFFFFFFF) == 0x07000000)) // En_Zf and En_Ny place a DL in segment 7
if ((!Globals::Instance->HasSegment(GETSEGNUM(data), res->parent->workerID)) || ((data & 0xFFFFFFFF) == 0x07000000)) // En_Zf and En_Ny place a DL in segment 7
{
uint32_t pp = (data & 0x000000FF00000000) >> 32;
uint32_t mm = (data & 0x00000000FFFFFFFF);
@ -370,7 +370,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
//std::string fName = StringHelper::Sprintf("%s\\%s", GetParentFolderName(res).c_str(), dListDecl2->varName.c_str());
std::string fName = OTRExporter_DisplayList::GetPathToRes(res, dListDecl2->varName.c_str());
if (!File::Exists("Extract\\" + fName))
if (files.find(fName) == files.end() && !File::Exists("Extract\\" + fName))
{
MemoryStream* dlStream = new MemoryStream();
BinaryWriter dlWriter = BinaryWriter(dlStream);
@ -382,7 +382,10 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
//otrArchive->RemoveFile(fName);
#endif
File::WriteAllBytes("Extract\\" + fName, dlStream->ToVector());
if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
File::WriteAllBytes("Extract\\" + fName, dlStream->ToVector());
else
files[fName] = dlStream->ToVector();
//otrArchive->AddFile(fName, (uintptr_t)dlStream->ToVector().data(), dlWriter.GetBaseAddress());
}
@ -401,7 +404,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
//case G_BRANCH_Z:
case G_DL:
{
if ((!Globals::Instance->HasSegment(GETSEGNUM(data)) && (int)opF3D != G_BRANCH_Z)
if ((!Globals::Instance->HasSegment(GETSEGNUM(data), res->parent->workerID) && (int)opF3D != G_BRANCH_Z)
|| ((data & 0xFFFFFFFF) == 0x07000000)) // En_Zf and En_Ny place a DL in segment 7
{
int32_t pp = (data & 0x00FF000000000000) >> 56;
@ -464,14 +467,17 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
//std::string fName = StringHelper::Sprintf("%s\\%s", GetParentFolderName(res).c_str(), dListDecl2->varName.c_str());
std::string fName = OTRExporter_DisplayList::GetPathToRes(res, dListDecl2->varName.c_str());
if (!File::Exists("Extract\\" + fName))
if (files.find(fName) == files.end() && !File::Exists("Extract\\" + fName))
{
MemoryStream* dlStream = new MemoryStream();
BinaryWriter dlWriter = BinaryWriter(dlStream);
Save(dList->otherDLists[i], outPath, &dlWriter);
File::WriteAllBytes("Extract\\" + fName, dlStream->ToVector());
if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
File::WriteAllBytes("Extract\\" + fName, dlStream->ToVector());
else
files[fName] = dlStream->ToVector();
}
}
else
@ -675,7 +681,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
uint32_t seg = data & 0xFFFFFFFF;
int32_t texAddress = Seg2Filespace(data, dList->parent->baseAddress);
if (!Globals::Instance->HasSegment(GETSEGNUM(seg)))
if (!Globals::Instance->HasSegment(GETSEGNUM(seg), res->parent->workerID))
{
int32_t __ = (data & 0x00FF000000000000) >> 48;
int32_t www = (data & 0x00000FFF00000000) >> 32;
@ -693,7 +699,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
else
{
std::string texName = "";
bool foundDecl = Globals::Instance->GetSegmentedPtrName(seg, dList->parent, "", texName);
bool foundDecl = Globals::Instance->GetSegmentedPtrName(seg, dList->parent, "", texName, res->parent->workerID);
int32_t __ = (data & 0x00FF000000000000) >> 48;
int32_t www = (data & 0x00000FFF00000000) >> 32;
@ -712,7 +718,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
if (foundDecl)
{
ZFile* assocFile = Globals::Instance->GetSegment(GETSEGNUM(seg));
ZFile* assocFile = Globals::Instance->GetSegment(GETSEGNUM(seg), res->parent->workerID);
std::string assocFileName = assocFile->GetName();
std::string fName = "";
@ -750,42 +756,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
word1 = value.words.w1 | 0xF0000000;
}
else
//if (dList->vertices.size() > 0)
{
// Connect neighboring vertex arrays
std::vector<std::pair<uint32_t, std::vector<ZVtx>>> vertsKeys(dList->vertices.begin(),
dList->vertices.end());
if (vertsKeys.size() > 0)
{
auto lastItem = vertsKeys[0];
for (size_t i = 1; i < vertsKeys.size(); i++)
{
auto curItem = vertsKeys[i];
int32_t sizeDiff = curItem.first - (lastItem.first + (lastItem.second.size() * 16));
// Make sure there isn't an unaccounted inbetween these two
if (sizeDiff == 0)
{
for (auto v : curItem.second)
{
dList->vertices[lastItem.first].push_back(v);
lastItem.second.push_back(v);
}
dList->vertices.erase(curItem.first);
vertsKeys.erase(vertsKeys.begin() + i);
i--;
continue;
}
lastItem = curItem;
}
}
// Write CRC64 of vtx file name
uint32_t addr = data & 0xFFFFFFFF;
@ -793,10 +764,7 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
addr -= dList->parent->baseAddress;
auto segOffset = GETSEGOFFSET(addr);
//uint32_t seg = data & 0xFFFFFFFF;
Declaration* vtxDecl = dList->parent->GetDeclarationRanged(segOffset);
//std::string vtxName = "";
//bool foundDecl = Globals::Instance->GetSegmentedPtrName(seg, dList->parent, "", vtxName);
int32_t aa = (data & 0x000000FF00000000ULL) >> 32;
int32_t nn = (data & 0x000FF00000000000ULL) >> 44;
@ -822,9 +790,8 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
word0 = hash >> 32;
word1 = hash & 0xFFFFFFFF;
if (!File::Exists("Extract\\" + fName))
if (files.find(fName) == files.end() && !File::Exists("Extract\\" + fName))
{
//printf("Exporting VTX Data %s\n", fName.c_str());
// Write vertices to file
MemoryStream* vtxStream = new MemoryStream();
BinaryWriter vtxWriter = BinaryWriter(vtxStream);
@ -847,44 +814,40 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
vtxWriter.Write((uint32_t)ZResourceType::Vertex);
vtxWriter.Write((uint32_t)arrCnt);
size_t sz = dList->vertices[vtxDecl->address].size();
auto start = std::chrono::steady_clock::now();
//if (sz > 0)
// God dammit this is so dumb
for (size_t i = 0; i < split.size(); i++)
{
auto start = std::chrono::steady_clock::now();
std::string line = split[i];
// God dammit this is so dumb
for (size_t i = 0; i < split.size(); i++)
if (StringHelper::Contains(line, "VTX("))
{
std::string line = split[i];
auto split2 = StringHelper::Split(StringHelper::Split(StringHelper::Split(line, "VTX(")[1], ")")[0], ",");
if (StringHelper::Contains(line, "VTX("))
{
auto split2 = StringHelper::Split(StringHelper::Split(StringHelper::Split(line, "VTX(")[1], ")")[0], ",");
vtxWriter.Write((int16_t)std::stoi(split2[0], nullptr, 10)); // v.x
vtxWriter.Write((int16_t)std::stoi(split2[1], nullptr, 10)); // v.y
vtxWriter.Write((int16_t)std::stoi(split2[2], nullptr, 10)); // v.z
vtxWriter.Write((int16_t)std::stoi(split2[0], nullptr, 10)); // v.x
vtxWriter.Write((int16_t)std::stoi(split2[1], nullptr, 10)); // v.y
vtxWriter.Write((int16_t)std::stoi(split2[2], nullptr, 10)); // v.z
vtxWriter.Write((int16_t)0); // v.flag
vtxWriter.Write((int16_t)0); // v.flag
vtxWriter.Write((int16_t)std::stoi(split2[3], nullptr, 10)); // v.s
vtxWriter.Write((int16_t)std::stoi(split2[4], nullptr, 10)); // v.t
vtxWriter.Write((uint8_t)std::stoi(split2[5], nullptr, 10)); // v.r
vtxWriter.Write((uint8_t)std::stoi(split2[6], nullptr, 10)); // v.g
vtxWriter.Write((uint8_t)std::stoi(split2[7], nullptr, 10)); // v.b
vtxWriter.Write((uint8_t)std::stoi(split2[8], nullptr, 10)); // v.a
}
vtxWriter.Write((int16_t)std::stoi(split2[3], nullptr, 10)); // v.s
vtxWriter.Write((int16_t)std::stoi(split2[4], nullptr, 10)); // v.t
vtxWriter.Write((uint8_t)std::stoi(split2[5], nullptr, 10)); // v.r
vtxWriter.Write((uint8_t)std::stoi(split2[6], nullptr, 10)); // v.g
vtxWriter.Write((uint8_t)std::stoi(split2[7], nullptr, 10)); // v.b
vtxWriter.Write((uint8_t)std::stoi(split2[8], nullptr, 10)); // v.a
}
File::WriteAllBytes("Extract\\" + fName, vtxStream->ToVector());
auto end = std::chrono::steady_clock::now();
size_t diff = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
//printf("Exported VTX Array %s in %zums\n", fName.c_str(), diff);
}
if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
File::WriteAllBytes("Extract\\" + fName, vtxStream->ToVector());
else
files[fName] = vtxStream->ToVector();
auto end = std::chrono::steady_clock::now();
size_t diff = std::chrono::duration_cast<std::chrono::milliseconds>(end - start).count();
}
}
else
@ -892,15 +855,6 @@ void OTRExporter_DisplayList::Save(ZResource* res, const fs::path& outPath, Bina
spdlog::error("vtxDecl == nullptr!");
}
}
/*else
{
writer->Write(word0);
writer->Write(word1);
word0 = 0;
word1 = 0;
spdlog::error("dList->vertices.size() <= 0!");
}*/
}
break;
}

View File

@ -25,6 +25,7 @@ std::string otrFileName = "oot.otr";
std::shared_ptr<Ship::Archive> otrArchive;
BinaryWriter* fileWriter;
std::chrono::steady_clock::time_point fileStart, resStart;
std::map<std::string, std::vector<char>> files;
void InitVersionInfo();
@ -39,6 +40,8 @@ static void ExporterParseFileMode(const std::string& buildMode, ZFileMode& fileM
{
fileMode = (ZFileMode)ExporterFileMode::BuildOTR;
printf("BOTR: Generating OTR Archive...\n");
if (File::Exists(otrFileName))
otrArchive = std::shared_ptr<Ship::Archive>(new Ship::Archive(otrFileName, true));
else
@ -54,6 +57,31 @@ static void ExporterParseFileMode(const std::string& buildMode, ZFileMode& fileM
}
}
static void ExporterProgramEnd()
{
if (Globals::Instance->fileMode == ZFileMode::ExtractDirectory)
{
printf("Generating OTR Archive...\n");
otrArchive = Ship::Archive::CreateArchive(otrFileName, 65536 / 2);
for (auto item : files)
{
auto fileData = item.second;
otrArchive->AddFile(item.first, (uintptr_t)fileData.data(), fileData.size());
}
// Add any additional files that need to be manually copied...
auto lst = Directory::ListFiles("Extract");
for (auto item : lst)
{
auto fileData = File::ReadAllBytes(item);
otrArchive->AddFile(StringHelper::Split(item, "Extract\\")[1], (uintptr_t)fileData.data(), fileData.size());
}
}
}
static void ExporterParseArgs(int argc, char* argv[], int& i)
{
std::string arg = argv[i];
@ -85,6 +113,7 @@ static void ExporterFileBegin(ZFile* file)
static void ExporterFileEnd(ZFile* file)
{
int bp = 0;
}
static void ExporterResourceEnd(ZResource* res, BinaryWriter& writer)
@ -124,7 +153,10 @@ static void ExporterResourceEnd(ZResource* res, BinaryWriter& writer)
else
fName = StringHelper::Sprintf("%s\\%s", oName.c_str(), rName.c_str());
File::WriteAllBytes("Extract\\" + fName, strem->ToVector());
if (Globals::Instance->fileMode == ZFileMode::ExtractDirectory)
files[fName] = strem->ToVector();
else
File::WriteAllBytes("Extract\\" + fName, strem->ToVector());
}
auto end = std::chrono::steady_clock::now();
@ -155,6 +187,8 @@ static void ImportExporters()
exporterSet->beginXMLFunc = ExporterXMLBegin;
exporterSet->endXMLFunc = ExporterXMLEnd;
exporterSet->resSaveFunc = ExporterResourceEnd;
exporterSet->endProgramFunc = ExporterProgramEnd;
exporterSet->exporters[ZResourceType::Background] = new OTRExporter_Background();
exporterSet->exporters[ZResourceType::Texture] = new OTRExporter_Texture();
exporterSet->exporters[ZResourceType::Room] = new OTRExporter_Room();

View File

@ -2,4 +2,5 @@
#include <Archive.h>
extern std::shared_ptr<Ship::Archive> otrArchive;
extern std::shared_ptr<Ship::Archive> otrArchive;
extern std::map<std::string, std::vector<char>> files;

View File

@ -63,6 +63,12 @@
<ClCompile Include="VersionInfo.cpp" />
<ClCompile Include="VtxExporter.cpp" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\OTRGui\build\ZERO_CHECK.vcxproj">
<Project>{02d10590-9542-3f55-aaf8-6055677e2a2a}</Project>
<ReferenceOutputAssembly>false</ReferenceOutputAssembly>
</ProjectReference>
</ItemGroup>
<PropertyGroup Label="Globals">
<VCProjectVersion>16.0</VCProjectVersion>
<Keyword>Win32Proj</Keyword>
@ -118,19 +124,31 @@
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(SolutionDir)otrlib;$(SolutionDir)\ZAPD\ZAPD\;$(SolutionDir)\ZAPD\lib\tinyxml2;$(SolutionDir)\ZAPD\lib\libgfxd;$(SolutionDir)\ZAPD\lib\elfio;$(SolutionDir)\ZAPD\lib\assimp\include;$(SolutionDir)\ZAPD\lib\stb;$(ProjectDir);$(IncludePath)</IncludePath>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">
<LinkIncremental>false</LinkIncremental>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">
<LinkIncremental>true</LinkIncremental>
<IncludePath>$(ProjectDir)..\..\ZAPDTR\ZAPD;$(ProjectDir)..\..\ZAPDTR\lib\tinyxml2;$(ProjectDir)..\..\ZAPDTR\lib\libgfxd;$(ProjectDir)..\..\ZAPDTR\ZAPDUtils;$(ProjectDir)..\..\libultraship\libultraship;$(ProjectDir)..\..\libultraship\libultraship\lib\spdlog\include;$(ProjectDir)..\..\libultraship\libultraship\Lib\Fast3D\U64;$(IncludePath)</IncludePath>
<LibraryPath>$(ProjectDir)..\..\libultraship\libultraship;$(LibraryPath)</LibraryPath>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'">
<LinkIncremental>false</LinkIncremental>
<IncludePath>$(ProjectDir)..\..\ZAPDTR\ZAPD;$(ProjectDir)..\..\ZAPDTR\lib\tinyxml2;$(ProjectDir)..\..\ZAPDTR\lib\libgfxd;$(ProjectDir)..\..\ZAPDTR\ZAPDUtils;$(ProjectDir)..\..\libultraship\libultraship;$(ProjectDir)..\..\libultraship\libultraship\lib\spdlog\include;$(ProjectDir)..\..\libultraship\libultraship\Lib\Fast3D\U64;$(IncludePath)</IncludePath>
<LibraryPath>$(ProjectDir)..\..\libultraship\libultraship;$(LibraryPath)</LibraryPath>
<CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRules />
<CodeAnalysisRuleAssemblies />
</PropertyGroup>
<ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">
<ClCompile>

View File

@ -407,7 +407,7 @@ void OTRExporter_Room::Save(ZResource* res, const fs::path& outPath, BinaryWrite
{
uint32_t seg = cmdHeaders->headers[i] & 0xFFFFFFFF;
std::string headerName = "";
bool foundDecl = Globals::Instance->GetSegmentedPtrName(seg, room->parent, "", headerName);
bool foundDecl = Globals::Instance->GetSegmentedPtrName(seg, room->parent, "", headerName, res->parent->workerID);
if (headerName == "NULL")
writer->Write("");
else
@ -443,7 +443,7 @@ void OTRExporter_Room::Save(ZResource* res, const fs::path& outPath, BinaryWrite
SetCutscenes* cmdSetCutscenes = (SetCutscenes*)cmd;
std::string listName;
Globals::Instance->GetSegmentedPtrName(cmdSetCutscenes->cmdArg2, room->parent, "CutsceneData", listName);
Globals::Instance->GetSegmentedPtrName(cmdSetCutscenes->cmdArg2, room->parent, "CutsceneData", listName, res->parent->workerID);
std::string fName = OTRExporter_DisplayList::GetPathToRes(room, listName);
//std::string fName = StringHelper::Sprintf("%s\\%s", OTRExporter_DisplayList::GetParentFolderName(room).c_str(), listName.c_str());
writer->Write(fName);
@ -452,8 +452,11 @@ void OTRExporter_Room::Save(ZResource* res, const fs::path& outPath, BinaryWrite
BinaryWriter csWriter = BinaryWriter(csStream);
OTRExporter_Cutscene cs;
cs.Save(cmdSetCutscenes->cutscenes[0], "", &csWriter);
File::WriteAllBytes("Extract\\" + fName, csStream->ToVector());
if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
File::WriteAllBytes("Extract\\" + fName, csStream->ToVector());
else
files[fName] = csStream->ToVector();
//std::string fName = OTRExporter_DisplayList::GetPathToRes(res, vtxDecl->varName);
//otrArchive->AddFile(fName, (uintptr_t)csStream->ToVector().data(), csWriter.GetBaseAddress());
@ -477,7 +480,10 @@ void OTRExporter_Room::Save(ZResource* res, const fs::path& outPath, BinaryWrite
OTRExporter_Path pathExp;
pathExp.Save(&cmdSetPathways->pathwayList, outPath, &pathWriter);
File::WriteAllBytes("Extract\\" + path, pathStream->ToVector());
if (Globals::Instance->fileMode != ZFileMode::ExtractDirectory)
File::WriteAllBytes("Extract\\" + path, pathStream->ToVector());
else
files[path] = pathStream->ToVector();
//otrArchive->AddFile(path, (uintptr_t)pathStream->ToVector().data(), pathWriter.GetBaseAddress());

View File

@ -23,7 +23,7 @@ void OTRExporter_Skeleton::Save(ZResource* res, const fs::path& outPath, BinaryW
Declaration* skelDecl = skel->parent->GetDeclarationRanged(GETSEGOFFSET(skel->limbsTable.limbsAddresses[i]));
std::string name;
bool foundDecl = Globals::Instance->GetSegmentedPtrName(skel->limbsTable.limbsAddresses[i], skel->parent, "", name);
bool foundDecl = Globals::Instance->GetSegmentedPtrName(skel->limbsTable.limbsAddresses[i], skel->parent, "", name, res->parent->workerID);
if (foundDecl)
{
if (name.at(0) == '&')

View File

@ -86,7 +86,7 @@ void OTRExporter_SkeletonLimb::Save(ZResource* res, const fs::path& outPath, Bin
if (limb->childPtr != 0)
{
std::string name;
bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->childPtr, limb->parent, "", name);
bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->childPtr, limb->parent, "", name, res->parent->workerID);
if (foundDecl)
{
if (name.at(0) == '&')
@ -107,7 +107,7 @@ void OTRExporter_SkeletonLimb::Save(ZResource* res, const fs::path& outPath, Bin
if (limb->siblingPtr != 0)
{
std::string name;
bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->siblingPtr, limb->parent, "", name);
bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->siblingPtr, limb->parent, "", name, res->parent->workerID);
if (foundDecl)
{
if (name.at(0) == '&')
@ -128,7 +128,7 @@ void OTRExporter_SkeletonLimb::Save(ZResource* res, const fs::path& outPath, Bin
if (limb->dListPtr != 0)
{
std::string name;
bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->dListPtr, limb->parent, "", name);
bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->dListPtr, limb->parent, "", name, res->parent->workerID);
if (foundDecl)
{
if (name.at(0) == '&')
@ -149,7 +149,7 @@ void OTRExporter_SkeletonLimb::Save(ZResource* res, const fs::path& outPath, Bin
if (limb->dList2Ptr != 0)
{
std::string name;
bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->dList2Ptr, limb->parent, "", name);
bool foundDecl = Globals::Instance->GetSegmentedPtrName(limb->dList2Ptr, limb->parent, "", name, res->parent->workerID);
if (foundDecl)
{
if (name.at(0) == '&')

View File

@ -48,7 +48,7 @@ def ExtractFunc(fullPath):
*pathList, xmlName = fullPath.split(os.sep)
objectName = os.path.splitext(xmlName)[0]
outPath = os.path.join("..\\soh\\assets\\", *pathList[4:], objectName)
outPath = os.path.join("..\\soh\\assets\\", *pathList[5:], objectName)
os.makedirs(outPath, exist_ok=True)
outSourcePath = outPath
@ -64,6 +64,7 @@ def main():
parser.add_argument("-s", "--single", help="asset path relative to assets/, e.g. objects/gameplay_keep")
parser.add_argument("-f", "--force", help="Force the extraction of every xml instead of checking the touched ones.", action="store_true")
parser.add_argument("-u", "--unaccounted", help="Enables ZAPD unaccounted detector warning system.", action="store_true")
parser.add_argument("-v", "--version", help="Sets game version.")
args = parser.parse_args()
global mainAbort
@ -73,6 +74,13 @@ def main():
extractedAssetsTracker = manager.dict()
xmlVer = "GC_NMQ_D"
if (args.version == "gc_pal_nmpq"):
xmlVer = "GC_NMQ_PAL_F"
elif (args.version == "dbg_mq"):
xmlVer = "GC_MQ_D"
asset_path = args.single
if asset_path is not None:
fullPath = os.path.join("..\\soh\\assets", "xml", asset_path + ".xml")
@ -90,7 +98,7 @@ def main():
extract_staff_text_path = None
xmlFiles = []
for currentPath, _, files in os.walk(os.path.join("..\\soh\\assets", "xml")):
for currentPath, _, files in os.walk(os.path.join("..\\soh\\assets\\xml\\", xmlVer)):
for file in files:
fullPath = os.path.join(currentPath, file)
if file.endswith(".xml"):

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -2,7 +2,7 @@
<SymbolMap File="symbols/SymbolMap_OoTMqDbg.txt"/>
<ActorList File="symbols/ActorList_OoTMqDbg.txt"/>
<ObjectList File="symbols/ObjectList_OoTMqDbg.txt"/>
<ExternalXMLFolder Path="assets/extractor/xmls/"/>
<ExternalXMLFolder Path="assets/extractor/xmls/GC_MQ_D/"/>
<TexturePool File="TexturePool.xml"/>
<ExternalFile XmlPath="objects/gameplay_keep.xml" OutPath="objects/gameplay_keep/"/>
</Root>

View File

@ -0,0 +1,8 @@
<Root>
<SymbolMap File="symbols/SymbolMap_OoTMqDbg.txt"/>
<ActorList File="symbols/ActorList_OoTMqDbg.txt"/>
<ObjectList File="symbols/ObjectList_OoTMqDbg.txt"/>
<ExternalXMLFolder Path="assets/extractor/xmls/GC_NMQ_D/"/>
<TexturePool File="TexturePool.xml"/>
<ExternalFile XmlPath="objects/gameplay_keep.xml" OutPath="objects/gameplay_keep/"/>
</Root>

View File

@ -0,0 +1,8 @@
<Root>
<SymbolMap File="symbols/SymbolMap_OoTMqDbg.txt"/>
<ActorList File="symbols/ActorList_OoTMqDbg.txt"/>
<ObjectList File="symbols/ObjectList_OoTMqDbg.txt"/>
<ExternalXMLFolder Path="assets/extractor/xmls/GC_NMQ_PAL_F/"/>
<TexturePool File="TexturePool.xml"/>
<ExternalFile XmlPath="objects/gameplay_keep.xml" OutPath="objects/gameplay_keep/"/>
</Root>

File diff suppressed because it is too large Load Diff

View File

@ -26,6 +26,7 @@ bool single_thread = false;
bool hide_second_btn = false;
RomVersion version;
const char* patched_rom = "tmp/rom.z64";
extern bool oldExtractMode;
static std::string currentStep = "None";
@ -72,11 +73,28 @@ void OTRGame::init(){
}
}
void ExtractRom() {
const WriteResult result = ExtractBaserom(patched_rom);
void ExtractRom()
{
WriteResult result;
if (oldExtractMode)
ExtractBaserom(patched_rom);
else
result.error = NULLSTR;
if (result.error == NULLSTR) {
if (MoonUtils::exists("oot.otr")) MoonUtils::rm("oot.otr");
startWorker();
if (MoonUtils::exists("Extract")) MoonUtils::rm("Extract");
MoonUtils::mkdir("Extract");
MoonUtils::copy("tmp/baserom/Audiobank", "Extract/Audiobank");
MoonUtils::copy("tmp/baserom/Audioseq", "Extract/Audioseq");
MoonUtils::copy("tmp/baserom/Audiotable", "Extract/Audiotable");
MoonUtils::copy("tmp/baserom/version", "Extract/version");
MoonUtils::copy("assets/game/", "Extract/assets/");
startWorker(version);
extracting = true;
}
}
@ -131,7 +149,7 @@ void OTRGame::draw() {
}
// Clamp the window to the borders of the monitors
if (wndPos.x < vsX1) wndPos.x = vsX1;
if (wndPos.x < vsX1) wndPos.x = vsX1;
if (wndPos.y < vsY1) wndPos.y = vsY1;
if (wndPos.x + windowSize.x > vsX2) wndPos.x = vsX2 - windowSize.x;
@ -160,7 +178,9 @@ void OTRGame::draw() {
UIUtils::GuiShadowText(("Rom Type: " + version.version).c_str(), 32, text_y, 10, WHITE, BLACK);
UIUtils::GuiShadowText("Tool Version: 1.0", 32, text_y + 15, 10, WHITE, BLACK);
UIUtils::GuiShadowText("OTR Version: 1.0", 32, text_y + 30, 10, WHITE, BLACK);
UIUtils::GuiToggle(&single_thread, "Single Thread", 32, text_y + 40, currentStep != NULLSTR);
if (oldExtractMode)
UIUtils::GuiToggle(&single_thread, "Single Thread", 32, text_y + 40, currentStep != NULLSTR);
if (!hide_second_btn && UIUtils::GuiIconButton("Folder", "Open\nShip Folder", 109, 50, currentStep != NULLSTR, "Select your Ship of Harkinian Folder\n\nYou could use another folder\nfor development purposes")) {
const std::string path = NativeFS->LaunchFileExplorer(LaunchType::FOLDER);

View File

@ -83,37 +83,37 @@ RomVersion GetVersion(FILE* rom) {
break;
case OOT_NTSC_JP_GC:
version.version = "JP GameCube (MQ Disk)";
version.listPath = "gamecube_mq.txt";
version.listPath = "gamecube.txt";
version.offset = OOT_OFF_JP_GC;
break;
case OOT_NTSC_JP_GC_CE:
version.version = "GameCube (Collectors Edition Disk)";
version.listPath = "gamecube_mq.txt";
version.listPath = "gamecube.txt";
version.offset = OOT_OFF_JP_GC_CE;
break;
case OOT_NTSC_JP_MQ:
version.version = "JP Master Quest";
version.listPath = "gamecube_mq.txt";
version.listPath = "gamecube.txt";
version.offset = OOT_OFF_JP_MQ;
break;
case OOT_NTSC_US_MQ:
version.version = "NTSC Master Quest";
version.listPath = "gamecube_mq.txt";
version.listPath = "gamecube.txt";
version.offset = OOT_OFF_JP_MQ;
break;
case OOT_NTSC_US_GC:
version.version = "NTSC GameCube";
version.listPath = "gamecube_mq.txt";
version.listPath = "gamecube.txt";
version.offset = OOT_OFF_US_MQ;
break;
case OOT_PAL_GC:
version.version = "PAL GameCube";
version.listPath = "gamecube_mq.txt";
version.listPath = "gamecube_pal.txt";
version.offset = OOT_OFF_PAL_GC;
break;
case OOT_PAL_MQ:
version.version = "PAL Master Quest";
version.listPath = "pal_mq.txt";
version.listPath = "gamecube_pal.txt";
version.offset = OOT_OFF_PAL_MQ;
break;
case OOT_PAL_GC_DBG1:
@ -179,6 +179,8 @@ WriteResult ExtractBaserom(const char* romPath) {
const std::vector<std::string> lines = MoonUtils::split(read(MoonUtils::join("assets/extractor/filelists", version.listPath)), '\n');
std::vector<uint8_t> decompressedData(1);
for (int i = 0; i < lines.size(); i++) {
FILE* outFile = fopen(MoonUtils::join("tmp/baserom", lines[i]).c_str(), "wb");
const int romOffset = version.offset + (DMA_ENTRY_SIZE * i);
@ -196,10 +198,13 @@ WriteResult ExtractBaserom(const char* romPath) {
auto outData = new uint8_t[size];
memcpy(outData, romData + physStart, size);
if (compressed) {
std::vector<uint8_t> compressedData = yaz0_encode(outData, size);
outData = compressedData.data();
size = compressedData.size();
int decSize = virtEnd - virtStart;
decompressedData = std::vector<uint8_t>(decSize);
yaz0_decode(outData, decompressedData.data(), decSize);
outData = decompressedData.data();
size = decSize;
}
fwrite(outData, sizeof(char), size, outFile);

View File

@ -1,23 +1,7 @@
#ifndef EXTRACT_BASEROM_H_
#define EXTRACT_BASEROM_H_
#define OOT_NTSC_10 0xEC7011B7
#define OOT_NTSC_11 0xD43DA81F
#define OOT_NTSC_12 0x693BA2AE
#define OOT_PAL_10 0xB044B569
#define OOT_PAL_11 0xB2055FBD
#define OOT_NTSC_JP_GC_CE 0xF7F52DB8
#define OOT_NTSC_JP_GC 0xF611F4BA
#define OOT_NTSC_US_GC 0xF3DD35BA
#define OOT_PAL_GC 0x09465AC3
#define OOT_NTSC_JP_MQ 0xF43B45BA
#define OOT_NTSC_US_MQ 0xF034001A
#define OOT_PAL_MQ 0x1D4136F3
#define OOT_PAL_GC_DBG1 0x871E1C92 // 03-21-2002 build
#define OOT_PAL_GC_DBG2 0x87121EFE // 03-13-2002 build
#define OOT_PAL_GC_MQ_DBG 0x917D18F6
#define OOT_IQUE_TW 0x3D81FB3E
#define OOT_IQUE_CN 0xB1E1E07B
#include "../../libultraship/libultraship/GameVersions.h"
#include <cstdio>
#include <string>

View File

@ -5,6 +5,7 @@
#include "utils/mutils.h"
#include "ctpl/ctpl_stl.h"
#include <thread>
#include <impl/baserom_extractor/baserom_extractor.h>
#ifdef _WIN32
#define PLATFORM Platforms::WINDOWS
@ -13,6 +14,7 @@
#endif
namespace Util = MoonUtils;
bool oldExtractMode = false;
static int maxResources = 0;
static int extractedResources = 0;
bool buildingOtr = false;
@ -22,19 +24,29 @@ bool isWindows() {
return (PLATFORM == Platforms::WINDOWS);
}
void BuildOTR(const std::string output) {
Util::copy("tmp/baserom/Audiobank", "Extract/Audiobank");
Util::copy("tmp/baserom/Audioseq", "Extract/Audioseq");
Util::copy("tmp/baserom/Audiotable", "Extract/Audiotable");
Util::copy("assets/game/", "Extract/assets/");
std::string execStr = Util::format("assets/extractor/%s", isWindows() ? "ZAPD.exe" : "ZAPD.out") + " botr -se OTR";
ProcessResult result = NativeFS->LaunchProcess(execStr);
if(result.exitCode != 0) {
std::cout << "\nError when building the OTR file with error code: " << result.exitCode << " !" << std::endl;
std::cout << "Aborting...\n" << std::endl;
std::string GetXMLVersion(RomVersion version)
{
switch (version.crc)
{
case OOT_PAL_GC_DBG1: return "GC_NMQ_D";
case OOT_PAL_GC_DBG2: return "GC_MQ_D";
case OOT_PAL_GC: return "GC_NMQ_PAL_F";
}
return "ERROR";
}
void BuildOTR(const std::string output) {
if (oldExtractMode)
{
std::string execStr = Util::format("assets/extractor/%s", isWindows() ? "ZAPD.exe" : "ZAPD.out") + " botr -se OTR";
ProcessResult result = NativeFS->LaunchProcess(execStr);
if (result.exitCode != 0) {
std::cout << "\nError when building the OTR file with error code: " << result.exitCode << " !" << std::endl;
std::cout << "Aborting...\n" << std::endl;
}
}
setCurrentStep("Done!");
if (output == ".") return;
@ -44,9 +56,9 @@ void BuildOTR(const std::string output) {
MoonUtils::copy("oot.otr", outputPath);
}
void ExtractFile(std::string xmlPath, std::string outPath, std::string outSrcPath) {
void ExtractFile(std::string xmlPath, std::string outPath, std::string outSrcPath, RomVersion version) {
std::string execStr = Util::format("assets/extractor/%s", isWindows() ? "ZAPD.exe" : "ZAPD.out");
std::string args = Util::format(" e -eh -i %s -b tmp/baserom/ -o %s -osf %s -gsf 1 -rconf assets/extractor/Config.xml -se OTR %s", xmlPath.c_str(), outPath.c_str(), outSrcPath.c_str(), xmlPath.find("overlays") != std::string::npos ? "--static" : "");
std::string args = Util::format(" e -eh -i %s -b tmp/baserom/ -o %s -osf %s -gsf 1 -rconf assets/extractor/Config_%s.xml -se OTR %s", xmlPath.c_str(), outPath.c_str(), outSrcPath.c_str(), GetXMLVersion(version).c_str(), xmlPath.find("overlays") != std::string::npos ? "--static" : "");
ProcessResult result = NativeFS->LaunchProcess(execStr + args);
if (result.exitCode != 0) {
@ -55,49 +67,78 @@ void ExtractFile(std::string xmlPath, std::string outPath, std::string outSrcPat
}
}
void ExtractFunc(std::string fullPath) {
void ExtractFunc(std::string fullPath, RomVersion version) {
std::vector<std::string> path = Util::split(fullPath, Util::pathSeparator());
std::string outPath = Util::join(Util::join("assets/extractor/xmls/output", path[4]), Util::basename(fullPath));
Util::mkdir(outPath);
ExtractFile(fullPath, outPath, outPath);
ExtractFile(fullPath, outPath, outPath, version);
setCurrentStep("Extracting: " + Util::basename(fullPath));
extractedResources++;
}
void startWorker() {
std::string path = "assets/extractor/xmls";
std::vector<std::string> files;
Util::dirscan(path, files);
std::vector<std::string> xmlFiles;
void startWorker(RomVersion version) {
std::string path = "assets/extractor/xmls/";
const int num_threads = std::thread::hardware_concurrency();
ctpl::thread_pool pool(num_threads / 2);
for(auto &file : files) {
if (file.find(".xml") != std::string::npos) xmlFiles.push_back(file);
}
path += GetXMLVersion(version);
for (auto& file : xmlFiles) {
if(single_thread) {
ExtractFunc(file);
} else {
pool.push([file](int) {
ExtractFunc(file);
});
Util::write("tmp/baserom/version", (char*)&version.crc, sizeof(version.crc));
if (oldExtractMode)
{
std::vector<std::string> files;
Util::dirscan(path, files);
std::vector<std::string> xmlFiles;
const int num_threads = std::thread::hardware_concurrency();
ctpl::thread_pool pool(num_threads / 2);
for (auto& file : files) {
if (file.find(".xml") != std::string::npos) xmlFiles.push_back(file);
}
}
maxResources = xmlFiles.size();
for (auto& file : xmlFiles) {
if (single_thread) {
ExtractFunc(file, version);
}
else {
pool.push([file, version](int) {
ExtractFunc(file, version);
});
}
}
maxResources = xmlFiles.size();
}
else
{
std::string execStr = Util::format("assets/extractor/%s", isWindows() ? "ZAPD.exe" : "ZAPD.out");
std::string args = Util::format(" ed -eh -i %s -b tmp/rom.z64 -fl assets/extractor/filelists -o %s -osf %s -gsf 1 -rconf assets/extractor/Config_%s.xml -se OTR %s", path.c_str(), path + "../", path + "../", GetXMLVersion(version).c_str(), "");
ProcessResult result = NativeFS->LaunchProcess(execStr + args);
if (result.exitCode != 0) {
std::cout << "\nError when extracting the ROM with error code: " << result.exitCode << " !" << std::endl;
std::cout << "Aborting...\n" << std::endl;
}
else
{
printf("All done?\n");
}
maxResources = 1;
}
}
void updateWorker(const std::string& output) {
if (maxResources > 0 && !buildingOtr && extractedResources >= maxResources) {
if (maxResources > 0 && !buildingOtr && (extractedResources >= maxResources || !oldExtractMode))
{
setCurrentStep("Building OTR...");
if (skipFrames < 3) {
skipFrames++;
return;
}
buildingOtr = true;
if (single_thread){
if (single_thread || !oldExtractMode){
BuildOTR(output);
return;
}

View File

@ -5,5 +5,7 @@ enum Platforms {
WINDOWS, LINUX
};
void startWorker();
struct RomVersion;
void startWorker(RomVersion version);
void updateWorker(const std::string& output);

View File

@ -72,7 +72,11 @@ namespace MoonUtils {
vector<string> result;
stringstream ss (s);
string item;
while (getline(ss, item, delim)) {
while (getline(ss, item, delim))
{
if (item.at(item.size() - 1) == '\r')
item = item.substr(0, item.size() - 1);
result.push_back (item);
}
return result;

View File

@ -1,6 +1,6 @@
#pragma once
static uint32_t CRC32B(unsigned char* message, int32_t size)
static uint32_t CRC32B(const unsigned char* message, int32_t size)
{
int32_t byte, crc;
int32_t mask;

View File

@ -1,6 +1,7 @@
#include "Declaration.h"
#include "Globals.h"
#include "ZVtx.h"
#include "Utils/StringHelper.h"
Declaration::Declaration(offset_t nAddress, DeclarationAlignment nAlignment, size_t nSize,
@ -61,6 +62,12 @@ Declaration::Declaration(offset_t nAddress, const std::string& nIncludePath, siz
varName = nVarName;
}
Declaration::~Declaration()
{
//for (auto item : vertexHack)
//delete item;
}
bool Declaration::IsStatic() const
{
switch (staticConf)

View File

@ -22,6 +22,8 @@ enum class StaticConfig
On
};
class ZVtx;
class Declaration
{
public:
@ -38,6 +40,8 @@ public:
std::string varName;
std::string includePath;
std::vector<ZVtx*> vertexHack;
bool isExternal = false;
bool isArray = false;
bool forceArrayCnt = false;
@ -65,6 +69,8 @@ public:
Declaration(offset_t nAddress, const std::string& nIncludePath, size_t nSize,
const std::string& nVarType, const std::string& nVarName);
~Declaration();
bool IsStatic() const;
std::string GetNormalDeclarationStr() const;

View File

15
ZAPDTR/ZAPD/FileWorker.h Normal file
View File

@ -0,0 +1,15 @@
#pragma once
#include <map>
#include <string>
#include <vector>
#include "ZFile.h"
class FileWorker
{
public:
std::vector<ZFile*> files;
std::vector<ZFile*> externalFiles;
std::vector<int32_t> segments;
std::map<int32_t, std::vector<ZFile*>> segmentRefFiles;
};

View File

@ -34,30 +34,88 @@ Globals::~Globals()
}
}
void Globals::AddSegment(int32_t segment, ZFile* file)
void Globals::AddSegment(int32_t segment, ZFile* file, int workerID)
{
if (std::find(segments.begin(), segments.end(), segment) == segments.end())
segments.push_back(segment);
if (cfg.segmentRefFiles.find(segment) == cfg.segmentRefFiles.end())
cfg.segmentRefFiles[segment] = std::vector<ZFile*>();
cfg.segmentRefFiles[segment].push_back(file);
}
bool Globals::HasSegment(int32_t segment)
{
return std::find(segments.begin(), segments.end(), segment) != segments.end();
}
ZFile* Globals::GetSegment(int32_t segment)
{
if (HasSegment(segment))
if (!Globals::Instance->singleThreaded)
{
int idx = std::find(segments.begin(), segments.end(), segment) - segments.begin();
return files[idx];
auto worker = workerData[workerID];
if (std::find(worker->segments.begin(), worker->segments.end(), segment) ==
worker->segments.end())
worker->segments.push_back(segment);
if (worker->segmentRefFiles.find(segment) == worker->segmentRefFiles.end())
worker->segmentRefFiles[segment] = std::vector<ZFile*>();
worker->segmentRefFiles[segment].push_back(file);
}
else
return nullptr;
{
if (std::find(segments.begin(), segments.end(), segment) == segments.end())
segments.push_back(segment);
if (cfg.segmentRefFiles.find(segment) == cfg.segmentRefFiles.end())
cfg.segmentRefFiles[segment] = std::vector<ZFile*>();
cfg.segmentRefFiles[segment].push_back(file);
}
}
bool Globals::HasSegment(int32_t segment, int workerID)
{
if (!Globals::Instance->singleThreaded)
return std::find(workerData[workerID]->segments.begin(),
workerData[workerID]->segments.end(), segment) != workerData[workerID]->segments.end();
else
return std::find(segments.begin(), segments.end(), segment) != segments.end();
}
ZFile* Globals::GetSegment(int32_t segment, int workerID)
{
if (!Globals::Instance->singleThreaded)
{
if (HasSegment(segment, workerID))
{
int idx = std::find(workerData[workerID]->segments.begin(),
workerData[workerID]->segments.end(), segment) -
workerData[workerID]->segments.begin();
return workerData[workerID]->files[idx];
}
else
return nullptr;
}
else
{
if (HasSegment(segment, workerID))
{
int idx = std::find(segments.begin(), segments.end(), segment) - segments.begin();
return files[idx];
}
else
return nullptr;
}
}
std::map<int32_t, std::vector<ZFile*>> Globals::GetSegmentRefFiles(int workerID)
{
if (!Globals::Instance->singleThreaded)
return workerData[workerID]->segmentRefFiles;
else
return cfg.segmentRefFiles;
}
void Globals::AddFile(ZFile* file, int workerID)
{
if (singleThreaded)
files.push_back(file);
else
workerData[workerID]->files.push_back(file);
}
void Globals::AddExternalFile(ZFile* file, int workerID)
{
if (singleThreaded)
externalFiles.push_back(file);
else
workerData[workerID]->externalFiles.push_back(file);
}
std::map<std::string, ExporterSet*>& Globals::GetExporterMap()
@ -93,8 +151,22 @@ ExporterSet* Globals::GetExporterSet()
return nullptr;
}
std::vector<uint8_t> Globals::GetBaseromFile(std::string fileName)
{
if (fileMode == ZFileMode::ExtractDirectory)
{
if (StringHelper::Contains(fileName, "baserom/"))
fileName = StringHelper::Split(fileName, "baserom/")[1];
return rom->GetFile(fileName);
}
else
return File::ReadAllBytes(fileName);