// Copyright Epic Games, Inc. All Rights Reserved. #include #include #include #include #include #include #include ZEN_THIRD_PARTY_INCLUDES_START #include ZEN_THIRD_PARTY_INCLUDES_END #if ZEN_WITH_TESTS # include # include #endif // ZEN_WITH_TESTS namespace zen { using namespace std::literals; namespace { std::filesystem::path ZenStateDownloadFolder(const std::filesystem::path& SystemRootDir) { return SystemRootDir / "builds" / "downloads"; } } // namespace ///////////////////////// BuildsSelection void BuildsSelection::Write(const BuildsSelection& Selection, CbWriter& Output) { Output.BeginArray("builds"sv); for (const BuildsSelection::Build& Build : Selection.Builds) { Output.BeginObject(); { ZEN_ASSERT(Build.Id != Oid::Zero); Output.AddObjectId("buildId"sv, Build.Id); compactbinary_helpers::WriteArray(Build.IncludeWildcards, "includeWildcards"sv, Output); compactbinary_helpers::WriteArray(Build.ExcludeWildcards, "excludeWildcards"sv, Output); Output.BeginArray("parts"sv); for (const BuildsSelection::BuildPart& BuildPart : Build.Parts) { Output.BeginObject(); { ZEN_ASSERT(BuildPart.Id != Oid::Zero); Output.AddObjectId("partId"sv, BuildPart.Id); Output.AddString("partName"sv, BuildPart.Name); } Output.EndObject(); } Output.EndArray(); // parts } Output.EndObject(); } Output.EndArray(); // builds } BuildsSelection BuildsSelection::Read(CbObjectView& Input, std::vector* OptionalOutLegacyPartsContent) { std::vector Builds; CbArrayView BuildsArray = Input["builds"sv].AsArrayView(); Builds.reserve(BuildsArray.Num()); for (CbFieldView BuildView : BuildsArray) { std::vector Parts; CbObjectView BuildObjectView = BuildView.AsObjectView(); Oid BuildId = BuildObjectView["buildId"sv].AsObjectId(); if (BuildId == Oid::Zero) { throw std::runtime_error(fmt::format("BuildsSelection build id is invalid '{}'", BuildId)); } std::vector IncludeWildcards = compactbinary_helpers::ReadArray("includeWildcards"sv, BuildObjectView); std::vector ExcludeWildcards = compactbinary_helpers::ReadArray("excludeWildcards"sv, BuildObjectView); CbArrayView BuildPartsArray = BuildObjectView["parts"sv].AsArrayView(); Parts.reserve(BuildPartsArray.Num()); for (CbFieldView PartView : BuildPartsArray) { CbObjectView PartObjectView = PartView.AsObjectView(); Oid BuildPartId = PartObjectView["partId"sv].AsObjectId(); if (BuildPartId == Oid::Zero) { throw std::runtime_error(fmt::format("BuildsSelection build part id in build '{}' is invalid '{}'", BuildId, BuildPartId)); } std::string_view BuildPartName = PartObjectView["partName"sv].AsString(); if (OptionalOutLegacyPartsContent != nullptr) { CbObjectView PartContentObjectView = PartObjectView["content"sv].AsObjectView(); if (PartContentObjectView) { OptionalOutLegacyPartsContent->push_back(LoadChunkedFolderContentFromCompactBinary(PartContentObjectView)); } } Parts.push_back(BuildsSelection::BuildPart{.Id = BuildPartId, .Name = std::string(BuildPartName)}); } Builds.push_back(BuildsSelection::Build{.Id = BuildId, .Parts = std::move(Parts), .IncludeWildcards = std::move(IncludeWildcards), .ExcludeWildcards = std::move(ExcludeWildcards)}); } return BuildsSelection{.Builds = std::move(Builds)}; } ///////////////////////// BuildState void BuildState::Write(const BuildState& State, CbWriter& Output) { BuildsSelection::Write(State.Selection, Output); Output.BeginObject("content"); { SaveChunkedFolderContentToCompactBinary(State.ChunkedContent, Output); } Output.EndObject(); // content } BuildState BuildState::Read(CbObjectView& Input) { std::vector LegacyPartsContent; BuildsSelection Selection = BuildsSelection::Read(Input, &LegacyPartsContent); ChunkedFolderContent ChunkedContent; CbObjectView ContentObjectView = Input["content"sv].AsObjectView(); if (ContentObjectView) { ZEN_ASSERT(LegacyPartsContent.empty()); ChunkedContent = LoadChunkedFolderContentFromCompactBinary(ContentObjectView); } else { if (LegacyPartsContent.size() == 1) { ChunkedContent = std::move(LegacyPartsContent.front()); } else { ChunkedContent = MergeChunkedFolderContents(LegacyPartsContent[0], std::span(LegacyPartsContent).subspan(1)); } } return BuildState{.Selection = std::move(Selection), .ChunkedContent = std::move(ChunkedContent)}; } ///////////////////////// BuildSaveState void BuildSaveState::Write(const BuildSaveState& SaveState, CbWriter& Output) { ZEN_ASSERT(!SaveState.LocalPath.empty()); Output.AddString("path", (const char*)SaveState.LocalPath.u8string().c_str()); BuildsSelection::Write(SaveState.State.Selection, Output); Output.BeginObject("content"); { SaveChunkedFolderContentToCompactBinary(SaveState.State.ChunkedContent, Output); } Output.EndObject(); // content Output.BeginObject("localFolderState"sv); { SaveFolderContentToCompactBinary(SaveState.FolderState, Output); } Output.EndObject(); // localFolderState } BuildSaveState BuildSaveState::Read(CbObjectView& Input) { BuildState State = BuildState::Read(Input); CbObjectView LocalFolderStateObject = Input["localFolderState"sv].AsObjectView(); FolderContent FolderState = LoadFolderContentToCompactBinary(LocalFolderStateObject); std::filesystem::path LocalPath = Input["path"sv].AsU8String(); if (LocalPath.empty()) { throw std::runtime_error("BuildSaveState is invalid, 'path' field is empty"); } return BuildSaveState{.State = std::move(State), .FolderState = std::move(FolderState), .LocalPath = std::move(LocalPath)}; } CbObject CreateBuildSaveStateObject(const BuildSaveState& SaveState) { CbObjectWriter CurrentStateWriter; BuildSaveState::Write(SaveState, CurrentStateWriter); return CurrentStateWriter.Save(); } BuildSaveState ReadBuildSaveStateFile(const std::filesystem::path& StateFilePath) { ZEN_TRACE_CPU("ReadStateFile"); IoBuffer DataBuffer; { BasicFile Source(StateFilePath, BasicFile::Mode::kRead); DataBuffer = Source.ReadAll(); } CbValidateError ValidateError; if (CbObject CurrentStateObject = ValidateAndReadCompactBinaryObject(std::move(DataBuffer), ValidateError); ValidateError == CbValidateError::None) { if (CurrentStateObject) { return BuildSaveState::Read(CurrentStateObject); } else { throw std::runtime_error(fmt::format("Compact binary object read from '{}' is empty", StateFilePath)); } } else { throw std::runtime_error( fmt::format("Failed to read compact binary object from '{}', reason: {}", StateFilePath, ToString(ValidateError))); } } ///////////////////////// BuildsDownloadInfo void BuildsDownloadInfo::Write(const BuildsDownloadInfo& Info, CbWriter& Output) { ZEN_ASSERT(!Info.LocalPath.empty()); ZEN_ASSERT(!Info.StateFilePath.empty()); ZEN_ASSERT(!Info.Iso8601Date.empty()); Output.AddString("path", (const char*)Info.LocalPath.u8string().c_str()); Output.AddString("statePath", (const char*)Info.StateFilePath.u8string().c_str()); Output.AddString("date", Info.Iso8601Date); BuildsSelection::Write(Info.Selection, Output); } BuildsDownloadInfo BuildsDownloadInfo::Read(CbObjectView& Input) { BuildsSelection Selection = BuildsSelection::Read(Input, /*OptionalOutLegacyPartsContent*/ nullptr); std::filesystem::path LocalPath = Input["path"sv].AsU8String(); if (LocalPath.empty()) { throw std::runtime_error("BuildsDownloadInfo is invalid, 'path' field is empty"); } std::filesystem::path StateFilePath = Input["statePath"sv].AsU8String(); if (StateFilePath.empty()) { throw std::runtime_error("BuildsDownloadInfo is invalid, 'statePath' field is empty"); } std::string_view Iso8601Date = Input["date"sv].AsString(); return BuildsDownloadInfo{.Selection = std::move(Selection), .LocalPath = std::move(LocalPath), .StateFilePath = std::move(StateFilePath), .Iso8601Date = std::string(Iso8601Date)}; } void AddDownloadedPath(const std::filesystem::path& SystemRootDir, const BuildsDownloadInfo& Info) { ZEN_TRACE_CPU("AddDownloadedPath"); ZEN_ASSERT(!SystemRootDir.empty()); ZEN_ASSERT(!Info.StateFilePath.empty()); ZEN_ASSERT(!Info.LocalPath.empty()); const std::u8string LocalPathString = Info.LocalPath.generic_u8string(); IoHash PathHash = IoHash::HashBuffer(LocalPathString.data(), LocalPathString.length()); const std::filesystem::path StateDownloadFolder = ZenStateDownloadFolder(SystemRootDir); CreateDirectories(StateDownloadFolder); std::filesystem::path WritePath = StateDownloadFolder / (PathHash.ToHexString() + ".json"); CbObjectWriter Writer; BuildsDownloadInfo::Write(Info, Writer); CbObject Payload = Writer.Save(); ExtendableStringBuilder<512> SB; CompactBinaryToJson(Payload.GetView(), SB); MemoryView JsonPayload(SB.Data(), SB.Size()); TemporaryFile::SafeWriteFile(WritePath, JsonPayload); } BuildsDownloadInfo ReadDownloadedInfoFile(const std::filesystem::path& DownloadInfoPath) { IoBuffer MetaDataJson = ReadFile(DownloadInfoPath).Flatten(); std::string_view Json(reinterpret_cast(MetaDataJson.GetData()), MetaDataJson.GetSize()); std::string JsonError; CbObject DownloadInfo = LoadCompactBinaryFromJson(Json, JsonError).AsObject(); if (!JsonError.empty()) { throw std::runtime_error(fmt::format("Invalid download state file at {}. '{}'", DownloadInfoPath, JsonError)); } return BuildsDownloadInfo::Read(DownloadInfo); } std::vector GetDownloadedStatePaths(const std::filesystem::path& SystemRootDir) { ZEN_ASSERT(!SystemRootDir.empty()); ZEN_TRACE_CPU("GetDownloadedPaths"); std::vector Result; const std::filesystem::path StateDownloadFolder = ZenStateDownloadFolder(SystemRootDir); if (IsDir(StateDownloadFolder)) { DirectoryContent Content; GetDirectoryContent(ZenStateDownloadFolder(SystemRootDir), DirectoryContentFlags::IncludeFiles, Content); for (const std::filesystem::path& EntryPath : Content.Files) { IoHash EntryPathHash; if (IoHash::TryParse(EntryPath.stem().string(), EntryPathHash)) { Result.push_back(EntryPath); } } } return Result; } #if ZEN_WITH_TESTS void buildsavedstate_forcelink() { } namespace buildsavestate_test { bool Equal(const BuildsSelection& Lhs, const BuildsSelection& Rhs) { if (Lhs.Builds.size() != Rhs.Builds.size()) { return false; } for (size_t BuildIndex = 0; BuildIndex < Lhs.Builds.size(); BuildIndex++) { const BuildsSelection::Build LhsBuild = Lhs.Builds[BuildIndex]; const BuildsSelection::Build RhsBuild = Rhs.Builds[BuildIndex]; if (LhsBuild.Id != RhsBuild.Id) { return false; } if (LhsBuild.Parts.size() != RhsBuild.Parts.size()) { return false; } for (size_t PartIndex = 0; PartIndex < LhsBuild.Parts.size(); PartIndex++) { const BuildsSelection::BuildPart LhsPart = LhsBuild.Parts[PartIndex]; const BuildsSelection::BuildPart RhsPart = RhsBuild.Parts[PartIndex]; if (LhsPart.Id != RhsPart.Id) { return false; } if (LhsPart.Name != RhsPart.Name) { return false; } } if (LhsBuild.IncludeWildcards != RhsBuild.IncludeWildcards) { return false; } if (LhsBuild.ExcludeWildcards != RhsBuild.ExcludeWildcards) { return false; } } return true; } bool Equal(const ChunkedContentData& Lhs, const ChunkedContentData& Rhs) { if (Lhs.SequenceRawHashes != Rhs.SequenceRawHashes) { return false; } if (Lhs.ChunkCounts != Rhs.ChunkCounts) { return false; } if (Lhs.ChunkOrders != Rhs.ChunkOrders) { return false; } if (Lhs.ChunkHashes != Rhs.ChunkHashes) { return false; } if (Lhs.ChunkRawSizes != Rhs.ChunkRawSizes) { return false; } return true; } bool Equal(const ChunkedFolderContent& Lhs, const ChunkedFolderContent& Rhs) { if (Lhs.Platform != Rhs.Platform) { return false; } if (Lhs.Paths != Rhs.Paths) { return false; } if (Lhs.RawSizes != Rhs.RawSizes) { return false; } if (Lhs.Attributes != Rhs.Attributes) { return false; } if (Lhs.RawHashes != Rhs.RawHashes) { return false; } return Equal(Lhs.ChunkedContent, Rhs.ChunkedContent); } bool Equal(const BuildState& Lhs, const BuildState& Rhs) { if (!Equal(Lhs.Selection, Rhs.Selection)) { return false; } if (!Equal(Lhs.ChunkedContent, Rhs.ChunkedContent)) { return false; } return true; } bool Equal(const FolderContent& Lhs, const FolderContent& Rhs) { if (Lhs.Platform != Rhs.Platform) { return false; } if (Lhs.Paths != Rhs.Paths) { return false; } if (Lhs.RawSizes != Rhs.RawSizes) { return false; } if (Lhs.Attributes != Rhs.Attributes) { return false; } if (Lhs.ModificationTicks != Rhs.ModificationTicks) { return false; } return true; } bool Equal(const BuildSaveState& Lhs, const BuildSaveState& Rhs) { if (!Equal(Lhs.State, Rhs.State)) { return false; } if (!Equal(Lhs.FolderState, Rhs.FolderState)) { return false; } if (Lhs.LocalPath != Rhs.LocalPath) { return false; } return true; } bool Equal(const BuildsDownloadInfo& Lhs, const BuildsDownloadInfo& Rhs) { if (!Equal(Lhs.Selection, Rhs.Selection)) { return false; } if (Lhs.LocalPath != Rhs.LocalPath) { return false; } if (Lhs.StateFilePath != Rhs.StateFilePath) { return false; } if (Lhs.Iso8601Date != Rhs.Iso8601Date) { return false; } return true; } BuildsSelection MakeBuildsSelectionA() { return BuildsSelection{ .Builds = std::vector{ BuildsSelection::Build{ .Id = Oid::NewOid(), .Parts = std::vector{BuildsSelection::BuildPart{.Id = Oid::NewOid(), .Name = "default"}}}, BuildsSelection::Build{ .Id = Oid::NewOid(), .Parts = std::vector{BuildsSelection::BuildPart{.Id = Oid::NewOid(), .Name = "first_part"}, BuildsSelection::BuildPart{.Id = Oid::NewOid(), .Name = "second_part"}}, .IncludeWildcards = std::vector{std::string{"*.exe"}, std::string{"*.dll"}}, .ExcludeWildcards = std::vector{std::string{"*.pdb"}}}, BuildsSelection::Build{ .Id = Oid::NewOid(), .Parts = std::vector{BuildsSelection::BuildPart{.Id = Oid::NewOid(), .Name = "default"}}, .IncludeWildcards = std::vector{std::string{"*.ucas"}, std::string{"*.utok"}}, .ExcludeWildcards = std::vector{std::string{"*.pak"}}}}}; } BuildsSelection MakeBuildsSelectionB() { return BuildsSelection{ .Builds = std::vector{ BuildsSelection::Build{ .Id = Oid::NewOid(), .Parts = std::vector{BuildsSelection::BuildPart{.Id = Oid::NewOid(), .Name = "default"}}}, BuildsSelection::Build{ .Id = Oid::NewOid(), .Parts = std::vector{BuildsSelection::BuildPart{.Id = Oid::NewOid(), .Name = "default"}}, .IncludeWildcards = std::vector{std::string{"*.exe"}, std::string{"*.dll"}}, .ExcludeWildcards = std::vector{std::string{"*.pdb"}, std::string{"*.xSYM"}}}}}; } BuildState MakeBuildStateA(FastRandom& Random) { BuildsSelection Selection = MakeBuildsSelectionA(); std::vector Chunks; ChunkedFolderContent Content = chunkedcontent_testutils::CreateChunkedFolderContent( Random, std::vector>{ std::pair{"file_1", 6u * 1024u}, std::pair{"file_2.exe", 0}, std::pair{"file_3.txt", 798}, std::pair{"dir_1/dir1_file_1.exe", 19u * 1024u}, std::pair{"dir_1/dir1_file_2.pdb", 7u * 1024u}, std::pair{"dir_1/dir1_file_3.txt", 93}, std::pair{"dir_2/dir2_dir1/dir2_dir1_file_1.exe", 31u * 1024u}, std::pair{"dir_2/dir2_dir1/dir2_dir1_file_2.pdb", 17u * 1024u}, std::pair{"dir_2/dir2_dir1/dir2_dir1_file_3.dll", 13u * 1024u}, std::pair{"dir_2/dir2_dir2/dir2_dir2_file_1.txt", 2u * 1024u}, std::pair{"dir_2/dir2_dir2/dir2_dir2_file_2.json", 3u * 1024u}}, 4u * 1024u, Chunks); return BuildState{.Selection = std::move(Selection), .ChunkedContent = std::move(Content)}; } FolderContent MakeFolderContent(ChunkedFolderContent& Content) { FolderContent Result = {.Platform = Content.Platform}; Result.Paths = Content.Paths; Result.RawSizes = Content.RawSizes; Result.Attributes = Content.Attributes; Result.ModificationTicks.resize(Result.Paths.size(), 0); uint64_t AttributeIndex = 0; for (uint64_t& ModificationTick : Result.ModificationTicks) { ModificationTick = ++AttributeIndex; } return Result; } } // namespace buildsavestate_test TEST_CASE("buildsavestate.BuildsSelection") { using namespace buildsavestate_test; const BuildsSelection Selection = MakeBuildsSelectionA(); CbObjectWriter Writer; BuildsSelection::Write(Selection, Writer); CbObject SavedObject = Writer.Save(); BuildsSelection ReadbackSelection = BuildsSelection::Read(SavedObject, nullptr); CHECK(Equal(Selection, ReadbackSelection)); } TEST_CASE("buildsavestate.BuildsState") { using namespace buildsavestate_test; FastRandom Random; BuildState State = MakeBuildStateA(Random); CbObjectWriter Writer; BuildState::Write(State, Writer); CbObject SavedObject = Writer.Save(); BuildState ReadbackState = BuildState::Read(SavedObject); CHECK(Equal(State, ReadbackState)); } TEST_CASE("buildsavestate.BuildsSaveState") { using namespace buildsavestate_test; FastRandom Random; BuildState State = MakeBuildStateA(Random); FolderContent FolderState = MakeFolderContent(State.ChunkedContent); BuildSaveState SaveState = {.State = std::move(State), .FolderState = std::move(FolderState), .LocalPath = "\\\\?\\E:\\temp\\localpath"}; CbObjectWriter Writer; BuildSaveState::Write(SaveState, Writer); CbObject SavedObject = Writer.Save(); BuildSaveState ReadbackSavedState = BuildSaveState::Read(SavedObject); CHECK(Equal(SaveState, ReadbackSavedState)); } TEST_CASE("buildsavestate.BuildsDownloadInfo") { using namespace buildsavestate_test; BuildsDownloadInfo DownloadInfo = {.Selection = MakeBuildsSelectionA(), .LocalPath = "\\\\?\\E:\\temp\\localpath", .StateFilePath = "\\\\?\\E:\\temp\\localpath\\.zen\\state.cbo", .Iso8601Date = DateTime::Now().ToIso8601()}; CbObjectWriter Writer; BuildsDownloadInfo::Write(DownloadInfo, Writer); CbObject SavedObject = Writer.Save(); BuildsDownloadInfo ReadbackDownloadInfo = BuildsDownloadInfo::Read(SavedObject); CHECK(Equal(DownloadInfo, ReadbackDownloadInfo)); } TEST_CASE("buildsavestate.DownloadedPaths") { using namespace buildsavestate_test; ScopedTemporaryDirectory SystemDir; BuildsDownloadInfo DownloadInfoA = {.Selection = MakeBuildsSelectionA(), .LocalPath = "\\\\?\\E:\\temp\\localpathA", .StateFilePath = "\\\\?\\E:\\temp\\localpathA\\.zen\\state.cbo", .Iso8601Date = DateTime(DateTime::Now().GetTicks() - 200).ToIso8601()}; BuildsDownloadInfo DownloadInfoB = {.Selection = MakeBuildsSelectionB(), .LocalPath = "\\\\?\\E:\\temp\\localpathB", .StateFilePath = "\\\\?\\E:\\temp\\localpathB\\.zen\\state.cbo", .Iso8601Date = DateTime(DateTime::Now().GetTicks() - 100).ToIso8601()}; AddDownloadedPath(SystemDir.Path(), DownloadInfoA); AddDownloadedPath(SystemDir.Path(), DownloadInfoB); std::vector DownloadedPaths = GetDownloadedStatePaths(SystemDir.Path()); CHECK_EQ(2u, DownloadedPaths.size()); BuildsDownloadInfo ReadBackDownloadInfo0 = ReadDownloadedInfoFile(DownloadedPaths[0]); BuildsDownloadInfo ReadBackDownloadInfo1 = ReadDownloadedInfoFile(DownloadedPaths[1]); if (DownloadInfoA.LocalPath == ReadBackDownloadInfo0.LocalPath) { CHECK(Equal(DownloadInfoA, ReadBackDownloadInfo0)); CHECK(Equal(DownloadInfoB, ReadBackDownloadInfo1)); } else { CHECK(Equal(DownloadInfoA, ReadBackDownloadInfo1)); CHECK(Equal(DownloadInfoB, ReadBackDownloadInfo0)); } } #endif // ZEN_WITH_TESTS } // namespace zen