aboutsummaryrefslogtreecommitdiff
path: root/src/zen/cmds/projectstore_cmd.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/zen/cmds/projectstore_cmd.cpp')
-rw-r--r--src/zen/cmds/projectstore_cmd.cpp68
1 files changed, 51 insertions, 17 deletions
diff --git a/src/zen/cmds/projectstore_cmd.cpp b/src/zen/cmds/projectstore_cmd.cpp
index 4885fd363..5ff591b54 100644
--- a/src/zen/cmds/projectstore_cmd.cpp
+++ b/src/zen/cmds/projectstore_cmd.cpp
@@ -41,12 +41,10 @@ ZEN_THIRD_PARTY_INCLUDES_END
namespace zen {
-namespace {
+namespace projectstore_impl {
using namespace std::literals;
-#define ZEN_CLOUD_STORAGE "Cloud Storage"
-
void WriteAuthOptions(CbObjectWriter& Writer,
std::string_view JupiterOpenIdProvider,
std::string_view JupiterAccessToken,
@@ -500,7 +498,7 @@ namespace {
return {};
}
-} // namespace
+} // namespace projectstore_impl
///////////////////////////////////////
@@ -522,6 +520,7 @@ DropProjectCommand::~DropProjectCommand()
void
DropProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
ZEN_UNUSED(GlobalOptions);
if (!ParseOptions(argc, argv))
@@ -611,6 +610,7 @@ ProjectInfoCommand::~ProjectInfoCommand()
void
ProjectInfoCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
ZEN_UNUSED(GlobalOptions);
if (!ParseOptions(argc, argv))
@@ -697,6 +697,7 @@ CreateProjectCommand::~CreateProjectCommand() = default;
void
CreateProjectCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
ZEN_UNUSED(GlobalOptions);
using namespace std::literals;
@@ -766,6 +767,7 @@ CreateOplogCommand::~CreateOplogCommand() = default;
void
CreateOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
ZEN_UNUSED(GlobalOptions);
using namespace std::literals;
@@ -989,6 +991,7 @@ ExportOplogCommand::~ExportOplogCommand()
void
ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
using namespace std::literals;
ZEN_UNUSED(GlobalOptions);
@@ -1469,6 +1472,20 @@ ImportOplogCommand::ImportOplogCommand()
"Enables both 'boost-worker-count' and 'boost-worker-memory' - may cause computer to be less responsive",
cxxopts::value(m_BoostWorkers),
"<boostworkermemory>");
+ m_Options.add_option(
+ "",
+ "",
+ "allow-partial-block-requests",
+ "Allow request for partial chunk blocks.\n"
+ " false = only full block requests allowed\n"
+ " mixed = multiple partial block ranges requests per block allowed to zen cache, single partial block range "
+ "request per block to host\n"
+ " zencacheonly = multiple partial block ranges requests per block allowed to zen cache, only full block requests "
+ "allowed to host\n"
+ " true = multiple partial block ranges requests per block allowed to zen cache and host\n"
+ "Defaults to 'mixed'.",
+ cxxopts::value(m_AllowPartialBlockRequests),
+ "<allowpartialblockrequests>");
m_Options.parse_positional({"project", "oplog", "gcpath"});
m_Options.positional_help("[<projectid> <oplogid> [<gcpath>]]");
@@ -1481,6 +1498,7 @@ ImportOplogCommand::~ImportOplogCommand()
void
ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
using namespace std::literals;
ZEN_UNUSED(GlobalOptions);
@@ -1513,6 +1531,13 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
throw OptionParseException("'--oplog' is required", m_Options.help());
}
+ EPartialBlockRequestMode Mode = PartialBlockRequestModeFromString(m_AllowPartialBlockRequests);
+ if (Mode == EPartialBlockRequestMode::Invalid)
+ {
+ throw OptionParseException(fmt::format("'--allow-partial-block-requests' ('{}') is invalid", m_AllowPartialBlockRequests),
+ m_Options.help());
+ }
+
HttpClient Http(m_HostName);
m_ProjectName = ResolveProject(Http, m_ProjectName);
if (m_ProjectName.empty())
@@ -1649,6 +1674,9 @@ ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg
{
Writer.AddBool("boostworkermemory"sv, true);
}
+
+ Writer.AddString("partialblockrequestmode", m_AllowPartialBlockRequests);
+
if (!m_FileDirectoryPath.empty())
{
Writer.BeginObject("file"sv);
@@ -1764,6 +1792,7 @@ SnapshotOplogCommand::~SnapshotOplogCommand()
void
SnapshotOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
using namespace std::literals;
ZEN_UNUSED(GlobalOptions);
@@ -1828,6 +1857,7 @@ ProjectStatsCommand::~ProjectStatsCommand()
void
ProjectStatsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
ZEN_UNUSED(GlobalOptions);
if (!ParseOptions(argc, argv))
@@ -1880,6 +1910,7 @@ ProjectOpDetailsCommand::~ProjectOpDetailsCommand()
void
ProjectOpDetailsCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
ZEN_UNUSED(GlobalOptions);
if (!ParseOptions(argc, argv))
@@ -1995,6 +2026,7 @@ OplogMirrorCommand::~OplogMirrorCommand()
void
OplogMirrorCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
ZEN_UNUSED(GlobalOptions);
if (!ParseOptions(argc, argv))
@@ -2262,6 +2294,7 @@ OplogValidateCommand::~OplogValidateCommand()
void
OplogValidateCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
ZEN_UNUSED(GlobalOptions);
if (!ParseOptions(argc, argv))
@@ -2413,6 +2446,7 @@ OplogDownloadCommand::~OplogDownloadCommand()
void
OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv)
{
+ using namespace projectstore_impl;
ZEN_UNUSED(GlobalOptions);
if (!ParseOptions(argc, argv))
@@ -2430,7 +2464,7 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
{
m_SystemRootDir = PickDefaultSystemRootDirectory();
}
- MakeSafeAbsolutePathÍnPlace(m_SystemRootDir);
+ MakeSafeAbsolutePathInPlace(m_SystemRootDir);
};
ParseSystemOptions();
@@ -2568,36 +2602,37 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
StorageInstance Storage;
- ClientSettings.AssumeHttp2 = ResolveRes.HostAssumeHttp2;
+ ClientSettings.AssumeHttp2 = ResolveRes.Cloud.AssumeHttp2;
ClientSettings.MaximumInMemoryDownloadSize = m_BoostWorkerMemory ? RemoteStoreOptions::DefaultMaxBlockSize : 1024u * 1024u;
- Storage.BuildStorageHttp = std::make_unique<HttpClient>(ResolveRes.HostUrl, ClientSettings);
+ Storage.BuildStorageHttp = std::make_unique<HttpClient>(ResolveRes.Cloud.Address, ClientSettings);
+ Storage.BuildStorageHost = ResolveRes.Cloud;
BuildStorageCache::Statistics StorageCacheStats;
std::atomic<bool> AbortFlag(false);
- if (!ResolveRes.CacheUrl.empty())
+ if (!ResolveRes.Cache.Address.empty())
{
Storage.CacheHttp = std::make_unique<HttpClient>(
- ResolveRes.CacheUrl,
+ ResolveRes.Cache.Address,
HttpClientSettings{
.LogCategory = "httpcacheclient",
.ConnectTimeout = std::chrono::milliseconds{3000},
.Timeout = std::chrono::milliseconds{30000},
- .AssumeHttp2 = ResolveRes.CacheAssumeHttp2,
+ .AssumeHttp2 = ResolveRes.Cache.AssumeHttp2,
.AllowResume = true,
.RetryCount = 0,
.MaximumInMemoryDownloadSize = m_BoostWorkerMemory ? RemoteStoreOptions::DefaultMaxBlockSize : 1024u * 1024u},
[&AbortFlag]() { return AbortFlag.load(); });
- Storage.CacheName = ResolveRes.CacheName;
+ Storage.CacheHost = ResolveRes.Cache;
}
if (!m_Quiet)
{
std::string StorageDescription =
fmt::format("Cloud {}{}. SessionId {}. Namespace '{}', Bucket '{}'",
- ResolveRes.HostName,
- (ResolveRes.HostUrl == ResolveRes.HostName) ? "" : fmt::format(" {}", ResolveRes.HostUrl),
+ ResolveRes.Cloud.Name,
+ (ResolveRes.Cloud.Address == ResolveRes.Cloud.Name) ? "" : fmt::format(" {}", ResolveRes.Cloud.Address),
Storage.BuildStorageHttp->GetSessionId(),
m_Namespace,
m_Bucket);
@@ -2608,8 +2643,8 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
{
std::string CacheDescription =
fmt::format("Zen {}{}. SessionId {}. Namespace '{}', Bucket '{}'",
- ResolveRes.CacheName,
- (ResolveRes.CacheUrl == ResolveRes.CacheName) ? "" : fmt::format(" {}", ResolveRes.CacheUrl),
+ ResolveRes.Cache.Name,
+ (ResolveRes.Cache.Address == ResolveRes.Cache.Name) ? "" : fmt::format(" {}", ResolveRes.Cache.Address),
Storage.CacheHttp->GetSessionId(),
m_Namespace,
m_Bucket);
@@ -2625,11 +2660,10 @@ OplogDownloadCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** a
Storage.BuildStorage =
CreateJupiterBuildStorage(Log(), *Storage.BuildStorageHttp, StorageStats, m_Namespace, m_Bucket, m_AllowRedirect, StorageTempPath);
- Storage.StorageName = ResolveRes.HostName;
if (Storage.CacheHttp)
{
- Storage.BuildCacheStorage = CreateZenBuildStorageCache(
+ Storage.CacheStorage = CreateZenBuildStorageCache(
*Storage.CacheHttp,
StorageCacheStats,
m_Namespace,
generated by cgit v1.2.3 (git 2.25.1) at 2026-06-06 19:32:14 +0000