diff options
| author | Dan Engelbrecht <[email protected]> | 2023-02-09 16:49:51 +0100 |
|---|---|---|
| committer | GitHub <[email protected]> | 2023-02-09 07:49:51 -0800 |
| commit | 2f872e432d4a77d1c2dd082cb97a0cbfddb3cc97 (patch) | |
| tree | d631da0746b78cad7140784de4e637bcfb4e1cac /zen/cmds/projectstore.cpp | |
| parent | Update README.md (diff) | |
| download | archived-zen-2f872e432d4a77d1c2dd082cb97a0cbfddb3cc97.tar.xz archived-zen-2f872e432d4a77d1c2dd082cb97a0cbfddb3cc97.zip | |
oplog upload/download (#214)
- Feature: Zen server endpoint `prj/{project}/oplog/{log}/chunks` to post multiple attachments in one request.
- Feature: Zen server endpoint `prj/{project}/oplog/{log}/save` to save an oplog container. Accepts `CbObject` containing a compressed oplog and attachment references organized in blocks.
- Feature: Zen server endpoint `prj/{project}/oplog/{log}/load` to request an oplog container. Responds with an `CbObject` containing a compressed oplog and attachment references organized in blocks.
- Feature: Zen server endpoint `{project}/oplog/{log}/rpc` to initiate an import to or export from an external location and other operations. Use either JSon or CbPackage as payload.
- CbObject/JSon RPC format for `import` and `export` methods:
- CbObject RPC format for `getchunks` method, returns CbPackage with the found chunks, if all chunks are found the number of attachments matches number of chunks requested.
- Feature: Zen server `{project}/oplog/{log}/{hash}` now accepts `HttpVerb::kPost` as well as `HttpVerb::kGet`.
- Feature: Zen command line tool `oplog-export` to export an oplog to an external target using the zenserver oplog export endpoint.
- Feature: Zen command line tool `oplog-import` to import an oplog from an external source using the zenserver oplog import endpoint.
Diffstat (limited to 'zen/cmds/projectstore.cpp')
| -rw-r--r-- | zen/cmds/projectstore.cpp | 509 |
1 files changed, 488 insertions, 21 deletions
diff --git a/zen/cmds/projectstore.cpp b/zen/cmds/projectstore.cpp index c53f0bc35..8ecdecaaa 100644 --- a/zen/cmds/projectstore.cpp +++ b/zen/cmds/projectstore.cpp @@ -2,28 +2,13 @@ #include "projectstore.h" -#include <zencore/compactbinary.h> #include <zencore/compactbinarybuilder.h> -#include <zencore/compactbinarypackage.h> -#include <zencore/compactbinaryvalue.h> -#include <zencore/compress.h> -#include <zencore/filesystem.h> -#include <zencore/fmtutils.h> -#include <zencore/iohash.h> #include <zencore/logging.h> #include <zencore/stream.h> -#include <zencore/uid.h> -#include <zencore/workthreadpool.h> #include <zenhttp/httpcommon.h> -#include <zenhttp/httpshared.h> -#include <zenutil/basicfile.h> -#include <zenutil/zenserverprocess.h> - -#include <memory> ZEN_THIRD_PARTY_INCLUDES_START #include <cpr/cpr.h> -#include <gsl/gsl-lite.hpp> ZEN_THIRD_PARTY_INCLUDES_END /////////////////////////////////////// @@ -32,9 +17,9 @@ DropProjectCommand::DropProjectCommand() { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value("http://localhost:1337"), "<hosturl>"); - m_Options.add_option("", "p", "project", "Namnspace name", cxxopts::value(m_ProjectName), "<projectname>"); - m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogName), "<oplogname>"); - m_Options.parse_positional({"{project}", "{oplog}"}); + m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectName), "<projectid>"); + m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogName), "<oplogid>"); + m_Options.parse_positional({"project", "oplog"}); } DropProjectCommand::~DropProjectCommand() = default; @@ -92,9 +77,9 @@ ProjectInfoCommand::ProjectInfoCommand() { m_Options.add_options()("h,help", "Print help"); m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value("http://localhost:1337"), "<hosturl>"); - m_Options.add_option("", "p", "project", "Namnspace name", cxxopts::value(m_ProjectName), "<projectname>"); - m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogName), "<oplogname>"); - m_Options.parse_positional({"{project}", "{oplog}"}); + m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectName), "<projectid>"); + m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogName), "<oplogid>"); + m_Options.parse_positional({"project", "oplog"}); } ProjectInfoCommand::~ProjectInfoCommand() = default; @@ -279,3 +264,485 @@ CreateOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** arg return GetReturnCode(Response); } + +/////////////////////////////////////// + +ExportOplogCommand::ExportOplogCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value("http://localhost:1337"), "<hosturl>"); + m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectName), "<projectid>"); + m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogName), "<oplogid>"); + m_Options.add_option("", "", "maxblocksize", "Max size for bundled attachments", cxxopts::value(m_MaxBlockSize), "<blocksize>"); + m_Options.add_option("", + "", + "maxchunkembedsize", + "Max size for attachment to be bundled", + cxxopts::value(m_MaxChunkEmbedSize), + "<chunksize>"); + m_Options.add_option("", "f", "force", "Force export of all attachments", cxxopts::value(m_Force), "<force>"); + m_Options.add_option("", + "", + "disableblocks", + "Disable block creation and save all attachments individually (applies to file and cloud target)", + cxxopts::value(m_DisableBlocks), + "<disable>"); + + m_Options.add_option("", "", "cloud", "Cloud Storage URL", cxxopts::value(m_CloudUrl), "<url>"); + m_Options.add_option("cloud", "", "namespace", "Cloud Storage namespace", cxxopts::value(m_CloudNamespace), "<namespace>"); + m_Options.add_option("cloud", "", "bucket", "Cloud Storage bucket", cxxopts::value(m_CloudBucket), "<bucket>"); + m_Options.add_option("cloud", "", "key", "Cloud Storage key", cxxopts::value(m_CloudKey), "<key>"); + m_Options + .add_option("cloud", "", "openid-provider", "Cloud Storage openid provider", cxxopts::value(m_CloudOpenIdProvider), "<provider>"); + m_Options.add_option("cloud", "", "access-token", "Cloud Storage access token", cxxopts::value(m_CloudAccessToken), "<accesstoken>"); + m_Options.add_option("cloud", + "", + "disabletempblocks", + "Disable temp block creation and upload blocks without waiting for oplog container to be uploaded", + cxxopts::value(m_CloudDisableTempBlocks), + "<disable>"); + + m_Options.add_option("", "", "zen", "Zen service upload address", cxxopts::value(m_ZenUrl), "<url>"); + m_Options.add_option("zen", "", "target-project", "Zen target project name", cxxopts::value(m_ZenProjectName), "<targetprojectid>"); + m_Options.add_option("zen", "", "target-oplog", "Zen target oplog name", cxxopts::value(m_ZenOplogName), "<targetoplogid>"); + m_Options.add_option("zen", "", "clean", "Delete existing target Zen oplog", cxxopts::value(m_ZenClean), "<clean>"); + + m_Options.add_option("", "", "file", "Local folder path", cxxopts::value(m_FileDirectoryPath), "<path>"); + m_Options.add_option("file", "", "name", "Local file name", cxxopts::value(m_FileName), "<filename>"); + m_Options.add_option("file", + "", + "forcetempblocks", + "Force creation of temp attachment blocks", + cxxopts::value(m_FileForceEnableTempBlocks), + "<forcetempblocks>"); + + m_Options.parse_positional({"project", "oplog"}); +} + +ExportOplogCommand::~ExportOplogCommand() = default; + +int +ExportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + using namespace std::literals; + + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + if (m_ProjectName.empty()) + { + ZEN_ERROR("Project name must be given"); + return 1; + } + + if (m_OplogName.empty()) + { + ZEN_ERROR("Oplog name must be given"); + return 1; + } + + size_t TargetCount = 0; + TargetCount += m_CloudUrl.empty() ? 0 : 1; + TargetCount += m_ZenUrl.empty() ? 0 : 1; + TargetCount += m_FileDirectoryPath.empty() ? 0 : 1; + if (TargetCount != 1) + { + ZEN_ERROR("Provide one target only"); + ZEN_CONSOLE("{}", m_Options.help({""}).c_str()); + return 1; + } + + cpr::Session Session; + + if (!m_CloudUrl.empty()) + { + if (m_CloudNamespace.empty() || m_CloudBucket.empty()) + { + ZEN_ERROR("Options for cloud target are missing"); + ZEN_CONSOLE("{}", m_Options.help({"cloud"}).c_str()); + return 1; + } + if (m_CloudKey.empty()) + { + std::string KeyString = fmt::format("{}/{}/{}/{}", m_ProjectName, m_OplogName, m_CloudNamespace, m_CloudBucket); + zen::IoHash Key = zen::IoHash::HashBuffer(KeyString.data(), KeyString.size()); + m_CloudKey = Key.ToHexString(); + ZEN_WARN("Using auto generated cloud key '{}'", m_CloudKey); + } + } + + if (!m_ZenUrl.empty()) + { + if (m_ZenProjectName.empty()) + { + m_ZenProjectName = m_ProjectName; + ZEN_WARN("Using default zen target project id '{}'", m_ZenProjectName); + } + if (m_ZenOplogName.empty()) + { + m_ZenOplogName = m_OplogName; + ZEN_WARN("Using default zen target oplog id '{}'", m_ZenOplogName); + } + + std::string TargetUrlBase = fmt::format("{}/prj", m_ZenUrl); + if (TargetUrlBase.find("://") == std::string::npos) + { + // Assume https URL + TargetUrlBase = fmt::format("http://{}", TargetUrlBase); + } + + Session.SetUrl({fmt::format("{}/{}/oplog/{}", TargetUrlBase, m_ZenProjectName, m_ZenOplogName)}); + cpr::Response Response = Session.Get(); + if (Response.status_code == static_cast<long>(zen::HttpResponseCode::NotFound)) + { + ZEN_WARN("Automatically creating oplog '{}/{}'", m_ZenProjectName, m_ZenOplogName) + Response = Session.Post(); + if (!zen::IsHttpSuccessCode(Response.status_code)) + { + ZEN_CONSOLE("{}", FormatResponse(Response)); + return GetReturnCode(Response); + } + } + else if (!zen::IsHttpSuccessCode(Response.status_code)) + { + ZEN_CONSOLE("{}", FormatResponse(Response)); + return GetReturnCode(Response); + } + else if (m_ZenClean) + { + ZEN_WARN("Cleaning oplog '{}/{}'", m_ZenProjectName, m_ZenOplogName) + Response = Session.Delete(); + if (!zen::IsHttpSuccessCode(Response.status_code)) + { + ZEN_CONSOLE("{}", FormatResponse(Response)); + return GetReturnCode(Response); + } + Response = Session.Post(); + if (!zen::IsHttpSuccessCode(Response.status_code)) + { + ZEN_CONSOLE("{}", FormatResponse(Response)); + return GetReturnCode(Response); + } + } + } + + if (!m_FileDirectoryPath.empty()) + { + if (m_FileName.empty()) + { + m_FileName = m_OplogName; + ZEN_WARN("Using default file name '{}'", m_FileName); + } + } + + const std::string SourceUrlBase = fmt::format("{}/prj", m_HostName); + std::string TargetDescription; + Session.SetUrl({fmt::format("{}/{}/oplog/{}/rpc", SourceUrlBase, m_ProjectName, m_OplogName)}); + Session.SetHeader({{"Content-Type", std::string(zen::MapContentTypeToString(zen::HttpContentType::kCbObject))}}); + zen::CbObjectWriter Writer; + Writer.AddString("method"sv, "export"sv); + Writer.BeginObject("params"sv); + { + if (m_MaxBlockSize != 0) + { + Writer.AddInteger("maxblocksize"sv, m_MaxBlockSize); + } + if (m_MaxChunkEmbedSize != 0) + { + Writer.AddInteger("maxchunkembedsize"sv, m_MaxChunkEmbedSize); + } + if (m_Force) + { + Writer.AddBool("force"sv, true); + } + if (!m_FileDirectoryPath.empty()) + { + Writer.BeginObject("file"sv); + { + Writer.AddString("file"sv, m_FileDirectoryPath); + Writer.AddString("name"sv, m_FileName); + if (m_DisableBlocks) + { + Writer.AddBool("disableblocks"sv, true); + } + if (m_FileForceEnableTempBlocks) + { + Writer.AddBool("enabletempblocks"sv, true); + } + } + Writer.EndObject(); // "file" + TargetDescription = fmt::format("[file] '{}/{}'", m_FileDirectoryPath, m_FileName); + } + if (!m_CloudUrl.empty()) + { + Writer.BeginObject("cloud"sv); + { + Writer.AddString("url"sv, m_CloudUrl); + Writer.AddString("namespace"sv, m_CloudNamespace); + Writer.AddString("bucket"sv, m_CloudBucket); + Writer.AddString("key"sv, m_CloudKey); + if (!m_CloudOpenIdProvider.empty()) + { + Writer.AddString("openid-provider"sv, m_CloudOpenIdProvider); + } + if (!m_CloudAccessToken.empty()) + { + Writer.AddString("access-token"sv, m_CloudAccessToken); + } + if (m_DisableBlocks) + { + Writer.AddBool("disableblocks"sv, true); + } + if (m_CloudDisableTempBlocks) + { + Writer.AddBool("disabletempblocks"sv, true); + } + } + Writer.EndObject(); // "cloud" + TargetDescription = fmt::format("[cloud] '{}/{}/{}/{}'", m_CloudUrl, m_CloudNamespace, m_CloudBucket, m_CloudKey); + } + if (!m_ZenUrl.empty()) + { + Writer.BeginObject("zen"sv); + { + Writer.AddString("url"sv, m_ZenUrl); + Writer.AddString("project"sv, m_ZenProjectName); + Writer.AddString("oplog"sv, m_ZenOplogName); + } + Writer.EndObject(); // "zen" + + TargetDescription = fmt::format("[zen] '{}/{}/{}'", m_ZenUrl, m_ZenProjectName, m_ZenOplogName); + } + } + Writer.EndObject(); // "params" + + zen::BinaryWriter MemOut; + Writer.Save(MemOut); + Session.SetBody(cpr::Body{(const char*)MemOut.GetData(), MemOut.GetSize()}); + + ZEN_CONSOLE("Saving oplog '{}/{}' from '{}' to {}", m_ProjectName, m_OplogName, m_HostName, TargetDescription); + cpr::Response Response = Session.Post(); + ZEN_CONSOLE("{}", FormatResponse(Response)); + return GetReturnCode(Response); +} + +//////////////////////////// + +ImportOplogCommand::ImportOplogCommand() +{ + m_Options.add_options()("h,help", "Print help"); + m_Options.add_option("", "u", "hosturl", "Host URL", cxxopts::value(m_HostName)->default_value("http://localhost:1337"), "<hosturl>"); + m_Options.add_option("", "p", "project", "Project name", cxxopts::value(m_ProjectName), "<projectid>"); + m_Options.add_option("", "o", "oplog", "Oplog name", cxxopts::value(m_OplogName), "<oplogid>"); + m_Options.add_option("", "", "maxblocksize", "Max size for bundled attachments", cxxopts::value(m_MaxBlockSize), "<blocksize>"); + m_Options.add_option("", + "", + "maxchunkembedsize", + "Max size for attachment to be bundled", + cxxopts::value(m_MaxChunkEmbedSize), + "<chunksize>"); + m_Options.add_option("", "f", "force", "Force import of all attachments", cxxopts::value(m_Force), "<force>"); + + m_Options.add_option("", "", "cloud", "Cloud Storage URL", cxxopts::value(m_CloudUrl), "<url>"); + m_Options.add_option("cloud", "", "namespace", "Cloud Storage namespace", cxxopts::value(m_CloudNamespace), "<namespace>"); + m_Options.add_option("cloud", "", "bucket", "Cloud Storage bucket", cxxopts::value(m_CloudBucket), "<bucket>"); + m_Options.add_option("cloud", "", "key", "Cloud Storage key", cxxopts::value(m_CloudKey), "<key>"); + m_Options + .add_option("cloud", "", "openid-provider", "Cloud Storage openid provider", cxxopts::value(m_CloudOpenIdProvider), "<provider>"); + m_Options.add_option("cloud", "", "access-token", "Cloud Storage access token", cxxopts::value(m_CloudAccessToken), "<accesstoken>"); + + m_Options.add_option("", "", "zen", "Zen service upload address", cxxopts::value(m_ZenUrl), "<url>"); + m_Options.add_option("zen", "", "source-project", "Zen source project name", cxxopts::value(m_ZenProjectName), "<sourceprojectid>"); + m_Options.add_option("zen", "", "source-oplog", "Zen source oplog name", cxxopts::value(m_ZenOplogName), "<sourceoplogid>"); + m_Options.add_option("zen", "", "clean", "Delete existing target Zen oplog", cxxopts::value(m_ZenClean), "<clean>"); + + m_Options.add_option("", "", "file", "Local folder path", cxxopts::value(m_FileDirectoryPath), "<path>"); + m_Options.add_option("file", "", "name", "Local file name", cxxopts::value(m_FileName), "<filename>"); + + m_Options.parse_positional({"project", "oplog"}); +} + +ImportOplogCommand::~ImportOplogCommand() = default; + +int +ImportOplogCommand::Run(const ZenCliOptions& GlobalOptions, int argc, char** argv) +{ + using namespace std::literals; + + ZEN_UNUSED(GlobalOptions); + + if (!ParseOptions(argc, argv)) + { + return 0; + } + + if (m_ProjectName.empty()) + { + ZEN_ERROR("Project name must be given"); + return 1; + } + + if (m_OplogName.empty()) + { + ZEN_ERROR("Oplog name must be given"); + return 1; + } + + size_t TargetCount = 0; + TargetCount += m_CloudUrl.empty() ? 0 : 1; + TargetCount += m_ZenUrl.empty() ? 0 : 1; + TargetCount += m_FileDirectoryPath.empty() ? 0 : 1; + if (TargetCount != 1) + { + ZEN_ERROR("Provide one source only"); + ZEN_CONSOLE("{}", m_Options.help({""}).c_str()); + return 1; + } + + cpr::Session Session; + + if (!m_CloudUrl.empty()) + { + if (m_CloudNamespace.empty() || m_CloudBucket.empty()) + { + ZEN_ERROR("Options for cloud source are missing"); + ZEN_CONSOLE("{}", m_Options.help({"cloud"}).c_str()); + return 1; + } + if (m_CloudKey.empty()) + { + std::string KeyString = fmt::format("{}/{}/{}/{}", m_ProjectName, m_OplogName, m_CloudNamespace, m_CloudBucket); + zen::IoHash Key = zen::IoHash::HashBuffer(KeyString.data(), KeyString.size()); + m_CloudKey = Key.ToHexString(); + ZEN_WARN("Using auto generated cloud key '{}'", m_CloudKey); + } + } + + if (!m_ZenUrl.empty()) + { + if (m_ZenProjectName.empty()) + { + m_ZenProjectName = m_ProjectName; + ZEN_WARN("Using default zen target project id '{}'", m_ZenProjectName); + } + if (m_ZenOplogName.empty()) + { + m_ZenOplogName = m_OplogName; + ZEN_WARN("Using default zen target oplog id '{}'", m_ZenOplogName); + } + } + + if (!m_FileDirectoryPath.empty()) + { + if (m_FileName.empty()) + { + m_FileName = m_OplogName; + ZEN_WARN("Using auto generated file name '{}'", m_FileName); + } + } + + const std::string TargetUrlBase = fmt::format("{}/prj", m_HostName); + Session.SetUrl({fmt::format("{}/{}/oplog/{}", TargetUrlBase, m_ProjectName, m_OplogName)}); + cpr::Response Response = Session.Get(); + if (Response.status_code == static_cast<long>(zen::HttpResponseCode::NotFound)) + { + ZEN_WARN("Automatically creating oplog '{}/{}'", m_ProjectName, m_OplogName) + Response = Session.Post(); + if (!zen::IsHttpSuccessCode(Response.status_code)) + { + ZEN_CONSOLE("{}", FormatResponse(Response)); + return GetReturnCode(Response); + } + } + else if (!zen::IsHttpSuccessCode(Response.status_code)) + { + ZEN_CONSOLE("{}", FormatResponse(Response)); + return GetReturnCode(Response); + } + else if (m_ZenClean) + { + ZEN_WARN("Cleaning oplog '{}/{}'", m_ProjectName, m_OplogName) + Response = Session.Delete(); + if (!zen::IsHttpSuccessCode(Response.status_code)) + { + ZEN_CONSOLE("{}", FormatResponse(Response)); + return GetReturnCode(Response); + } + Response = Session.Post(); + if (!zen::IsHttpSuccessCode(Response.status_code)) + { + ZEN_CONSOLE("{}", FormatResponse(Response)); + return GetReturnCode(Response); + } + } + + std::string SourceDescription; + Session.SetUrl(fmt::format("{}/{}/oplog/{}/rpc", TargetUrlBase, m_ProjectName, m_OplogName)); + Session.SetHeader({{"Content-Type", std::string(zen::MapContentTypeToString(zen::HttpContentType::kCbObject))}}); + + zen::CbObjectWriter Writer; + Writer.AddString("method"sv, "import"sv); + Writer.BeginObject("params"sv); + { + if (m_Force) + { + Writer.AddBool("force"sv, true); + } + if (!m_FileDirectoryPath.empty()) + { + Writer.BeginObject("file"sv); + { + Writer.AddString("file"sv, m_FileDirectoryPath); + Writer.AddString("name"sv, m_FileName); + } + Writer.EndObject(); // "file" + SourceDescription = fmt::format("[file] '{}/{}'", m_FileDirectoryPath, m_FileName); + } + if (!m_CloudUrl.empty()) + { + Writer.BeginObject("cloud"sv); + { + Writer.AddString("url"sv, m_CloudUrl); + Writer.AddString("namespace"sv, m_CloudNamespace); + Writer.AddString("bucket"sv, m_CloudBucket); + Writer.AddString("key"sv, m_CloudKey); + if (!m_CloudOpenIdProvider.empty()) + { + Writer.AddString("openid-provider"sv, m_CloudOpenIdProvider); + } + if (!m_CloudAccessToken.empty()) + { + Writer.AddString("access-token"sv, m_CloudAccessToken); + } + } + Writer.EndObject(); // "cloud" + SourceDescription = fmt::format("[cloud] '{}/{}/{}/{}'", m_CloudUrl, m_CloudNamespace, m_CloudBucket, m_CloudKey); + } + if (!m_ZenUrl.empty()) + { + Writer.BeginObject("zen"sv); + { + Writer.AddString("url"sv, m_ZenUrl); + Writer.AddString("project"sv, m_ZenProjectName); + Writer.AddString("oplog"sv, m_ZenOplogName); + } + Writer.EndObject(); // "zen" + SourceDescription = fmt::format("[zen] '{}'", m_ZenUrl); + } + } + Writer.EndObject(); // "params" + + zen::BinaryWriter MemOut; + Writer.Save(MemOut); + Session.SetBody(cpr::Body{(const char*)MemOut.GetData(), MemOut.GetSize()}); + + ZEN_CONSOLE("Loading oplog '{}/{}' from '{}' to {}", m_ProjectName, m_OplogName, SourceDescription, m_HostName); + Response = Session.Post(); + + ZEN_CONSOLE("{}", FormatResponse(Response)); + return GetReturnCode(Response); +} |