aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/zencore/include/zencore/xxhash.h31
-rw-r--r--src/zencore/xxhash.cpp51
-rw-r--r--src/zencore/zencore.cpp5
-rw-r--r--src/zenserver-test/zenserver-test.cpp20
-rw-r--r--src/zenserver/projectstore/projectstore.cpp108
5 files changed, 189 insertions, 26 deletions
diff --git a/src/zencore/include/zencore/xxhash.h b/src/zencore/include/zencore/xxhash.h
index 94d98be98..1616e5f93 100644
--- a/src/zencore/include/zencore/xxhash.h
+++ b/src/zencore/include/zencore/xxhash.h
@@ -61,8 +61,10 @@ struct XXH3_128
struct XXH3_128Stream
{
+ XXH3_128Stream() { Reset(); }
+
/// Begin streaming hash compute (not needed on freshly constructed instance)
- void Reset() { memset(&m_State, 0, sizeof m_State); }
+ void Reset() { XXH3_128bits_reset(&m_State); }
/// Append another chunk
XXH3_128Stream& Append(const void* Data, size_t ByteCount)
@@ -83,6 +85,33 @@ struct XXH3_128Stream
}
private:
+ XXH3_state_s m_State;
+};
+
+struct XXH3_128Stream_deprecated
+{
+ /// Begin streaming hash compute (not needed on freshly constructed instance)
+ void Reset() { memset(&m_State, 0, sizeof m_State); }
+
+ /// Append another chunk
+ XXH3_128Stream_deprecated& Append(const void* Data, size_t ByteCount)
+ {
+ XXH3_128bits_update(&m_State, Data, ByteCount);
+ return *this;
+ }
+
+ /// Append another chunk
+ XXH3_128Stream_deprecated& Append(MemoryView Data) { return Append(Data.GetData(), Data.GetSize()); }
+
+ /// Obtain final hash. If you wish to reuse the instance call reset()
+ XXH3_128 GetHash()
+ {
+ XXH3_128 Hash;
+ XXH128_canonicalFromHash((XXH128_canonical_t*)Hash.Hash, XXH3_128bits_digest(&m_State));
+ return Hash;
+ }
+
+private:
XXH3_state_s m_State{};
};
diff --git a/src/zencore/xxhash.cpp b/src/zencore/xxhash.cpp
index 450131d19..6d1050531 100644
--- a/src/zencore/xxhash.cpp
+++ b/src/zencore/xxhash.cpp
@@ -47,4 +47,55 @@ XXH3_128::ToHexString(StringBuilderBase& OutBuilder) const
return OutBuilder;
}
+//////////////////////////////////////////////////////////////////////////
+//
+// Unit tests
+//
+
+#if ZEN_WITH_TESTS
+
+void
+xxhash_forcelink()
+{
+}
+
+TEST_CASE("XXH3_128")
+{
+ using namespace std::literals;
+
+ const std::string_view ShortString{"1234"};
+ const std::string_view LongString{
+ "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"};
+
+ SUBCASE("short_deprecated")
+ {
+ XXH3_128Stream_deprecated x;
+ x.Append(ShortString.data(), ShortString.size());
+ const XXH3_128 Hash = x.GetHash();
+ CHECK(Hash == XXH3_128::FromHexString("0d44dd7fde8ea2b4ba961e1a26f71f21"sv));
+ }
+
+ SUBCASE("short")
+ {
+ XXH3_128Stream x;
+ x.Append(ShortString.data(), ShortString.size());
+ const XXH3_128 Hash = x.GetHash();
+ CHECK(Hash == XXH3_128::FromHexString("9a4dea864648af82823c8c03e6dd2202"sv));
+ CHECK(Hash == XXH3_128::HashMemory(ShortString.data(), ShortString.size()));
+ }
+
+ SUBCASE("long")
+ {
+ XXH3_128Stream x;
+ x.Append(LongString.data(), LongString.size());
+ const XXH3_128 Hash = x.GetHash();
+ CHECK(Hash == XXH3_128::FromHexString("fbd5e72f7a5894590d1ef49dfcc58b7d"sv));
+ CHECK(Hash == XXH3_128::HashMemory(LongString.data(), LongString.size()));
+ }
+}
+
+#endif
+
} // namespace zen
diff --git a/src/zencore/zencore.cpp b/src/zencore/zencore.cpp
index e813a3e6c..db821bff8 100644
--- a/src/zencore/zencore.cpp
+++ b/src/zencore/zencore.cpp
@@ -36,6 +36,10 @@
#include <atomic>
+namespace zen {
+extern void xxhash_forcelink();
+}
+
namespace zen::assert {
void
@@ -264,6 +268,7 @@ zencore_forcelinktests()
zen::cbjson_forcelink();
zen::cbyaml_forcelink();
zen::workthreadpool_forcelink();
+ xxhash_forcelink();
}
} // namespace zen
diff --git a/src/zenserver-test/zenserver-test.cpp b/src/zenserver-test/zenserver-test.cpp
index 1eba769d2..ec288f1dc 100644
--- a/src/zenserver-test/zenserver-test.cpp
+++ b/src/zenserver-test/zenserver-test.cpp
@@ -2864,12 +2864,22 @@ TEST_CASE("project.remote")
ZEN_ASSERT(It == AttachmentSizes.end());
}
- auto AddOp = [](const CbObject& Op, std::unordered_map<Oid, uint32_t, Oid::Hasher>& Ops) {
- XXH3_128Stream KeyHasher;
+ // Note: This is a clone of the function in projectstore.cpp
+ auto ComputeOpKey = [](const CbObjectView& Op) -> Oid {
+ using namespace std::literals;
+
+ XXH3_128Stream_deprecated KeyHasher;
Op["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); });
- XXH3_128 KeyHash = KeyHasher.GetHash();
- Oid Id;
- memcpy(Id.OidBits, &KeyHash, sizeof Id.OidBits);
+ XXH3_128 KeyHash128 = KeyHasher.GetHash();
+
+ Oid KeyHash;
+ memcpy(&KeyHash, KeyHash128.Hash, sizeof KeyHash);
+
+ return KeyHash;
+ };
+
+ auto AddOp = [ComputeOpKey](const CbObject& Op, std::unordered_map<Oid, uint32_t, Oid::Hasher>& Ops) {
+ const Oid Id = ComputeOpKey(Op);
IoBuffer Buffer = Op.GetBuffer().AsIoBuffer();
const uint32_t OpCoreHash = uint32_t(XXH3_64bits(Buffer.GetData(), Buffer.GetSize()) & 0xffffFFFF);
Ops.insert({Id, OpCoreHash});
diff --git a/src/zenserver/projectstore/projectstore.cpp b/src/zenserver/projectstore/projectstore.cpp
index 63a80fbd8..ef2c2e4ab 100644
--- a/src/zenserver/projectstore/projectstore.cpp
+++ b/src/zenserver/projectstore/projectstore.cpp
@@ -407,6 +407,39 @@ namespace {
//////////////////////////////////////////////////////////////////////////
+Oid
+ComputeOpKey(const CbObjectView& Op)
+{
+ using namespace std::literals;
+
+ BinaryWriter KeyStream;
+
+ Op["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyStream.Write(Data, Size); });
+
+ XXH3_128 KeyHash128;
+
+ // This logic currently exists to work around a problem caused by misusing the xxhash
+ // functions in the past. Short keys are evaluated using the old and buggy
+ // path but longer paths are evaluated properly. In the future all key lengths
+ // should be evaluated using the proper path, this is a temporary workaround to
+ // maintain compatibility with existing disk state.
+ if (KeyStream.GetSize() < 240)
+ {
+ XXH3_128Stream_deprecated KeyHasher;
+ KeyHasher.Append(KeyStream.Data(), KeyStream.Size());
+ KeyHash128 = KeyHasher.GetHash();
+ }
+ else
+ {
+ KeyHash128 = XXH3_128::HashMemory(KeyStream.GetView());
+ }
+
+ Oid KeyHash;
+ memcpy(&KeyHash, KeyHash128.Hash, sizeof KeyHash);
+
+ return KeyHash;
+}
+
struct ProjectStore::OplogStorage : public RefCounted
{
OplogStorage(ProjectStore::Oplog* OwnerOplog, std::filesystem::path BasePath) : m_OwnerOplog(OwnerOplog), m_OplogStoragePath(BasePath)
@@ -900,10 +933,7 @@ struct ProjectStore::OplogStorage : public RefCounted
ZEN_ASSERT(WriteSize != 0);
- XXH3_128Stream KeyHasher;
- Core["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); });
- XXH3_128 KeyHash128 = KeyHasher.GetHash();
- memcpy(&OpData.KeyHash, KeyHash128.Hash, sizeof OpData.KeyHash);
+ OpData.KeyHash = ComputeOpKey(Core);
return OpData;
}
@@ -1107,11 +1137,7 @@ ProjectStore::Oplog::ScrubStorage(ScrubContext& Ctx)
Op.IterateAttachments([&](CbFieldView Visitor) { Cids.emplace_back(Visitor.AsAttachment()); });
{
- XXH3_128Stream KeyHasher;
- Op["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); });
- XXH3_128 KeyHash128 = KeyHasher.GetHash();
- Oid KeyHash;
- memcpy(&KeyHash, KeyHash128.Hash, sizeof KeyHash);
+ const Oid KeyHash = ComputeOpKey(Op);
ZEN_ASSERT_FORMAT(KeyHash == Key, "oplog data does not match information from index (op:{} != index:{})", KeyHash, Key);
}
@@ -2366,11 +2392,7 @@ ProjectStore::Oplog::GetAttachmentsLocked(std::vector<IoHash>& OutAttachments, b
Op.IterateAttachments([&](CbFieldView Visitor) { OutAttachments.emplace_back(Visitor.AsAttachment()); });
if (StoreMetaDataOnDisk)
{
- XXH3_128Stream KeyHasher;
- Op["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); });
- XXH3_128 KeyHash128 = KeyHasher.GetHash();
- Oid KeyHash;
- memcpy(&KeyHash, KeyHash128.Hash, sizeof KeyHash);
+ const Oid KeyHash = ComputeOpKey(Op);
Keys.push_back(KeyHash);
AttachmentCounts.push_back(gsl::narrow<uint32_t>(OutAttachments.size() - CurrentAttachmentCount));
}
@@ -6541,12 +6563,7 @@ OpKeyStringAsOid(std::string_view OpKey)
CbObjectWriter Writer;
Writer << "key"sv << OpKey;
- XXH3_128Stream KeyHasher;
- Writer.Save()["key"sv].WriteToStream([&](const void* Data, size_t Size) { KeyHasher.Append(Data, Size); });
- XXH3_128 KeyHash = KeyHasher.GetHash();
-
- Oid OpId;
- memcpy(OpId.OidBits, &KeyHash, sizeof(OpId.OidBits));
+ const Oid OpId = ComputeOpKey(Writer.Save());
return OpId;
}
@@ -6708,6 +6725,57 @@ namespace testutils {
} // namespace testutils
+TEST_CASE("project.opkeys")
+{
+ using namespace std::literals;
+
+ const std::string_view LongKey =
+ "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"sv;
+
+ // Not a test per se, this code just exercises the key computation logic to ensure all
+ // edge cases are handled by the bug workaround logic
+
+ for (int i = 1; i < 300; ++i)
+ {
+ CbObjectWriter Cbo;
+ Cbo << "key"sv << LongKey.substr(0, i);
+
+ const Oid KeyId = ComputeOpKey(Cbo.Save());
+ }
+
+ {
+ CbObjectWriter Cbo;
+ Cbo << "key"sv
+ << "abcdef";
+
+ const Oid KeyId = ComputeOpKey(Cbo.Save());
+ const Oid CorrectId = Oid::FromHexString(
+ "7a03540e"
+ "ecb0daa9"
+ "00f2949e");
+
+ CHECK(KeyId == CorrectId);
+ }
+
+ {
+ CbObjectWriter Cbo;
+ Cbo << "key"sv
+ << "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"
+ "1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890";
+
+ const Oid KeyId = ComputeOpKey(Cbo.Save());
+ const Oid CorrectId = Oid::FromHexString(
+ "c5e88c79"
+ "06b7fa38"
+ "7b0d2efd");
+
+ CHECK(KeyId == CorrectId);
+ }
+}
+
TEST_CASE("project.store.create")
{
using namespace std::literals;
generated by cgit v1.2.3 (git 2.25.1) at 2026-04-30 02:46:11 +0000