add support for OTLP logging/tracing (#599)

- adds `zentelemetry` project which houses new functionality for serializing logs and traces in OpenTelemetry Protocol format (OTLP)
- moved existing stats functionality from `zencore` to `zentelemetry`
- adds `TRefCounted<T>` for vtable-less refcounting
- adds `MemoryArena` class which allows for linear allocation of memory from chunks
- adds `protozero` which is used to encode OTLP protobuf messages

1 files changed, 474 insertions, 0 deletions

diff --git a/src/zentelemetry/oteltraceprotozero.h b/src/zentelemetry/oteltraceprotozero.h
new file mode 100644
index 000000000..8b80c8f7f
--- /dev/null
+++ b/src/zentelemetry/oteltraceprotozero.h
@@ -0,0 +1,474 @@
+// Copyright Epic Games, Inc. All Rights Reserved.
+
+#pragma once
+
+#include "otelprotozero.h"
+
+//////////////////////////////////////////////////////////////////////////
+//
+//  OTEL .proto definitions, for reference
+//
+
+#if 0
+
+// clang-format off
+
+/////////////////////////////////////////////////////////////////////////
+// trace/v1/trace.proto
+//
+
+// TracesData represents the traces data that can be stored in a persistent storage,
+// OR can be embedded by other protocols that transfer OTLP traces data but do
+// not implement the OTLP protocol.
+//
+// The main difference between this message and collector protocol is that
+// in this message there will not be any "control" or "metadata" specific to
+// OTLP protocol.
+//
+// When new fields are added into this message, the OTLP request MUST be updated
+// as well.
+message TracesData {
+  // An array of ResourceSpans.
+  // For data coming from a single resource this array will typically contain
+  // one element. Intermediary nodes that receive data from multiple origins
+  // typically batch the data before forwarding further and in that case this
+  // array will contain multiple elements.
+  repeated ResourceSpans resource_spans = 1;
+}
+
+// A collection of ScopeSpans from a Resource.
+message ResourceSpans {
+  reserved 1000;
+
+  // The resource for the spans in this message.
+  // If this field is not set then no resource info is known.
+  opentelemetry.proto.resource.v1.Resource resource = 1;
+
+  // A list of ScopeSpans that originate from a resource.
+  repeated ScopeSpans scope_spans = 2;
+
+  // The Schema URL, if known. This is the identifier of the Schema that the resource data
+  // is recorded in. Notably, the last part of the URL path is the version number of the
+  // schema: http[s]://server[:port]/path/<version>. To learn more about Schema URL see
+  // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url
+  // This schema_url applies to the data in the "resource" field. It does not apply
+  // to the data in the "scope_spans" field which have their own schema_url field.
+  string schema_url = 3;
+}
+
+// A collection of Spans produced by an InstrumentationScope.
+message ScopeSpans {
+  // The instrumentation scope information for the spans in this message.
+  // Semantically when InstrumentationScope isn't set, it is equivalent with
+  // an empty instrumentation scope name (unknown).
+  opentelemetry.proto.common.v1.InstrumentationScope scope = 1;
+
+  // A list of Spans that originate from an instrumentation scope.
+  repeated Span spans = 2;
+
+  // The Schema URL, if known. This is the identifier of the Schema that the span data
+  // is recorded in. Notably, the last part of the URL path is the version number of the
+  // schema: http[s]://server[:port]/path/<version>. To learn more about Schema URL see
+  // https://opentelemetry.io/docs/specs/otel/schemas/#schema-url
+  // This schema_url applies to all spans and span events in the "spans" field.
+  string schema_url = 3;
+}
+
+// A Span represents a single operation performed by a single component of the system.
+//
+// The next available field id is 17.
+message Span {
+  // A unique identifier for a trace. All spans from the same trace share
+  // the same `trace_id`. The ID is a 16-byte array. An ID with all zeroes OR
+  // of length other than 16 bytes is considered invalid (empty string in OTLP/JSON
+  // is zero-length and thus is also invalid).
+  //
+  // This field is required.
+  bytes trace_id = 1;
+
+  // A unique identifier for a span within a trace, assigned when the span
+  // is created. The ID is an 8-byte array. An ID with all zeroes OR of length
+  // other than 8 bytes is considered invalid (empty string in OTLP/JSON
+  // is zero-length and thus is also invalid).
+  //
+  // This field is required.
+  bytes span_id = 2;
+
+  // trace_state conveys information about request position in multiple distributed tracing graphs.
+  // It is a trace_state in w3c-trace-context format: https://www.w3.org/TR/trace-context/#tracestate-header
+  // See also https://github.com/w3c/distributed-tracing for more details about this field.
+  string trace_state = 3;
+
+  // The `span_id` of this span's parent span. If this is a root span, then this
+  // field must be empty. The ID is an 8-byte array.
+  bytes parent_span_id = 4;
+
+  // Flags, a bit field.
+  //
+  // Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace
+  // Context specification. To read the 8-bit W3C trace flag, use
+  // `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`.
+  //
+  // See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions.
+  //
+  // Bits 8 and 9 represent the 3 states of whether a span's parent
+  // is remote. The states are (unknown, is not remote, is remote).
+  // To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`.
+  // To read whether the span is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`.
+  //
+  // When creating span messages, if the message is logically forwarded from another source
+  // with an equivalent flags fields (i.e., usually another OTLP span message), the field SHOULD
+  // be copied as-is. If creating from a source that does not have an equivalent flags field
+  // (such as a runtime representation of an OpenTelemetry span), the high 22 bits MUST
+  // be set to zero.
+  // Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero.
+  //
+  // [Optional].
+  fixed32 flags = 16;
+
+  // A description of the span's operation.
+  //
+  // For example, the name can be a qualified method name or a file name
+  // and a line number where the operation is called. A best practice is to use
+  // the same display name at the same call point in an application.
+  // This makes it easier to correlate spans in different traces.
+  //
+  // This field is semantically required to be set to non-empty string.
+  // Empty value is equivalent to an unknown span name.
+  //
+  // This field is required.
+  string name = 5;
+
+  // SpanKind is the type of span. Can be used to specify additional relationships between spans
+  // in addition to a parent/child relationship.
+  enum SpanKind {
+    // Unspecified. Do NOT use as default.
+    // Implementations MAY assume SpanKind to be INTERNAL when receiving UNSPECIFIED.
+    SPAN_KIND_UNSPECIFIED = 0;
+
+    // Indicates that the span represents an internal operation within an application,
+    // as opposed to an operation happening at the boundaries. Default value.
+    SPAN_KIND_INTERNAL = 1;
+
+    // Indicates that the span covers server-side handling of an RPC or other
+    // remote network request.
+    SPAN_KIND_SERVER = 2;
+
+    // Indicates that the span describes a request to some remote service.
+    SPAN_KIND_CLIENT = 3;
+
+    // Indicates that the span describes a producer sending a message to a broker.
+    // Unlike CLIENT and SERVER, there is often no direct critical path latency relationship
+    // between producer and consumer spans. A PRODUCER span ends when the message was accepted
+    // by the broker while the logical processing of the message might span a much longer time.
+    SPAN_KIND_PRODUCER = 4;
+
+    // Indicates that the span describes consumer receiving a message from a broker.
+    // Like the PRODUCER kind, there is often no direct critical path latency relationship
+    // between producer and consumer spans.
+    SPAN_KIND_CONSUMER = 5;
+  }
+
+  // Distinguishes between spans generated in a particular context. For example,
+  // two spans with the same name may be distinguished using `CLIENT` (caller)
+  // and `SERVER` (callee) to identify queueing latency associated with the span.
+  SpanKind kind = 6;
+
+  // The start time of the span. On the client side, this is the time
+  // kept by the local machine where the span execution starts. On the server side, this
+  // is the time when the server's application handler starts running.
+  // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970.
+  //
+  // This field is semantically required and it is expected that end_time >= start_time.
+  fixed64 start_time_unix_nano = 7;
+
+  // The end time of the span. On the client side, this is the time
+  // kept by the local machine where the span execution ends. On the server side, this
+  // is the time when the server application handler stops running.
+  // Value is UNIX Epoch time in nanoseconds since 00:00:00 UTC on 1 January 1970.
+  //
+  // This field is semantically required and it is expected that end_time >= start_time.
+  fixed64 end_time_unix_nano = 8;
+
+  // A collection of key/value pairs. Note, global attributes
+  // like server name can be set using the resource API. Examples of attributes:
+  //
+  //     "/http/user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
+  //     "/http/server_latency": 300
+  //     "example.com/myattribute": true
+  //     "example.com/score": 10.239
+  //
+  // Attribute keys MUST be unique (it is not allowed to have more than one
+  // attribute with the same key).
+  //
+  // The attribute values SHOULD NOT contain empty values.
+  // The attribute values SHOULD NOT contain bytes values.
+  // The attribute values SHOULD NOT contain array values different than array of string values, bool values, int values,
+  // double values.
+  // The attribute values SHOULD NOT contain kvlist values.
+  // The behavior of software that receives attributes containing such values can be unpredictable.
+  // These restrictions can change in a minor release.
+  // The restrictions take origin from the OpenTelemetry specification:
+  // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.47.0/specification/common/README.md#attribute.
+  repeated opentelemetry.proto.common.v1.KeyValue attributes = 9;
+
+  // The number of attributes that were discarded. Attributes
+  // can be discarded because their keys are too long or because there are too many
+  // attributes. If this value is 0, then no attributes were dropped.
+  uint32 dropped_attributes_count = 10;
+
+  // Event is a time-stamped annotation of the span, consisting of user-supplied
+  // text description and key-value pairs.
+  message Event {
+    // The time the event occurred.
+    fixed64 time_unix_nano = 1;
+
+    // The name of the event.
+    // This field is semantically required to be set to non-empty string.
+    string name = 2;
+
+    // A collection of attribute key/value pairs on the event.
+    // Attribute keys MUST be unique (it is not allowed to have more than one
+    // attribute with the same key).
+    //
+    // The attribute values SHOULD NOT contain empty values.
+    // The attribute values SHOULD NOT contain bytes values.
+    // The attribute values SHOULD NOT contain array values different than array of string values, bool values, int values,
+    // double values.
+    // The attribute values SHOULD NOT contain kvlist values.
+    // The behavior of software that receives attributes containing such values can be unpredictable.
+    // These restrictions can change in a minor release.
+    // The restrictions take origin from the OpenTelemetry specification:
+    // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.47.0/specification/common/README.md#attribute.
+    repeated opentelemetry.proto.common.v1.KeyValue attributes = 3;
+
+    // The number of dropped attributes. If the value is 0,
+    // then no attributes were dropped.
+    uint32 dropped_attributes_count = 4;
+  }
+
+  // A collection of Event items.
+  repeated Event events = 11;
+
+  // The number of dropped events. If the value is 0, then no
+  // events were dropped.
+  uint32 dropped_events_count = 12;
+
+  // A pointer from the current span to another span in the same trace or in a
+  // different trace. For example, this can be used in batching operations,
+  // where a single batch handler processes multiple requests from different
+  // traces or when the handler receives a request from a different project.
+  message Link {
+    // A unique identifier of a trace that this linked span is part of. The ID is a
+    // 16-byte array.
+    bytes trace_id = 1;
+
+    // A unique identifier for the linked span. The ID is an 8-byte array.
+    bytes span_id = 2;
+
+    // The trace_state associated with the link.
+    string trace_state = 3;
+
+    // A collection of attribute key/value pairs on the link.
+    // Attribute keys MUST be unique (it is not allowed to have more than one
+    // attribute with the same key).
+    //
+    // The attribute values SHOULD NOT contain empty values.
+    // The attribute values SHOULD NOT contain bytes values.
+    // The attribute values SHOULD NOT contain array values different than array of string values, bool values, int values,
+    // double values.
+    // The attribute values SHOULD NOT contain kvlist values.
+    // The behavior of software that receives attributes containing such values can be unpredictable.
+    // These restrictions can change in a minor release.
+    // The restrictions take origin from the OpenTelemetry specification:
+    // https://github.com/open-telemetry/opentelemetry-specification/blob/v1.47.0/specification/common/README.md#attribute.
+    repeated opentelemetry.proto.common.v1.KeyValue attributes = 4;
+
+    // The number of dropped attributes. If the value is 0,
+    // then no attributes were dropped.
+    uint32 dropped_attributes_count = 5;
+
+    // Flags, a bit field.
+    //
+    // Bits 0-7 (8 least significant bits) are the trace flags as defined in W3C Trace
+    // Context specification. To read the 8-bit W3C trace flag, use
+    // `flags & SPAN_FLAGS_TRACE_FLAGS_MASK`.
+    //
+    // See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions.
+    //
+    // Bits 8 and 9 represent the 3 states of whether the link is remote.
+    // The states are (unknown, is not remote, is remote).
+    // To read whether the value is known, use `(flags & SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK) != 0`.
+    // To read whether the link is remote, use `(flags & SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK) != 0`.
+    //
+    // Readers MUST NOT assume that bits 10-31 (22 most significant bits) will be zero.
+    // When creating new spans, bits 10-31 (most-significant 22-bits) MUST be zero.
+    //
+    // [Optional].
+    fixed32 flags = 6;
+  }
+
+  // A collection of Links, which are references from this span to a span
+  // in the same or different trace.
+  repeated Link links = 13;
+
+  // The number of dropped links after the maximum size was
+  // enforced. If this value is 0, then no links were dropped.
+  uint32 dropped_links_count = 14;
+
+  // An optional final status for this span. Semantically when Status isn't set, it means
+  // span's status code is unset, i.e. assume STATUS_CODE_UNSET (code = 0).
+  Status status = 15;
+}
+
+// The Status type defines a logical error model that is suitable for different
+// programming environments, including REST APIs and RPC APIs.
+message Status {
+  reserved 1;
+
+  // A developer-facing human readable error message.
+  string message = 2;
+
+  // For the semantics of status codes see
+  // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#set-status
+  enum StatusCode {
+    // The default status.
+    STATUS_CODE_UNSET               = 0;
+    // The Span has been validated by an Application developer or Operator to 
+    // have completed successfully.
+    STATUS_CODE_OK                  = 1;
+    // The Span contains an error.
+    STATUS_CODE_ERROR               = 2;
+  };
+
+  // The status code.
+  StatusCode code = 3;
+}
+
+// SpanFlags represents constants used to interpret the
+// Span.flags field, which is protobuf 'fixed32' type and is to
+// be used as bit-fields. Each non-zero value defined in this enum is
+// a bit-mask.  To extract the bit-field, for example, use an
+// expression like:
+//
+//   (span.flags & SPAN_FLAGS_TRACE_FLAGS_MASK)
+//
+// See https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions.
+//
+// Note that Span flags were introduced in version 1.1 of the
+// OpenTelemetry protocol.  Older Span producers do not set this
+// field, consequently consumers should not rely on the absence of a
+// particular flag bit to indicate the presence of a particular feature.
+enum SpanFlags {
+  // The zero value for the enum. Should not be used for comparisons.
+  // Instead use bitwise "and" with the appropriate mask as shown above.
+  SPAN_FLAGS_DO_NOT_USE = 0;
+
+  // Bits 0-7 are used for trace flags.
+  SPAN_FLAGS_TRACE_FLAGS_MASK = 0x000000FF;
+
+  // Bits 8 and 9 are used to indicate that the parent span or link span is remote.
+  // Bit 8 (`HAS_IS_REMOTE`) indicates whether the value is known.
+  // Bit 9 (`IS_REMOTE`) indicates whether the span or link is remote.
+  SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 0x00000100;
+  SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK = 0x00000200;
+
+  // Bits 10-31 are reserved for future use.
+}
+
+// clang-format on
+#endif
+
+namespace zen::otel::pbf {
+
+// Traces
+
+enum class TracesData : protozero::pbf_tag_type
+{
+	repeated_ResourceSpans_resource_spans = 1
+};
+
+enum class ResourceSpans : protozero::pbf_tag_type
+{
+	Resource_resource				= 1,
+	repeated_ScopeSpans_scope_spans = 2,
+	string_schema_url				= 3
+};
+
+enum class ScopeSpans : protozero::pbf_tag_type
+{
+	InstrumentationScope_scope = 1,
+	repeated_Span_spans		   = 2,
+	string_schema_url		   = 3
+};
+
+enum SpanFlags
+{
+	SPAN_FLAGS_DO_NOT_USE				  = 0,
+	SPAN_FLAGS_TRACE_FLAGS_MASK			  = 0x000000FF,
+	SPAN_FLAGS_CONTEXT_HAS_IS_REMOTE_MASK = 0x00000100,
+	SPAN_FLAGS_CONTEXT_IS_REMOTE_MASK	  = 0x00000200,
+};
+
+enum SpanKind
+{
+	SPAN_KIND_UNSPECIFIED = 0,
+	SPAN_KIND_INTERNAL	  = 1,
+	SPAN_KIND_SERVER	  = 2,
+	SPAN_KIND_CLIENT	  = 3,
+	SPAN_KIND_PRODUCER	  = 4,
+	SPAN_KIND_CONSUMER	  = 5
+};
+
+enum class Span_Event : protozero::pbf_tag_type
+{
+	fixed64_time_unix_nano			= 1,
+	string_name						= 2,
+	repeated_KeyValue_attributes	= 3,
+	uint32_dropped_attributes_count = 4
+};
+
+enum class Span_Link : protozero::pbf_tag_type
+{
+	bytes_trace_id					= 1,
+	bytes_span_id					= 2,
+	string_trace_state				= 3,
+	repeated_KeyValue_attributes	= 4,
+	uint32_dropped_attributes_count = 5,
+	fixed32_flags					= 6
+};
+
+enum class Span : protozero::pbf_tag_type
+{
+	required_bytes_trace_id				  = 1,
+	required_bytes_span_id				  = 2,
+	string_trace_state					  = 3,
+	bytes_parent_span_id				  = 4,
+	fixed32_flags						  = 16,
+	required_string_name				  = 5,
+	SpanKind_kind						  = 6,
+	required_fixed64_start_time_unix_nano = 7,
+	required_fixed64_end_time_unix_nano	  = 8,
+	repeated_KeyValue_attributes		  = 9,
+	uint32_dropped_attributes_count		  = 10,
+	repeated_Event_events				  = 11,
+	uint32_dropped_events_count			  = 12,
+	repeated_Link_links					  = 13,
+	uint32_dropped_links_count			  = 14,
+	Status_status						  = 15
+};
+
+enum StatusCode
+{
+	STATUS_CODE_UNSET = 0,
+	STATUS_CODE_OK	  = 1,
+	STATUS_CODE_ERROR = 2
+};
+
+enum class Status : protozero::pbf_tag_type
+{
+	string_message	= 2,
+	StatusCode_code = 3
+};
+
+}  // namespace zen::otel::pbf