Skip to content

Commit

Permalink
feat(core): trace data type (vectordotdev#10483)
Browse files Browse the repository at this point in the history
  • Loading branch information
@prognant
prognant authored Feb 16, 2022
1 parent e3c5817 commit 93c08b3
Show file tree
Hide file tree
Showing 52 changed files with 649 additions and 193 deletions.
22 changes: 21 additions & 1 deletion Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,7 @@ arc-swap = { version = "1.5", default-features = false }
async-compression = { version = "0.3.7", default-features = false, features = ["tokio", "gzip", "zstd"] }
avro-rs = { version = "0.13.0", default-features = false, optional = true }
base64 = { version = "0.13.0", default-features = false, optional = true }
bitmask-enum = { version = "1.1.2", default-features = false }
bloom = { version = "0.3.2", default-features = false, optional = true }
bollard = { version = "0.11.1", default-features = false, features = ["ssl"], optional = true }
bytes = { version = "1.1.0", default-features = false, features = ["serde"] }
Expand Down Expand Up @@ -291,6 +292,8 @@ socket2 = { version = "0.4.4", default-features = false }
stream-cancel = { version = "0.8.1", default-features = false }
strip-ansi-escapes = { version = "0.1.1", default-features = false }
structopt = { version = "0.3.26", default-features = false }
strum = { version = "0.23", default-features = false }
strum_macros = { version = "0.23", default-features = false }
syslog = { version = "6.0.1", default-features = false, optional = true }
syslog_loose = { version = "0.16.0", default-features = false, optional = true }
tikv-jemallocator = { version = "0.4.1", default-features = false, optional = true }
Expand Down
6 changes: 3 additions & 3 deletions benches/remap.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ fn benchmark_remap(c: &mut Criterion) {

let add_fields_runner = |tform: &mut Box<dyn SyncTransform>, event: Event| {
let mut outputs =
TransformOutputsBuf::new_with_capacity(vec![Output::default(DataType::Any)], 1);
TransformOutputsBuf::new_with_capacity(vec![Output::default(DataType::all())], 1);
tform.transform(event, &mut outputs);
let result = outputs.take_primary();
let output_1 = result.first().unwrap().as_log();
Expand Down Expand Up @@ -101,7 +101,7 @@ fn benchmark_remap(c: &mut Criterion) {

let json_parser_runner = |tform: &mut Box<dyn SyncTransform>, event: Event| {
let mut outputs =
TransformOutputsBuf::new_with_capacity(vec![Output::default(DataType::Any)], 1);
TransformOutputsBuf::new_with_capacity(vec![Output::default(DataType::all())], 1);
tform.transform(event, &mut outputs);
let result = outputs.take_primary();
let output_1 = result.first().unwrap().as_log();
Expand Down Expand Up @@ -176,7 +176,7 @@ fn benchmark_remap(c: &mut Criterion) {
let coerce_runner =
|tform: &mut Box<dyn SyncTransform>, event: Event, timestamp: DateTime<Utc>| {
let mut outputs =
TransformOutputsBuf::new_with_capacity(vec![Output::default(DataType::Any)], 1);
TransformOutputsBuf::new_with_capacity(vec![Output::default(DataType::all())], 1);
tform.transform(event, &mut outputs);
let result = outputs.take_primary();
let output_1 = result.first().unwrap().as_log();
Expand Down
150 changes: 142 additions & 8 deletions lib/vector-api-client/graphql/schema.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2973,6 +2973,11 @@
"kind": "OBJECT",
"name": "EventNotification",
"ofType": null
},
{
"kind": "OBJECT",
"name": "Trace",
"ofType": null
}
]
},
Expand Down Expand Up @@ -4149,16 +4154,24 @@
"deprecationReason": null
},
{
"name": "outputType",
"name": "outputTypes",
"description": "Source output type",
"args": [],
"type": {
"kind": "NON_NULL",
"name": null,
"ofType": {
"kind": "ENUM",
"name": "SourceOutputType",
"ofType": null
"kind": "LIST",
"name": null,
"ofType": {
"kind": "NON_NULL",
"name": null,
"ofType": {
"kind": "ENUM",
"name": "SourceOutputType",
"ofType": null
}
}
}
},
"isDeprecated": false,
Expand Down Expand Up @@ -4469,19 +4482,19 @@
"interfaces": null,
"enumValues": [
{
"name": "ANY",
"name": "LOG",
"description": null,
"isDeprecated": false,
"deprecationReason": null
},
{
"name": "LOG",
"name": "METRIC",
"description": null,
"isDeprecated": false,
"deprecationReason": null
},
{
"name": "METRIC",
"name": "TRACE",
"description": null,
"isDeprecated": false,
"deprecationReason": null
Expand All @@ -4498,6 +4511,42 @@
{
"name": "equals",
"description": null,
"type": {
"kind": "LIST",
"name": null,
"ofType": {
"kind": "NON_NULL",
"name": null,
"ofType": {
"kind": "ENUM",
"name": "SourceOutputType",
"ofType": null
}
}
},
"defaultValue": null
},
{
"name": "notEquals",
"description": null,
"type": {
"kind": "LIST",
"name": null,
"ofType": {
"kind": "NON_NULL",
"name": null,
"ofType": {
"kind": "ENUM",
"name": "SourceOutputType",
"ofType": null
}
}
},
"defaultValue": null
},
{
"name": "contains",
"description": null,
"type": {
"kind": "ENUM",
"name": "SourceOutputType",
Expand All @@ -4506,7 +4555,7 @@
"defaultValue": null
},
{
"name": "notEquals",
"name": "notContains",
"description": null,
"type": {
"kind": "ENUM",
Expand Down Expand Up @@ -5676,6 +5725,91 @@
"enumValues": null,
"possibleTypes": null
},
{
"kind": "OBJECT",
"name": "Trace",
"description": "Log event with fields for querying log data",
"fields": [
{
"name": "componentId",
"description": "Id of the component associated with the log event",
"args": [],
"type": {
"kind": "NON_NULL",
"name": null,
"ofType": {
"kind": "SCALAR",
"name": "String",
"ofType": null
}
},
"isDeprecated": false,
"deprecationReason": null
},
{
"name": "string",
"description": "Trace event as an encoded string format",
"args": [
{
"name": "encoding",
"description": null,
"type": {
"kind": "NON_NULL",
"name": null,
"ofType": {
"kind": "ENUM",
"name": "EventEncodingType",
"ofType": null
}
},
"defaultValue": null
}
],
"type": {
"kind": "NON_NULL",
"name": null,
"ofType": {
"kind": "SCALAR",
"name": "String",
"ofType": null
}
},
"isDeprecated": false,
"deprecationReason": null
},
{
"name": "json",
"description": "Get JSON field data on the trace event, by field name",
"args": [
{
"name": "field",
"description": null,
"type": {
"kind": "NON_NULL",
"name": null,
"ofType": {
"kind": "SCALAR",
"name": "String",
"ofType": null
}
},
"defaultValue": null
}
],
"type": {
"kind": "SCALAR",
"name": "Json",
"ofType": null
},
"isDeprecated": false,
"deprecationReason": null
}
],
"inputFields": null,
"interfaces": [],
"enumValues": null,
"possibleTypes": null
},
{
"kind": "OBJECT",
"name": "Transform",
Expand Down
1 change: 1 addition & 0 deletions lib/vector-core/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ publish = false
async-graphql = { version = "3.0.29", default-features = false, optional = true }
async-trait = { version = "0.1", default-features = false }
atomig = { version = "0.3.3", features = ["derive", "serde"] }
bitmask-enum = { version = "1.1.2", default-features = false }
bytes = { version = "1.1.0", default-features = false, features = ["serde"] }
chrono = { version = "0.4.19", default-features = false, features = ["serde"] }
db-key = { version = "0.0.5", default-features = false, optional = true }
Expand Down
5 changes: 5 additions & 0 deletions lib/vector-core/proto/event.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,18 @@ message EventWrapper {
oneof event {
Log log = 1;
Metric metric = 2;
Trace trace = 3;
}
}

message Log {
map<string, Value> fields = 1;
}

message Trace {
map<string, Value> fields = 1;
}

message ValueMap {
map<string, Value> fields = 1;
}
Expand Down
Loading

0 comments on commit 93c08b3

Please sign in to comment.