Add support for otel_with_suffixes metricSource (grafana#1331)

* Add otel_with_suffixes for golang

* add otel_with_suffixes for process-observ-lib

* add otel_with_suffixes for jvm-observ-lib

golang-observ-lib/README.md

@@ -12,9 +12,9 @@ jb install https://github.com/grafana/jsonnet-libs/golang-observ-lib
 
 ## Examples
 
-Prometheus metrics' source:
+Prometheus metrics' source (`prometheus`):
 
 <img width="1737" alt="image" src="https://github.com/grafana/jsonnet-libs/assets/14870891/f1587d2f-b79b-4284-a4a8-e45b4e0ad183">
 
-OTel metrics' source:
+OTel metrics' source (`otel` or `otel_with_suffixes`):
 <img width="2047" alt="image" src="https://github.com/grafana/jsonnet-libs/assets/14870891/8df6283f-5ec7-427d-b99d-b19b8aa78d97">

golang-observ-lib/config.libsonnet

@@ -19,7 +19,8 @@
   dashboardRefresh: '1m',
 
   // otel: https://pkg.go.dev/go.opentelemetry.io/contrib/instrumentation/runtime
+  // otel_with_suffixes: when add_metric_suffixes=true in otelcollector
   // or prometheus
-  metricsSource: ['prometheus'],
+  metricsSource: ['prometheus', 'otel', 'otel_with_suffixes'],
   signals: (import './signals/golang.libsonnet')(this),
 }

golang-observ-lib/dashboards.libsonnet

@@ -20,7 +20,10 @@ local g = import './g.libsonnet';
         + g.dashboard.withPanels(
           g.util.grid.wrapPanels(
             g.util.panel.resolveCollapsedFlagOnRows(
-              std.objectValues(this.grafana.rows)
+              [
+                this.grafana.rows.golangRuntime,
+                this.grafana.rows.golangMemory,
+              ],
             )
           )
         )

golang-observ-lib/mixin.libsonnet

@@ -5,7 +5,6 @@ local golang =
   + golanglib.withConfigMixin(
     {
       filteringSelector: 'job!=""',
-      metricsSource: ['otel', 'prometheus'],
     }
   );
 

golang-observ-lib/signals/golang.libsonnet

@@ -3,6 +3,7 @@ function(this)
     discoveryMetric: {
       prometheus: 'go_info',
       otel: 'process_runtime_go_goroutines',
+      otel_with_suffixes: self.otel,
     },
     filteringSelector: this.filteringSelector,
     groupLabels: this.groupLabels,
@@ -24,6 +25,9 @@ function(this)
           otel: {
             expr: 'runtime_uptime{%(queriesSelector)s}/1000',
           },
+          otel_with_suffixes: {
+            expr: 'runtime_uptime_milliseconds_total{%(queriesSelector)s}/1000',
+          },
         },
       },
       version: {
@@ -54,6 +58,7 @@ function(this)
             {
               expr: 'process_runtime_go_goroutines{%(queriesSelector)s}',
             },
+          otel_with_suffixes: self.otel,
         },
       },
       cgoCalls: {
@@ -67,6 +72,7 @@ function(this)
           otel: {
             expr: 'process_runtime_go_cgo_calls{%(queriesSelector)s}',
           },
+          otel_with_suffixes: self.otel,
         },
       },
       // gc duration:
@@ -119,6 +125,7 @@ function(this)
           otel: {
             expr: 'process_runtime_go_gc_pause_ns_bucket{%(queriesSelector)s}',
           },
+          otel_with_suffixes: self.otel,
         },
       },
       goThreads: {
@@ -192,6 +199,9 @@ function(this)
           otel: {
             expr: 'process_runtime_go_mem_heap_alloc{%(queriesSelector)s}',
           },
+          otel_with_suffixes: {
+            expr: 'process_runtime_go_mem_heap_alloc_bytes{%(queriesSelector)s}',
+          },
         },
       },
       memHeapIdleBytes: {
@@ -208,6 +218,10 @@ function(this)
           otel: {
             expr: 'process_runtime_go_mem_heap_idle{%(queriesSelector)s}',
           },
+          otel_with_suffixes: {
+            expr: 'process_runtime_go_mem_heap_idle_bytes{%(queriesSelector)s}',
+          },
+
         },
       },
       memHeapInUseBytes: {
@@ -224,6 +238,9 @@ function(this)
           otel: {
             expr: 'process_runtime_go_mem_heap_inuse{%(queriesSelector)s}',
           },
+          otel_with_suffixes: {
+            expr: 'process_runtime_go_mem_heap_inuse_bytes{%(queriesSelector)s}',
+          },
         },
       },
       memHeapReleasedBytes: {
@@ -240,6 +257,9 @@ function(this)
           otel: {
             expr: 'process_runtime_go_mem_heap_released{%(queriesSelector)s}',
           },
+          otel_with_suffixes: {
+            expr: 'process_runtime_go_mem_heap_released_bytes{%(queriesSelector)s}',
+          },
         },
       },
 
@@ -258,6 +278,7 @@ function(this)
           otel: {
             expr: 'process_runtime_go_mem_heap_objects{%(queriesSelector)s}',
           },
+          otel_with_suffixes: self.otel,
         },
       },
 

jvm-observ-lib/README.md

@@ -6,6 +6,7 @@ Supports the following sources:
 
 - `prometheus` (https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-memory-metrics). This also works for jmx_exporter (javaagent mode) starting from 1.0.1 release.
 - `otel` (https://github.com/open-telemetry/opentelemetry-java-contrib/blob/main/jmx-metrics/docs/target-systems/jvm.md)
+- `otel_with_suffixes` same as otel with add_metric_suffixes=true in otelcollector
 - `java_micrometer` (springboot) (https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmMemoryMetrics.java)
 - `prometheus_old` client_java instrumentation prior to 1.0.0 release: (https://github.com/prometheus/client_java/releases/tag/v1.0.0-alpha-4). This also works for jmx_exporter (javaagent mode) prior to 1.0.1 release.
 - `jmx_exporter`. Works with jmx_exporter (both http and javaagent modes) and the folllowing snippet:

jvm-observ-lib/config.libsonnet

@@ -6,7 +6,7 @@
   uid: 'jvm',
   dashboardNamePrefix: '',
   dashboardTags: ['java', 'jvm'],
-  metricsSource: ['java_micrometer', 'prometheus', 'prometheus_old', 'otel', 'jmx_exporter'],
+  metricsSource: ['java_micrometer', 'prometheus', 'prometheus_old', 'otel', 'otel_with_suffixes', 'jmx_exporter'],
   signals+:
     {
       memory: (import './signals/memory.libsonnet')(this),

jvm-observ-lib/main.libsonnet

@@ -21,6 +21,7 @@ local processlib = import 'process-observ-lib/main.libsonnet';
           metricsSource:
             []
             + (if std.member(this.config.metricsSource, 'otel') then ['java_otel'] else [])
+            + (if std.member(this.config.metricsSource, 'otel_with_suffixes') then ['java_otel_with_suffixes'] else [])
             + (if std.member(this.config.metricsSource, 'prometheus') then ['prometheus'] else [])
             + (if std.member(this.config.metricsSource, 'jmx_exporter') then ['jmx_exporter'] else [])
             + (if std.member(this.config.metricsSource, 'prometheus_old') then ['prometheus'] else [])

jvm-observ-lib/signals/buffers.libsonnet

@@ -11,6 +11,7 @@ function(this)
       java_micrometer: 'jvm_buffer_memory_used_bytes',  // https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/JvmMemoryMetrics.java
       prometheus: 'jvm_buffer_pool_used_bytes',  // https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-buffer-pool-metrics
       otel: 'process_runtime_jvm_buffer_usage',
+      otel_with_suffixes: 'process_runtime_jvm_buffer_usage_bytes',
       prometheus_old: 'jvm_buffer_pool_used_bytes',
     },
     signals: {
@@ -30,6 +31,9 @@ function(this)
           otel: {
             expr: 'process_runtime_jvm_buffer_usage{pool="direct", %(queriesSelector)s}',
           },
+          otel_with_suffixes: {
+            expr: 'process_runtime_jvm_buffer_usage_bytes{pool="direct", %(queriesSelector)s}',
+          },
           prometheus_old: {
             expr: 'jvm_buffer_pool_used_bytes{pool="direct", %(queriesSelector)s}',
           },
@@ -51,6 +55,9 @@ function(this)
           otel: {
             expr: 'process_runtime_jvm_buffer_limit{pool="direct", %(queriesSelector)s}',
           },
+          otel_with_suffixes: {
+            expr: 'process_runtime_jvm_buffer_limit_bytes{pool="direct", %(queriesSelector)s}',
+          },
           prometheus_old: {
             expr: 'jvm_buffer_pool_capacity_bytes{pool="direct", %(queriesSelector)s}',
           },
@@ -72,6 +79,9 @@ function(this)
           otel: {
             expr: 'process_runtime_jvm_buffer_usage{pool="mapped", %(queriesSelector)s}',
           },
+          otel_with_suffixes: {
+            expr: 'process_runtime_jvm_buffer_usage_bytes{pool="mapped", %(queriesSelector)s}',
+          },
           prometheus_old: {
             expr: 'jvm_buffer_pool_used_bytes{pool="mapped", %(queriesSelector)s}',
           },
@@ -93,6 +103,9 @@ function(this)
           otel: {
             expr: 'process_runtime_jvm_buffer_limit{pool="mapped", %(queriesSelector)s}',
           },
+          otel_with_suffixes: {
+            expr: 'process_runtime_jvm_buffer_limit_bytes{pool="mapped", %(queriesSelector)s}',
+          },
           prometheus_old: {
             expr: 'jvm_buffer_pool_capacity_bytes{pool="mapped", %(queriesSelector)s}',
           },

jvm-observ-lib/signals/classes.libsonnet

@@ -11,6 +11,7 @@ function(this)
       java_micrometer: 'jvm_classes_loaded_classes',  // https://github.com/micrometer-metrics/micrometer/blob/main/micrometer-core/src/main/java/io/micrometer/core/instrument/binder/jvm/ClassLoaderMetrics.java
       prometheus: 'jvm_classes_loaded',  // https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-class-loading-metrics
       otel: 'process_runtime_jvm_classes_loaded',
+      otel_with_suffixes: 'process_runtime_jvm_classes_loaded_total',
       prometheus_old: 'jvm_classes_loaded',
       jmx_exporter: 'java_lang_classloading_loadedclasscount',
     },
@@ -30,6 +31,9 @@ function(this)
           otel: {
             expr: 'process_runtime_jvm_classes_loaded{%(queriesSelector)s}',
           },
+          otel_with_suffixes: {
+            expr: 'process_runtime_jvm_classes_loaded_total{%(queriesSelector)s}',
+          },
           prometheus_old: {
             expr: 'jvm_classes_loaded{%(queriesSelector)s}',
           },

jvm-observ-lib/signals/gc.libsonnet

@@ -10,6 +10,7 @@ function(this)
       prometheus: 'jvm_memory_used_bytes',  // https://prometheus.github.io/client_java/instrumentation/jvm/#jvm-memory-metrics
       prometheus_old: 'jvm_memory_bytes_used',
       otel: 'process_runtime_jvm_memory_usage',  //https://opentelemetry.io/docs/specs/semconv/runtime/jvm-metrics/
+      otel_with_suffixes: 'process_runtime_jvm_memory_usage_bytes',
     },
     signals: {
 
@@ -48,6 +49,9 @@ function(this)
               actionLabelPrettify,
             ],
           },
+          otel_with_suffixes: self.otel {
+            expr: 'process_runtime_jvm_gc_duration_seconds_count{%(queriesSelector)s}',
+          },
         },
       },
       //gc
@@ -117,6 +121,11 @@ function(this)
               actionLabelPrettify,
             ],
           },
+          otel_with_suffixes: self.otel {
+            expr: |||
+              histogram_quantile(0.95, sum(rate(process_runtime_jvm_gc_duration_seconds_bucket{%(queriesSelector)s}[$__rate_interval])) by (le,%(agg)s,action,gc))
+            |||,
+          },
         },
       },
 
@@ -166,6 +175,7 @@ function(this)
             |||,
             legendCustomTemplate: '{{ %(agg)s }}: Allocated ({{ thread_name }}, {{ arena }})',
           },
+          otel_with_suffixes: self.otel,
         },
       },
 
@@ -191,6 +201,9 @@ function(this)
           otel: {
             expr: 'process_runtime_jvm_memory_usage{type="heap", pool=~"(G1 |PS )?Eden Space", %(queriesSelector)s}',
           },
+          otel_with_suffixes: {
+            expr: 'process_runtime_jvm_memory_usage_bytes{type="heap", pool=~"(G1 |PS )?Eden Space", %(queriesSelector)s}',
+          },
         },
       },
       memoryMaxEden: {
@@ -227,6 +240,9 @@ function(this)
               ['', ' != -1'],
             ],
           },
+          otel_with_suffixes: self.otel {
+            expr: 'process_runtime_jvm_memory_limit_bytes{pool=~"(G1 |PS )?Eden Space", type="heap", %(queriesSelector)s}',
+          },
         },
       },
       memoryCommittedEden: {
@@ -250,6 +266,9 @@ function(this)
           otel: {
             expr: 'process_runtime_jvm_memory_committed{pool=~"(G1 |PS )?Eden Space", type="heap", %(queriesSelector)s}',
           },
+          otel_with_suffixes: {
+            expr: 'process_runtime_jvm_memory_committed_bytes{pool=~"(G1 |PS )?Eden Space", type="heap", %(queriesSelector)s}',
+          },
         },
       },
 
@@ -275,6 +294,9 @@ function(this)
           otel: {
             expr: 'process_runtime_jvm_memory_usage{type="heap", pool=~"(G1 |PS )?Survivor Space", %(queriesSelector)s}',
           },
+          otel_with_suffixes: {
+            expr: 'process_runtime_jvm_memory_usage_bytes{type="heap", pool=~"(G1 |PS )?Survivor Space", %(queriesSelector)s}',
+          },
         },
       },
       memoryMaxSurvival: {
@@ -305,6 +327,9 @@ function(this)
           otel: {
             expr: 'process_runtime_jvm_memory_limit{pool=~"(G1 |PS )?Survivor Space", type="heap", %(queriesSelector)s} != -1',
           },
+          otel_with_suffixes: {
+            expr: 'process_runtime_jvm_memory_limit_bytes{pool=~"(G1 |PS )?Survivor Space", type="heap", %(queriesSelector)s} != -1',
+          },
         },
       },
       memoryCommittedSurvival: {
@@ -328,6 +353,9 @@ function(this)
           otel: {
             expr: 'process_runtime_jvm_memory_committed{pool=~"(G1 |PS )?Survivor Space", type="heap", %(queriesSelector)s}',
           },
+          otel_with_suffixes: {
+            expr: 'process_runtime_jvm_memory_committed_bytes{pool=~"(G1 |PS )?Survivor Space", type="heap", %(queriesSelector)s}',
+          },
         },
       },
 
@@ -352,6 +380,9 @@ function(this)
           otel: {
             expr: 'process_runtime_jvm_memory_usage{type="heap", pool=~"G1 Old Gen|Tenured Gen", %(queriesSelector)s}',
           },
+          otel_with_suffixes: {
+            expr: 'process_runtime_jvm_memory_usage_bytes{type="heap", pool=~"G1 Old Gen|Tenured Gen", %(queriesSelector)s}',
+          },
         },
       },
       memoryMaxTenured: {
@@ -371,20 +402,32 @@ function(this)
         optional: true,
         sources: {
           java_micrometer: {
-            expr: 'jvm_memory_max_bytes{id="Tenured Gen", area="heap", %(queriesSelector)s} != -1',
+            expr: 'jvm_memory_max_bytes{id="Tenured Gen", area="heap", %(queriesSelector)s}',
+            exprWrappers: [
+              ['', ' != -1'],
+            ],
           },
           prometheus: {
-            expr: 'jvm_memory_pool_used_bytes{pool="PS Old Gen", %(queriesSelector)s} != -1',
+            expr: 'jvm_memory_pool_used_bytes{pool="PS Old Gen", %(queriesSelector)s}',
+            exprWrappers: [
+              ['', ' != -1'],
+            ],
           },
           prometheus_old: {
-            expr: 'jvm_memory_pool_bytes_max{pool="G1 Old Gen", %(queriesSelector)s} != -1',
+            expr: 'jvm_memory_pool_bytes_max{pool="G1 Old Gen", %(queriesSelector)s}',
+            exprWrappers: [
+              ['', ' != -1'],
+            ],
           },
           otel: {
-            expr: 'process_runtime_jvm_memory_limit{pool=~"G1 Old Gen|Tenured Gen", type="heap", %(queriesSelector)s} != -1',
+            expr: 'process_runtime_jvm_memory_limit{pool=~"G1 Old Gen|Tenured Gen", type="heap", %(queriesSelector)s}',
             exprWrappers: [
               ['', ' != -1'],
             ],
           },
+          otel_with_suffixes: self.otel {
+            expr: 'process_runtime_jvm_memory_limit_bytes{pool=~"G1 Old Gen|Tenured Gen", type="heap", %(queriesSelector)s}',
+          },
         },
       },
       memoryCommittedTenured: {
@@ -408,6 +451,9 @@ function(this)
           otel: {
             expr: 'process_runtime_jvm_memory_committed{pool=~"G1 Old Gen|Tenured Gen", type="heap", %(queriesSelector)s}',
           },
+          otel_with_suffixes: {
+            expr: 'process_runtime_jvm_memory_committed_bytes{pool=~"G1 Old Gen|Tenured Gen", type="heap", %(queriesSelector)s}',
+          },
         },
       },
     },