Use MAL to calculate JVM metrics, remove OAL dependency. (apache#7091)

CHANGES.md

@@ -12,6 +12,7 @@ Release Notes.
 
 #### OAP-Backend
 * Disable Spring sleuth meter analyzer by default.
+* Use MAL to calculate JVM metrics, remove OAL dependency.
 
 #### UI
 

apm-dist-es7/src/main/assembly/binary-es7.xml

@@ -67,6 +67,7 @@
                 <include>ui-initialized-templates/*</include>
                 <include>lal/*</include>
                 <include>log-mal-rules/*</include>
+                <include>jvm-metrics-rules/*.yaml</include>
             </includes>
             <outputDirectory>config</outputDirectory>
         </fileSet>

apm-dist/src/main/assembly/binary.xml

@@ -67,6 +67,7 @@
                 <include>ui-initialized-templates/*</include>
                 <include>lal/*</include>
                 <include>log-mal-rules/*</include>
+                <include>jvm-metrics-rules/*.yaml</include>
             </includes>
             <outputDirectory>config</outputDirectory>
         </fileSet>

docs/en/concepts-and-designs/scope-definitions.md

@@ -52,62 +52,6 @@ This calculates the metrics data from each request of the service instance.
 | tcpInfo.receivedBytes | The received bytes of the TCP traffic, if this request is a TCP call. | | long |
 | tcpInfo.sentBytes | The sent bytes of the TCP traffic, if this request is a TCP call. | | long |
 
-#### Secondary scopes of `ServiceInstance` 
-
-This calculates the metrics data if the service instance is a JVM and collects through javaagent.
-
-1. SCOPE `ServiceInstanceJVMCPU`
-
-| Name | Remarks | Group Key | Type | 
-|---|---|---|---|
-| name |  The name of the service instance, such as `ip:port@Service Name`.  **Note**: Currently, the native agent uses `uuid@ipv4` as the instance name, which does not assist in setting up a filter in aggregation. | | string|
-| serviceName | The name of the service. | | string |
-| usePercent | The percentage of CPU time spent.| | double|
-
-2. SCOPE `ServiceInstanceJVMMemory`
-
-| Name | Remarks | Group Key | Type | 
-|---|---|---|---|
-| name |  The name of the service instance, such as `ip:port@Service Name`.  **Note**: Currently, the native agent uses `uuid@ipv4` as the instance name, which does not assist in setting up a filter in aggregation. | | string|
-| serviceName | The name of the service. | | string |
-| heapStatus | Indicates whether the metric has a heap property or not. | | bool |
-| init | See the JVM documentation. | | long |
-| max | See the JVM documentation. | | long |
-| used | See the JVM documentation. | | long |
-| committed | See the JVM documentation. | | long |
-
-3. SCOPE `ServiceInstanceJVMMemoryPool`
-
-| Name | Remarks | Group Key | Type | 
-|---|---|---|---|
-| name |  The name of the service instance, such as `ip:port@Service Name`.  **Note**: Currently, the native agent uses `uuid@ipv4` as the instance name, which does not assist in setting up a filter in aggregation. | | string|
-| serviceName | The name of the service. | | string |
-| poolType | The type may be CODE_CACHE_USAGE, NEWGEN_USAGE, OLDGEN_USAGE, SURVIVOR_USAGE, PERMGEN_USAGE, or METASPACE_USAGE based on different versions of JVM. | | enum |
-| init | See the JVM documentation. | | long |
-| max | See the JVM documentation. | | long |
-| used | See the JVM documentation. | | long |
-| committed | See the JVM documentation. | | long |
-
-4. SCOPE `ServiceInstanceJVMGC`
-
-| Name | Remarks | Group Key | Type | 
-|---|---|---|---|
-| name |  The name of the service instance, such as `ip:port@Service Name`.  **Note**: Currently, the native agent uses `uuid@ipv4` as the instance name, which does not assist in setting up a filter in aggregation. | | string|
-| serviceName | The name of the service. | | string |
-| phrase | Includes both NEW and OLD. | | Enum |
-| time | The time spent in GC. | | long |
-| count | The count in GC operations. | | long |
-
-5. SCOPE `ServiceInstanceJVMThread`
-
-| Name | Remarks | Group Key | Type | 
-|---|---|---|---|
-| name |  The name of the service instance, such as `ip:port@Service Name`.  **Note**: Currently, the native agent uses `uuid@ipv4` as the instance name, which does not assist in setting up a filter in aggregation. | | string|
-| serviceName | The name of the service. | | string |
-| liveCount | The current number of live threads. | | int |
-| daemonCount | The current number of daemon threads. | | int |
-| peakCount | The current number of peak threads. | | int |
-
 ### SCOPE `Endpoint`
 
 This calculates the metrics data from each request of the endpoint in the service. 

oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4

@@ -34,11 +34,6 @@ SRC_ENDPOINT: 'Endpoint';
 SRC_SERVICE_RELATION: 'ServiceRelation';
 SRC_SERVICE_INSTANCE_RELATION: 'ServiceInstanceRelation';
 SRC_ENDPOINT_RELATION: 'EndpointRelation';
-SRC_SERVICE_INSTANCE_JVM_CPU: 'ServiceInstanceJVMCPU';
-SRC_SERVICE_INSTANCE_JVM_MEMORY: 'ServiceInstanceJVMMemory';
-SRC_SERVICE_INSTANCE_JVM_MEMORY_POOL: 'ServiceInstanceJVMMemoryPool';
-SRC_SERVICE_INSTANCE_JVM_GC: 'ServiceInstanceJVMGC';
-SRC_SERVICE_INSTANCE_JVM_THREAD: 'ServiceInstanceJVMThread';
 SRC_DATABASE_ACCESS: 'DatabaseAccess';
 SRC_SERVICE_INSTANCE_CLR_CPU: 'ServiceInstanceCLRCPU';
 SRC_SERVICE_INSTANCE_CLR_GC: 'ServiceInstanceCLRGC';

oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4

@@ -52,7 +52,6 @@ filterExpression
 source
     : SRC_ALL | SRC_SERVICE | SRC_DATABASE_ACCESS | SRC_SERVICE_INSTANCE | SRC_ENDPOINT |
       SRC_SERVICE_RELATION | SRC_SERVICE_INSTANCE_RELATION | SRC_ENDPOINT_RELATION |
-      SRC_SERVICE_INSTANCE_JVM_CPU | SRC_SERVICE_INSTANCE_JVM_MEMORY | SRC_SERVICE_INSTANCE_JVM_MEMORY_POOL | SRC_SERVICE_INSTANCE_JVM_GC | SRC_SERVICE_INSTANCE_JVM_THREAD |// JVM source of service instance
       SRC_SERVICE_INSTANCE_CLR_CPU | SRC_SERVICE_INSTANCE_CLR_GC | SRC_SERVICE_INSTANCE_CLR_THREAD |
       SRC_ENVOY_INSTANCE_METRIC |
       SRC_BROWSER_APP_PERF | SRC_BROWSER_APP_PAGE_PERF | SRC_BROWSER_APP_SINGLE_VERSION_PERF |

oap-server/server-bootstrap/pom.xml

@@ -285,6 +285,7 @@
                         <exclude>zabbix-rules/</exclude>
                         <exclude>lal/</exclude>
                         <exclude>log-mal-rules/</exclude>
+                        <exclude>jvm-metrics-rules/</exclude>
                     </excludes>
                 </configuration>
             </plugin>

oap-server/server-bootstrap/src/main/resources/jvm-metrics-rules/jvm.yaml

@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This will parse a textual representation of a duration. The formats
+# accepted are based on the ISO-8601 duration format {@code PnDTnHnMn.nS}
+# with days considered to be exactly 24 hours.
+# <p>
+# Examples:
+# <pre>
+#    "PT20.345S" -- parses as "20.345 seconds"
+#    "PT15M"     -- parses as "15 minutes" (where a minute is 60 seconds)
+#    "PT10H"     -- parses as "10 hours" (where an hour is 3600 seconds)
+#    "P2D"       -- parses as "2 days" (where a day is 24 hours or 86400 seconds)
+#    "P2DT3H4M"  -- parses as "2 days, 3 hours and 4 minutes"
+#    "P-6H3M"    -- parses as "-6 hours and +3 minutes"
+#    "-P6H3M"    -- parses as "-6 hours and -3 minutes"
+#    "-P-6H+3M"  -- parses as "+6 hours and -3 minutes"
+# </pre>
+expSuffix: instance(['service'], ['instance'])
+metricPrefix: instance_jvm
+metricsRules:
+  - name: cpu
+    exp: sw_jvm_cpu
+
+  - name: memory_heap
+    exp: sw_jvm_memory.tagEqual('memory_type', 'used', 'heap_status', 'true').avg(['service', 'instance'])
+  - name: memory_noheap
+    exp: sw_jvm_memory.tagEqual('memory_type', 'used', 'heap_status', 'false').avg(['service', 'instance'])
+  - name: memory_heap_max
+    exp: sw_jvm_memory.tagEqual('memory_type', 'max', 'heap_status', 'true').avg(['service', 'instance'])
+  - name: memory_noheap_max
+    exp: sw_jvm_memory.tagEqual('memory_type', 'max', 'heap_status', 'false').avg(['service', 'instance'])
+
+  - name: young_gc_time
+    exp: sw_jvm_gc_time.tagEqual('gc_phrase', 'new').sum(['service', 'instance'])
+  - name: old_gc_time
+    exp: sw_jvm_gc_time.tagEqual('gc_phrase', 'old').sum(['service', 'instance'])
+
+  - name: young_gc_count
+    exp: sw_jvm_gc_count.tagEqual('gc_phrase', 'new').sum(['service', 'instance'])
+  - name: old_gc_count
+    exp: sw_jvm_gc_count.tagEqual('gc_phrase', 'old').sum(['service', 'instance'])
+
+  - name: thread_live_count
+    exp: sw_jvm_thread.tagEqual('thread_type', 'live').avg(['service', 'instance'])
+  - name: thread_daemon_count
+    exp: sw_jvm_thread.tagEqual('thread_type', 'daemon').avg(['service', 'instance'])
+  - name: thread_peak_count
+    exp: sw_jvm_thread.tagEqual('thread_type', 'peak').avg(['service', 'instance'])