Skip to content

Commit

Permalink
perf: optimize Groovy-based LAL DSL with static compilation (apache#7263
Browse files Browse the repository at this point in the history
)

Groovy naturally supports many dynamic features that we don't benefit for now but cost performance loss, in this patch we compile our Groovy-based DSL scripts statically to optimize performance.
  • Loading branch information
kezhenxu94 authored Jul 9, 2021
1 parent 58dce44 commit fd92f46
Show file tree
Hide file tree
Showing 23 changed files with 456 additions and 63 deletions.
1 change: 1 addition & 0 deletions CHANGES.md
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ Release Notes.
* Upgrade etcd cluster coordinator and dynamic configuration to v3.x.
* Configuration: Allow to configure server maximum request header size.
* Add thread state metric and class loaded info metric to JVMMetric.
* Performance: compile LAL DSL statically and run with type checked.

#### UI

Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ docker.agent: $(SW_ROOT)/docker/agent/Dockerfile.agent
# 4. This rule runs $(BUILD_PRE) prior to any docker build and only if specified as a dependency variable
# 5. This rule finally runs docker build passing $(BUILD_ARGS) to docker if they are specified as a dependency variable

DOCKER_RULE=time (mkdir -p $(DOCKER_BUILD_TOP)/$@ && cp -r $^ $(DOCKER_BUILD_TOP)/$@ && cd $(DOCKER_BUILD_TOP)/$@ && $(BUILD_PRE) docker build $(BUILD_ARGS) -t $(HUB)/$(subst docker.,,$@):$(TAG) -f Dockerfile$(suffix $@) .)
DOCKER_RULE=time (mkdir -p $(DOCKER_BUILD_TOP)/$@ && cp -r $^ $(DOCKER_BUILD_TOP)/$@ && cd $(DOCKER_BUILD_TOP)/$@ && $(BUILD_PRE) docker build --no-cache $(BUILD_ARGS) -t $(HUB)/$(subst docker.,,$@):$(TAG) -f Dockerfile$(suffix $@) .)

# for each docker.XXX target create a push.docker.XXX target that pushes
# the local docker image to another hub
Expand Down
2 changes: 1 addition & 1 deletion dist-material/release-docs/LICENSE
Original file line number Diff line number Diff line change
Expand Up @@ -455,7 +455,7 @@ Apache 2.0 licenses
========================================
echarts 5.0.2: https://github.com/apache/echarts Apache-2.0
Material Icons 3.0.1 https://github.com/google/material-design-icons Apache-2.0
groovy 3.0.3 https://github.com/apache/groovy Apache-2.0
groovy 3.0.8 https://github.com/apache/groovy Apache-2.0

========================================
BSD licenses
Expand Down
2 changes: 1 addition & 1 deletion oap-server-bom/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@
<freemarker.version>2.3.28</freemarker.version>
<javaassist.version>3.25.0-GA</javaassist.version>
<vavr.version>0.10.3</vavr.version>
<groovy.version>3.0.3</groovy.version>
<groovy.version>3.0.8</groovy.version>
<mvel.version>2.4.8.Final</mvel.version>
<commons-beanutils.version>1.9.4</commons-beanutils.version>
<flatbuffers-java.version>1.12.0</flatbuffers-java.version>
Expand Down
4 changes: 4 additions & 0 deletions oap-server/analyzer/log-analyzer/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,10 @@
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
</dependencies>

</project>
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,20 @@
package org.apache.skywalking.oap.log.analyzer.dsl;

import groovy.lang.GroovyShell;
import groovy.transform.CompileStatic;
import groovy.util.DelegatingScript;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import org.apache.skywalking.oap.log.analyzer.dsl.spec.LALDelegatingScript;
import org.apache.skywalking.oap.log.analyzer.dsl.spec.filter.FilterSpec;
import org.apache.skywalking.oap.log.analyzer.provider.LogAnalyzerModuleConfig;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
import org.apache.skywalking.oap.server.library.module.ModuleStartException;
import org.codehaus.groovy.control.CompilerConfiguration;
import org.codehaus.groovy.control.customizers.ASTTransformationCustomizer;

import static java.util.Collections.singletonList;
import static java.util.Collections.singletonMap;

@RequiredArgsConstructor(access = AccessLevel.PRIVATE)
public class DSL {
Expand All @@ -38,7 +44,16 @@ public static DSL of(final ModuleManager moduleManager,
final LogAnalyzerModuleConfig config,
final String dsl) throws ModuleStartException {
final CompilerConfiguration cc = new CompilerConfiguration();
cc.setScriptBaseClass(DelegatingScript.class.getName());
final ASTTransformationCustomizer customizer =
new ASTTransformationCustomizer(
singletonMap(
"extensions",
singletonList(LALPrecompiledExtension.class.getName())
),
CompileStatic.class
);
cc.addCompilationCustomizers(customizer);
cc.setScriptBaseClass(LALDelegatingScript.class.getName());

final GroovyShell sh = new GroovyShell(cc);
final DelegatingScript script = (DelegatingScript) sh.parse(dsl);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package org.apache.skywalking.oap.log.analyzer.dsl;

import org.apache.skywalking.apm.network.logging.v3.LogData;
import org.apache.skywalking.apm.network.logging.v3.LogDataBody;
import org.apache.skywalking.apm.network.logging.v3.LogTags;
import org.apache.skywalking.apm.network.logging.v3.TraceContext;
import org.codehaus.groovy.ast.expr.ConstantExpression;
import org.codehaus.groovy.ast.expr.Expression;
import org.codehaus.groovy.ast.expr.PropertyExpression;
import org.codehaus.groovy.ast.expr.VariableExpression;
import org.codehaus.groovy.transform.stc.AbstractTypeCheckingExtension;
import org.codehaus.groovy.transform.stc.StaticTypeCheckingVisitor;

import static org.codehaus.groovy.ast.ClassHelper.makeCached;

public class LALPrecompiledExtension extends AbstractTypeCheckingExtension {

public LALPrecompiledExtension(final StaticTypeCheckingVisitor typeCheckingVisitor) {
super(typeCheckingVisitor);
}

@Override
public boolean handleUnresolvedProperty(final PropertyExpression pexp) {
final Expression exp = pexp.getObjectExpression();

if (exp.getText().startsWith("parsed")) {
makeDynamic(pexp);
setHandled(true);
return true;
}

if (exp.getText().startsWith("log")) {
if (handleLogVariable(pexp)) {
return true;
}
}

return super.handleUnresolvedProperty(pexp);
}

private boolean handleLogVariable(final PropertyExpression pexp) {
final Expression exp = pexp.getObjectExpression();
final Expression p = pexp.getProperty();

if (exp instanceof VariableExpression) {
final VariableExpression v = (VariableExpression) exp;
if (v.getName().equals("log")) {
storeType(v, makeCached(LogData.Builder.class));
}
if (p instanceof ConstantExpression) {
final ConstantExpression c = (ConstantExpression) p;
switch (c.getText()) {
case "body":
storeType(pexp, makeCached(LogDataBody.class));
break;
case "traceContext":
storeType(pexp, makeCached(TraceContext.class));
break;
case "tags":
storeType(pexp, makeCached(LogTags.class));
break;
}
}
setHandled(true);
return true;
}

return false;
}
}

Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package org.apache.skywalking.oap.log.analyzer.dsl.spec;

import groovy.lang.Closure;
import groovy.lang.DelegatesTo;
import groovy.util.DelegatingScript;
import org.apache.skywalking.oap.log.analyzer.dsl.spec.filter.FilterSpec;

public class LALDelegatingScript extends DelegatingScript {
@Override
public Object run() {
return null;
}

public void filter(@DelegatesTo(value = FilterSpec.class, strategy = Closure.DELEGATE_ONLY) Closure<?> closure) {
closure.setDelegate(getDelegate());
closure.call();
}

public void json(@DelegatesTo(value = FilterSpec.class) Closure<?> closure) {
closure.setDelegate(getDelegate());
closure.call();
}

public void text(@DelegatesTo(value = FilterSpec.class) Closure<?> closure) {
closure.setDelegate(getDelegate());
closure.call();
}

public void extractor(@DelegatesTo(value = FilterSpec.class) Closure<?> closure) {
closure.setDelegate(getDelegate());
closure.call();
}

public void sink(@DelegatesTo(value = FilterSpec.class) Closure<?> closure) {
closure.setDelegate(getDelegate());
closure.call();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableMap;
import groovy.lang.Closure;
import groovy.lang.DelegatesTo;
import java.util.Collection;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -54,7 +55,8 @@ public ExtractorSpec(final ModuleManager moduleManager,
final LogAnalyzerModuleConfig moduleConfig) throws ModuleStartException {
super(moduleManager, moduleConfig);

final MeterSystem meterSystem = moduleManager.find(CoreModule.NAME).provider().getService(MeterSystem.class);
final MeterSystem meterSystem =
moduleManager.find(CoreModule.NAME).provider().getService(MeterSystem.class);

metricConverts = moduleConfig.malConfigs()
.stream()
Expand Down Expand Up @@ -93,7 +95,7 @@ public void endpoint(final String endpoint) {
}

@SuppressWarnings("unused")
public void tag(final Map<String, Object> kv) {
public void tag(final Map<String, ?> kv) {
if (BINDING.get().shouldAbort()) {
return;
}
Expand All @@ -108,7 +110,8 @@ public void tag(final Map<String, Object> kv) {
kv.entrySet()
.stream()
.filter(it -> isNotBlank(it.getKey()))
.filter(it -> nonNull(it.getValue()) && isNotBlank(Objects.toString(it.getValue())))
.filter(it -> nonNull(it.getValue()) &&
isNotBlank(Objects.toString(it.getValue())))
.map(it -> {
final Object val = it.getValue();
String valStr = Objects.toString(val);
Expand Down Expand Up @@ -176,7 +179,7 @@ public void timestamp(final String timestamp) {
}

@SuppressWarnings("unused")
public void metrics(final Closure<Void> cl) {
public void metrics(@DelegatesTo(SampleBuilder.class) final Closure<?> cl) {
if (BINDING.get().shouldAbort()) {
return;
}
Expand All @@ -188,8 +191,8 @@ public void metrics(final Closure<Void> cl) {

metricConverts.forEach(it -> it.toMeter(
ImmutableMap.<String, SampleFamily>builder()
.put(sample.getName(), SampleFamilyBuilder.newBuilder(sample).build())
.build()
.put(sample.getName(), SampleFamilyBuilder.newBuilder(sample).build())
.build()
));
}

Expand All @@ -198,11 +201,15 @@ public static class SampleBuilder {
private final Sample.SampleBuilder sampleBuilder = Sample.builder();

@SuppressWarnings("unused")
public Sample.SampleBuilder labels(final Map<String, String> labels) {
final Map<String, String> filtered = labels.entrySet()
.stream()
.filter(it -> isNotBlank(it.getKey()) && isNotBlank(it.getValue()))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
public Sample.SampleBuilder labels(final Map<String, ?> labels) {
final Map<String, String> filtered =
labels.entrySet()
.stream()
.filter(it -> isNotBlank(it.getKey()) && nonNull(it.getValue()))
.collect(
Collectors.toMap(Map.Entry::getKey,
it -> Objects.toString(it.getValue()))
);
return sampleBuilder.labels(ImmutableMap.copyOf(filtered));
}
}
Expand Down
Loading

0 comments on commit fd92f46

Please sign in to comment.