From 7e1cdeda85dc293efe8bd408444d7bbef8401be6 Mon Sep 17 00:00:00 2001 From: "Ma,Gang" Date: Thu, 18 Oct 2018 17:41:37 +0800 Subject: [PATCH] KYLIN-2894 add a new signature calculator FactTableRealizationSetCalculator --- .../apache/kylin/common/KylinConfigBase.java | 2 +- .../FactTableRealizationSetCalculator.java | 112 ++++++++++++++++++ .../rest/util/SQLResponseSignatureUtil.java | 6 +- 3 files changed, 116 insertions(+), 4 deletions(-) create mode 100644 server-base/src/main/java/org/apache/kylin/rest/signature/FactTableRealizationSetCalculator.java diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index 43c0831fc6e..b19f2e974fd 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -1555,7 +1555,7 @@ public String getQueryRealizationFilter() { public String getSQLResponseSignatureClass() { return this.getOptional("kylin.query.signature-class", - "org.apache.kylin.rest.signature.RealizationSetCalculator"); + "org.apache.kylin.rest.signature.FactTableRealizationSetCalculator"); } public boolean isQueryCacheSignatureEnabled() { diff --git a/server-base/src/main/java/org/apache/kylin/rest/signature/FactTableRealizationSetCalculator.java b/server-base/src/main/java/org/apache/kylin/rest/signature/FactTableRealizationSetCalculator.java new file mode 100644 index 00000000000..944cb1b2037 --- /dev/null +++ b/server-base/src/main/java/org/apache/kylin/rest/signature/FactTableRealizationSetCalculator.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.signature; + +import java.util.Set; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.cube.CubeManager; +import org.apache.kylin.metadata.model.DataModelDesc; +import org.apache.kylin.metadata.model.TableRef; +import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.metadata.project.RealizationEntry; +import org.apache.kylin.metadata.realization.IRealization; +import org.apache.kylin.metadata.realization.RealizationType; +import org.apache.kylin.storage.hybrid.HybridInstance; +import org.apache.kylin.storage.hybrid.HybridManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.Sets; + +public class FactTableRealizationSetCalculator extends RealizationSetCalculator { + + public static final Logger logger = LoggerFactory.getLogger(FactTableRealizationSetCalculator.class); + + /** + * In case that cube selection result changes after a new cube's data is ready, + * the cache result should be invalidated, which requires the related signature should be changed. + * To achieve this, we need to consider all of those cubes who shares the same fact table + */ + @Override + protected Set getRealizations(KylinConfig config, String cubes, ProjectInstance project) { + Set realizations = super.getRealizations(config, cubes, project); + if (realizations == null) { + return null; + } + Set factTables = Sets.newHashSet(); + for (String realName : realizations) { + IRealization realInstance = getRealization(config, realName); + String factTable = getRootFactTableForRealization(realInstance); + if (factTable != null) { + factTables.add(factTable); + } + } + + Set ret = Sets.newHashSet(realizations); + for (RealizationEntry entry : project.getRealizationEntries()) { + String realName = entry.getRealization(); + IRealization realInstance = getRealization(config, realName, entry.getType()); + String factTableForEntry = getRootFactTableForRealization(realInstance); + if (factTableForEntry != null) { + if (factTables.contains(factTableForEntry)) { + ret.add(realName); + } + } + } + return ret; + } + + private String getRootFactTableForRealization(IRealization realization) { + if (realization == null) { + logger.warn("Cannot find realization %s", realization); + return null; + } + DataModelDesc model = realization.getModel(); + if (model == null) { + logger.warn("The model for realization %s is null", realization.getName()); + return null; + } + TableRef rootFactTable = model.getRootFactTable(); + if (rootFactTable == null) { + logger.warn("The root table for model %s is null", model.getName()); + return null; + } + return rootFactTable.getTableIdentity(); + } + + private IRealization getRealization(KylinConfig config, String name, RealizationType type) { + switch (type) { + case CUBE: + return CubeManager.getInstance(config).getCube(name); + case HYBRID: + return HybridManager.getInstance(config).getHybridInstance(name); + default: + return getRealization(config, name); + } + } + + private IRealization getRealization(KylinConfig config, String name) { + HybridInstance hybridInstance = HybridManager.getInstance(config).getHybridInstance(name); + if (hybridInstance != null) { + return hybridInstance; + } + return CubeManager.getInstance(config).getCube(name); + } +} diff --git a/server-base/src/main/java/org/apache/kylin/rest/util/SQLResponseSignatureUtil.java b/server-base/src/main/java/org/apache/kylin/rest/util/SQLResponseSignatureUtil.java index c6d35073785..2a575546a72 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/util/SQLResponseSignatureUtil.java +++ b/server-base/src/main/java/org/apache/kylin/rest/util/SQLResponseSignatureUtil.java @@ -22,7 +22,7 @@ import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.project.ProjectManager; import org.apache.kylin.rest.response.SQLResponse; -import org.apache.kylin.rest.signature.RealizationSetCalculator; +import org.apache.kylin.rest.signature.FactTableRealizationSetCalculator; import org.apache.kylin.rest.signature.SignatureCalculator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,7 +52,7 @@ public static String createSignature(KylinConfig config, SQLResponse sqlResponse signatureCalculator = (SignatureCalculator) signatureClass.getConstructor().newInstance(); } catch (Exception e) { logger.warn("Will use default signature since fail to construct signature due to " + e); - signatureCalculator = new RealizationSetCalculator(); + signatureCalculator = new FactTableRealizationSetCalculator(); } return signatureCalculator.calculateSignature(config, sqlResponse, project); } @@ -62,7 +62,7 @@ private static Class getSignatureClass(KylinConfig config) { return Class.forName(config.getSQLResponseSignatureClass()); } catch (ClassNotFoundException e) { logger.warn("Will use default signature since cannot find class " + config.getSQLResponseSignatureClass()); - return RealizationSetCalculator.class; + return FactTableRealizationSetCalculator.class; } } }