Merge pull request apache#2517 from b-slim/adding_lookup_snapshot_uti…

…lity

[QTL][Lookup] lookup module with the snapshot utility

docs/content/querying/lookups.md

@@ -27,6 +27,12 @@ and such data belongs in the raw denormalized data for use in Druid.
 Very small lookups (count of keys on the order of a few dozen to a few hundred) can be passed at query time as a "map" 
 lookup as per [dimension specs](../querying/dimensionspecs.html).
 
+## Configuration
+
+|Property|Description|Default|
+|--------|-----------|-------|
+|`druid.lookup.snapshotWorkingDir`| Working path used to store snapshot of current lookup configuration, leaving this property null will disable snapshot/bootstrap utility|null|
+
 Namespaced lookups are appropriate for lookups which are not possible to pass at query time due to their size, 
 or are not desired to be passed at query time because the data is to reside in and be handled by the Druid servers. 
 Namespaced lookups can be specified as part of the runtime properties file. The property is a list of the namespaces 

extensions-core/namespace-lookup/src/main/java/io/druid/query/extraction/NamespacedExtractor.java

@@ -27,6 +27,7 @@
 import com.google.common.base.Preconditions;
 import com.google.inject.name.Named;
 import com.metamx.common.StringUtils;
+import io.druid.query.lookup.LookupExtractor;
 
 import javax.validation.constraints.NotNull;
 import java.nio.ByteBuffer;

processing/src/main/java/io/druid/query/dimension/LookupDimensionSpec.java

@@ -26,9 +26,9 @@
 import com.google.common.base.Strings;
 import com.metamx.common.StringUtils;
 import io.druid.query.extraction.ExtractionFn;
-import io.druid.query.extraction.LookupExtractionFn;
-import io.druid.query.extraction.LookupExtractor;
-import io.druid.query.extraction.LookupReferencesManager;
+import io.druid.query.lookup.LookupExtractionFn;
+import io.druid.query.lookup.LookupExtractor;
+import io.druid.query.lookup.LookupReferencesManager;
 import io.druid.query.filter.DimFilterCacheHelper;
 import io.druid.segment.DimensionSelector;
 

processing/src/main/java/io/druid/query/extraction/ExtractionFn.java

@@ -21,6 +21,7 @@
 
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import io.druid.query.lookup.LookupExtractionFn;
 
 /**
  */

processing/src/main/java/io/druid/query/extraction/MapLookupExtractor.java

@@ -30,6 +30,7 @@
 import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.metamx.common.StringUtils;
+import io.druid.query.lookup.LookupExtractor;
 
 import javax.annotation.Nullable;
 import javax.validation.constraints.NotNull;

processing/src/main/java/io/druid/query/filter/ExtractionDimFilter.java

@@ -26,8 +26,8 @@
 import com.google.common.collect.Lists;
 import com.metamx.common.StringUtils;
 import io.druid.query.extraction.ExtractionFn;
-import io.druid.query.extraction.LookupExtractionFn;
-import io.druid.query.extraction.LookupExtractor;
+import io.druid.query.lookup.LookupExtractionFn;
+import io.druid.query.lookup.LookupExtractor;
 
 import java.nio.ByteBuffer;
 import java.util.List;

processing/src/main/java/io/druid/query/lookup/LookupBean.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.query.lookup;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+//TODO merge this code to the same definition when pr/1576 is merged
+class LookupBean
+{
+  @JsonProperty
+  LookupExtractorFactory factory;
+  @JsonProperty
+  String name;
+
+  @Override
+  public boolean equals(Object o)
+  {
+    if (this == o) {
+      return true;
+    }
+    if (!(o instanceof LookupBean)) {
+      return false;
+    }
+
+    LookupBean that = (LookupBean) o;
+
+    if (!factory.equals(that.factory)) {
+      return false;
+    }
+    return name.equals(that.name);
+
+  }
+}

processing/src/main/java/io/druid/query/lookup/LookupConfig.java

@@ -0,0 +1,65 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.query.lookup;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Strings;
+
+public class LookupConfig
+{
+
+  @JsonProperty
+  private final String snapshotWorkingDir;
+
+  /**
+   * @param snapshotWorkingDir working directory to store lookups snapshot file, passing null or empty string will disable the snapshot utility
+   */
+  @JsonCreator
+  public LookupConfig(
+      @JsonProperty("snapshotWorkingDir") String snapshotWorkingDir
+  )
+  {
+    this.snapshotWorkingDir = Strings.nullToEmpty(snapshotWorkingDir);
+  }
+
+  public String getSnapshotWorkingDir()
+  {
+    return snapshotWorkingDir;
+  }
+
+
+  @Override
+  public boolean equals(Object o)
+  {
+    if (this == o) {
+      return true;
+    }
+    if (!(o instanceof LookupConfig)) {
+      return false;
+    }
+
+    LookupConfig that = (LookupConfig) o;
+
+    return getSnapshotWorkingDir().equals(that.getSnapshotWorkingDir());
+
+  }
+
+}

processing/src/main/java/io/druid/query/extraction/LookupExtractionFn.java → processing/src/main/java/io/druid/query/lookup/LookupExtractionFn.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package io.druid.query.extraction;
+package io.druid.query.lookup;
 
 
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -26,6 +26,8 @@
 import com.google.common.base.Strings;
 import com.google.common.base.Throwables;
 import com.metamx.common.StringUtils;
+import io.druid.query.extraction.ExtractionCacheHelper;
+import io.druid.query.extraction.FunctionalExtraction;
 
 import javax.annotation.Nullable;
 import java.io.ByteArrayOutputStream;

processing/src/main/java/io/druid/query/extraction/LookupExtractor.java → processing/src/main/java/io/druid/query/lookup/LookupExtractor.java

@@ -17,11 +17,12 @@
  * under the License.
  */
 
-package io.druid.query.extraction;
+package io.druid.query.lookup;
 
 
 import com.fasterxml.jackson.annotation.JsonSubTypes;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
+import io.druid.query.extraction.MapLookupExtractor;
 
 import javax.annotation.Nullable;
 import javax.validation.constraints.NotNull;

processing/src/main/java/io/druid/query/extraction/LookupExtractorFactory.java → processing/src/main/java/io/druid/query/lookup/LookupExtractorFactory.java

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package io.druid.query.extraction;
+package io.druid.query.lookup;
 
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.google.common.base.Supplier;

processing/src/main/java/io/druid/query/lookup/LookupModule.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed to Metamarkets Group Inc. (Metamarkets) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. Metamarkets licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package io.druid.query.lookup;
+
+
+import com.fasterxml.jackson.databind.Module;
+import com.google.inject.Binder;
+import io.druid.guice.JsonConfigProvider;
+import io.druid.guice.LifecycleModule;
+import io.druid.initialization.DruidModule;
+
+import java.util.Collections;
+import java.util.List;
+
+public class LookupModule implements DruidModule
+{
+  private static final String PROPERTY_BASE = "druid.lookup";
+
+  @Override
+  public List<? extends Module> getJacksonModules()
+  {
+    return Collections.emptyList();
+  }
+
+  @Override
+  public void configure(Binder binder)
+  {
+    JsonConfigProvider.bind(binder, PROPERTY_BASE, LookupConfig.class);
+    LifecycleModule.register(binder, LookupReferencesManager.class);
+  }
+}