HIVE-5543: Running the mini tez cluster for tez unit tests (Vikram Di…

…xit K via Gunther Hagleitner) git-svn-id: https://svn.apache.org/repos/asf/hive/branches/tez@1536926 13f79535-47bb-0310-9956-ffa450edef68
tanbokan · Oct 29, 2013 · 1274f9a · 1274f9a
1 parent d89648c
commit 1274f9a
Show file tree

Hide file tree

Showing 21 changed files with 3,243 additions and 38 deletions.
diff --git a/build-common.xml b/build-common.xml
@@ -61,6 +61,7 @@
   <property name="test.junit.output.usefile" value="true"/>
   <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q"/>
   <property name="minimr.query.negative.files" value="cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q" />
+  <property name="minitez.query.files" value="tez_join_tests.q,tez_joins_explain.q"/>
   <property name="test.silent" value="true"/>
   <property name="hadoopVersion" value="${hadoop.version.ant-internal}"/>
   <property name="test.serialize.qplan" value="false"/>
@@ -203,6 +204,7 @@
     <fileset dir="${hive.root}/lib" includes="*.jar"/>
     <fileset dir="${build.ivy.lib.dir}/default" includes="junit*.jar" />
     <fileset dir="${build.ivy.lib.dir}/hadoop0.${hadoop.mr.rev}.shim" includes="*.jar" erroronmissingdir="false" />
+    <fileset dir="${build.ivy.lib.dir}/tez.shim" includes="*.jar" erroronmissingdir="false" />
     <fileset dir="${build.ivy.lib.dir}/default" includes="*.jar"
              erroronmissingdir="false"/>
   </path>

diff --git a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseQTestUtil.java
@@ -24,14 +24,15 @@
  */
 public class HBaseQTestUtil extends QTestUtil {
   public HBaseQTestUtil(
-    String outDir, String logDir, boolean miniMr, HBaseTestSetup setup)
+    String outDir, String logDir, String miniMr, HBaseTestSetup setup)
     throws Exception {
 
     super(outDir, logDir, miniMr, null);
     setup.preTest(conf);
     super.init();
   }
 
+  @Override
   public void init() throws Exception {
     // defer
   }

diff --git a/hbase-handler/src/test/templates/TestHBaseCliDriver.vm b/hbase-handler/src/test/templates/TestHBaseCliDriver.vm
@@ -43,7 +43,7 @@ public class $className extends TestCase {
   @Override
   protected void setUp() {
     try {
-      boolean miniMR = "$clusterMode".equals("miniMR");
+      String miniMR = "$clusterMode";
       qt = new HBaseQTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, setup);
 
     } catch (Exception e) {

diff --git a/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm b/hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm
@@ -43,7 +43,7 @@ public class $className extends TestCase {
   @Override
   protected void setUp() {
     try {
-      boolean miniMR = "$clusterMode".equals("miniMR");
+      String miniMR = "$clusterMode";
 
       qt = new HBaseQTestUtil((HIVE_ROOT + "$resultsDir"), (HIVE_ROOT + "$logDir"), miniMR, setup);
 

diff --git a/ql/build.xml b/ql/build.xml
@@ -35,7 +35,7 @@
   <property name="ql.test.results.beelinepositive.dir" location="${ql.test.results.dir}/beelinepositive"/>
 
   <import file="../build-common.xml"/>
-  <property name="ql.test.clientpositive.exclude" value="${minimr.query.files}"/>
+  <property name="ql.test.clientpositive.exclude" value="${minimr.query.files},${minitez.query.files}"/>
 
   <property name="ql.test.beelinepositive.exclude" value="${test.beelinepositive.exclude}"/>
 
@@ -97,7 +97,7 @@
                   resultsDirectory="${ql.test.results.beelinepositive.dir}" className="TestBeeLineDriver"
                   logFile="${test.log.dir}/testbeelinedrivergen.log"
                   logDirectory="${test.log.dir}/beelinepositive"
-                  hadoopVersion="${hadoopVersion}" />
+                  hadoopVersion="${hadoopVersion}"/>
       </then>
     </if>
 
@@ -135,6 +135,31 @@
                   logDirectory="${test.log.dir}/clientnegative"
                   hadoopVersion="${hadoopVersion}"
                   />
+
+        <if>
+          <equals arg1="${hadoop.mr.rev}" arg2="23"/>
+          <then>
+            <echo message="hadoop mr rev: ${hadoop.mr.rev}"/>
+            <qtestgen hiveRootDirectory="${hive.root}"
+                  outputDirectory="${test.build.src}/org/apache/hadoop/hive/cli" 
+                  templatePath="${ql.test.template.dir}" template="TestCliDriver.vm" 
+                  queryDirectory="${ql.test.query.clientpositive.dir}" 
+                  queryFile="${qfile}"
+                  includeQueryFile="${minitez.query.files}"
+                  queryFileRegex="${qfile_regex}"
+                  clusterMode="tez"
+                  runDisabled="${run_disabled}"
+                  resultsDirectory="${ql.test.results.clientpositive.dir}" className="TestMiniTezCliDriver"
+                  logFile="${test.log.dir}/testminimrclidrivergen.log"
+                  logDirectory="${test.log.dir}/clientpositive"
+                  hadoopVersion="23"
+                  />
+          </then>
+          <else>
+          </else>
+        </if>
+
+
       </then>
     </if>
 

diff --git a/ql/ivy.xml b/ql/ivy.xml
@@ -54,6 +54,7 @@
     <dependency org="org.apache.tez" name="tez-common" rev="${tez.version}" conf="compile->master" />
     <dependency org="org.apache.tez" name="tez-runtime-library" rev="${tez.version}" conf="compile->master" />
     <dependency org="org.apache.tez" name="tez-mapreduce" rev="${tez.version}" conf="compile->master" />
+    <dependency org="org.apache.tez" name="tez-runtime-internals" rev="${tez.version}" conf="compile->master" />
     <dependency org="org.apache.hadoop" name="hadoop-yarn-api" rev="${hadoop-0.23.version}" 
                 conf="compile->master" transitive="false" />
     <dependency org="org.apache.hadoop" name="hadoop-yarn-common" rev="${hadoop-0.23.version}" 

diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezSessionState.java
@@ -228,7 +228,8 @@ private LocalResource createHiveExecLocalResource()
 
         // jar wasn't in the directory, copy the one in current use
         if (jarPath == null) {
-          return DagUtils.localizeResource(new Path(currentVersionPathStr), hiveJarDirPath, conf);
+          Path dest = new Path(hiveJarDir + "/" + currentJarName);
+          return DagUtils.localizeResource(new Path(currentVersionPathStr), dest, conf);
         }
       }
     }

diff --git a/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java b/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -67,7 +67,6 @@
 import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.exec.Utilities.StreamPrinter;
 import org.apache.hadoop.hive.ql.exec.vector.util.AllVectorTypesRecord;
-import org.apache.hadoop.hive.ql.exec.vector.util.OrcFileGenerator;
 import org.apache.hadoop.hive.ql.io.IgnoreKeyTextOutputFormat;
 import org.apache.hadoop.hive.ql.lockmgr.zookeeper.ZooKeeperHiveLockManager;
 import org.apache.hadoop.hive.ql.metadata.Hive;
@@ -83,6 +82,7 @@
 import org.apache.hadoop.hive.serde2.thrift.ThriftDeserializer;
 import org.apache.hadoop.hive.serde2.thrift.test.Complex;
 import org.apache.hadoop.hive.shims.HadoopShims;
+import org.apache.hadoop.hive.shims.TezShims;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.mapred.SequenceFileInputFormat;
 import org.apache.hadoop.mapred.SequenceFileOutputFormat;
@@ -127,6 +127,7 @@ public class QTestUtil {
   private boolean miniMr = false;
   private String hadoopVer = null;
   private QTestSetup setup = null;
+  private boolean miniTez = false;
 
   public boolean deleteDirectory(File path) {
     if (path.exists()) {
@@ -195,7 +196,7 @@ public void normalizeNames(File path) throws Exception {
   }
 
   public QTestUtil(String outDir, String logDir) throws Exception {
-    this(outDir, logDir, false, "0.20");
+    this(outDir, logDir, "", "0.20");
   }
 
   public String getOutputDirectory() {
@@ -233,9 +234,8 @@ public void initConf() throws Exception {
     conf.setVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL,
         "org.apache.hadoop.hive.metastore.VerifyingObjectStore");
 
-    if (miniMr) {
+    if (mr != null) {
       assert dfs != null;
-      assert mr != null;
 
       mr.setupConfiguration(conf);
 
@@ -288,23 +288,47 @@ private String getHdfsUriString(String uriStr) {
     return uriStr;
   }
 
-  public QTestUtil(String outDir, String logDir, boolean miniMr, String hadoopVer)
+  public enum MiniClusterType {
+    miniMr,
+    tez
+  }
+
+  public QTestUtil(String outDir, String logDir, String miniMr, String hadoopVer)
     throws Exception {
     this.outDir = outDir;
     this.logDir = logDir;
     conf = new HiveConf(Driver.class);
-    this.miniMr = miniMr;
+    this.miniMr = miniMr.equals("miniMr");
+    this.miniTez = miniMr.equals("tez");
     this.hadoopVer = getHadoopMainVersion(hadoopVer);
     qMap = new TreeMap<String, String>();
     qSkipSet = new HashSet<String>();
     qSortSet = new HashSet<String>();
 
-    if (miniMr) {
-      dfs = ShimLoader.getHadoopShims().getMiniDfs(conf, 4, true, null);
-      FileSystem fs = dfs.getFileSystem();
-      mr = ShimLoader.getHadoopShims().getMiniMrCluster(conf, 4, getHdfsUriString(fs.getUri().toString()), 1);
+    HadoopShims shims = null;
+    MiniClusterType clusterType = MiniClusterType.valueOf(miniMr);
+    switch (clusterType) {
+      case miniMr:
+        shims = ShimLoader.getHadoopShims();
+        break;
+
+      case tez:
+        if (!hadoopVer.equals("23")) {
+          throw new Exception("Unsupported version of hadoop for tez. Please use " + 
+              "-Dhadoop.mr.rev=23 on the command line");
+        }
+
+        shims = (HadoopShims) new TezShims();
+        break;
+
+      default:
+        throw new Exception("Unknown cluster type");
     }
 
+    dfs = shims.getMiniDfs(conf, 4, true, null);
+    FileSystem fs = dfs.getFileSystem();
+    mr = shims.getMiniMrCluster(conf, 4, getHdfsUriString(fs.getUri().toString()), 1);
+
     initConf();
 
     // Use the current directory if it is not specified
@@ -1430,7 +1454,7 @@ public static QTestUtil[] queryListRunnerSetup(File[] qfiles, String resDir,
   {
     QTestUtil[] qt = new QTestUtil[qfiles.length];
     for (int i = 0; i < qfiles.length; i++) {
-      qt[i] = new QTestUtil(resDir, logDir, false, "0.20");
+      qt[i] = new QTestUtil(resDir, logDir, "", "0.20");
       qt[i].addFile(qfiles[i]);
       qt[i].clearTestSideEffects();
     }

diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestLocationQueries.java b/ql/src/test/org/apache/hadoop/hive/ql/TestLocationQueries.java
@@ -82,7 +82,7 @@ public int checkCliDriverResults(String tname) throws Exception {
       return failedCount;
     }
 
-    public CheckResults(String outDir, String logDir, boolean miniMr,
+    public CheckResults(String outDir, String logDir, String miniMr,
         String hadoopVer, String locationSubdir)
       throws Exception
     {
@@ -103,7 +103,7 @@ public void testAlterTablePartitionLocation_alter5() throws Exception {
 
     QTestUtil[] qt = new QTestUtil[qfiles.length];
     for (int i = 0; i < qfiles.length; i++) {
-      qt[i] = new CheckResults(resDir, logDir, false, "0.20", "parta");
+      qt[i] = new CheckResults(resDir, logDir, "non-minimr", "0.20", "parta");
       qt[i].addFile(qfiles[i]);
       qt[i].clearTestSideEffects();
     }

diff --git a/ql/src/test/queries/clientpositive/tez_join_tests.q b/ql/src/test/queries/clientpositive/tez_join_tests.q
@@ -0,0 +1,13 @@
+set hive.optimize.tez=true;
+explain
+select * from (select b.key, b.value from src1 a left outer join src b on (a.key = b.key) order by b.key) x right outer join src c on (x.value = c.value) order by x.key;
+
+select * from (select b.key, b.value from src1 a left outer join src b on (a.key = b.key) order by b.key) x right outer join src c on (x.value = c.value) order by x.key;
+select * from (select b.key, b.value from src1 a left outer join src b on (a.key = b.key)) x right outer join src c on (x.value = c.value) order by x.key;
+select * from src1 a left outer join src b on (a.key = b.key) right outer join src c on (a.value = c.value) order by a.key;
+select * from src1 a left outer join src b on (a.key = b.key) left outer join src c on (a.value = c.value) order by a.key;
+select * from src1 a left outer join src b on (a.key = b.key) join src c on (a.key = c.key);
+select * from src1 a join src b on (a.key = b.key) join src c on (a.key = c.key);
+
+select count(*) from src1 a join src b on (a.key = b.key) join src c on (a.key = c.key);
+
diff --git a/ql/src/test/queries/clientpositive/tez_joins_explain.q b/ql/src/test/queries/clientpositive/tez_joins_explain.q
@@ -0,0 +1,6 @@
+set hive.optimize.tez=true;
+explain
+select * from (select b.key, b.value from src1 a left outer join src b on (a.key = b.key) order by b.key) x right outer join src c on (x.value = c.value) order by x.key;
+
+select * from (select b.key, b.value from src1 a left outer join src b on (a.key = b.key) order by b.key) x right outer join src c on (x.value = c.value) order by x.key;
+