HUE-449. tarball build changes after mavenization

- Preserve the java-lib directory in prod tarballs.
- Exclude the java source directory from prod tarballs.
- Exclude the mvn `target' directory from release builds.
- Added top-level `ext-clean' target, to clean the ext-pys.
- App clean should invoke `mvn clean' when applicable.

Makefile

@@ -130,7 +130,7 @@ docs:
 # Development use crepo to fetch thirdparty dependencies.
 
 .PHONY: crepo
-crepo: $(THIRDPARTY_JS_DIR)/manifest.json $(THIRDPARTY_JS_DIR)/*.hash parent-pom
+crepo: $(THIRDPARTY_JS_DIR)/manifest.json $(THIRDPARTY_JS_DIR)/*.hash
 	@echo "--- Synchronizing external dependencies with crepo"
 	@mkdir -p $(BLD_DIR)
 	@cd $(THIRDPARTY_JS_DIR) && $(CREPO) sync && \
@@ -164,7 +164,7 @@ $(BLD_DIR_ENV)/stamp:
 .PHONY: desktop
 
 # <<<< DEV ONLY
-desktop: crepo
+desktop: crepo parent-pom
 # END DEV ONLY >>>>
 desktop: virtual-env
 	@$(MAKE) -C desktop
@@ -257,6 +257,11 @@ distclean: clean
 	@$(MAKE) -C desktop distclean
 	@$(MAKE) -C apps distclean
 
+.PHONY: ext-clean
+ext-clean:
+	@$(MAKE) -C desktop ext-clean
+	@$(MAKE) -C apps ext-clean
+
 # <<<< DEV ONLY
 ###############################################
 # Misc (some used by automated test scripts)

Makefile.sdk

@@ -179,9 +179,11 @@ COMMON_EXCLUDES := \
 	  --exclude=.\*.sw[op] \
 	  --exclude=~\* \
 	  --exclude=.gitignore \
-	  --exclude=tag
+	  --exclude=tag \
+	  --exclude=target
 
-SDIST_EXCLUDES += $(COMMON_EXCLUDES)
+
+SDIST_EXCLUDES += $(COMMON_EXCLUDES) --exclude=java-lib
 BDIST_EXCLUDES += $(COMMON_EXCLUDES) --exclude=ext-py
 
 SDIST_DIR := $(BUILD_DIR)/sdist/$(APP_FULL_NAME)

Makefile.tarball

@@ -24,22 +24,29 @@ PROD_INCLUDES := \
 	ext/thirdparty \
 	LICENSE.txt \
 	Makefile* \
+	maven \
 	tools/app_reg \
 	tools/virtual-bootstrap \
-    maven \
 	VERSION_DATA
 
+# Check for things in BDIST_EXCLUDES in various apps
 PROD_EXCLUDES := \
-        Makefile.tarball \
-	desktop/apps/jframegallery \
-	desktop/apps/beeswax \
+	apps/beeswax/java \
+	apps/beeswax/thrift \
+	apps/hello \
+	apps/jframegallery \
+	apps/jobsub/src/jobsub/java \
+	build \
 	desktop/conf \
 	desktop/desktop.db \
-	desktop/apps/hello \
-	desktop/apps/visualizer \
 	desktop/devtools.mk \
+	desktop/libs/hadoop/java \
+	desktop/libs/hadoop/regenerate-thrift.sh \
+	desktop/libs/hadoop/static-group-mapping/src \
+	desktop/libs/hadoop/sudo-shell/src \
+	ext/thirdparty/js/manifest.json \
 	ext/thirdparty/README.md \
-	ext/thirdparty/js/manifest.json
+        Makefile.tarball
 
 # Macro to remove things we don't want to package
 define remove_devtree_exclusions
@@ -48,7 +55,10 @@ define remove_devtree_exclusions
 		      -name '.*~' -o \
 		      -name '.#*' -o \
 		      -name '.pylintrc' -o \
+		      -name 'build' -o \
+		      -name 'logs' -o \
 		      -name 'tags' -o \
+		      -name 'target' -o \
 		      -name 'desktop.db' -o \
 		      -name 'app.reg' -o \
 		      -name '*.py[co]' -o \
@@ -69,7 +79,7 @@ $(BLD_DIR_PROD_TGZ): $(BLD_DIR_PROD)
 	@echo "--- Generated $(BLD_DIR_PROD_TGZ)"
 
 .PHONY: $(BLD_DIR_PROD)
-$(BLD_DIR_PROD): crepo docs
+$(BLD_DIR_PROD): apps crepo docs
 	@echo "--- Preparing general distribution tree at $@"
 	@rm -rf $@
 	@mkdir -p $@
@@ -89,6 +99,8 @@ $(BLD_DIR_PROD): crepo docs
 	@$(call STRIP_DEV, $(ROOT)/Makefile.vars.priv, $(BLD_DIR_PROD)/Makefile.vars.priv)
 	@$(call STRIP_DEV, $(APPS_DIR)/Makefile, $(BLD_DIR_PROD)/apps/Makefile)
 	@$(call STRIP_DEV, $(DESKTOP_DIR)/Makefile, $(BLD_DIR_PROD)/desktop/Makefile)
-	@cd $(BLD_DIR_PROD) && make distclean
+	@echo "---- Removing build directories"
+	find $(BLD_DIR_PROD) -name build -prune -exec rm -rf {} \;
+	@$(MAKE) -C $(BLD_DIR_PROD) ext-clean
 
 # END DEV ONLY >>>>

apps/Makefile

@@ -83,6 +83,7 @@ install-source-parts:
 ################################################
 CLEAN_TARGETS := $(APPS:%=.recursive-clean/%)
 DISTCLEAN_TARGETS := $(APPS:%=.recursive-distclean/%)
+EXT_CLEAN_TARGETS := $(APPS:%=.recursive-ext-clean/%)
 
 clean: $(CLEAN_TARGETS)
 .recursive-clean/%:
@@ -93,3 +94,7 @@ distclean: $(DISTCLEAN_TARGETS)
 .recursive-distclean/%:
 	@$(MAKE) -C $* distclean
 	@$(APP_REG) --remove $* ||:
+
+ext-clean: $(EXT_CLEAN_TARGETS)
+.recursive-ext-clean/%:
+	@$(MAKE) -C $* ext-clean

apps/beeswax/Makefile

@@ -42,10 +42,10 @@ compile: $(BEESWAX)
 
 ifneq (,$(HAVE_JAVA_SRC))
 
-# If we have the source, remove it for clean
+# If we have the source, remove the java-lib
 clean::
 	rm -Rf $(BEESWAX_JAVA_LIB)
-
+	cd $(BEESWAX_JAVA_DIR) && mvn clean || :
 
 BEESWAX_OPTS := -Dbuild.dir=$(BLD_DIR_BEESWAX) \
 	        -Dhadoop.home=$(HADOOP_HOME) \
@@ -61,7 +61,6 @@ $(BEESWAX):
 	$(error Cannot build beeswax jars without source)
 endif
 
-
 #
 # Tell the `bdist' target to exclude our java source.
 #

desktop/Makefile

@@ -100,6 +100,10 @@ DISTCLEAN_TARGETS := $(APPS:%=.recursive-distclean/%)
 .recursive-distclean/%: %
 	$(MAKE) -C $< distclean
 
+EXT_CLEAN_TARGETS := $(APPS:%=.recursive-ext-clean/%)
+.recursive-ext-clean/%: %
+	$(MAKE) -C $< ext-clean
+
 .PHONY: env-install
 env-install: $(ENV_INSTALL_TARGETS)
 
@@ -110,6 +114,9 @@ clean: $(CLEAN_TARGETS)
 .PHONY: distclean
 distclean: clean $(DISTCLEAN_TARGETS)
 
+.PHONY: ext-clean
+ext-clean: $(EXT_CLEAN_TARGETS)
+
 
 #
 # hue target

desktop/libs/hadoop/Makefile

@@ -59,9 +59,6 @@ SUDO_SHELL := $(SUDO_SHELL_LIB)/sudo-shell-$(DESKTOP_VERSION).jar
 
 compile: $(PLUGIN) $(STATIC_GROUP_MAPPING) $(SUDO_SHELL)
 
-clean::
-	rm -Rf $(PLUGIN_JAVA_LIB) $(STATIC_GROUP_MAPPING_LIB) $(SUDO_SHELL_LIB)
-
 # Build the plugin jar iff the source is present
 ifneq (,$(wildcard $(PLUGIN_JAVA_DIR)))
 PLUGIN_OPTS := -Dbuild.dir=$(BLD_DIR_PLUGINS) \
@@ -73,6 +70,10 @@ $(PLUGIN): $(shell find $(PLUGIN_JAVA_DIR) )
 	@echo "--- Building Hadoop plugins"
 	cd $(PLUGIN_JAVA_DIR) && mvn clean install -DskipTests
 	cp $(BLD_DIR_PLUGINS)/hue-plugins-$(MAVEN_VERSION).jar $(PLUGIN)
+
+clean::
+	rm -rf $(PLUGIN_JAVA_LIB)
+	cd $(PLUGIN_JAVA_DIR) && mvn clean ||:
 else
 $(PLUGIN):
 	$(error Cannot build hadoop plugin without source)
@@ -89,6 +90,10 @@ $(STATIC_GROUP_MAPPING): $(shell find $(STATIC_GROUP_MAPPING_SRC))
 	@echo "--- Building static UserGroupMappingService"
 	cd $(STATIC_GROUP_MAPPING_DIR) && mvn clean install -DskipTests
 	cp $(STATIC_GROUP_MAPPING_BUILD)/static-group-mapping-$(MAVEN_VERSION).jar $(STATIC_GROUP_MAPPING)
+
+clean::
+	rm -rf $(STATIC_GROUP_MAPPING_LIB)
+	cd $(STATIC_GROUP_MAPPING_DIR) && mvn clean ||:
 else
 $(STATIC_GROUP_MAPPING):
 	$(error Cannot build static-group-mapping jar without source)
@@ -105,6 +110,10 @@ $(SUDO_SHELL): $(shell find $(SUDO_SHELL_SRC))
 	@echo "--- Building sudo fsshell tool"
 	cd $(SUDO_SHELL_DIR) && mvn clean install -DskipTests
 	cp $(SUDO_SHELL_BUILD)/sudo-shell-$(MAVEN_VERSION).jar $(SUDO_SHELL)
+
+clean::
+	rm -rf $(SUDO_SHELL_LIB)
+	cd $(SUDO_SHELL_DIR) && mvn clean ||:
 else
 $(SUDO_SHELL):
 	$(error cannot build sudo-shell jar without source)