quash non-stderr from ci logs (#9216)

.github/workflows/fe-validate-links.yml

@@ -33,7 +33,7 @@ jobs:
           EOF
 
       - name: Run :airbyte-webapp:validateLinks
-        run: ./gradlew --no-daemon :airbyte-webapp:validateLinks --scan
+        run: ./gradlew --no-daemon :airbyte-webapp:validateLinks --scan -DciMode
 
       - name: Publish Failures to dev-frontend channel
         uses: abinoda/slack-action@master

.gitignore

@@ -86,3 +86,6 @@ dd-java-agent.jar
 # Ignore airbyte.yml since this file contains user-provided information that is sensitive and should not be committed
 # See airbyte.sample.yml for an example of the expected file structure.
 /configs/airbyte.yml
+
+# ignore local gradle build log
+build.log

airbyte-commons/src/main/resources/log4j2.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <Configuration status="INFO" packages="io.airbyte.commons.logging">
     <Properties>
-
+        <Property name="ciMode">${sys:ciMode:-false}</Property>
         <!-- Mask the string apikey=<string> to apikey=***** to prevent secrets leaking. -->
         <Property name="default-pattern">%d{yyyy-MM-dd HH:mm:ss}{GMT+0} %highlight{%p} %C{1.}(%M):%L - %replace{%m}{apikey=[\w\-]*}{apikey=*****}%n</Property>
         <!--Logs the timestamp and log_source/application name in the beginning of the line if it exists with a > separator, and then always the rest of the line.-->
@@ -21,9 +21,13 @@
 
     <Appenders>
         <Console name="Default" target="SYSTEM_OUT">
-            <ContextMapFilter onMatch="DENY" onMismatch="ACCEPT">
-                <KeyValuePair key="simple" value="true"/>
-            </ContextMapFilter>
+            <Filters>
+                <!-- If ciMode is true, show only ERROR and above on Console -->
+                <ThresholdFilter level="ERROR" onMatch="ACCEPT" onMismatch="DENY"/>
+                <ContextMapFilter onMatch="DENY" onMismatch="ACCEPT">
+                    <KeyValuePair key="simple" value="true"/>
+                </ContextMapFilter>
+            </Filters>
             <PatternLayout pattern="${default-pattern}"/>
         </Console>
 
@@ -34,6 +38,16 @@
             <PatternLayout pattern="${simple-pattern}"/>
         </Console>
 
+        <File name="LogFile" fileName="build.log">
+            <Filters>
+                <!-- Only log to file when ciMode is true -->
+                <If condition="${sys:ciMode} == 'true'">
+                    <ThresholdFilter level="${log-level}" onMatch="ACCEPT" onMismatch="DENY"/>
+                </If>
+            </Filters>
+            <PatternLayout pattern="${default-pattern}"/>
+        </File>
+
         <Rewrite name="SecretMaskRewrite">
             <MaskedDataInterceptor></MaskedDataInterceptor>
             <AppenderRef ref="Default"/>
@@ -44,6 +58,7 @@
             <AppenderRef ref="SimpleLogSplitCloud"/>
             <AppenderRef ref="AppLogSplit"/>
             <AppenderRef ref="AppLogSplitCloud"/>
+            <AppenderRef ref="LogFile"/>
         </Rewrite>
 
         <Routing name="LogSplit">

build.gradle

@@ -1,6 +1,10 @@
 // The buildscript block defines dependencies in order for .gradle file evaluation.
 // This is separate from application dependencies.
 // See https://stackoverflow.com/questions/17773817/purpose-of-buildscript-block-in-gradle.
+
+
+
+
 buildscript {
     repositories {
         maven {
@@ -48,6 +52,22 @@ repositories {
     }
 }
 
+if (System.getProperty("ciMode") != null) {
+    gradle.startParameter.logLevel = LogLevel.QUIET
+
+    def logFile = new FileOutputStream('gradle.log', true)
+    gradle.services.get(LoggingManager).addStandardOutputListener(logFile)
+    gradle.services.get(LoggingManager).addStandardErrorListener(logFile)
+
+    allprojects {
+        tasks.withType(JavaCompile) {
+            options.deprecation = false
+            options.warnings = false
+        }
+    }
+}
+
+
 reporting {
     reports {
         testAggregateTestReport(JacocoCoverageReport) {

ci/oss/tasks.py

@@ -37,7 +37,7 @@ async def build_oss_backend_task(settings: OssSettings, ctx: PipelineContext, cl
     # This is the list of files needed to run. Keep this as minimal as possible to avoid accidentally invalidating the cache
     files_from_host=["airbyte-*/**/*"]
 
-    gradle_command = ["./gradlew", "assemble", "-x", "buildDockerImage", "-x", "dockerBuildImage", "publishtoMavenLocal", "--build-cache", "--no-daemon"]
+    gradle_command = ["./gradlew", "assemble", "-x", "buildDockerImage", "-x", "dockerBuildImage", "publishtoMavenLocal", "--build-cache", "--no-daemon", "-DciMode"]
 
     # Now we execute the build inside the container. Each step is analagous to a layer in buildkit
     # Be careful what is passed in here, as anything like a timestamp will invalidate the whole cache.
@@ -46,7 +46,7 @@ async def build_oss_backend_task(settings: OssSettings, ctx: PipelineContext, cl
                 .with_(load_settings(client, settings))
                 .with_env_variable("VERSION", "dev")
                 .with_workdir("/airbyte/oss" if base_dir == "oss" else "/airbyte")
-                .with_exec(["./gradlew", ":airbyte-config:specs:downloadConnectorRegistry", "--rerun", "--build-cache", "--no-daemon"])
+                .with_exec(["./gradlew", ":airbyte-config:specs:downloadConnectorRegistry", "--rerun", "--build-cache", "--no-daemon", "-DciMode"])
                 .with_exec(gradle_command + ["--scan"] if scan else gradle_command)
                 .with_(sync_to_gradle_cache_from_homedir(settings.GRADLE_CACHE_VOLUME_PATH, settings.GRADLE_HOMEDIR_PATH)) #TODO: Move this to a context manager
         )
@@ -123,7 +123,7 @@ async def build_storybook_oss_frontend_task(settings: OssSettings, ctx: Pipeline
 
 @task
 async def check_oss_backend_task(client: Client, oss_build_result: Container, settings: OssSettings, ctx: PipelineContext, scan: bool) -> Container:
-    gradle_command = ["./gradlew", "check", "-x", "test", "-x", ":airbyte-webapp:check","-x", "buildDockerImage", "-x", "dockerBuildImage", "--build-cache", "--no-daemon"]
+    gradle_command = ["./gradlew", "check", "-x", "test", "-x", ":airbyte-webapp:check","-x", "buildDockerImage", "-x", "dockerBuildImage", "--build-cache", "--no-daemon", "-DciMode"]
     files_from_result = [
                         "**/build.gradle", 
                         "**/gradle.properties", 
@@ -198,7 +198,7 @@ async def test_oss_backend_task(client: Client, oss_build_result: Container, set
         .with_exec(["./run.sh"])
     )
 
-    gradle_command = ["./gradlew", "test", "-x", ":airbyte-webapp:test","-x", "buildDockerImage", "-x", "dockerBuildImage", "--build-cache", "--no-daemon"]
+    gradle_command = ["./gradlew", "test", "-x", ":airbyte-webapp:test","-x", "buildDockerImage", "-x", "dockerBuildImage", "--build-cache", "--no-daemon", "-DciMode"]
 
     result = ( 
                 with_gradle(client, ctx, settings, directory=base_dir)

tools/bin/acceptance_test.sh

@@ -6,6 +6,11 @@ set -e
 
 assert_root
 
+
+if [[ ! -z "$CI" ]]; then
+    CI_MODE_FLAG='-DciMode'
+fi
+
 ## Helper functions
 
 get_epoch_time() {
@@ -53,4 +58,6 @@ check_success 'init'
 check_success 'airbyte-bootloader'
 
 echo "Running e2e tests via gradle"
-USE_EXTERNAL_DEPLOYMENT=true ./gradlew :airbyte-tests:acceptanceTest --rerun-tasks --scan
+
+USE_EXTERNAL_DEPLOYMENT=true ./gradlew :airbyte-tests:acceptanceTest --rerun-tasks --scan "$CI_MODE_FLAG"
+

tools/bin/e2e_test.sh

@@ -8,6 +8,10 @@ assert_root
 
 echo "Starting app..."
 
+if [[ ! -z "$CI" ]]; then
+    CI_MODE_FLAG='-DciMode'
+fi
+
 # todo (cgardens) - docker-compose 1.27.3 contained a bug that causes a failure if the volume path
 #  does not exist when the volume is created. It was fixed in 1.27.4. Github actions virtual envs,
 #  however, new ubuntu release upgraded to 1.27.3 on 09/24/20. Once github actions virtual envs
@@ -38,10 +42,10 @@ done
 
 echo "Running e2e tests via gradle without cypress key"
 
-if ./gradlew --no-daemon :airbyte-webapp:e2etest ; then
+if ./gradlew --no-daemon :airbyte-webapp:e2etest "$CI_MODE_FLAG"; then
   echo "Tests succeeded"
 else
   echo "Tests failed, retry and record"
-  ./gradlew --no-daemon :airbyte-webapp:e2etest -PcypressWebappKey=$CYPRESS_WEBAPP_KEY ;
+  ./gradlew --no-daemon :airbyte-webapp:e2etest -PcypressWebappKey=$CYPRESS_WEBAPP_KEY "$CI_MODE_FLAG";
 fi
 

tools/bin/gke-kube-helm-acceptance-test/acceptance_test_kube_gke.sh

@@ -5,7 +5,11 @@ set -e
 . tools/lib/lib.sh
 
 assert_root
-kubectl --help
+
+
+if [[ ! -z "$CI" ]]; then
+    CI_MODE_FLAG='-DciMode'
+fi
 
 NAMESPACE=$(openssl rand -hex 12)
 echo "Namespace" $NAMESPACE
@@ -74,4 +78,4 @@ kubectl port-forward svc/echo-server-svc 6000:8080 --namespace=$NAMESPACE &
 sleep 10s
 
 echo "Running e2e tests via gradle..."
-KUBE=true IS_GKE=true USE_EXTERNAL_DEPLOYMENT=true ./gradlew :airbyte-tests:acceptanceTest --scan -i
+KUBE=true IS_GKE=true USE_EXTERNAL_DEPLOYMENT=true ./gradlew :airbyte-tests:acceptanceTest --scan "$CI_MODE_FLAG"

tools/bin/release_version.sh

@@ -4,6 +4,10 @@ set -e
 
 . tools/lib/lib.sh
 
+if [[ ! -z "$CI" ]]; then
+    CI_MODE_FLAG='-DciMode'
+fi
+
 if [[ -z "${CLOUDREPO_USER}" ]]; then
   echo 'CLOUDREPO_USER env var not set. Please retrieve the user email from the CloudRepo lastpass secret and run export CLOUDREPO_USER=<user_from_secret>.';
   exit 1;
@@ -29,8 +33,8 @@ docker login -u "${DOCKER_HUB_USERNAME}" -p "${DOCKER_HUB_PASSWORD}"
 source ./tools/bin/bump_version.sh
 
 echo "Building and publishing PLATFORM version $NEW_VERSION for git revision $GIT_REVISION..."
-VERSION=$NEW_VERSION ./gradlew clean build --scan
-VERSION=$NEW_VERSION ./gradlew publish --scan
+VERSION=$NEW_VERSION ./gradlew clean build --scan "$CI_MODE_FLAG"
+VERSION=$NEW_VERSION ./gradlew publish --scan "$CI_MODE_FLAG"
 
 # Container should be running before build starts
 # It generates binaries to build images for different CPU architecture