Skip to content

Commit

Permalink
Local CI improvements + Fix async call finished error + rename ci job…
Browse files Browse the repository at this point in the history 
…s (#8836)
  • Loading branch information
Conor committed Sep 18, 2023
1 parent 3c9c2ed commit ab5003c
Show file tree
Hide file tree
Showing 6 changed files with 97 additions and 67 deletions.
32 changes: 1 addition & 31 deletions airbyte-webapp/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,23 +28,7 @@ node {
distBaseUrl = "https://nodejs.org/dist"
}

tasks.register("validateLockFiles") {
description "Validate only a pnpm-lock.yaml lock file exists"
inputs.files "pnpm-lock.yaml", "package-lock.json", "yarn.lock"

// The validateLockFiles has no outputs, thus we always treat the outputs up to date
// as long as the inputs have not changed
outputs.upToDateWhen { true }

doLast {
assert file("pnpm-lock.yaml").exists()
assert !file("package-lock.json").exists()
assert !file("yarn.lock").exists()
}
}

tasks.named("pnpmInstall") {
dependsOn tasks.named("validateLockFiles")
// Add patches folder to inputs of pnpmInstall task, since it has pnpm-lock.yml as an output
// thus wouldn't rerun in case a patch get changed
inputs.dir "patches"
Expand Down Expand Up @@ -125,19 +109,6 @@ tasks.register("cloudE2eTest", PnpmTask) {
outputs.upToDateWhen { false }
}

tasks.register("licenseCheck", PnpmTask) {
dependsOn tasks.named("pnpmInstall")

args = ['run', 'license-check']

inputs.files nodeModules
inputs.file 'package.json'
inputs.file 'scripts/license-check.js'

// The licenseCheck has no outputs, thus we always treat the outputs up to date
// as long as the inputs have not changed
outputs.upToDateWhen { true }
}
//
//tasks.register("validateLinks", PnpmTask) {
// dependsOn tasks.named("pnpmInstall")
Expand Down Expand Up @@ -189,10 +160,9 @@ tasks.register("copyNginx", Copy) {

// Those tasks should be run as part of the "check" task
tasks.named("check") {
dependsOn /*tasks.named("validateLinks"),*/ tasks.named("licenseCheck"), tasks.named("test")
dependsOn /*tasks.named("validateLinks"),*/ tasks.named("test")
}


tasks.named("build") {
dependsOn tasks.named("buildStorybook")
}
Expand Down
1 change: 1 addition & 0 deletions airbyte-webapp/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
"license-check": "node ./scripts/license-check.js",
"generate-client": "./scripts/load-declarative-schema.sh && orval",
"validate-links": "ts-node --skip-project ./scripts/validate-links.ts",
"validate-lock": "node ./scripts/validate-lock-files.js",
"preanalyze-lowcode": "TS_NODE_TRANSPILE_ONLY=true pnpm run generate-client",
"analyze-lowcode": "ts-node --skip-project ./scripts/analyze-low-code-manifests.ts",
"cypress:open": "cypress open --config-file cypress/cypress.config.ts",
Expand Down
25 changes: 25 additions & 0 deletions airbyte-webapp/scripts/validate-lock-files.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
const fs = require("node:fs");
const path = require("node:path");

function assertFileExists(filepath) {
if (!fs.existsSync(filepath)) {
throw new Error(`File ${filepath} does not exist`);
}
}

function assertFileNotExists(filepath) {
if (fs.existsSync(filepath)) {
throw new Error(`File ${filepath} exists but should not`);
}
}

try {
assertFileExists(path.join(__dirname, "..", "pnpm-lock.yaml"));
assertFileNotExists(path.join(__dirname, "..", "package-lock.json"));
assertFileNotExists(path.join(__dirname, "..", "yarn.lock"));

console.log("Lock file validation successful.");
} catch (error) {
console.error(`Lock file validation failed: ${error.message}`);
process.exit(1);
}
35 changes: 30 additions & 5 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,12 @@ buildscript {

plugins {
id "base"
id "io.airbyte.gradle.jvm" version "0.14.0" apply false
id "io.airbyte.gradle.jvm.app" version "0.14.0" apply false
id "io.airbyte.gradle.jvm.lib" version "0.14.0" apply false
id "io.airbyte.gradle.docker" version "0.14.0" apply false
id "io.airbyte.gradle.publish" version "0.14.0" apply false
id "com.dorongold.task-tree" version "2.1.1"
id "io.airbyte.gradle.jvm" version "0.15.0" apply false
id "io.airbyte.gradle.jvm.app" version "0.15.0" apply false
id "io.airbyte.gradle.jvm.lib" version "0.15.0" apply false
id "io.airbyte.gradle.docker" version "0.15.0" apply false
id "io.airbyte.gradle.publish" version "0.15.0" apply false
}

repositories {
Expand Down Expand Up @@ -71,6 +72,30 @@ allprojects {
version = rootProject.ext.version
}

tasks.register('archiveReports', Tar) {
dependsOn subprojects.collect { it.getTasksByName('checkstyleMain', true) }
dependsOn subprojects.collect { it.getTasksByName('checkstyleTest', true) }
dependsOn subprojects.collect { it.getTasksByName('jacocoTestReport', true) }
dependsOn subprojects.collect { it.getTasksByName('pmdMain', true) }
dependsOn subprojects.collect { it.getTasksByName('pmdTest', true) }
dependsOn subprojects.collect { it.getTasksByName('spotbugsMain', true) }
dependsOn subprojects.collect { it.getTasksByName('spotbugsTest', true) }
dependsOn subprojects.collect { it.getTasksByName('test', true) }
dependsOn subprojects.collect { it.getTasksByName('checkstyleAcceptanceTests', true) }
dependsOn subprojects.collect { it.getTasksByName('pmdAcceptanceTests', true) }
dependsOn subprojects.collect { it.getTasksByName('spotbugsAcceptanceTests', true) }

archiveFileName = "${project.name}-reports.tar"
destinationDirectory = layout.buildDirectory.dir('dist')

// Collect reports from each subproject
subprojects.each { subproject ->
from("${subproject.buildDir}/reports") {
into("${subproject.name}/reports")
}
}
}

subprojects { sp ->
// airbyte-webapp has not been converted to the gradle plugins
if (sp.name != "airbyte-webapp") {
Expand Down
16 changes: 9 additions & 7 deletions ci/oss/ci.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
build_oss_backend_task,
build_oss_frontend_task,
build_storybook_oss_frontend_task,
check_oss_backend_task,
test_oss_backend_task,
test_oss_frontend_task,
)
Expand Down Expand Up @@ -63,8 +64,6 @@ class BackendTestCommand(ClickCommandMetadata):
type = ParameterType.BOOL,
help = "Enables gradle scanning",
default = False)]
#options: List[ClickOption] = [ClickOption(name = "--event",type=ParameterType.STRING,help="GHA Event to simulate locally"),]


class FrontendBuildCommand(ClickCommandMetadata):
command_name: str = "build"
Expand All @@ -83,7 +82,7 @@ class TestCommand(ClickCommandMetadata):
@pass_global_settings
@pass_pipeline_context
@flow(validate_parameters=False, name="OSS Build")
async def build(settings: OssSettings, ctx: PipelineContext, client: Optional[Client] = None, scan: bool = False) -> List[Container]:
async def oss_build(settings: OssSettings, ctx: PipelineContext, client: Optional[Client] = None, scan: bool = False) -> List[Container]:
client = await ctx.get_dagger_client(client, ctx.prefect_flow_run_context.flow.name)
results: List[List[State]] = await gather(backend_build, frontend_build, args=[(), ()], kwargs=[{'scan': scan, 'client': client},{'client': client}])
return results
Expand Down Expand Up @@ -115,10 +114,11 @@ async def frontend_build(settings: OssSettings, ctx: PipelineContext, client: Op
@pass_global_settings
@pass_pipeline_context
@flow(validate_parameters=False, name="OSS Test")
async def test(settings: OssSettings, ctx: PipelineContext, client: Optional[Client] = None, scan: bool = False) -> List[Container]:
async def oss_test(settings: OssSettings, ctx: PipelineContext, build_results: Optional[List[Container]] = None, client: Optional[Client] = None, scan: bool = False) -> List[Container]:
test_client = await ctx.get_dagger_client(client, ctx.prefect_flow_run_context.flow.name)
build_results = await build(scan=scan, client=test_client)
build_results = await oss_build(scan=scan, client=test_client) if build_results is None else build_results
test_results = await test_oss_backend_task.submit(client=test_client, oss_build_result=build_results[0][0], settings=settings, ctx=quote(ctx), scan=scan)
await check_oss_backend_task.submit(client=test_client, oss_build_result=build_results[0][0], settings=settings, ctx=quote(ctx), scan=scan)
# TODO: add cypress E2E tests here
return [test_results]

Expand All @@ -130,15 +130,17 @@ async def backend_test(settings: OssSettings, ctx: PipelineContext, client: Opti
test_client = await ctx.get_dagger_client(client, ctx.prefect_flow_run_context.flow.name)
build_results = await backend_build(scan=scan, client=test_client)
test_results = await test_oss_backend_task.submit(client=test_client, oss_build_result=build_results[0], settings=settings, ctx=quote(ctx), scan=scan)
await check_oss_backend_task.submit(client=test_client, oss_build_result=build_results[0], settings=settings, ctx=quote(ctx), scan=scan)

return test_results

@oss_group.command(CICommand())
@pass_global_settings
@pass_pipeline_context
@flow(validate_parameters=False, name="OSS CI")
async def ci(settings: OssSettings, ctx: PipelineContext, client: Optional[Client] = None, scan: bool = False) -> List[Container]:
async def oss_ci(settings: OssSettings, ctx: PipelineContext, client: Optional[Client] = None, scan: bool = False) -> List[Container]:
ci_client = await ctx.get_dagger_client(client, ctx.prefect_flow_run_context.flow.name)
ci_results = await test(scan=scan, client=ci_client)
ci_results = await oss_test(scan=scan, client=ci_client)
return ci_results


Expand Down
55 changes: 31 additions & 24 deletions ci/oss/tasks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,10 @@
with_gradle,
with_node,
with_pnpm,
with_typescript_gha,
)
from aircmd.actions.pipelines import get_repo_dir
from aircmd.actions.pipelines import get_repo_dir, sync_to_gradle_cache_from_homedir
from aircmd.models.base import PipelineContext
from aircmd.models.settings import GithubActionsInputSettings, load_settings
from aircmd.models.settings import load_settings
from dagger import CacheSharingMode, CacheVolume, Client, Container
from prefect import task
from prefect.artifacts import create_link_artifact
Expand Down Expand Up @@ -49,7 +48,7 @@ async def build_oss_backend_task(settings: OssSettings, ctx: PipelineContext, cl
.with_workdir("/airbyte/oss" if base_dir == "oss" else "/airbyte")
.with_exec(["./gradlew", ":airbyte-config:specs:downloadConnectorRegistry", "--rerun", "--build-cache", "--no-daemon"])
.with_exec(gradle_command + ["--scan"] if scan else gradle_command)
.with_exec(["rsync", "-az", "/root/.gradle/", "/root/gradle-cache"]) #TODO: Move this to a context manager
.with_(sync_to_gradle_cache_from_homedir(settings.GRADLE_CACHE_VOLUME_PATH, settings.GRADLE_HOMEDIR_PATH)) #TODO: Move this to a context manager
)
await result.sync()
if scan:
Expand All @@ -75,6 +74,8 @@ async def build_oss_frontend_task(settings: OssSettings, ctx: PipelineContext, c
.with_(load_settings(client, settings))
.with_mounted_cache("./build/airbyte-repository", airbyte_repo_cache, sharing=CacheSharingMode.LOCKED)
.with_exec(["pnpm", "install"])
.with_exec(["pnpm", "run", "license-check"])
.with_exec(["pnpm", "run", "validate-lock"])
.with_exec(["pnpm", "build"]))
await result.sync()
return result
Expand Down Expand Up @@ -120,6 +121,28 @@ async def build_storybook_oss_frontend_task(settings: OssSettings, ctx: Pipeline
await result.sync()
return result

@task
async def check_oss_backend_task(client: Client, oss_build_result: Container, settings: OssSettings, ctx: PipelineContext, scan: bool) -> Container:
gradle_command = ["./gradlew", "check", "-x", "test", "-x", ":airbyte-webapp:check","-x", "buildDockerImage", "-x", "dockerBuildImage", "--build-cache", "--no-daemon"]
files_from_result = [
"**/build.gradle",
"**/gradle.properties",
"**/settings.gradle",
"**/airbyte-*/**/*"
]
result = (
with_gradle(client, ctx, settings, directory=base_dir)
.with_directory("/root/.m2/repository", oss_build_result.directory("/root/.m2/repository")) # published jar files from mavenLocal
.with_directory("/airbyte/oss", oss_build_result.directory("/airbyte/oss"), include=files_from_result)
.with_workdir("/airbyte/oss" if base_dir == "oss" else "/airbyte")
.with_(load_settings(client, settings))
.with_env_variable("VERSION", "dev")
.with_env_variable("METRIC_CLIENT", "") # override 'datadog' value for metrics-lib test
.with_exec(gradle_command + ["--scan"] if scan else gradle_command)
.with_(sync_to_gradle_cache_from_homedir(settings.GRADLE_CACHE_VOLUME_PATH, settings.GRADLE_HOMEDIR_PATH))
)
await result.sync()


@task
async def test_oss_backend_task(client: Client, oss_build_result: Container, settings: OssSettings, ctx: PipelineContext, scan: bool) -> Container:
Expand Down Expand Up @@ -175,9 +198,9 @@ async def test_oss_backend_task(client: Client, oss_build_result: Container, set
.with_exec(["./run.sh"])
)

gradle_command = ["./gradlew", "checK", "test", "-x", ":airbyte-webapp:test", "-x", "buildDockerImage", "--build-cache", "--no-daemon"]
gradle_command = ["./gradlew", "test", "-x", ":airbyte-webapp:test","-x", "buildDockerImage", "-x", "dockerBuildImage", "--build-cache", "--no-daemon"]

result = (
result = (
with_gradle(client, ctx, settings, directory=base_dir)
.with_service_binding("airbyte-proxy-test-container", airbyte_proxy_service_auth)
.with_service_binding("airbyte-proxy-test-container-newauth", airbyte_proxy_service_newauth)
Expand All @@ -189,11 +212,8 @@ async def test_oss_backend_task(client: Client, oss_build_result: Container, set
.with_env_variable("VERSION", "dev")
.with_env_variable("METRIC_CLIENT", "") # override 'datadog' value for metrics-lib test
.with_exec(gradle_command + ["--scan"] if scan else gradle_command)
.with_exec(["rsync", "-az", "/root/.gradle/", "/root/gradle-cache"]) #TODO: Move this to a context manager)
.with_exec(["rsync", "-azm", "--include='*/'", "--include='jacocoTestReport.xml", "--exclude='*'", "/airbyte/oss", "/jacoco"]) # Collect jacoco reports


)
.with_(sync_to_gradle_cache_from_homedir(settings.GRADLE_CACHE_VOLUME_PATH, settings.GRADLE_HOMEDIR_PATH))
)
await result.sync()
if scan:
scan_file_contents: str = await result.file("/airbyte/oss/scan-journal.log").contents()
Expand All @@ -203,17 +223,4 @@ async def test_oss_backend_task(client: Client, oss_build_result: Container, set
description="Gradle build scan for OSS Backend Tests",
)


# TODO: We should move as much of these actions that interface with PR's to prefect Artifacts as we can
# instead of relying on CI only marketplace actions that assume that they are running in a CI environment
# This will allow us to run these actions locally as well
if settings.CI:
jacoco_inputs = GithubActionsInputSettings(settings, **{
"paths": "/input/**/jacocoTestReport.xml",
"token": settings.GITHUB_TOKEN,
"debug-mode": "true"
})
# TODO: Upload buildpulse here as well
await with_typescript_gha(client, result.directory("/jacoco"), "Madrapps/jacoco-report", "v1.6.1", jacoco_inputs)

return result

0 comments on commit ab5003c

Please sign in to comment.