plugins {

id 'base'

// Enable publishing build scans

id 'com.gradle.build-scan' version '2.3' apply false

// This plugin provides a task to determine which dependencies have updates.

// Additionally, the plugin checks for updates to Gradle itself.

//

// See https://github.com/ben-manes/gradle-versions-plugin for further details.

id 'com.github.ben-manes.versions' version '0.20.0'

// Apply one top level rat plugin to perform any required license enforcement analysis

id 'org.nosphere.apache.rat' version '0.4.0'

// Enable gradle-based release management

id 'net.researchgate.release' version '2.6.0'

id 'org.apache.beam.module'

id "org.sonarqube" version "2.7"

}

if (!gradle.startParameter.isOffline()) {

apply plugin: "com.gradle.build-scan"

}

def isCIBuild = ['JENKINS_HOME', 'BUILD_ID'].every System.&getenv

if (isCIBuild) {

buildScan {

// Build Scan enabled and TOS accepted for Jenkins lab build. This does not apply to builds on

// non-Jenkins machines. Developers need to separately enable and accept TOS to use build scans.

termsOfServiceUrl = 'https://gradle.com/terms-of-service'

termsOfServiceAgree = 'yes'

publishAlways()

}

}

rat {

// Set input directory to that of the root project instead of the CWD. This

// makes .gitignore rules (added below) work properly.

inputDir = project.rootDir

def exclusions = [

// Ignore files we track but do not distribute

"**/.github/**/*",

"**/package-list",

"**/test.avsc",

"**/user.avsc",

"**/test/resources/**/*.txt",

"**/test/**/.placeholder",

// Default eclipse excludes neglect subprojects

// Proto/grpc generated wrappers

"**/apache_beam/portability/api/*_pb2*.py",

"**/go/pkg/beam/**/*.pb.go",

// Ignore go.sum files, which don't permit headers

"**/go.sum",

// Ignore Go test data files

"**/go/data/**",

// VCF test files

"**/apache_beam/testing/data/vcf/*",

// JDBC package config files

"**/META-INF/services/java.sql.Driver",

// Website build files

"**/Gemfile.lock",

"**/Rakefile",

"**/.htaccess",

"website/www/site/assets/scss/_bootstrap.scss",

"website/www/site/assets/scss/bootstrap/**/*",

"website/www/site/static/images/mascot/*.ai",

"website/www/site/static/js/bootstrap*.js",

"website/www/site/static/js/bootstrap/**/*",

"website/www/site/themes",

"website/www/yarn.lock",

"website/www/package.json",

// Ignore ownership files

"ownership/**/*",

"**/OWNERS",

// Json doesn't support comments.

"**/*.json",

// Katas files

"learning/katas/**/course-remote-info.yaml",

"learning/katas/**/section-remote-info.yaml",

"learning/katas/**/lesson-remote-info.yaml",

"learning/katas/**/task-remote-info.yaml",

"learning/katas/*/IO/**/*.txt",

// test p8 file for SnowflakeIO

"sdks/java/io/snowflake/src/test/resources/test_rsa_key.p8",

// Mockito extensions

"sdks/java/io/amazon-web-services2/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker",

"sdks/java/extensions/ml/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker"

]

// Add .gitignore excludes to the Apache Rat exclusion list. We re-create the behavior

// of the Apache Maven Rat plugin since the Apache Ant Rat plugin doesn't do this

// automatically.

def gitIgnore = project(':').file('.gitignore')

if (gitIgnore.exists()) {

def gitIgnoreExcludes = gitIgnore.readLines().findAll { !it.isEmpty() && !it.startsWith('#') }

exclusions.addAll(gitIgnoreExcludes)

}

failOnError = true

excludes = exclusions

}

check.dependsOn rat

task javaPreCommit() {

// We need to list the model/* builds since sdks/java/core doesn't

// depend on any of the model.

dependsOn ":model:pipeline:build"

dependsOn ":model:job-management:build"

dependsOn ":model:fn-execution:build"

dependsOn ":runners:google-cloud-dataflow-java:worker:legacy-worker:build"

dependsOn ":sdks:java:core:buildNeeded"

dependsOn ":sdks:java:core:buildDependents"

dependsOn ":examples:java:preCommit"

dependsOn ":sdks:java:extensions:sql:jdbc:preCommit"

dependsOn ":sdks:java:javadoc:allJavadoc"

dependsOn ":runners:direct-java:needsRunnerTests"

dependsOn ":sdks:java:container:docker"

}

task sqlPreCommit() {

dependsOn ":sdks:java:extensions:sql:runBasicExample"

dependsOn ":sdks:java:extensions:sql:runPojoExample"

dependsOn ":sdks:java:extensions:sql:build"

dependsOn ":sdks:java:extensions:sql:buildDependents"

}

task javaPreCommitBeamZetaSQL() {

dependsOn ":sdks:java:extensions:sql:zetasql:test"

}

task javaPreCommitPortabilityApi() {

dependsOn ":runners:google-cloud-dataflow-java:worker:build"

}

task javaPostCommit() {

dependsOn ":runners:google-cloud-dataflow-java:postCommit"

dependsOn ":sdks:java:extensions:google-cloud-platform-core:postCommit"

dependsOn ":sdks:java:extensions:zetasketch:postCommit"

dependsOn ":sdks:java:io:google-cloud-platform:postCommit"

}

task sqlPostCommit() {

dependsOn ":sdks:java:extensions:sql:postCommit"

dependsOn ":sdks:java:extensions:sql:jdbc:postCommit"

dependsOn ":sdks:java:extensions:sql:datacatalog:postCommit"

}

task goPreCommit() {

dependsOn ":sdks:go:goTest"

dependsOn ":sdks:go:examples:goBuild"

dependsOn ":sdks:go:test:goBuild"

// Ensure all container Go boot code builds as well.

dependsOn ":sdks:java:container:goBuild"

dependsOn ":sdks:python:container:goBuild"

dependsOn ":sdks:go:container:goBuild"

}

task goPostCommit() {

dependsOn ":goIntegrationTests"

}

task goIntegrationTests() {

doLast {

exec {

executable 'sh'

args '-c', './sdks/go/test/run_integration_tests.sh'

}

}

dependsOn ":sdks:go:test:build"

dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"

}

task pythonPreCommit() {

dependsOn ":sdks:python:test-suites:tox:pycommon:preCommitPyCommon"

dependsOn ":sdks:python:test-suites:tox:py2:preCommitPy2"

dependsOn ":sdks:python:test-suites:tox:py35:preCommitPy35"

dependsOn ":sdks:python:test-suites:tox:py36:preCommitPy36"

dependsOn ":sdks:python:test-suites:tox:py37:preCommitPy37"

dependsOn ":sdks:python:test-suites:tox:py38:preCommitPy38"

dependsOn ":sdks:python:test-suites:dataflow:py2:preCommitIT"

dependsOn ":sdks:python:test-suites:dataflow:py2:preCommitIT_V2"

dependsOn ":sdks:python:test-suites:dataflow:py37:preCommitIT"

dependsOn ":sdks:python:test-suites:dataflow:py37:preCommitIT_V2"

// We don't include Py35, Py36 precommit ITs to reduce quota footprint.

// We can reconsider if we ever see an issue that these suites would

// have caught. Note that the same tests will still run in postcommit.

}

task pythonDockerBuildPreCommit() {

dependsOn ":sdks:python:container:py2:docker"

dependsOn ":sdks:python:container:py35:docker"

dependsOn ":sdks:python:container:py36:docker"

dependsOn ":sdks:python:container:py37:docker"

dependsOn ":sdks:python:container:py38:docker"

}

task pythonLintPreCommit() {

dependsOn ":sdks:python:test-suites:tox:py2:lint"

dependsOn ":sdks:python:test-suites:tox:py37:lint"

}

task pythonFormatterPreCommit() {

dependsOn 'sdks:python:test-suites:tox:py38:formatter'

}

task python2PostCommit() {

dependsOn ":sdks:python:test-suites:portable:py2:crossLanguageTests"

dependsOn ":sdks:python:test-suites:dataflow:py2:postCommitIT"

dependsOn ":sdks:python:test-suites:direct:py2:directRunnerIT"

dependsOn ":sdks:python:test-suites:direct:py2:hdfsIntegrationTest"

dependsOn ":sdks:python:test-suites:direct:py2:mongodbioIT"

dependsOn ":sdks:python:test-suites:portable:py2:postCommitPy2"

}

task python35PostCommit() {

dependsOn ":sdks:python:test-suites:dataflow:py35:postCommitIT"

dependsOn ":sdks:python:test-suites:direct:py35:postCommitIT"

dependsOn ":sdks:python:test-suites:portable:py35:postCommitPy35"

}

task python36PostCommit() {

dependsOn ":sdks:python:test-suites:dataflow:py36:postCommitIT"

dependsOn ":sdks:python:test-suites:direct:py36:postCommitIT"

dependsOn ":sdks:python:test-suites:portable:py36:postCommitPy36"

}

task python37PostCommit() {

dependsOn ":sdks:python:test-suites:dataflow:py37:postCommitIT"

dependsOn ":sdks:python:test-suites:direct:py37:postCommitIT"

dependsOn ":sdks:python:test-suites:direct:py37:hdfsIntegrationTest"

dependsOn ":sdks:python:test-suites:portable:py37:postCommitPy37"

}

task portablePythonPreCommit() {

dependsOn ":sdks:python:test-suites:portable:py2:preCommitPy2"

dependsOn ":sdks:python:test-suites:portable:py35:preCommitPy35"

dependsOn ":sdks:python:test-suites:portable:py36:preCommitPy36"

dependsOn ":sdks:python:test-suites:portable:py37:preCommitPy37"

}

task websitePreCommit() {

dependsOn ":website:preCommit"

}

task communityMetricsPreCommit() {

dependsOn ":beam-test-infra-metrics:preCommit"

}

task communityMetricsProber() {

dependsOn ":beam-test-infra-metrics:checkProber"

}

task javaExamplesDataflowPrecommit() {

dependsOn ":runners:google-cloud-dataflow-java:examples:preCommit"

dependsOn ":runners:google-cloud-dataflow-java:examples-streaming:preCommit"

}

task runBeamDependencyCheck() {

dependsOn ":dependencyUpdates"

dependsOn ":sdks:python:dependencyUpdates"

}

release {

revertOnFail = false

tagTemplate = 'v${version}'

git {

requireBranch = 'release-.*|master'

pushToRemote = ''

}

}

if (project.hasProperty('javaLinkageArtifactIds')) {

if (!project.hasProperty('publishing')) {

throw new GradleException('You can only check linkage of Java artifacts if you specify -Ppublishing on the command line as well.')

}

configurations { linkageCheckerJava }

dependencies {

linkageCheckerJava "com.google.cloud.tools:dependencies:1.4.0"

}

// We need to evaluate all the projects first so that we can find depend on all the

// publishMavenJavaPublicationToMavenLocal tasks below.

for (p in rootProject.subprojects) {

if (!p.path.equals(project.path)) {

evaluationDependsOn(p.path)

}

}

project.task('checkJavaLinkage', type: JavaExec) {

dependsOn project.getTasksByName('publishMavenJavaPublicationToMavenLocal', true /* recursively */)

classpath = project.configurations.linkageCheckerJava

main = 'com.google.cloud.tools.opensource.classpath.LinkageCheckerMain'

def arguments = ['-a', project.javaLinkageArtifactIds.split(',').collect({

if (it.contains(':')) {

"${project.ext.mavenGroupId}:${it}"

} else {

// specify the version if not provided

"${project.ext.mavenGroupId}:${it}:${project.version}"

}

}).join(',')]

// Exclusion file filters out existing linkage errors before a change

if (project.hasProperty('javaLinkageWriteBaseline')) {

arguments += ['--output-exclusion-file', project.getProperty('javaLinkageWriteBaseline')]

} else if (project.hasProperty('javaLinkageReadBaseline')) {

arguments += ['--exclusion-file', project.getProperty('javaLinkageReadBaseline')]

}

args arguments

doLast {

println "NOTE: This task published artifacts into your local Maven repository. You may want to remove them manually."

}

}

}

if (project.hasProperty('compileAndRunTestsWithJava11')) {

project.javaPreCommitPortabilityApi.dependsOn ':sdks:java:testing:test-utils:verifyJavaVersion'

project.javaExamplesDataflowPrecommit.dependsOn ':sdks:java:testing:test-utils:verifyJavaVersion'

project.javaPreCommitBeamZetaSQL.dependsOn ':sdks:java:testing:test-utils:verifyJavaVersion'

project.sqlPreCommit.dependsOn ':sdks:java:testing:test-utils:verifyJavaVersion'

} else {

allprojects {

tasks.withType(Test) {

exclude '**/JvmVerification.class'

}

}

}