Repository navigation
284 lines (264 loc) · 10.6 KB
/
Copy pathbuild.yml
File metadata and controls
284 lines (264 loc) · 10.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: Check and Test
# This workflow should only be called from ci.yml which is triggered on
# the "pull_request" event type. We should never dispatch this workflow from
# a "pull_request_target" event.
on:
workflow_call:
inputs:
is-trunk:
description: "Is this a trunk build?"
default: true
type: boolean
is-public-fork:
description: "Is this CI run from a public fork?"
default: true
type: boolean
jobs:
load-catalog:
runs-on: ubuntu-latest
name: Load Test Catalog
steps:
- name: Checkout main
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Checkout test-catalog
uses: actions/checkout@v4
with:
ref: 'test-catalog'
persist-credentials: false
fetch-depth: 100 # Needs to be large enough to ensure we can fetch N days ago
path: test-catalog
- name: Checkout catalog at earlier date
run: |
cd test-catalog
SHA=$(git rev-list -1 --before 7.days.ago origin/test-catalog)
echo $SHA
git switch --detach $SHA
git show --no-patch
- name: Setup Python
uses: ./.github/actions/setup-python
# Prior to this step, we don't expect any errors. For the rest of this "load-catalog" job, we need to ensure
# we do not fail as this will fail the overall workflow.
- name: Combine Catalog into single file
id: combine-catalog
continue-on-error: true
run: |
python .github/scripts/format-test-catalog.py --path "test-catalog/test-catalog/**/*.yaml"
- name: Archive Combined Test Catalog
if: steps.combine-catalog.outcome == 'success'
uses: actions/upload-artifact@v4
with:
name: combined-test-catalog
path: combined-test-catalog.txt
compression-level: 9
validate:
runs-on: ubuntu-latest
name: Compile and Check Java
outputs:
is-draft: ${{ steps.check-draft-pr.outputs.is-draft }}
steps:
- name: Env
run: printenv
env:
GITHUB_CONTEXT: ${{ toJson(github) }}
- name: Check for Draft PR
id: check-draft-pr
if: |
github.event_name == 'pull_request' &&
github.event.pull_request.draft
run: echo "is-draft=true" >> "$GITHUB_OUTPUT"
- name: Checkout code
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Setup Gradle
uses: ./.github/actions/setup-gradle
with:
java-version: 23
gradle-cache-read-only: ${{ !inputs.is-trunk }}
gradle-cache-write-only: ${{ inputs.is-trunk }}
develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
- name: Compile and validate
env:
SCAN_ARG: ${{ inputs.is-public-fork && '--no-scan' || '--scan' }}
# Gradle flags
# --build-cache: Let Gradle restore the build cache
# --info: For now, we'll generate lots of logs while setting up the GH Actions
# --scan: Publish the build scan. This will only work on PRs from apache/kafka and trunk
# --no-scan: For public fork PRs, we won't attempt to publish the scan
run: |
./gradlew --build-cache --info $SCAN_ARG check siteDocTar -x test
- name: Archive check reports
if: always()
uses: actions/upload-artifact@v4
with:
name: check-reports
path: |
**/build/**/*.html
compression-level: 9
if-no-files-found: ignore
- name: Annotate checkstyle errors
if: failure()
run: python .github/scripts/checkstyle.py
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Annotate Rat errors
if: failure()
run: python .github/scripts/rat.py
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
- name: Check generated documentation
# Check if there are any empty files under ./site-docs/generated, If any empty files are found, print an error
# message and list the empty files
run: |
tar zxvf core/build/distributions/kafka_2.13-$(./gradlew properties | grep version: | awk '{print $NF}' | head -n 1)-site-docs.tgz
if find ./site-docs/generated -type f -exec grep -L "." {} \; | grep -q "."; then
echo "One or more documentation files are empty!" >&2
find ./site-docs/generated -type f -exec grep -L "." {} \; >&2
exit 1
fi
test:
needs: [validate, load-catalog]
if: ${{ ! needs.validate.outputs.is-draft }}
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
java: [ 23, 17 ] # If we change these, make sure to adjust ci-complete.yml
outputs:
timed-out: ${{ (steps.junit-test.outputs.gradle-exitcode == '124' || steps.junit-quarantined-test.outputs.gradle-exitcode == '124') }}
name: JUnit tests Java ${{ matrix.java }}
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Setup Python
uses: ./.github/actions/setup-python
- name: Setup Gradle
uses: ./.github/actions/setup-gradle
with:
java-version: ${{ matrix.java }}
gradle-cache-read-only: ${{ !inputs.is-trunk }}
gradle-cache-write-only: ${{ inputs.is-trunk }}
develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
# If the load-catalog job failed, we won't be able to download the artifact. Since we don't want this to fail
# the overall workflow, so we'll continue here without a test catalog.
- name: Load Test Catalog
id: load-test-catalog
uses: actions/download-artifact@v4
continue-on-error: true
with:
name: combined-test-catalog
- name: JUnit Quarantined Tests
id: junit-quarantined-test
uses: ./.github/actions/run-gradle
with:
test-task: quarantinedTest
timeout-minutes: 180
test-catalog-path: ${{ steps.load-test-catalog.outputs.download-path }}/combined-test-catalog.txt
build-scan-artifact-name: build-scan-quarantined-test-${{ matrix.java }}
- name: JUnit Tests
id: junit-test
uses: ./.github/actions/run-gradle
with:
test-task: test
timeout-minutes: 180 # 3 hours
test-catalog-path: ${{ steps.load-test-catalog.outputs.download-path }}/combined-test-catalog.txt
build-scan-artifact-name: build-scan-test-${{ matrix.java }}
- name: Archive JUnit HTML reports
uses: actions/upload-artifact@v4
id: junit-upload-artifact
with:
name: junit-reports-${{ matrix.java }}
path: |
**/build/reports/tests/*
compression-level: 9
if-no-files-found: ignore
- name: Archive JUnit XML
uses: actions/upload-artifact@v4
with:
name: junit-xml-${{ matrix.java }}
path: |
build/junit-xml/**/*.xml
compression-level: 9
if-no-files-found: ignore
- name: Archive Thread Dumps
id: thread-dump-upload-artifact
if: always() && (steps.junit-test.outputs.gradle-exitcode == '124' || steps.junit-quarantined-test.outputs.gradle-exitcode == '124')
uses: actions/upload-artifact@v4
with:
name: junit-thread-dumps-${{ matrix.java }}
path: |
thread-dumps/*
compression-level: 9
if-no-files-found: ignore
- name: Parse JUnit tests
run: python .github/scripts/junit.py --export-test-catalog ./test-catalog >> $GITHUB_STEP_SUMMARY
env:
GITHUB_WORKSPACE: ${{ github.workspace }}
JUNIT_REPORT_URL: ${{ steps.junit-upload-artifact.outputs.artifact-url }}
THREAD_DUMP_URL: ${{ steps.thread-dump-upload-artifact.outputs.artifact-url }}
GRADLE_TEST_EXIT_CODE: ${{ steps.junit-test.outputs.gradle-exitcode }}
GRADLE_QUARANTINED_TEST_EXIT_CODE: ${{ steps.junit-quarantined-test.outputs.gradle-exitcode }}
- name: Archive Test Catalog
if: ${{ always() && matrix.java == '23' }}
uses: actions/upload-artifact@v4
with:
name: test-catalog
path: test-catalog
compression-level: 9
if-no-files-found: ignore
update-test-catalog:
name: Update Test Catalog
needs: test
if: ${{ always() && inputs.is-trunk && needs.test.outputs.timed-out == 'false' }}
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- name: Checkout Test Catalog
uses: actions/checkout@v4
with:
persist-credentials: true # Needed to commit and push later
ref: test-catalog
- name: Reset Catalog
run: |
rm -rf test-catalog
- name: Download Test Catalog
uses: actions/download-artifact@v4
with:
name: test-catalog
path: test-catalog
- name: Push Test Catalog
# Git user.name and user.email come from https://github.com/actions/checkout?tab=readme-ov-file#push-a-commit-using-the-built-in-token
env:
COMMIT_MSG: |
Update test catalog data for GHA workflow run ${{ github.run_id }}
Commit: https://github.com/apache/kafka/commit/${{ github.sha }}
GitHub Run: https://github.com/apache/kafka/actions/runs/${{ github.run_id }}
run: |
pwd
ls -R
git config user.name 'github-actions[bot]'
git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
git add test-catalog
git diff --quiet && git diff --staged --quiet || git commit -m "$COMMIT_MSG"
git push