Visitar URL original
Implement fail-fast tests ordering for JUnit 5 (#8055) · DataDog/dd-trace-java@70a9d04 · GitHub
Skip to content

Commit 70a9d04

Browse files
nikita-tkachenko-datadognikita-tkachenko-datadog
authored
Implement fail-fast tests ordering for JUnit 5 (#8055)
1 parent 5b58772 commit 70a9d04

50 files changed

Lines changed: 654 additions & 78 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ConfigurationApiImpl.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
1616
import datadog.trace.api.civisibility.telemetry.tag.CoverageEnabled;
1717
import datadog.trace.api.civisibility.telemetry.tag.EarlyFlakeDetectionEnabled;
18+
import datadog.trace.api.civisibility.telemetry.tag.FlakyTestRetriesEnabled;
1819
import datadog.trace.api.civisibility.telemetry.tag.ItrEnabled;
1920
import datadog.trace.api.civisibility.telemetry.tag.ItrSkipEnabled;
2021
import datadog.trace.api.civisibility.telemetry.tag.RequireGit;
@@ -132,6 +133,7 @@ public CiVisibilitySettings getSettings(TracerEnvironment tracerEnvironment) thr
132133
settings.getEarlyFlakeDetectionSettings().isEnabled()
133134
? EarlyFlakeDetectionEnabled.TRUE
134135
: null,
136+
settings.isFlakyTestRetriesEnabled() ? FlakyTestRetriesEnabled.TRUE : null,
135137
settings.isGitUploadRequired() ? RequireGit.TRUE : null);
136138

137139
return settings;
@@ -185,6 +187,14 @@ public SkippableTests getSkippableTests(TracerEnvironment tracerEnvironment) thr
185187
@Override
186188
public Map<String, Collection<TestIdentifier>> getFlakyTestsByModule(
187189
TracerEnvironment tracerEnvironment) throws IOException {
190+
OkHttpUtils.CustomListener telemetryListener =
191+
new TelemetryListener.Builder(metricCollector)
192+
.requestCount(CiVisibilityCountMetric.FLAKY_TESTS_REQUEST)
193+
.requestErrors(CiVisibilityCountMetric.FLAKY_TESTS_REQUEST_ERRORS)
194+
.requestDuration(CiVisibilityDistributionMetric.FLAKY_TESTS_REQUEST_MS)
195+
.responseBytes(CiVisibilityDistributionMetric.FLAKY_TESTS_RESPONSE_BYTES)
196+
.build();
197+
188198
String uuid = uuidGenerator.get();
189199
EnvelopeDto<TracerEnvironment> request =
190200
new EnvelopeDto<>(
@@ -196,11 +206,12 @@ public Map<String, Collection<TestIdentifier>> getFlakyTestsByModule(
196206
FLAKY_TESTS_URI,
197207
requestBody,
198208
is -> testIdentifiersResponseAdapter.fromJson(Okio.buffer(Okio.source(is))).data,
199-
null,
209+
telemetryListener,
200210
false);
201211

202212
LOGGER.debug("Received {} flaky tests in total", response.size());
203213

214+
int flakyTestsCount = 0;
204215
Map<String, Collection<TestIdentifier>> testIdentifiers = new HashMap<>();
205216
for (DataDto<TestIdentifierJson> dataDto : response) {
206217
TestIdentifierJson testIdentifierJson = dataDto.getAttributes();
@@ -209,7 +220,10 @@ public Map<String, Collection<TestIdentifier>> getFlakyTestsByModule(
209220
testIdentifiers
210221
.computeIfAbsent(moduleName, k -> new HashSet<>())
211222
.add(testIdentifierJson.toTestIdentifier());
223+
flakyTestsCount++;
212224
}
225+
226+
metricCollector.add(CiVisibilityDistributionMetric.FLAKY_TESTS_RESPONSE_TESTS, flakyTestsCount);
213227
return testIdentifiers;
214228
}
215229

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettings.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,6 @@ public Collection<TestIdentifier> getKnownTests() {
114114

115115
@Nullable
116116
public Collection<TestIdentifier> getFlakyTests() {
117-
// backend does not store module info for flaky tests yet
118117
return flakyTests;
119118
}
120119

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ExecutionSettingsFactoryImpl.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package datadog.trace.civisibility.config;
22

33
import datadog.trace.api.Config;
4+
import datadog.trace.api.civisibility.CIConstants;
45
import datadog.trace.api.civisibility.CiVisibilityWellKnownTags;
56
import datadog.trace.api.civisibility.config.TestIdentifier;
67
import datadog.trace.api.civisibility.config.TestMetadata;
@@ -139,11 +140,17 @@ private TracerEnvironment buildTracerEnvironment(
139140

140141
Map<String, Collection<TestIdentifier>> flakyTestsByModule =
141142
flakyTestRetriesEnabled && config.isCiVisibilityFlakyRetryOnlyKnownFlakes()
143+
|| CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(
144+
config.getCiVisibilityTestOrder())
142145
? getFlakyTestsByModule(tracerEnvironment)
143146
: null;
144147

145148
Map<String, Collection<TestIdentifier>> knownTestsByModule =
146-
earlyFlakeDetectionEnabled ? getKnownTestsByModule(tracerEnvironment) : null;
149+
earlyFlakeDetectionEnabled
150+
|| CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(
151+
config.getCiVisibilityTestOrder())
152+
? getKnownTestsByModule(tracerEnvironment)
153+
: null;
147154

148155
Set<String> moduleNames = new HashSet<>(Collections.singleton(DEFAULT_SETTINGS));
149156
moduleNames.addAll(skippableTestIdentifiers.keySet());

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/TestIdentifierSerializer.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,10 @@ public static void serialize(Serializer serializer, TestIdentifier testIdentifie
1313
}
1414

1515
public static TestIdentifier deserialize(ByteBuffer buffer) {
16+
String suiteName = Serializer.readString(buffer);
1617
return new TestIdentifier(
17-
Serializer.readString(buffer),
18+
// suite name repeats a lot; interning it to save memory
19+
suiteName != null ? suiteName.intern() : null,
1820
Serializer.readString(buffer),
1921
Serializer.readString(buffer));
2022
}

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/AbstractTestSession.java‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,14 @@
55
import datadog.trace.api.Config;
66
import datadog.trace.api.DDTraceId;
77
import datadog.trace.api.IdGenerationStrategy;
8+
import datadog.trace.api.civisibility.CIConstants;
89
import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric;
910
import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
1011
import datadog.trace.api.civisibility.telemetry.TagValue;
12+
import datadog.trace.api.civisibility.telemetry.tag.AgentlessLogSubmissionEnabled;
1113
import datadog.trace.api.civisibility.telemetry.tag.AutoInjected;
1214
import datadog.trace.api.civisibility.telemetry.tag.EventType;
15+
import datadog.trace.api.civisibility.telemetry.tag.FailFastTestOrderEnabled;
1316
import datadog.trace.api.civisibility.telemetry.tag.HasCodeowner;
1417
import datadog.trace.api.civisibility.telemetry.tag.IsHeadless;
1518
import datadog.trace.api.civisibility.telemetry.tag.IsUnsupportedCI;
@@ -109,7 +112,11 @@ public AbstractTestSession(
109112
CiVisibilityCountMetric.TEST_SESSION,
110113
1,
111114
ciProvider,
112-
config.isCiVisibilityAutoInjected() ? AutoInjected.TRUE : null);
115+
config.isCiVisibilityAutoInjected() ? AutoInjected.TRUE : null,
116+
config.isAgentlessLogSubmissionEnabled() ? AgentlessLogSubmissionEnabled.TRUE : null,
117+
CIConstants.FAIL_FAST_TEST_ORDER.equalsIgnoreCase(config.getCiVisibilityTestOrder())
118+
? FailFastTestOrderEnabled.TRUE
119+
: null);
113120

114121
if (instrumentationType == InstrumentationType.MANUAL_API) {
115122
metricCollector.add(CiVisibilityCountMetric.MANUAL_API_EVENTS, 1, EventType.SESSION);

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ TestSuiteImpl testSuiteStart(
2525
*/
2626
boolean isNew(TestIdentifier test);
2727

28+
boolean isFlaky(TestIdentifier test);
29+
2830
/**
2931
* Checks if a given test should be skipped with Intelligent Test Runner or not
3032
*

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,11 @@ public boolean isNew(TestIdentifier test) {
8787
return executionStrategy.isNew(test);
8888
}
8989

90+
@Override
91+
public boolean isFlaky(TestIdentifier test) {
92+
return executionStrategy.isFlaky(test);
93+
}
94+
9095
@Override
9196
public boolean shouldBeSkipped(TestIdentifier test) {
9297
return executionStrategy.shouldBeSkipped(test);

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,11 @@ public boolean isNew(TestIdentifier test) {
7272
return executionStrategy.isNew(test);
7373
}
7474

75+
@Override
76+
public boolean isFlaky(TestIdentifier test) {
77+
return executionStrategy.isFlaky(test);
78+
}
79+
7580
@Override
7681
public boolean shouldBeSkipped(TestIdentifier test) {
7782
return executionStrategy.shouldBeSkipped(test);

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,16 @@ public TestRetryPolicy retryPolicy(TestIdentifier test) {
272272
return testModule.retryPolicy(test);
273273
}
274274

275+
@Override
276+
public boolean isNew(TestIdentifier test) {
277+
return testModule.isNew(test);
278+
}
279+
280+
@Override
281+
public boolean isFlaky(TestIdentifier test) {
282+
return testModule.isFlaky(test);
283+
}
284+
275285
@Override
276286
public void close() {
277287
testModule.end(null);

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,11 @@ public boolean isNew(TestIdentifier test) {
4343
return knownTests != null && !knownTests.contains(test.withoutParameters());
4444
}
4545

46+
public boolean isFlaky(TestIdentifier test) {
47+
Collection<TestIdentifier> flakyTests = executionSettings.getFlakyTests();
48+
return flakyTests != null && flakyTests.contains(test.withoutParameters());
49+
}
50+
4651
public boolean shouldBeSkipped(TestIdentifier test) {
4752
if (test == null) {
4853
return false;

0 commit comments

Comments
 (0)