Skip to content

Commit c95c1f9

Browse files
javier-garcia-sonarsourcesonartech
javier-garcia-sonarsource
authored and
sonartech
committed
SQRP-187 Add SCA steps telemetry
1 parent 47edcce commit c95c1f9

25 files changed

+525
-69
lines changed

‎server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/step/ProjectNclocComputationStepIT.java‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.junit.Rule;
2323
import org.junit.Test;
2424
import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolderRule;
25+
import org.sonar.ce.task.step.ComputationStep;
2526
import org.sonar.ce.task.step.TestComputationStepContext;
2627
import org.sonar.db.DbClient;
2728
import org.sonar.db.DbTester;
@@ -54,7 +55,17 @@ public void test_computing_branch_ncloc() {
5455
BranchDto branch2 = db.components().insertProjectBranch(project, b -> b.setBranchType(BranchType.BRANCH));
5556
db.measures().insertMeasure(branch2, m -> m.addValue(ncloc.getKey(), 10d));
5657
analysisMetadataHolder.setProject(new Project(project.getUuid(), project.getKey(), project.getName(), project.getDescription(), emptyList()));
57-
step.execute(TestComputationStepContext.TestStatistics::new);
58+
step.execute(new ComputationStep.Context() {
59+
@Override
60+
public ComputationStep.Statistics getStatistics() {
61+
return new TestComputationStepContext.TestStatistics();
62+
}
63+
64+
@Override
65+
public void addTelemetryMetric(String key, Object value) {
66+
// do nothing
67+
}
68+
});
5869

5970
assertThat(dbClient.projectDao().getNclocSum(db.getSession())).isEqualTo(200L);
6071
}

‎server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/taskprocessor/AuditPurgeStepIT.java‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,6 @@
1919
*/
2020
package org.sonar.ce.task.projectanalysis.taskprocessor;
2121

22-
import java.time.Instant;
23-
import java.time.ZoneId;
24-
import java.time.ZonedDateTime;
2522
import org.junit.Before;
2623
import org.junit.Rule;
2724
import org.junit.Test;
@@ -37,16 +34,15 @@
3734
import static org.mockito.ArgumentMatchers.anyString;
3835
import static org.mockito.Mockito.mock;
3936
import static org.mockito.Mockito.when;
37+
import static org.sonar.ce.task.projectanalysis.taskprocessor.ContextUtils.EMPTY_CONTEXT;
4038
import static org.sonar.core.config.Frequency.MONTHLY;
4139
import static org.sonar.core.config.PurgeConstants.AUDIT_HOUSEKEEPING_FREQUENCY;
4240

4341
public class AuditPurgeStepIT {
44-
private final static long NOW = 1_400_000_000_000L;
45-
private final static long BEFORE = 1_300_000_000_000L;
46-
private final static long LATER = 1_500_000_000_000L;
47-
private final static ZonedDateTime thresholdDate = Instant.ofEpochMilli(NOW)
48-
.atZone(ZoneId.systemDefault());
49-
private final static PropertyDto FREQUENCY_PROPERTY = new PropertyDto()
42+
private static final long NOW = 1_400_000_000_000L;
43+
private static final long BEFORE = 1_300_000_000_000L;
44+
private static final long LATER = 1_500_000_000_000L;
45+
private static final PropertyDto FREQUENCY_PROPERTY = new PropertyDto()
5046
.setKey(AUDIT_HOUSEKEEPING_FREQUENCY)
5147
.setValue(MONTHLY.name());
5248

@@ -75,7 +71,7 @@ public void executeDeletesOlderAudits() {
7571
prepareRowsWithDeterministicCreatedAt();
7672
assertThat(dbClient.auditDao().selectOlderThan(db.getSession(), LATER + 1)).hasSize(3);
7773

78-
underTest.execute(() -> null);
74+
underTest.execute(EMPTY_CONTEXT);
7975

8076
assertThat(dbClient.auditDao().selectOlderThan(db.getSession(), LATER + 1)).hasSize(2);
8177
}

‎server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/taskprocessor/ContextUtils.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/*
2+
* SonarQube
3+
* Copyright (C) 2009-2025 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the GNU Lesser General Public
8+
* License as published by the Free Software Foundation; either
9+
* version 3 of the License, or (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
* Lesser General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Lesser General Public License
17+
* along with this program; if not, write to the Free Software Foundation,
18+
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19+
*/
20+
package org.sonar.ce.task.projectanalysis.taskprocessor;
21+
22+
import org.sonar.ce.task.step.ComputationStep;
23+
24+
class ContextUtils {
25+
26+
private ContextUtils() {
27+
// utility class
28+
}
29+
30+
static final ComputationStep.Context EMPTY_CONTEXT = new ComputationStep.Context() {
31+
@Override
32+
public ComputationStep.Statistics getStatistics() {
33+
return null;
34+
}
35+
36+
@Override
37+
public void addTelemetryMetric(String key, Object value) {
38+
// do nothing
39+
}
40+
};
41+
}

‎server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/taskprocessor/IgnoreOrphanBranchStepIT.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333

3434
import static org.assertj.core.api.Assertions.assertThat;
3535
import static org.assertj.core.api.Assertions.assertThatThrownBy;
36+
import static org.sonar.ce.task.projectanalysis.taskprocessor.ContextUtils.EMPTY_CONTEXT;
3637
import static org.sonar.db.component.BranchType.BRANCH;
3738

3839
public class IgnoreOrphanBranchStepIT {
@@ -66,7 +67,7 @@ public void execute() {
6667
dbClient.branchDao().insert(dbTester.getSession(), branchDto);
6768
dbTester.commit();
6869

69-
underTest.execute(() -> null);
70+
underTest.execute(EMPTY_CONTEXT);
7071

7172
Optional<BranchDto> branch = dbClient.branchDao().selectByUuid(dbTester.getSession(), BRANCH_UUID);
7273
assertThat(branch.get().isNeedIssueSync()).isFalse();
@@ -85,7 +86,7 @@ public void execute_on_already_indexed_branch() {
8586
dbClient.branchDao().insert(dbTester.getSession(), branchDto);
8687
dbTester.commit();
8788

88-
underTest.execute(() -> null);
89+
underTest.execute(EMPTY_CONTEXT);
8990

9091
Optional<BranchDto> branch = dbClient.branchDao().selectByUuid(dbTester.getSession(), BRANCH_UUID);
9192
assertThat(branch.get().isNeedIssueSync()).isFalse();
@@ -102,7 +103,7 @@ public void fail_if_missing_main_component_in_task() {
102103
.build();
103104
IgnoreOrphanBranchStep underTest = new IgnoreOrphanBranchStep(ceTask, dbClient);
104105

105-
assertThatThrownBy(() -> underTest.execute(() -> null))
106+
assertThatThrownBy(() -> underTest.execute(EMPTY_CONTEXT))
106107
.isInstanceOf(UnsupportedOperationException.class)
107108
.hasMessage("entity not found in task");
108109
}
@@ -125,7 +126,7 @@ public void execute_givenMainBranchWithDifferentUuidFromProject_shouldNotInclude
125126
dbClient.projectDao().insert(dbTester.getSession(), projectDto, false);
126127
dbTester.commit();
127128

128-
underTest.execute(() -> null);
129+
underTest.execute(EMPTY_CONTEXT);
129130

130131
Optional<BranchDto> branch = dbClient.branchDao().selectByUuid(dbTester.getSession(), BRANCH_UUID);
131132
assertThat(branch.get().isNeedIssueSync()).isTrue();
@@ -145,7 +146,7 @@ public void execute_givenNotExistingEntityUuid_shouldIncludeItInPurge() {
145146
dbClient.branchDao().insert(dbTester.getSession(), branchDto);
146147
dbTester.commit();
147148

148-
underTest.execute(() -> null);
149+
underTest.execute(EMPTY_CONTEXT);
149150

150151
Optional<BranchDto> branch = dbClient.branchDao().selectByUuid(dbTester.getSession(), BRANCH_UUID);
151152
assertThat(branch.get().isNeedIssueSync()).isFalse();

‎server/sonar-ce-task-projectanalysis/src/it/java/org/sonar/ce/task/projectanalysis/taskprocessor/IndexIssuesStepIT.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import static org.mockito.Mockito.mock;
3636
import static org.mockito.Mockito.times;
3737
import static org.mockito.Mockito.verify;
38+
import static org.sonar.ce.task.projectanalysis.taskprocessor.ContextUtils.EMPTY_CONTEXT;
3839
import static org.sonar.db.component.BranchType.BRANCH;
3940

4041
public class IndexIssuesStepIT {
@@ -72,7 +73,7 @@ public void execute() {
7273
dbClient.branchDao().insert(dbTester.getSession(), branchDto);
7374
dbTester.commit();
7475

75-
underTest.execute(() -> null);
76+
underTest.execute(EMPTY_CONTEXT);
7677

7778
verify(issueIndexer, times(1)).indexOnAnalysis(BRANCH_UUID);
7879
Optional<BranchDto> branch = dbClient.branchDao().selectByUuid(dbTester.getSession(), BRANCH_UUID);
@@ -91,7 +92,7 @@ public void execute_on_already_indexed_branch() {
9192
dbClient.branchDao().insert(dbTester.getSession(), branchDto);
9293
dbTester.commit();
9394

94-
underTest.execute(() -> null);
95+
underTest.execute(EMPTY_CONTEXT);
9596

9697
verify(issueIndexer, times(0)).indexOnAnalysis(BRANCH_UUID);
9798
}
@@ -106,7 +107,7 @@ public void fail_if_missing_component_in_task() {
106107
.build();
107108
IndexIssuesStep underTest = new IndexIssuesStep(ceTask, dbClient, issueIndexer);
108109

109-
assertThatThrownBy(() -> underTest.execute(() -> null))
110+
assertThatThrownBy(() -> underTest.execute(EMPTY_CONTEXT))
110111
.isInstanceOf(UnsupportedOperationException.class)
111112
.hasMessage("component not found in task");
112113
}

‎server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/ProjectAnalysisTaskModule.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.sonar.ce.task.projectanalysis.taskprocessor.ReportTaskProcessor;
2424
import org.sonar.ce.task.projectexport.taskprocessor.ProjectExportTaskProcessor;
2525
import org.sonar.ce.task.step.ComputationStepExecutor;
26+
import org.sonar.ce.task.telemetry.StepsTelemetryUnavailableHolderImpl;
2627
import org.sonar.core.platform.Module;
2728

2829
public class ProjectAnalysisTaskModule extends Module {
@@ -31,6 +32,7 @@ protected void configureModule() {
3132
add(
3233
// task
3334
ContainerFactoryImpl.class,
35+
StepsTelemetryUnavailableHolderImpl.class,
3436
ComputationStepExecutor.class,
3537
ReportTaskProcessor.class,
3638
ProjectExportTaskProcessor.class);

‎server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/container/ProjectAnalysisTaskContainerPopulator.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,7 @@
154154
import org.sonar.ce.task.step.ComputationStepExecutor;
155155
import org.sonar.ce.task.step.ComputationSteps;
156156
import org.sonar.ce.task.taskprocessor.MutableTaskResultHolderImpl;
157+
import org.sonar.ce.task.telemetry.StepsTelemetryHolderImpl;
157158
import org.sonar.core.issue.tracking.Tracker;
158159
import org.sonar.core.platform.ContainerPopulator;
159160
import org.sonar.server.issue.TaintChecker;
@@ -233,6 +234,7 @@ private static List<Object> componentClasses() {
233234
SiblingComponentsWithOpenIssues.class,
234235
DefaultScaStepImpl.class,
235236
DefaultPersistScaStepImpl.class,
237+
StepsTelemetryHolderImpl.class,
236238

237239
// repositories
238240
PreviousSourceHashRepositoryImpl.class,

‎server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/ReportComputationSteps.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,6 @@ public class ReportComputationSteps extends AbstractComputationSteps {
5252
ValidateProjectStep.class,
5353
LoadQualityProfilesStep.class,
5454

55-
SendAnalysisTelemetryStep.class,
56-
5755
// Dependencies
5856
ScaStep.class,
5957

@@ -136,7 +134,11 @@ public class ReportComputationSteps extends AbstractComputationSteps {
136134

137135
ExecuteStatelessOnFinishStep.class,
138136
PublishTaskResultStep.class,
139-
TriggerViewRefreshStep.class);
137+
TriggerViewRefreshStep.class,
138+
139+
// send analysis and steps statistics telemetry
140+
SendAnalysisTelemetryStep.class
141+
);
140142

141143
public ReportComputationSteps(TaskContainer taskContainer) {
142144
super(taskContainer);

‎server/sonar-ce-task-projectanalysis/src/main/java/org/sonar/ce/task/projectanalysis/step/SendAnalysisTelemetryStep.java‎

Lines changed: 53 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,15 @@
2020
package org.sonar.ce.task.projectanalysis.step;
2121

2222
import java.util.HashSet;
23+
import java.util.Map;
2324
import java.util.Set;
25+
import java.util.stream.Collectors;
2426
import org.sonar.api.config.Configuration;
2527
import org.sonar.api.platform.Server;
26-
import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder;
2728
import org.sonar.ce.common.scanner.ScannerReportReader;
29+
import org.sonar.ce.task.projectanalysis.analysis.AnalysisMetadataHolder;
2830
import org.sonar.ce.task.step.ComputationStep;
31+
import org.sonar.ce.task.telemetry.StepsTelemetryHolder;
2932
import org.sonar.core.util.CloseableIterator;
3033
import org.sonar.core.util.UuidFactory;
3134
import org.sonar.scanner.protocol.output.ScannerReport;
@@ -40,54 +43,83 @@
4043

4144
public class SendAnalysisTelemetryStep implements ComputationStep {
4245

46+
private static final int MAX_METRICS = 1000;
47+
4348
private final TelemetryClient telemetryClient;
4449
private final ScannerReportReader scannerReportReader;
4550
private final Server server;
4651
private final UuidFactory uuidFactory;
4752
private final Configuration config;
4853
private final AnalysisMetadataHolder analysisMetadataHolder;
54+
private final StepsTelemetryHolder stepsTelemetryHolder;
4955

5056
public SendAnalysisTelemetryStep(TelemetryClient telemetryClient, ScannerReportReader scannerReportReader,
51-
UuidFactory uuidFactory, Server server, Configuration configuration, AnalysisMetadataHolder analysisMetadataHolder) {
57+
UuidFactory uuidFactory, Server server, Configuration configuration, AnalysisMetadataHolder analysisMetadataHolder,
58+
StepsTelemetryHolder stepsTelemetryHolder) {
5259
this.telemetryClient = telemetryClient;
5360
this.scannerReportReader = scannerReportReader;
5461
this.server = server;
5562
this.uuidFactory = uuidFactory;
5663
this.config = configuration;
5764
this.analysisMetadataHolder = analysisMetadataHolder;
65+
this.stepsTelemetryHolder = stepsTelemetryHolder;
5866
}
5967

6068
@Override
6169
public void execute(Context context) {
6270
if (!config.getBoolean(SONAR_TELEMETRY_ENABLE.getKey()).orElse(false)) {
6371
return;
6472
}
73+
74+
String projectUuid = analysisMetadataHolder.getProject().getUuid();
75+
String analysisType = analysisMetadataHolder.isPullRequest() ? "pull_request" : "branch";
76+
77+
// it was agreed to limit the number of telemetry entries to 1000 per one analysis among scanner report metrics and step metrics
78+
Set<Metric> scannerReportMetrics = getScannerReportMetrics(projectUuid, analysisType);
79+
Set<Metric> stepsStatisticsMetrics = getStepsTelemetryMetrics(projectUuid, analysisType, MAX_METRICS - scannerReportMetrics.size(),
80+
stepsTelemetryHolder.getTelemetryMetrics());
81+
82+
Set<Metric> metrics = new HashSet<>();
83+
metrics.addAll(scannerReportMetrics);
84+
metrics.addAll(stepsStatisticsMetrics);
85+
sendMetrics(metrics);
86+
}
87+
88+
private void sendMetrics(Set<Metric> metrics) {
89+
if (metrics.isEmpty()) {
90+
return;
91+
}
92+
BaseMessage baseMessage = new BaseMessage.Builder()
93+
.setMessageUuid(uuidFactory.create())
94+
.setInstallationId(server.getId())
95+
.setDimension(Dimension.ANALYSIS)
96+
.setMetrics(metrics)
97+
.build();
98+
99+
String jsonString = MessageSerializer.serialize(baseMessage);
100+
telemetryClient.uploadMetricAsync(jsonString);
101+
}
102+
103+
private Set<Metric> getScannerReportMetrics(String projectUuid, String analysisType) {
104+
Set<Metric> metrics = new HashSet<>();
65105
try (CloseableIterator<ScannerReport.TelemetryEntry> it = scannerReportReader.readTelemetryEntries()) {
66-
Set<Metric> metrics = new HashSet<>();
67-
String projectUuid = analysisMetadataHolder.getProject().getUuid();
68-
String analysisType = analysisMetadataHolder.isPullRequest() ? "pull_request" : "branch";
69-
// it was agreed to limit the number of telemetry entries to 1000 per one analysis
70-
final int limit = 1000;
71106
int count = 0;
72-
while (it.hasNext() && count++ < limit) {
107+
while (it.hasNext() && count < MAX_METRICS) {
73108
ScannerReport.TelemetryEntry telemetryEntry = it.next();
74109
metrics.add(new AnalysisMetric(telemetryEntry.getKey(), telemetryEntry.getValue(), projectUuid, analysisType));
110+
count++;
75111
}
76-
77-
if (metrics.isEmpty()) {
78-
return;
79-
}
80-
BaseMessage baseMessage = new BaseMessage.Builder()
81-
.setMessageUuid(uuidFactory.create())
82-
.setInstallationId(server.getId())
83-
.setDimension(Dimension.ANALYSIS)
84-
.setMetrics(metrics)
85-
.build();
86-
87-
String jsonString = MessageSerializer.serialize(baseMessage);
88-
telemetryClient.uploadMetricAsync(jsonString);
89112
}
113+
return metrics;
114+
}
90115

116+
private static Set<Metric> getStepsTelemetryMetrics(String projectUuid, String analysisType, int maxMetrics, Map<String, Object> telemetryMetrics) {
117+
return telemetryMetrics.entrySet().stream()
118+
.map(entry ->
119+
new AnalysisMetric(entry.getKey(), String.valueOf(entry.getValue()), projectUuid, analysisType)
120+
)
121+
.limit(maxMetrics)
122+
.collect(Collectors.toSet());
91123
}
92124

93125
@Override

0 commit comments

Comments
 (0)