Skip to content

Commit 0eafb59

Browse files
Copilottig
Copilot
and
tig
authored
Move parallel test repeat logic to stress tests (#4455)
* Initial plan * Move parallel test repeat logic to stress tests Co-authored-by: tig <[email protected]> * Simplify unit tests workflow and improve comments Co-authored-by: tig <[email protected]> --------- Co-authored-by: copilot-swe-agent[bot] <[email protected]> Co-authored-by: tig <[email protected]>
1 parent 7a8b6e4 commit 0eafb59

File tree

2 files changed

+138
-70
lines changed

2 files changed

+138
-70
lines changed

‎.github/workflows/stress-tests.yml‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,3 +49,103 @@ jobs:
4949
logs/
5050
TestResults/StressTests
5151
52+
parallel_unittests_stress:
53+
name: Parallel Unit Tests Stress (3 iterations)
54+
runs-on: ${{ matrix.os }}
55+
strategy:
56+
fail-fast: false
57+
matrix:
58+
os: [ ubuntu-latest, windows-latest, macos-latest ]
59+
60+
timeout-minutes: 90
61+
steps:
62+
- name: Checkout code
63+
uses: actions/checkout@v4
64+
65+
- name: Setup .NET Core
66+
uses: actions/setup-dotnet@v4
67+
with:
68+
dotnet-version: 8.x
69+
dotnet-quality: 'ga'
70+
71+
- name: Install dependencies
72+
run: dotnet restore
73+
74+
- name: Build UnitTestsParallelizable
75+
run: dotnet build Tests/UnitTestsParallelizable --configuration Debug --no-restore
76+
77+
- name: Disable Windows Defender (Windows only)
78+
if: runner.os == 'Windows'
79+
shell: powershell
80+
run: |
81+
Add-MpPreference -ExclusionPath "${{ github.workspace }}"
82+
Add-MpPreference -ExclusionProcess "dotnet.exe"
83+
Add-MpPreference -ExclusionProcess "testhost.exe"
84+
Add-MpPreference -ExclusionProcess "VSTest.Console.exe"
85+
86+
- name: Set VSTEST_DUMP_PATH
87+
shell: bash
88+
run: echo "VSTEST_DUMP_PATH=logs/UnitTestsParallelizable/${{ runner.os }}/" >> $GITHUB_ENV
89+
90+
- name: Run UnitTestsParallelizable (3 iterations with varying parallelization)
91+
shell: bash
92+
run: |
93+
# Run tests 3 times with different parallelization settings to expose concurrency issues
94+
# Run 1: Default parallelization (2x) - standard test execution
95+
# Run 2: Maximum parallelization (unlimited) - stress test with high concurrency
96+
# Run 3: Single-threaded execution (1) - deterministic execution to expose ordering issues
97+
for RUN in {1..3}; do
98+
echo "============================================"
99+
echo "Starting test run $RUN of 3"
100+
echo "============================================"
101+
102+
# Use a combination of run number and timestamp to create different execution patterns
103+
SEED=$((1000 + $RUN + $(date +%s) % 1000))
104+
echo "Using randomization seed: $SEED"
105+
106+
# Vary the xUnit parallelization based on run number to expose race conditions
107+
if [ $RUN -eq 1 ]; then
108+
XUNIT_MAX_PARALLEL_THREADS="2x"
109+
echo "Run $RUN: Using default parallelization (2x)"
110+
elif [ $RUN -eq 2 ]; then
111+
XUNIT_MAX_PARALLEL_THREADS="unlimited"
112+
echo "Run $RUN: Using maximum parallelization (unlimited)"
113+
else
114+
XUNIT_MAX_PARALLEL_THREADS="1"
115+
echo "Run $RUN: Using single-threaded execution"
116+
fi
117+
118+
dotnet test Tests/UnitTestsParallelizable \
119+
--no-build \
120+
--verbosity normal \
121+
--settings Tests/UnitTestsParallelizable/runsettings.xml \
122+
--diag:logs/UnitTestsParallelizable/${{ runner.os }}/run${RUN}-logs.txt \
123+
--blame \
124+
--blame-crash \
125+
--blame-hang \
126+
--blame-hang-timeout 60s \
127+
--blame-crash-collect-always \
128+
-- xUnit.MaxParallelThreads=${XUNIT_MAX_PARALLEL_THREADS}
129+
130+
if [ $? -ne 0 ]; then
131+
echo "ERROR: Test run $RUN failed!"
132+
exit 1
133+
fi
134+
135+
echo "Test run $RUN completed successfully"
136+
echo ""
137+
done
138+
139+
echo "============================================"
140+
echo "All 3 test runs completed successfully!"
141+
echo "============================================"
142+
143+
- name: Upload UnitTestsParallelizable Logs
144+
if: always()
145+
uses: actions/upload-artifact@v4
146+
with:
147+
name: parallel_unittests_stress-logs-${{ runner.os }}
148+
path: |
149+
logs/UnitTestsParallelizable/
150+
TestResults/
151+

‎.github/workflows/unit-tests.yml‎

Lines changed: 38 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -154,80 +154,48 @@ jobs:
154154
shell: bash
155155
run: echo "VSTEST_DUMP_PATH=logs/UnitTestsParallelizable/${{ runner.os }}/" >> $GITHUB_ENV
156156

157-
- name: Run UnitTestsParallelizable (10 iterations with varying parallelization)
157+
- name: Run UnitTestsParallelizable
158158
shell: bash
159159
run: |
160-
# Run tests 3 times with different parallelization settings to expose concurrency issues
161-
for RUN in {1..3}; do
162-
echo "============================================"
163-
echo "Starting test run $RUN of 3"
164-
echo "============================================"
165-
166-
# Use a combination of run number and timestamp to create different execution patterns
167-
SEED=$((1000 + $RUN + $(date +%s) % 1000))
168-
echo "Using randomization seed: $SEED"
169-
170-
# Vary the xUnit parallelization based on run number to expose race conditions
171-
# Runs 1-3: Default parallelization (2x CPU cores)
172-
# Runs 4-6: Max parallelization (unlimited)
173-
# Runs 7-9: Single threaded (1)
174-
# Run 10: Random (1-4 threads)
175-
if [ $RUN -le 3 ]; then
176-
XUNIT_MAX_PARALLEL_THREADS="2x"
177-
echo "Run $RUN: Using default parallelization (2x)"
178-
elif [ $RUN -le 6 ]; then
179-
XUNIT_MAX_PARALLEL_THREADS="unlimited"
180-
echo "Run $RUN: Using maximum parallelization (unlimited)"
181-
elif [ $RUN -le 9 ]; then
182-
XUNIT_MAX_PARALLEL_THREADS="1"
183-
echo "Run $RUN: Using single-threaded execution"
184-
else
185-
# Random parallelization based on seed
186-
PROC_COUNT=$(( ($SEED % 4) + 1 ))
187-
XUNIT_MAX_PARALLEL_THREADS="$PROC_COUNT"
188-
echo "Run $RUN: Using random parallelization with $PROC_COUNT threads"
189-
fi
190-
191-
# Run tests with or without coverage based on OS and run number
192-
if [ "${{ runner.os }}" == "Linux" ] && [ $RUN -eq 1 ]; then
193-
echo "Run $RUN: Running with coverage collection"
194-
dotnet test Tests/UnitTestsParallelizable \
195-
--no-build \
196-
--verbosity normal \
197-
--collect:"XPlat Code Coverage" \
198-
--settings Tests/UnitTests/runsettings.coverage.xml \
199-
--diag:logs/UnitTestsParallelizable/${{ runner.os }}/run${RUN}-logs.txt \
200-
--blame \
201-
--blame-crash \
202-
--blame-hang \
203-
--blame-hang-timeout 60s \
204-
--blame-crash-collect-always \
205-
-- xUnit.MaxParallelThreads=${XUNIT_MAX_PARALLEL_THREADS}
206-
else
207-
dotnet test Tests/UnitTestsParallelizable \
208-
--no-build \
209-
--verbosity normal \
210-
--settings Tests/UnitTestsParallelizable/runsettings.xml \
211-
--diag:logs/UnitTestsParallelizable/${{ runner.os }}/run${RUN}-logs.txt \
212-
--blame \
213-
--blame-crash \
214-
--blame-hang \
215-
--blame-hang-timeout 60s \
216-
--blame-crash-collect-always \
217-
-- xUnit.MaxParallelThreads=${XUNIT_MAX_PARALLEL_THREADS}
218-
fi
219-
220-
if [ $? -ne 0 ]; then
221-
echo "ERROR: Test run $RUN failed!"
222-
exit 1
223-
fi
224-
225-
echo "Test run $RUN completed successfully"
226-
echo ""
227-
done
160+
# Run tests once for regular workflow runs (stress tests will run multiple iterations)
161+
echo "============================================"
162+
echo "Starting parallel unit tests"
163+
echo "============================================"
164+
165+
# Use default parallelization (2x CPU cores)
166+
echo "Using default parallelization (2x)"
167+
168+
# Run tests with or without coverage based on OS
169+
if [ "${{ runner.os }}" == "Linux" ]; then
170+
echo "Running with coverage collection on Linux"
171+
dotnet test Tests/UnitTestsParallelizable \
172+
--no-build \
173+
--verbosity normal \
174+
--collect:"XPlat Code Coverage" \
175+
--settings Tests/UnitTests/runsettings.coverage.xml \
176+
--diag:logs/UnitTestsParallelizable/${{ runner.os }}/logs.txt \
177+
--blame \
178+
--blame-crash \
179+
--blame-hang \
180+
--blame-hang-timeout 60s \
181+
--blame-crash-collect-always \
182+
-- xUnit.MaxParallelThreads=2x
183+
else
184+
dotnet test Tests/UnitTestsParallelizable \
185+
--no-build \
186+
--verbosity normal \
187+
--settings Tests/UnitTestsParallelizable/runsettings.xml \
188+
--diag:logs/UnitTestsParallelizable/${{ runner.os }}/logs.txt \
189+
--blame \
190+
--blame-crash \
191+
--blame-hang \
192+
--blame-hang-timeout 60s \
193+
--blame-crash-collect-always \
194+
-- xUnit.MaxParallelThreads=2x
195+
fi
228196
229197
echo "============================================"
230-
echo "All 10 test runs completed successfully!"
198+
echo "Parallel unit tests completed successfully!"
231199
echo "============================================"
232200
233201
- name: Upload UnitTestsParallelizable Logs

0 commit comments

Comments
 (0)