Skip to content

Commit cd12487

Browse files
davidmorgandavidmorgan
authored
Add playback from logs to watcher e2e test, start collection of test cases. (#2257)
1 parent 3e2ee13 commit cd12487

File tree

3 files changed

+319
-137
lines changed

3 files changed

+319
-137
lines changed

‎pkgs/watcher/test/directory_watcher/end_to_end_test_runner.dart‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
// ignore_for_file: unreachable_from_main
5+
6+
import 'dart:io';
7+
8+
import 'package:path/path.dart' as p;
9+
import 'package:test/test.dart' as package_test;
10+
import 'package:watcher/src/utils.dart';
11+
import 'package:watcher/watcher.dart';
12+
13+
import '../utils.dart' as utils;
14+
import 'client_simulator.dart';
15+
import 'end_to_end_tests.dart';
16+
import 'file_changer.dart';
17+
18+
/// Runs the test with [name] for logging.
19+
///
20+
/// To run without `package:test`, pass [addTearDown], [createWatcher], [fail]
21+
/// and [printOnFailure] replacements.
22+
///
23+
/// Runs [repeats] times, by default 50. Set to -1 to run until failure.
24+
///
25+
/// By default, runs through a fixed series of pseudo-random test cases. Or,
26+
/// pass [fixSeed] to run at a fixed seed. Or, pass [replayLog] to replay from a
27+
/// test log.
28+
Future<void> runTest({
29+
required String name,
30+
void Function(void Function())? addTearDown,
31+
Watcher Function({required String path})? createWatcher,
32+
void Function(String)? fail,
33+
void Function(String)? printOnFailure,
34+
int repeats = 50,
35+
int? fixSeed,
36+
String? replayLog,
37+
}) async {
38+
addTearDown ??= package_test.addTearDown;
39+
createWatcher ??= utils.createWatcher;
40+
fail ??= package_test.fail;
41+
printOnFailure ??= package_test.printOnFailure;
42+
43+
final temp = Directory.systemTemp.createTempSync();
44+
addTearDown(() => temp.deleteSync(recursive: true));
45+
46+
// Turn on logging of the watchers.
47+
final log = <LogEntry>[];
48+
logForTesting = (message) => log.add(LogEntry('W $message'));
49+
50+
// Create the watcher and [ClientSimulator].
51+
final watcher = createWatcher(path: temp.path);
52+
final client = await ClientSimulator.watch(
53+
watcher: watcher, log: (message) => log.add(LogEntry('C $message')));
54+
addTearDown(client.close);
55+
56+
// Making changes, waiting for events to settle, check for consistency.
57+
final changer = FileChanger(temp.path);
58+
for (var i = 0; i != repeats; ++i) {
59+
log.clear();
60+
if (repeats < 0) stdout.write('.');
61+
for (final entity in temp.listSync()) {
62+
entity.deleteSync(recursive: true);
63+
}
64+
65+
// File changes.
66+
int? seed;
67+
if (replayLog == null) {
68+
seed ??= fixSeed ?? i;
69+
log.addAll(await changer.changeFiles(times: 200, seed: seed));
70+
} else {
71+
log.addAll(await changer.replayLog(replayLog));
72+
}
73+
74+
// Give time for events to arrive. To allow tests to run quickly when the
75+
// events are handled quickly, poll and continue if verification passes.
76+
var succeeded = false;
77+
for (var waits = 0; waits != 20; ++waits) {
78+
if (client.verify()) {
79+
succeeded = true;
80+
break;
81+
}
82+
await client.waitForNoEvents(const Duration(milliseconds: 100));
83+
}
84+
85+
// Fail the test if still not consistent.
86+
if (!succeeded) {
87+
if (repeats < 0) print('');
88+
client.verify(printOnFailure: printOnFailure);
89+
// Write the file operations before the failure to a log, fail the test.
90+
final logTemp = Directory.systemTemp.createTempSync();
91+
final logPath = p.join(logTemp.path, 'log.txt');
92+
93+
// Sort the log entries by timestamp.
94+
log.sort();
95+
96+
File(logPath).writeAsStringSync(log.map((m) => '$m\n').join(''));
97+
final failMessage = StringBuffer('\n');
98+
99+
if (seed != null) {
100+
failMessage.write('''
101+
Failed `$name` on run $i. Run in a loop with that seed using:
102+
103+
dart test/directory_watcher/end_to_end_test_runner.dart seed $seed
104+
''');
105+
} else {
106+
failMessage.write('''
107+
Failed `$name` on run $i.
108+
''');
109+
}
110+
111+
failMessage.write('''
112+
113+
Changes/watcher/client log: $logPath
114+
''');
115+
116+
fail(failMessage.toString());
117+
}
118+
}
119+
}
120+
121+
/// Main method for running the e2e test without `package:test`.
122+
///
123+
/// By default runs endlessly with seeds starting at 0.
124+
///
125+
/// Pass `seed <number>` to fix the seed.
126+
///
127+
/// Pass `replay` to run endlessly with hardcoded test cases from
128+
/// `end_to_end_tests.dart`.
129+
///
130+
/// Exits on failure, or runs forever.
131+
Future<void> main(List<String> arguments) async {
132+
final command = arguments.isEmpty ? null : arguments[0];
133+
final fixSeed = command == 'seed' ? int.parse(arguments[1]) : null;
134+
final replay = command == 'replay';
135+
136+
final teardowns = <void Function()>[];
137+
try {
138+
if (replay) {
139+
while (true) {
140+
stdout.write('.');
141+
for (final testCase in testCases) {
142+
await runTest(
143+
name: testCase.name,
144+
addTearDown: teardowns.add,
145+
createWatcher: ({required String path}) => DirectoryWatcher(path),
146+
fail: (message) {
147+
print(message);
148+
exit(1);
149+
},
150+
printOnFailure: print,
151+
// Repeat a few times before moving onto the next test case to get
152+
// any effect of multiple consecutive runs.
153+
repeats: 3,
154+
replayLog: testCase.log,
155+
);
156+
}
157+
}
158+
} else {
159+
await runTest(
160+
name: fixSeed == null ? 'random' : 'fixed seed $fixSeed',
161+
addTearDown: teardowns.add,
162+
createWatcher: ({required String path}) => DirectoryWatcher(path),
163+
fail: (message) {
164+
print(message);
165+
exit(1);
166+
},
167+
printOnFailure: print,
168+
repeats: -1,
169+
fixSeed: fixSeed,
170+
);
171+
}
172+
} finally {
173+
for (final teardown in teardowns) {
174+
teardown();
175+
}
176+
}
177+
}
178+
179+
/// Log entry with timestamp.
180+
///
181+
/// Because file events happen on a different isolate the merged log uses
182+
/// timestamps to put entries in the correct order.
183+
class LogEntry implements Comparable<LogEntry> {
184+
final DateTime timestamp;
185+
final String message;
186+
187+
LogEntry(this.message) : timestamp = DateTime.now();
188+
189+
@override
190+
int compareTo(LogEntry other) => timestamp.compareTo(other.timestamp);
191+
192+
@override
193+
String toString() => message;
194+
}
195+
196+
/// Test case using log replay.
197+
class TestCase {
198+
final String name;
199+
final String log;
200+
final bool skipOnLinux;
201+
202+
TestCase(this.name, this.log, {this.skipOnLinux = false});
203+
}

‎pkgs/watcher/test/directory_watcher/end_to_end_tests.dart‎

Lines changed: 35 additions & 133 deletions
Original file line numberDiff line numberDiff line change
@@ -5,149 +5,51 @@
55

66
import 'dart:io';
77

8-
import 'package:path/path.dart' as p;
9-
import 'package:test/test.dart' as package_test;
10-
import 'package:watcher/src/utils.dart';
11-
import 'package:watcher/watcher.dart';
8+
import 'package:test/test.dart';
129

13-
import '../utils.dart' as utils;
1410
import 'client_simulator.dart';
11+
import 'end_to_end_test_runner.dart';
1512
import 'file_changer.dart';
1613

17-
/// End to end test using a [FileChanger] that randomly changes files, then a
14+
/// End to end test using a [FileChanger] to change files then a
1815
/// [ClientSimulator] that tracks state using a Watcher.
1916
///
20-
/// The test passes if the [ClientSimulator] tracking matches what's actually on
17+
/// The tests pass if the [ClientSimulator] tracking matches what's actually on
2118
/// disk.
19+
///
20+
/// `end_to_end_test_runner` can be run as a binary to try random file changes
21+
/// until a failure, it outputs a log which can be turned into a test case here.
2222
void endToEndTests() {
23-
package_test.test('end to end test',
24-
timeout: const package_test.Timeout(Duration(minutes: 5)), () async {
25-
await _runTest();
23+
// Random test to cover a wide range of cases.
24+
test('end to end test: random', timeout: const Timeout(Duration(minutes: 5)),
25+
() async {
26+
await runTest(name: 'random', repeats: 100);
2627
});
27-
}
28-
29-
/// Runs the test.
30-
///
31-
/// To run without `package:test`, pass [addTearDown], [createWatcher], [fail]
32-
/// and [printOnFailure] replacements.
33-
///
34-
/// To run until failure, set [endlessMode] to `true`.
35-
///
36-
/// To fix the seed, set [seed]. The failure message prints the seed, so this
37-
/// can be used to run just the events that triggered the failure.
38-
Future<void> _runTest({
39-
void Function(void Function())? addTearDown,
40-
Watcher Function({required String path})? createWatcher,
41-
void Function(String)? fail,
42-
void Function(String)? printOnFailure,
43-
bool endlessMode = false,
44-
int? seed,
45-
}) async {
46-
addTearDown ??= package_test.addTearDown;
47-
createWatcher ??= utils.createWatcher;
48-
fail ??= package_test.fail;
49-
printOnFailure ??= package_test.printOnFailure;
50-
51-
final temp = Directory.systemTemp.createTempSync();
52-
addTearDown(() => temp.deleteSync(recursive: true));
53-
54-
// Turn on logging of the watchers.
55-
final log = <LogEntry>[];
56-
logForTesting = (message) => log.add(LogEntry('W $message'));
57-
58-
// Create the watcher and [ClientSimulator].
59-
final watcher = createWatcher(path: temp.path);
60-
final client = await ClientSimulator.watch(
61-
watcher: watcher, log: (message) => log.add(LogEntry('C $message')));
62-
addTearDown(client.close);
63-
64-
// 40 iterations of making changes, waiting for events to settle, and
65-
// checking for consistency.
66-
final changer = FileChanger(temp.path);
67-
for (var i = 0; endlessMode || i != 40; ++i) {
68-
final runSeed = seed ?? i;
69-
log.clear();
70-
if (endlessMode) stdout.write('.');
71-
for (final entity in temp.listSync()) {
72-
entity.deleteSync(recursive: true);
73-
}
74-
// File changes.
75-
log.addAll(await changer.changeFiles(times: 200, seed: runSeed));
76-
77-
// Give time for events to arrive. To allow tests to run quickly when the
78-
// events are handled quickly, poll and continue if verification passes.
79-
var succeeded = false;
80-
for (var waits = 0; waits != 20; ++waits) {
81-
if (client.verify()) {
82-
succeeded = true;
83-
break;
84-
}
85-
await client.waitForNoEvents(const Duration(milliseconds: 100));
86-
}
87-
88-
// Fail the test if still not consistent.
89-
if (!succeeded) {
90-
if (endlessMode) print('');
91-
client.verify(printOnFailure: printOnFailure);
92-
// Write the file operations before the failure to a log, fail the test.
93-
final logTemp = Directory.systemTemp.createTempSync();
94-
final logPath = p.join(logTemp.path, 'log.txt');
95-
96-
// Sort the log entries by timestamp.
97-
log.sort();
9828

99-
File(logPath).writeAsStringSync(log.map((m) => '$m\n').join(''));
100-
fail('''
101-
Failed on run $i, seed $runSeed. Run in a loop with that seed using:
102-
103-
dart test/directory_watcher/end_to_end_tests.dart $runSeed
104-
105-
Changes/watcher/client log: $logPath
106-
''');
107-
}
108-
}
109-
}
110-
111-
/// Main method for running the e2e test without `package:test`.
112-
///
113-
/// Optionally, pass the seed to run with as the only argument.
114-
///
115-
/// Exits on failure, or runs forever.
116-
Future<void> main(List<String> arguments) async {
117-
final seed = arguments.isNotEmpty ? int.parse(arguments.first) : null;
118-
final teardowns = <void Function()>[];
119-
try {
120-
await _runTest(
121-
addTearDown: teardowns.add,
122-
createWatcher: ({required String path}) => DirectoryWatcher(path),
123-
fail: (message) {
124-
print(message);
125-
exit(1);
126-
},
127-
printOnFailure: print,
128-
endlessMode: true,
129-
seed: seed,
130-
);
131-
} finally {
132-
for (final teardown in teardowns) {
133-
teardown();
134-
}
29+
// Specific test cases that have caught bugs.
30+
for (final testCase in testCases) {
31+
test('end to end test: ${testCase.name}',
32+
timeout: const Timeout(Duration(minutes: 5)), () async {
33+
await runTest(name: testCase.name, replayLog: testCase.log, repeats: 100);
34+
}, skip: testCase.skipOnLinux && Platform.isLinux);
13535
}
13636
}
13737

138-
/// Log entry with timestamp.
139-
///
140-
/// Because file events happen on a different isolate the merged log uses
141-
/// timestamps to put entries in the correct order.
142-
class LogEntry implements Comparable<LogEntry> {
143-
final DateTime timestamp;
144-
final String message;
145-
146-
LogEntry(this.message) : timestamp = DateTime.now();
147-
148-
@override
149-
int compareTo(LogEntry other) => timestamp.compareTo(other.timestamp);
150-
151-
@override
152-
String toString() => message;
153-
}
38+
final testCases = [
39+
TestCase(
40+
'move over, modify, delete in new directory',
41+
'''
42+
F create,62543,809
43+
F wait
44+
F wait
45+
F wait
46+
F wait
47+
F create directory,a
48+
F create,a/63090,758
49+
F move file over file,62543,a/63090
50+
F modify,a/63090,439
51+
F delete,a/63090
52+
''',
53+
skipOnLinux: true,
54+
)
55+
];

0 commit comments

Comments
 (0)