Skip to content

Commit 667db80

Browse files
duxiao1212facebook-github-bot
authored andcommitted
misc: Refactor ShuffleTest (prestodb#26643)
Summary: Completely replace TestShuffleWriter/Reader with local shuffle, and refactor the ShuffleTest class Differential Revision: D87160791
1 parent c87b839 commit 667db80

File tree

3 files changed

+873
-1034
lines changed

3 files changed

+873
-1034
lines changed

presto-native-execution/presto_cpp/main/operators/LocalShuffle.cpp

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,10 @@ void LocalShuffleWriter::collect(
356356
"key '{}' must be empty for non-sorted shuffle",
357357
key);
358358

359+
// Testvalue injection point for exception testing
360+
velox::common::testutil::TestValue::adjust(
361+
"facebook::presto::operators::LocalShuffleWriter::collect", this);
362+
359363
const auto rowSize = this->rowSize(key.size(), data.size());
360364
auto& buffer = inProgressPartitions_[partition];
361365
if (buffer == nullptr) {
@@ -520,6 +524,10 @@ LocalShuffleReader::next(uint64_t maxBytes) {
520524
initialized_,
521525
"LocalShuffleReader::initialize() must be called before next()");
522526

527+
// Testvalue injection point for exception testing
528+
velox::common::testutil::TestValue::adjust(
529+
"facebook::presto::operators::LocalShuffleReader::next", this);
530+
523531
return folly::makeSemiFuture(
524532
sortedShuffle_ ? nextSorted(maxBytes) : nextUnsorted(maxBytes));
525533
}
@@ -617,13 +625,4 @@ std::shared_ptr<ShuffleWriter> LocalPersistentShuffleFactory::createWriter(
617625
sortedShuffle,
618626
pool);
619627
}
620-
621-
// Testing function to expose extractRowMetadata for tests.
622-
// This will be removed after reader changes.
623-
std::vector<RowMetadata> testingExtractRowMetadata(
624-
const char* buffer,
625-
size_t bufferSize,
626-
bool sortedShuffle) {
627-
return extractRowMetadata(buffer, bufferSize, sortedShuffle);
628-
}
629628
} // namespace facebook::presto::operators

presto-native-execution/presto_cpp/main/operators/LocalShuffle.h

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -53,12 +53,6 @@ inline bool compareKeys(std::string_view key1, std::string_view key2) noexcept {
5353
return key1.size() < key2.size();
5454
}
5555

56-
// Testing function to expose extractRowMetadata for tests.
57-
std::vector<RowMetadata> testingExtractRowMetadata(
58-
const char* buffer,
59-
size_t bufferSize,
60-
bool sortedShuffle);
61-
6256
// LocalShuffleWriteInfo is used for containing shuffle write information.
6357
// This struct is a 1:1 strict API mapping to
6458
// presto-spark-base/src/main/java/com/facebook/presto/spark/execution/PrestoSparkLocalShuffleWriteInfo.java
@@ -75,7 +69,6 @@ struct LocalShuffleWriteInfo {
7569
/// Structures are assumed to be encoded in JSON format.
7670
static LocalShuffleWriteInfo deserialize(const std::string& info);
7771
};
78-
7972
// LocalShuffleReadInfo is used for containing shuffle read metadata
8073
// This struct is a 1:1 strict API mapping to
8174
// presto-spark-base/src/main/java/com/facebook/presto/spark/execution/PrestoSparkLocalShuffleReadInfo.java.
@@ -87,8 +80,6 @@ struct LocalShuffleReadInfo {
8780
std::string queryId;
8881
std::vector<std::string> partitionIds;
8982

90-
/// Deserializes shuffle information that is used by LocalPersistentShuffle.
91-
/// Structures are assumed to be encoded in JSON format.
9283
static LocalShuffleReadInfo deserialize(const std::string& info);
9384
};
9485

0 commit comments

Comments
 (0)