Skip to content

Commit 347896e

Browse files
duxiao1212facebook-github-bot
authored andcommitted
misc: Refactor ShuffleTest (#26643)
Summary: Completely replace TestShuffleWriter/Reader with local shuffle, and refactor the ShuffleTest class Differential Revision: D87160791
1 parent 262eac7 commit 347896e

File tree

3 files changed

+850
-1034
lines changed

3 files changed

+850
-1034
lines changed

presto-native-execution/presto_cpp/main/operators/LocalShuffle.cpp

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -355,6 +355,8 @@ void LocalShuffleWriter::collect(
355355
sortedShuffle_ || key.empty(),
356356
"key '{}' must be empty for non-sorted shuffle",
357357
key);
358+
velox::common::testutil::TestValue::adjust(
359+
"facebook::presto::operators::LocalShuffleWriter::collect", this);
358360

359361
const auto rowSize = this->rowSize(key.size(), data.size());
360362
auto& buffer = inProgressPartitions_[partition];
@@ -519,6 +521,8 @@ LocalShuffleReader::next(uint64_t maxBytes) {
519521
VELOX_CHECK(
520522
initialized_,
521523
"LocalShuffleReader::initialize() must be called before next()");
524+
velox::common::testutil::TestValue::adjust(
525+
"facebook::presto::operators::LocalShuffleReader::next", this);
522526

523527
return folly::makeSemiFuture(
524528
sortedShuffle_ ? nextSorted(maxBytes) : nextUnsorted(maxBytes));
@@ -617,13 +621,4 @@ std::shared_ptr<ShuffleWriter> LocalPersistentShuffleFactory::createWriter(
617621
sortedShuffle,
618622
pool);
619623
}
620-
621-
// Testing function to expose extractRowMetadata for tests.
622-
// This will be removed after reader changes.
623-
std::vector<RowMetadata> testingExtractRowMetadata(
624-
const char* buffer,
625-
size_t bufferSize,
626-
bool sortedShuffle) {
627-
return extractRowMetadata(buffer, bufferSize, sortedShuffle);
628-
}
629624
} // namespace facebook::presto::operators

presto-native-execution/presto_cpp/main/operators/LocalShuffle.h

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -53,12 +53,6 @@ inline bool compareKeys(std::string_view key1, std::string_view key2) noexcept {
5353
return key1.size() < key2.size();
5454
}
5555

56-
// Testing function to expose extractRowMetadata for tests.
57-
std::vector<RowMetadata> testingExtractRowMetadata(
58-
const char* buffer,
59-
size_t bufferSize,
60-
bool sortedShuffle);
61-
6256
// LocalShuffleWriteInfo is used for containing shuffle write information.
6357
// This struct is a 1:1 strict API mapping to
6458
// presto-spark-base/src/main/java/com/facebook/presto/spark/execution/PrestoSparkLocalShuffleWriteInfo.java
@@ -75,7 +69,6 @@ struct LocalShuffleWriteInfo {
7569
/// Structures are assumed to be encoded in JSON format.
7670
static LocalShuffleWriteInfo deserialize(const std::string& info);
7771
};
78-
7972
// LocalShuffleReadInfo is used for containing shuffle read metadata
8073
// This struct is a 1:1 strict API mapping to
8174
// presto-spark-base/src/main/java/com/facebook/presto/spark/execution/PrestoSparkLocalShuffleReadInfo.java.
@@ -87,8 +80,6 @@ struct LocalShuffleReadInfo {
8780
std::string queryId;
8881
std::vector<std::string> partitionIds;
8982

90-
/// Deserializes shuffle information that is used by LocalPersistentShuffle.
91-
/// Structures are assumed to be encoded in JSON format.
9283
static LocalShuffleReadInfo deserialize(const std::string& info);
9384
};
9485

0 commit comments

Comments
 (0)