Skip to content

Commit 8814621

Browse files
MryangeMryange
committed
upd
1 parent c80dd34 commit 8814621

File tree

4 files changed

+193
-0
lines changed

4 files changed

+193
-0
lines changed

‎be/src/vec/columns/column.cpp‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,63 @@ std::string IColumn::dump_structure() const {
4343
return res.str();
4444
}
4545

46+
int IColumn::count_const_column() const {
47+
int count = is_column_const(*this) ? 1 : 0;
48+
ColumnCallback callback = [&](ColumnPtr& subcolumn) {
49+
count += subcolumn->count_const_column();
50+
};
51+
// simply read using for_each_subcolumn without modification; const_cast can be used.
52+
const_cast<IColumn*>(this)->for_each_subcolumn(callback);
53+
return count;
54+
}
55+
56+
bool IColumn::const_nested_check() const {
57+
auto const_cnt = count_const_column();
58+
if (const_cnt == 0) {
59+
return true;
60+
}
61+
// A const column is not allowed to be nested; it may only appear as the outermost (top-level) column.
62+
return const_cnt == 1 && is_column_const(*this);
63+
}
64+
65+
bool IColumn::null_map_check() const {
66+
auto check_null_map_is_zero_or_one = [&](const IColumn& subcolumn) {
67+
if (is_column_nullable(subcolumn)) {
68+
const auto& nullable_col = assert_cast<const ColumnNullable&>(subcolumn);
69+
const auto& null_map = nullable_col.get_null_map_data();
70+
for (size_t i = 0; i < null_map.size(); ++i) {
71+
if (null_map[i] != 0 && null_map[i] != 1) {
72+
return false;
73+
}
74+
}
75+
}
76+
return true;
77+
};
78+
79+
bool is_valid = check_null_map_is_zero_or_one(*this);
80+
ColumnCallback callback = [&](ColumnPtr& subcolumn) {
81+
if (!check_null_map_is_zero_or_one(*subcolumn)) {
82+
is_valid = false;
83+
}
84+
};
85+
// simply read using for_each_subcolumn without modification; const_cast can be used.
86+
const_cast<IColumn*>(this)->for_each_subcolumn(callback);
87+
return is_valid;
88+
}
89+
90+
Status IColumn::column_self_check() const {
91+
// check const nested
92+
if (!const_nested_check()) {
93+
return Status::InvalidArgument("const nested check failed for column: {} , {}", get_name(),
94+
dump_structure());
95+
}
96+
// check null map
97+
if (!null_map_check()) {
98+
return Status::InvalidArgument("null map check failed for column: {}", get_name());
99+
}
100+
return Status::OK();
101+
}
102+
46103
void IColumn::insert_from(const IColumn& src, size_t n) {
47104
insert(src[n]);
48105
}

‎be/src/vec/columns/column.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -647,6 +647,17 @@ class IColumn : public COW<IColumn> {
647647
*/
648648
String dump_structure() const;
649649

650+
// count how many const column including self
651+
int count_const_column() const;
652+
653+
bool null_map_check() const;
654+
655+
// const column nested check, eg. const(nullable(...)) is allowed
656+
// const(array(const(...))) is not allowed
657+
bool const_nested_check() const;
658+
659+
Status column_self_check() const;
660+
650661
// only used in agg value replace for column which is not variable length, eg.BlockReader::_copy_value_data
651662
// usage: self_column.replace_column_data(other_column, other_column's row index, self_column's row index)
652663
virtual void replace_column_data(const IColumn&, size_t row, size_t self_row = 0) = 0;

‎be/src/vec/core/block.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -399,6 +399,8 @@ Status Block::check_type_and_column() const {
399399
const auto& type = elem.type;
400400
const auto& column = elem.column;
401401

402+
RETURN_IF_ERROR(column->column_self_check());
403+
402404
auto st = type->check_column(*column);
403405
if (!st.ok()) {
404406
return Status::InternalError(

‎be/test/vec/columns/column_self_check.cpp‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#include <gtest/gtest.h>
19+
20+
#include "runtime/primitive_type.h"
21+
#include "testutil/column_helper.h"
22+
#include "vec/columns/column.h"
23+
#include "vec/columns/column_array.h"
24+
#include "vec/columns/column_const.h"
25+
#include "vec/data_types/data_type_number.h"
26+
#include "vec/functions/function.h"
27+
#include "vec/functions/simple_function_factory.h"
28+
29+
namespace doris::vectorized {
30+
31+
TEST(ColumnSelfCheckTest, const_check_test) {
32+
{
33+
ColumnPtr col = ColumnHelper::create_column<DataTypeInt32>({1, 2, 3});
34+
EXPECT_EQ(col->const_nested_check(), true);
35+
}
36+
37+
{
38+
ColumnPtr col = ColumnHelper::create_column<DataTypeInt32>({1});
39+
ColumnPtr const_col = ColumnConst::create(col, 3);
40+
EXPECT_EQ(const_col->const_nested_check(), true);
41+
}
42+
43+
{
44+
ColumnPtr col = ColumnHelper::create_column<DataTypeInt32>({});
45+
46+
auto array_const_col = ColumnArray::create(col);
47+
48+
array_const_col->data = ColumnConst::create(col, 3, true);
49+
50+
auto const_array = ColumnConst::create(std::move(array_const_col), 2, true);
51+
52+
std::cout << const_array->dump_structure() << std::endl;
53+
54+
std::cout << const_array->count_const_column() << std::endl;
55+
56+
EXPECT_EQ(const_array->const_nested_check(), false);
57+
}
58+
59+
{
60+
ColumnPtr col = ColumnHelper::create_column<DataTypeInt32>({});
61+
62+
auto array_const_col = ColumnArray::create(col);
63+
64+
array_const_col->data = ColumnConst::create(col, 3, true);
65+
66+
auto const_array = ColumnConst::create(std::move(array_const_col), 2, true);
67+
68+
Block block;
69+
block.insert({std::move(const_array),
70+
std::make_shared<DataTypeArray>(std::make_shared<DataTypeInt32>()),
71+
"array_col"});
72+
73+
EXPECT_FALSE(block.check_type_and_column());
74+
}
75+
}
76+
77+
TEST(ColumnSelfCheckTest, nullmap_check_test) {
78+
{
79+
auto col = ColumnHelper::create_column<DataTypeInt32>({1, 2, 3});
80+
EXPECT_EQ(col->null_map_check(), true);
81+
}
82+
83+
{
84+
auto col = ColumnHelper::create_nullable_column<DataTypeInt32>({1, 2, 3}, {0, 0, 0});
85+
EXPECT_EQ(col->null_map_check(), true);
86+
}
87+
88+
{
89+
auto col = ColumnHelper::create_nullable_column<DataTypeInt32>({1, 2, 3}, {0, 1, 0});
90+
EXPECT_EQ(col->null_map_check(), true);
91+
}
92+
93+
{
94+
auto col = ColumnHelper::create_nullable_column<DataTypeInt32>({1, 2, 3}, {0, 2, 0});
95+
EXPECT_EQ(col->null_map_check(), false);
96+
}
97+
98+
{
99+
auto col = ColumnHelper::create_nullable_column<DataTypeInt32>(
100+
{
101+
1,
102+
},
103+
{3});
104+
105+
auto col_const = ColumnConst::create(col, 2);
106+
107+
EXPECT_EQ(col_const->null_map_check(), false);
108+
}
109+
110+
{
111+
auto col = ColumnHelper::create_nullable_column<DataTypeInt32>(
112+
{
113+
1,
114+
},
115+
{3});
116+
117+
auto col_const = ColumnConst::create(col, 2);
118+
119+
EXPECT_EQ(col_const->column_self_check(), false);
120+
}
121+
}
122+
123+
} // namespace doris::vectorized

0 commit comments

Comments
 (0)