Skip to content

Commit 5ba7c4e

Browse files
SpadeA-TangSpadeA-Tang
authored
fix: fix false negative panic on missing fields [2.6] (#45903)
pr: #45902 issue: #45834 Signed-off-by: SpadeA <[email protected]>
1 parent eaed105 commit 5ba7c4e

File tree

2 files changed

+287
-2
lines changed

2 files changed

+287
-2
lines changed

‎internal/storage/utils.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -550,11 +550,13 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
550550
Data: make(map[FieldID]FieldData),
551551
}
552552
length := 0
553+
hasMissingFields := false
553554
getFieldData := func(field *schemapb.FieldSchema) (FieldData, error) {
554555
srcField, ok := srcFields[field.GetFieldID()]
555556
if !ok && field.GetFieldID() >= common.StartOfUserFieldID {
556-
err := fillMissingFields(collSchema, idata)
557-
return nil, err
557+
// Field not found in incoming message, will be handled by fillMissingFields later
558+
hasMissingFields = true
559+
return nil, nil
558560
}
559561
var fieldData FieldData
560562
switch field.DataType {
@@ -818,6 +820,12 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
818820
}
819821
}
820822
}
823+
if hasMissingFields {
824+
// Fill missing fields after all fields are processed
825+
if err := fillMissingFields(collSchema, idata); err != nil {
826+
return nil, err
827+
}
828+
}
821829

822830
idata.Infos = []BlobInfo{
823831
{Length: length},

‎internal/storage/utils_test.go‎

Lines changed: 277 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2558,3 +2558,280 @@ func TestFillMissingFields(t *testing.T) {
25582558
assert.Equal(t, 0, field101.RowNum())
25592559
})
25602560
}
2561+
2562+
func TestInsertDataWithMissingFieldBeforeRequired(t *testing.T) {
2563+
dim := 8
2564+
numRows := 3
2565+
2566+
// Schema where nullable field (101) comes BEFORE required field (102)
2567+
schema := &schemapb.CollectionSchema{
2568+
Name: "test_missing_field_order",
2569+
Fields: []*schemapb.FieldSchema{
2570+
{
2571+
FieldID: common.RowIDField,
2572+
Name: common.RowIDFieldName,
2573+
DataType: schemapb.DataType_Int64,
2574+
IsPrimaryKey: false,
2575+
},
2576+
{
2577+
FieldID: common.TimeStampField,
2578+
Name: common.TimeStampFieldName,
2579+
DataType: schemapb.DataType_Int64,
2580+
IsPrimaryKey: false,
2581+
},
2582+
{
2583+
FieldID: 100,
2584+
Name: "pk",
2585+
DataType: schemapb.DataType_Int64,
2586+
IsPrimaryKey: true,
2587+
},
2588+
{
2589+
// This nullable field will be MISSING from message
2590+
// It comes BEFORE the required field in schema order
2591+
FieldID: 101,
2592+
Name: "nullable_field",
2593+
DataType: schemapb.DataType_Int64,
2594+
Nullable: true,
2595+
},
2596+
{
2597+
// This required field EXISTS in message but comes AFTER nullable field
2598+
// Old bug: fillMissingFields was called when processing field 101,
2599+
// at which point field 102 was not yet in insertData, causing false error
2600+
FieldID: 102,
2601+
Name: "required_field",
2602+
DataType: schemapb.DataType_Int64,
2603+
Nullable: false,
2604+
},
2605+
{
2606+
FieldID: 103,
2607+
Name: "vec",
2608+
TypeParams: []*commonpb.KeyValuePair{
2609+
{Key: common.DimKey, Value: strconv.Itoa(dim)},
2610+
},
2611+
DataType: schemapb.DataType_FloatVector,
2612+
},
2613+
},
2614+
}
2615+
2616+
// Message has field 102 (required) but NOT field 101 (nullable)
2617+
msg := &msgstream.InsertMsg{
2618+
BaseMsg: msgstream.BaseMsg{},
2619+
InsertRequest: &msgpb.InsertRequest{
2620+
Base: &commonpb.MsgBase{MsgType: commonpb.MsgType_Insert},
2621+
NumRows: uint64(numRows),
2622+
Version: msgpb.InsertDataVersion_ColumnBased,
2623+
RowIDs: []int64{1, 2, 3},
2624+
Timestamps: []uint64{100, 200, 300},
2625+
FieldsData: []*schemapb.FieldData{
2626+
// pk field (100)
2627+
{
2628+
Type: schemapb.DataType_Int64,
2629+
FieldName: "pk",
2630+
FieldId: 100,
2631+
Field: &schemapb.FieldData_Scalars{
2632+
Scalars: &schemapb.ScalarField{
2633+
Data: &schemapb.ScalarField_LongData{
2634+
LongData: &schemapb.LongArray{Data: []int64{10, 20, 30}},
2635+
},
2636+
},
2637+
},
2638+
},
2639+
// Note: nullable_field (101) is intentionally MISSING
2640+
// required_field (102) - EXISTS in message
2641+
{
2642+
Type: schemapb.DataType_Int64,
2643+
FieldName: "required_field",
2644+
FieldId: 102,
2645+
Field: &schemapb.FieldData_Scalars{
2646+
Scalars: &schemapb.ScalarField{
2647+
Data: &schemapb.ScalarField_LongData{
2648+
LongData: &schemapb.LongArray{Data: []int64{100, 200, 300}},
2649+
},
2650+
},
2651+
},
2652+
},
2653+
// vec field (103)
2654+
{
2655+
Type: schemapb.DataType_FloatVector,
2656+
FieldName: "vec",
2657+
FieldId: 103,
2658+
Field: &schemapb.FieldData_Vectors{
2659+
Vectors: &schemapb.VectorField{
2660+
Dim: int64(dim),
2661+
Data: &schemapb.VectorField_FloatVector{
2662+
FloatVector: &schemapb.FloatArray{Data: make([]float32, numRows*dim)},
2663+
},
2664+
},
2665+
},
2666+
},
2667+
},
2668+
},
2669+
}
2670+
2671+
insertData, err := ColumnBasedInsertMsgToInsertData(msg, schema)
2672+
assert.NoError(t, err)
2673+
assert.NotNil(t, insertData)
2674+
2675+
field102, exists102 := insertData.Data[102]
2676+
assert.True(t, exists102, "required_field should be present")
2677+
assert.Equal(t, numRows, field102.RowNum())
2678+
2679+
// Verify nullable field was filled with nulls
2680+
field101, exists101 := insertData.Data[101]
2681+
assert.True(t, exists101, "nullable_field should be filled")
2682+
int64Field := field101.(*Int64FieldData)
2683+
assert.Equal(t, numRows, len(int64Field.ValidData))
2684+
for _, valid := range int64Field.ValidData {
2685+
assert.False(t, valid, "nullable field should have all null values")
2686+
}
2687+
}
2688+
2689+
func TestInsertDataWithStructAndMissingField(t *testing.T) {
2690+
dim := 8
2691+
numRows := 3
2692+
2693+
schema := &schemapb.CollectionSchema{
2694+
Name: "test_struct_missing_field",
2695+
Fields: []*schemapb.FieldSchema{
2696+
{
2697+
FieldID: common.RowIDField,
2698+
Name: common.RowIDFieldName,
2699+
DataType: schemapb.DataType_Int64,
2700+
IsPrimaryKey: false,
2701+
},
2702+
{
2703+
FieldID: common.TimeStampField,
2704+
Name: common.TimeStampFieldName,
2705+
DataType: schemapb.DataType_Int64,
2706+
IsPrimaryKey: false,
2707+
},
2708+
{
2709+
FieldID: 100,
2710+
Name: "pk",
2711+
DataType: schemapb.DataType_Int64,
2712+
IsPrimaryKey: true,
2713+
},
2714+
{
2715+
FieldID: 101,
2716+
Name: "vec",
2717+
TypeParams: []*commonpb.KeyValuePair{
2718+
{Key: common.DimKey, Value: strconv.Itoa(dim)},
2719+
},
2720+
DataType: schemapb.DataType_FloatVector,
2721+
},
2722+
{
2723+
// This nullable field will be missing from the message
2724+
FieldID: 102,
2725+
Name: "nullable_field",
2726+
DataType: schemapb.DataType_Int64,
2727+
Nullable: true,
2728+
},
2729+
},
2730+
StructArrayFields: []*schemapb.StructArrayFieldSchema{
2731+
{
2732+
FieldID: 103,
2733+
Name: "struct_array",
2734+
Fields: []*schemapb.FieldSchema{
2735+
{
2736+
FieldID: 104,
2737+
Name: "struct_array[name]",
2738+
DataType: schemapb.DataType_VarChar,
2739+
},
2740+
{
2741+
FieldID: 105,
2742+
Name: "struct_array[age]",
2743+
DataType: schemapb.DataType_Int64,
2744+
},
2745+
},
2746+
},
2747+
},
2748+
}
2749+
2750+
// Create message with struct subfields but WITHOUT the nullable field (102)
2751+
msg := &msgstream.InsertMsg{
2752+
BaseMsg: msgstream.BaseMsg{},
2753+
InsertRequest: &msgpb.InsertRequest{
2754+
Base: &commonpb.MsgBase{MsgType: commonpb.MsgType_Insert},
2755+
NumRows: uint64(numRows),
2756+
Version: msgpb.InsertDataVersion_ColumnBased,
2757+
RowIDs: []int64{1, 2, 3},
2758+
Timestamps: []uint64{100, 200, 300},
2759+
FieldsData: []*schemapb.FieldData{
2760+
// pk field (100)
2761+
{
2762+
Type: schemapb.DataType_Int64,
2763+
FieldName: "pk",
2764+
FieldId: 100,
2765+
Field: &schemapb.FieldData_Scalars{
2766+
Scalars: &schemapb.ScalarField{
2767+
Data: &schemapb.ScalarField_LongData{
2768+
LongData: &schemapb.LongArray{Data: []int64{10, 20, 30}},
2769+
},
2770+
},
2771+
},
2772+
},
2773+
// vec field (101)
2774+
{
2775+
Type: schemapb.DataType_FloatVector,
2776+
FieldName: "vec",
2777+
FieldId: 101,
2778+
Field: &schemapb.FieldData_Vectors{
2779+
Vectors: &schemapb.VectorField{
2780+
Dim: int64(dim),
2781+
Data: &schemapb.VectorField_FloatVector{
2782+
FloatVector: &schemapb.FloatArray{Data: make([]float32, numRows*dim)},
2783+
},
2784+
},
2785+
},
2786+
},
2787+
// Note: nullable_field (102) is intentionally missing
2788+
// struct_array[name] (104)
2789+
{
2790+
Type: schemapb.DataType_VarChar,
2791+
FieldName: "struct_array[name]",
2792+
FieldId: 104,
2793+
Field: &schemapb.FieldData_Scalars{
2794+
Scalars: &schemapb.ScalarField{
2795+
Data: &schemapb.ScalarField_StringData{
2796+
StringData: &schemapb.StringArray{Data: []string{"alice", "bob", "charlie"}},
2797+
},
2798+
},
2799+
},
2800+
},
2801+
// struct_array[age] (105)
2802+
{
2803+
Type: schemapb.DataType_Int64,
2804+
FieldName: "struct_array[age]",
2805+
FieldId: 105,
2806+
Field: &schemapb.FieldData_Scalars{
2807+
Scalars: &schemapb.ScalarField{
2808+
Data: &schemapb.ScalarField_LongData{
2809+
LongData: &schemapb.LongArray{Data: []int64{25, 30, 35}},
2810+
},
2811+
},
2812+
},
2813+
},
2814+
},
2815+
},
2816+
}
2817+
2818+
insertData, err := ColumnBasedInsertMsgToInsertData(msg, schema)
2819+
assert.NoError(t, err)
2820+
assert.NotNil(t, insertData)
2821+
2822+
// Verify struct subfields are present
2823+
_, exists104 := insertData.Data[104]
2824+
assert.True(t, exists104, "struct_array[name] should be present")
2825+
_, exists105 := insertData.Data[105]
2826+
assert.True(t, exists105, "struct_array[age] should be present")
2827+
2828+
// Verify nullable field was filled with nulls
2829+
field102, exists102 := insertData.Data[102]
2830+
assert.True(t, exists102, "nullable_field should be filled")
2831+
int64Field := field102.(*Int64FieldData)
2832+
assert.Equal(t, numRows, len(int64Field.ValidData))
2833+
// All values should be null
2834+
for _, valid := range int64Field.ValidData {
2835+
assert.False(t, valid, "nullable field should have all null values")
2836+
}
2837+
}

0 commit comments

Comments
 (0)