Fix column range cardinality crash when histogram is null

Signed-off-by: Michael Okoko <[email protected]>

Author: idoqo

mysql-test/main/statistics.result

@@ -1,10 +1,11 @@
 set @SINGLE_PREC_TYPE='single_prec_hb';
 set @DOUBLE_PREC_TYPE='double_prec_hb';
+set @DEFAULT_HIST_TYPE=@@histogram_type;
 drop table if exists t1,t2;
 set @save_use_stat_tables=@@use_stat_tables;
 set @save_histogram_size=@@global.histogram_size;
 set @@global.histogram_size=0,@@local.histogram_size=0;
-set @save_hist_type=@@histogram_type;
+set @save_hist_type=@DEFAULT_HIST_TYPE;
 set histogram_type=@SINGLE_PREC_TYPE;
 DELETE FROM mysql.table_stats;
 DELETE FROM mysql.column_stats;

mysql-test/main/statistics_json.result

@@ -2441,7 +2441,7 @@ Warnings:
 Note	1003	select `test`.`t1_json`.`a` AS `a` from `test`.`t1_json` where `test`.`t1_json`.`a` < 'b-1a'
 analyze select * from t1_json where a > 'zzzzzzzzz';
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	r_rows	filtered	r_filtered	Extra
-1	SIMPLE	t1_json	ALL	NULL	NULL	NULL	NULL	10	10.00	10.00	0.00	Using where
+1	SIMPLE	t1_json	ALL	NULL	NULL	NULL	NULL	10	10.00	0.00	0.00	Using where
 UPDATE mysql.column_stats SET histogram='["a-1", "a-2", {"a": "b"}, "a-3"]' WHERE table_name='t1_json';
 FLUSH TABLES;
 explain extended select * from t1_json where a between 'a-3a' and 'zzzzzzzzz';
@@ -2475,7 +2475,7 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	r_rows	filtered	r_f
 1	SIMPLE	users	ALL	NULL	NULL	NULL	NULL	101	101.00	2.00	1.98	Using where
 explain extended select * from users where city < 'Lagos';
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
-1	SIMPLE	users	ALL	NULL	NULL	NULL	NULL	101	50.00	Using where
+1	SIMPLE	users	ALL	NULL	NULL	NULL	NULL	101	3.58	Using where
 Warnings:
 Note	1003	select `test`.`users`.`city` AS `city` from `test`.`users` where `test`.`users`.`city` < 'Lagos'
 drop table t1_bin;

sql/sql_statistics.cc

@@ -4364,10 +4364,37 @@ double get_column_range_cardinality(Field *field,
   {
     if (col_stats->min_max_values_are_provided())
     {
-      Histogram_base *hist = col_stats->histogram_;
-      double sel= hist->range_selectivity(field, min_endp, max_endp);
+      Histogram_base *hist= col_stats->histogram_;
+      double sel;
+      if (hist && hist->is_usable(thd))
+      {
+        sel= hist->range_selectivity(field, min_endp, max_endp);
+        set_if_bigger(res, col_stats->get_avg_frequency());
+      } else
+      {
+        double min_mp_pos, max_mp_pos;
+        if (min_endp && !(field->null_ptr && min_endp->key[0]))
+        {
+          store_key_image_to_rec(field, (uchar *) min_endp->key,
+                                 field->key_length());
+          min_mp_pos=
+              field->pos_in_interval(col_stats->min_value, col_stats->max_value);
+        }
+        else
+          min_mp_pos= 0.0;
+        if (max_endp)
+        {
+          store_key_image_to_rec(field, (uchar *) max_endp->key,
+                                 field->key_length());
+          max_mp_pos=
+              field->pos_in_interval(col_stats->min_value, col_stats->max_value);
+        }
+        else
+          max_mp_pos= 1.0;
+
+        sel = (max_mp_pos - min_mp_pos);
+      }
       res= col_non_nulls * sel;
-      set_if_bigger(res, col_stats->get_avg_frequency());
     }
     else
       res= col_non_nulls;
@@ -4525,23 +4552,16 @@ double Histogram_binary::range_selectivity(Field *field,
   else
     max_mp_pos= 1.0;
 
-  if (is_usable(field->table->in_use))
-  {
-    double bucket_sel= 1.0 / (get_width() + 1);
-    uint min= find_bucket(min_mp_pos, TRUE);
-    uint max= find_bucket(max_mp_pos, FALSE);
-    sel= bucket_sel * (max - min + 1);
+  double bucket_sel= 1.0 / (get_width() + 1);
+  uint min= find_bucket(min_mp_pos, TRUE);
+  uint max= find_bucket(max_mp_pos, FALSE);
+  sel= bucket_sel * (max - min + 1);
 
-    /*fprintf(stderr, "bucket_sel =%g\n", bucket_sel);
-    fprintf(stderr, "min pos_in_interval =%g\n", min_mp_pos);
-    fprintf(stderr, "max pos_in_interval =%g\n", max_mp_pos);
-    fprintf(stderr, "min =%d\n", min);
-    fprintf(stderr, "max =%d\n", max);*/
-  }
-  else
-  {
-    sel= (max_mp_pos - min_mp_pos);
-  }
+  /*fprintf(stderr, "bucket_sel =%g\n", bucket_sel);
+  fprintf(stderr, "min pos_in_interval =%g\n", min_mp_pos);
+  fprintf(stderr, "max pos_in_interval =%g\n", max_mp_pos);
+  fprintf(stderr, "min =%d\n", min);
+  fprintf(stderr, "max =%d\n", max);*/
   /*fprintf(stderr, "final sel =%g\n", sel);
   fprintf(stderr, "Histogram_binary::range_selectivity ends\n");*/
   return sel;