MDEV-28882: Assertion `tmp >= 0' failed in best_access_path

Histogram_json_hb::range_selectivity() may return small negative
numbers due to rounding errors in the histogram.

Make sure the returned value is non-negative.
Add an assert to catch negative values that are not small.

(attempt #2)

Author: spetrunia

mysql-test/main/statistics_json.result

@@ -8322,3 +8322,34 @@ histogram
   ]
 }
 drop table t1;
+#
+# MDEV-28882: Assertion `tmp >= 0' failed in best_access_path
+#
+CREATE TABLE t1 (a varchar(1));
+INSERT INTO t1 VALUES ('o'),('s'),('j'),('s'),('y'),('s'),('l'),
+('q'),('x'),('m'),('t'),('d'),('v'),('j'),('p'),('t'),('b'),('q');
+set histogram_type=json_hb;
+analyze table t1 persistent for all;
+Table	Op	Msg_type	Msg_text
+test.t1	analyze	status	Engine-independent statistics collected
+test.t1	analyze	status	OK
+# filtered must not be negative:
+explain format=json select * from t1 where a > 'y';
+EXPLAIN
+{
+  "query_block": {
+    "select_id": 1,
+    "nested_loop": [
+      {
+        "table": {
+          "table_name": "t1",
+          "access_type": "ALL",
+          "rows": 18,
+          "filtered": 5.555555344,
+          "attached_condition": "t1.a > 'y'"
+        }
+      }
+    ]
+  }
+}
+drop table t1;

mysql-test/main/statistics_json.test

@@ -460,3 +460,19 @@ from mysql.column_stats where table_name='t1' and db_name=database();
 
 drop table t1;
 
+--echo #
+--echo # MDEV-28882: Assertion `tmp >= 0' failed in best_access_path
+--echo #
+
+CREATE TABLE t1 (a varchar(1));
+
+INSERT INTO t1 VALUES ('o'),('s'),('j'),('s'),('y'),('s'),('l'),
+  ('q'),('x'),('m'),('t'),('d'),('v'),('j'),('p'),('t'),('b'),('q');
+
+set histogram_type=json_hb;
+analyze table t1 persistent for all;
+--echo # filtered must not be negative:
+explain format=json select * from t1 where a > 'y';
+
+drop table t1;
+

sql/opt_histogram_json.cc

@@ -959,9 +959,16 @@ std::string& Histogram_json_hb::get_end_value(int idx)
   @param field    The table field histogram is for.  We don't care about the
                   field's current value, we only need its virtual functions to
                   perform various operations
-
   @param min_endp Left endpoint, or NULL if there is none
   @param max_endp Right endpoint, or NULL if there is none
+  @param avg_sel  Average selectivity of "field=const" equality for this field
+
+  @return
+     Range selectivity: a number between 0.0 and 1.0.
+
+  @note
+     This may return 0.0. Adjustments to avoid multiply-by-zero meltdown are
+     made elsewhere.
 */
 
 double Histogram_json_hb::range_selectivity(Field *field, key_range *min_endp,
@@ -1062,6 +1069,18 @@ double Histogram_json_hb::range_selectivity(Field *field, key_range *min_endp,
   else
     max= 1.0;
 
+  if (min > max)
+  {
+    /*
+      This can happen due to rounding errors.
+
+      What is the acceptable error size? Json_writer::add_double() uses
+      %.11lg format. This gives 9 digits after the dot. A histogram may have
+      hundreds of buckets, let's multiply the error by 1000. 9-3=6
+    */
+    DBUG_ASSERT(max < min + 1e-6);
+    max= min;
+  }
   return max - min;
 }