diff --git a/mysql-test/main/func_equal.result b/mysql-test/main/func_equal.result index 4f876b4f27bfe..90048f13e6934 100644 --- a/mysql-test/main/func_equal.result +++ b/mysql-test/main/func_equal.result @@ -54,3 +54,17 @@ c1 0 DROP TABLE t0; # End of 10.5 tests +# +# MDEV-36964: Stack looping, SIGSEGVs in `Item_hybrid_func::type_handler` and `Item::check_cols` on SELECT when using a function +# +CREATE FUNCTION f() RETURNS ROW TYPE OF t RETURN 1; +CREATE TABLE t (c INT KEY,c2 INT,UNIQUE (c2)); +SELECT ROW(COALESCE(f()),1)=ROW(1,1) AS eq; +ERROR HY000: Illegal parameter data type row for operation 'coalesce' +SELECT ROW(COALESCE(f()),1)=ROW(COALESCE(f()),1) AS eq; +ERROR HY000: Illegal parameter data type row for operation 'coalesce' +SELECT ROW(1, COALESCE(f()), 1)=ROW(1, COALESCE(f()), 1) AS eq; +ERROR HY000: Illegal parameter data type row for operation 'coalesce' +DROP FUNCTION f; +DROP TABLE t; +# diff --git a/mysql-test/main/func_equal.test b/mysql-test/main/func_equal.test index 88cab9e95a0f2..47b13a7058fa4 100644 --- a/mysql-test/main/func_equal.test +++ b/mysql-test/main/func_equal.test @@ -59,3 +59,24 @@ DROP TABLE t0; --echo # End of 10.5 tests + +--echo # +--echo # MDEV-36964: Stack looping, SIGSEGVs in `Item_hybrid_func::type_handler` and `Item::check_cols` on SELECT when using a function +--echo # + +CREATE FUNCTION f() RETURNS ROW TYPE OF t RETURN 1; +CREATE TABLE t (c INT KEY,c2 INT,UNIQUE (c2)); + +--error ER_ILLEGAL_PARAMETER_DATA_TYPE_FOR_OPERATION +SELECT ROW(COALESCE(f()),1)=ROW(1,1) AS eq; + +--error ER_ILLEGAL_PARAMETER_DATA_TYPE_FOR_OPERATION +SELECT ROW(COALESCE(f()),1)=ROW(COALESCE(f()),1) AS eq; + +--error ER_ILLEGAL_PARAMETER_DATA_TYPE_FOR_OPERATION +SELECT ROW(1, COALESCE(f()), 1)=ROW(1, COALESCE(f()), 1) AS eq; + +DROP FUNCTION f; +DROP TABLE t; + +--echo # diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc index eaa705642d843..aec3d1d9ff51b 100644 --- a/sql/item_cmpfunc.cc +++ b/sql/item_cmpfunc.cc @@ -60,9 +60,16 @@ static int cmp_row_type(Item* item1, Item* item2) return 1; for (uint i=0; ielement_index(i)->check_cols(item1->element_index(i)->cols()) || - (item1->element_index(i)->result_type() == ROW_RESULT && - cmp_row_type(item1->element_index(i), item2->element_index(i)))) + Item *left_item= item1->element_index(i); + Item *right_item= item2->element_index(i); + + if (right_item->check_cols(left_item->cols())) + return 1; + + if (!(left_item->cmp_type() == ROW_RESULT && + right_item->cmp_type() == ROW_RESULT) || + (item1 == left_item || item2 == right_item) || + cmp_row_type(left_item, right_item)) return 1; } return 0; @@ -8129,4 +8136,3 @@ Item *Item_equal::multiple_equality_transformer(THD *thd, uchar *arg) break; } } -