Fix "element <@ range" cost estimation.

The statistics-based cost estimation patch for range types broke that, by
incorrectly assuming that the left operand of all range oeprators is a
range. That lead to a "type x is not a range type" error. Because it took so
long for anyone to notice, add a regression test for that case.

We still don't do proper statistics-based cost estimation for that, so you
just get a default constant estimate. We should look into implementing that,
but this patch at least fixes the regression.

Spotted by Tom Lane, when testing query from Josh Berkus.

3 files changed, 24 insertions, 2 deletions

diff --git a/src/backend/utils/adt/rangetypes_selfuncs.c b/src/backend/utils/adt/rangetypes_selfuncs.c
index 76dc913811..c450c6a158 100644
--- a/src/backend/utils/adt/rangetypes_selfuncs.c
+++ b/src/backend/utils/adt/rangetypes_selfuncs.c
@@ -154,8 +154,6 @@ rangesel(PG_FUNCTION_ARGS)
 		}
 	}
 
-	typcache = range_get_typcache(fcinfo, vardata.vartype);
-
 	/*
 	 * OK, there's a Var and a Const we're dealing with here.  We need the
 	 * Const to be of same range type as the column, else we can't do anything
@@ -169,6 +167,8 @@ rangesel(PG_FUNCTION_ARGS)
 	 */
 	if (operator == OID_RANGE_CONTAINS_ELEM_OP)
 	{
+		typcache = range_get_typcache(fcinfo, vardata.vartype);
+
 		if (((Const *) other)->consttype == typcache->rngelemtype->type_id)
 		{
 			RangeBound lower, upper;
@@ -185,6 +185,8 @@ rangesel(PG_FUNCTION_ARGS)
 	}
 	else
 	{
+		typcache = range_get_typcache(fcinfo, ((Const *) other)->consttype);
+
 		if (((Const *) other)->consttype == vardata.vartype)
 			constrange = DatumGetRangeType(((Const *) other)->constvalue);
 	}
diff --git a/src/test/regress/expected/rangetypes.out b/src/test/regress/expected/rangetypes.out
index 0cb6e53bf0..39db992729 100644
--- a/src/test/regress/expected/rangetypes.out
+++ b/src/test/regress/expected/rangetypes.out
@@ -1043,6 +1043,17 @@ select count(*) from test_range_spgist where ir -|- int4range(100,500);
 RESET enable_seqscan;
 RESET enable_indexscan;
 RESET enable_bitmapscan;
+-- test elem <@ range operator
+create table test_range_elem(i int4);
+create index test_range_elem_idx on test_range_elem (i);
+insert into test_range_elem select i from generate_series(1,100) i;
+select count(*) from test_range_elem where i <@ int4range(10,50);
+ count 
+-------
+    40
+(1 row)
+
+drop table test_range_elem;
 --
 -- Btree_gist is not included by default, so to test exclusion
 -- constraints with range types, use singleton int ranges for the "="
diff --git a/src/test/regress/sql/rangetypes.sql b/src/test/regress/sql/rangetypes.sql
index 035fceca19..fad843a405 100644
--- a/src/test/regress/sql/rangetypes.sql
+++ b/src/test/regress/sql/rangetypes.sql
@@ -286,6 +286,15 @@ RESET enable_seqscan;
 RESET enable_indexscan;
 RESET enable_bitmapscan;
 
+-- test elem <@ range operator
+create table test_range_elem(i int4);
+create index test_range_elem_idx on test_range_elem (i);
+insert into test_range_elem select i from generate_series(1,100) i;
+
+select count(*) from test_range_elem where i <@ int4range(10,50);
+
+drop table test_range_elem;
+
 --
 -- Btree_gist is not included by default, so to test exclusion
 -- constraints with range types, use singleton int ranges for the "="