Fix erroneous range-union logic for varlena types in contrib/btree_gist.

tglsfdc · tglsfdc · commit 500889a9d27c · 2013-02-07T18:22:32.000-05:00
gbt_var_bin_union() failed to do the right thing when the existing range
needed to be widened at both ends rather than just one end.  This could
result in an invalid index in which keys that are present would not be
found by searches, because the searches would not think they need to
descend to the relevant leaf pages.  This error affected all the varlena
datatypes supported by btree_gist (text, bytea, bit, numeric).

Per investigation of a trouble report from Tomas Vondra.  (There is also
an issue in gbt_var_penalty(), but that should only result in inefficiency
not wrong answers.  I'm committing this separately so that we have a git
state in which it can be tested that bad penalty results don't produce
invalid indexes.)  Back-patch to all supported branches.
diff --git a/contrib/btree_gist/btree_utils_var.c b/contrib/btree_gist/btree_utils_var.c
@@ -223,39 +223,40 @@ void
 gbt_var_bin_union(Datum *u, GBT_VARKEY *e, Oid collation,
 				  const gbtree_vinfo *tinfo)
 {
-	GBT_VARKEY *nk = NULL;
-	GBT_VARKEY *tmp = NULL;
-	GBT_VARKEY_R nr;
 	GBT_VARKEY_R eo = gbt_var_key_readable(e);
+	GBT_VARKEY_R nr;
 
 	if (eo.lower == eo.upper)	/* leaf */
 	{
+		GBT_VARKEY *tmp;
+
 		tmp = gbt_var_leaf2node(e, tinfo);
 		if (tmp != e)
 			eo = gbt_var_key_readable(tmp);
 	}
 
 	if (DatumGetPointer(*u))
 	{
-
 		GBT_VARKEY_R ro = gbt_var_key_readable((GBT_VARKEY *) DatumGetPointer(*u));
+		bool		update = false;
+
+		nr.lower = ro.lower;
+		nr.upper = ro.upper;
 
 		if ((*tinfo->f_cmp) (ro.lower, eo.lower, collation) > 0)
 		{
 			nr.lower = eo.lower;
-			nr.upper = ro.upper;
-			nk = gbt_var_key_copy(&nr, TRUE);
+			update = true;
 		}
 
 		if ((*tinfo->f_cmp) (ro.upper, eo.upper, collation) < 0)
 		{
 			nr.upper = eo.upper;
-			nr.lower = ro.lower;
-			nk = gbt_var_key_copy(&nr, TRUE);
+			update = true;
 		}
 
-		if (nk)
-			*u = PointerGetDatum(nk);
+		if (update)
+			*u = PointerGetDatum(gbt_var_key_copy(&nr, TRUE));
 	}
 	else
 	{

Original file line number	Diff line number	Diff line change
`@@ -223,39 +223,40 @@ void`
`223`	`223`	`gbt_var_bin_union(Datum u, GBT_VARKEY e, Oid collation,`
`224`	`224`	`const gbtree_vinfo *tinfo)`
`225`	`225`	`{`
`226`		`- GBT_VARKEY *nk = NULL;`
`227`		`- GBT_VARKEY *tmp = NULL;`
`228`		`- GBT_VARKEY_R nr;`
`229`	`226`	`GBT_VARKEY_R eo = gbt_var_key_readable(e);`
	`227`	`+ GBT_VARKEY_R nr;`
`230`	`228`
`231`	`229`	`if (eo.lower == eo.upper) /* leaf */`
`232`	`230`	`{`
	`231`	`+ GBT_VARKEY *tmp;`
	`232`	`+`
`233`	`233`	`tmp = gbt_var_leaf2node(e, tinfo);`
`234`	`234`	`if (tmp != e)`
`235`	`235`	`eo = gbt_var_key_readable(tmp);`
`236`	`236`	`}`
`237`	`237`
`238`	`238`	`if (DatumGetPointer(*u))`
`239`	`239`	`{`
`240`		`-`
`241`	`240`	`GBT_VARKEY_R ro = gbt_var_key_readable((GBT_VARKEY ) DatumGetPointer(u));`
	`241`	`+ bool update = false;`
	`242`	`+`
	`243`	`+ nr.lower = ro.lower;`
	`244`	`+ nr.upper = ro.upper;`
`242`	`245`
`243`	`246`	`if ((*tinfo->f_cmp) (ro.lower, eo.lower, collation) > 0)`
`244`	`247`	`{`
`245`	`248`	`nr.lower = eo.lower;`
`246`		`- nr.upper = ro.upper;`
`247`		`- nk = gbt_var_key_copy(&nr, TRUE);`
	`249`	`+ update = true;`
`248`	`250`	`}`
`249`	`251`
`250`	`252`	`if ((*tinfo->f_cmp) (ro.upper, eo.upper, collation) < 0)`
`251`	`253`	`{`
`252`	`254`	`nr.upper = eo.upper;`
`253`		`- nr.lower = ro.lower;`
`254`		`- nk = gbt_var_key_copy(&nr, TRUE);`
	`255`	`+ update = true;`
`255`	`256`	`}`
`256`	`257`
`257`		`- if (nk)`
`258`		`- *u = PointerGetDatum(nk);`
	`258`	`+ if (update)`
	`259`	`+ *u = PointerGetDatum(gbt_var_key_copy(&nr, TRUE));`
`259`	`260`	`}`
`260`	`261`	`else`
`261`	`262`	`{`