Revert "Permit dump/reload of not-too-large >1GB tuples"

alvherre · alvherre · commit b66adb7b0c83 · 2017-05-10T18:41:27.000-03:00
This reverts commits fa2fa99 and 42f50cb. While the functionality that was intended to be provided by these commits is desired, the patch didn't actually solve as many of the problematic situations as we hoped, and it created a bunch of its own problems. Since we're going to require more extensive changes soon for other reasons and users have been working around these problems for a long time already, there is no point in spending effort in fixing this halfway measure. Per complaint from Tom Lane. Discussion: https://postgr.es/m/21407.1484606922@sss.pgh.pa.us (Commit fa2fa99 had already been reverted in branches 9.5 as f858524 and 9.6 as e9e44a0, so this touches master only. Commit 42f50cb was not present in the older branches.)
diff --git a/src/backend/access/common/heaptuple.c b/src/backend/access/common/heaptuple.c
@@ -741,9 +741,7 @@ heap_form_tuple(TupleDesc tupleDescriptor,
 	 * Allocate and zero the space needed.  Note that the tuple body and
 	 * HeapTupleData management structure are allocated in one chunk.
 	 */
-	tuple = MemoryContextAllocExtended(CurrentMemoryContext,
-									   HEAPTUPLESIZE + len,
-									   MCXT_ALLOC_HUGE | MCXT_ALLOC_ZERO);
+	tuple = (HeapTuple) palloc0(HEAPTUPLESIZE + len);
 	tuple->t_data = td = (HeapTupleHeader) ((char *) tuple + HEAPTUPLESIZE);
 
 	/*
diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c
@@ -394,7 +394,7 @@ ReceiveCopyBegin(CopyState cstate)
 			pq_sendint(&buf, format, 2);		/* per-column formats */
 		pq_endmessage(&buf);
 		cstate->copy_dest = COPY_NEW_FE;
-		cstate->fe_msgbuf = makeLongStringInfo();
+		cstate->fe_msgbuf = makeStringInfo();
 	}
 	else
 	{
@@ -1954,7 +1954,7 @@ CopyTo(CopyState cstate)
 	cstate->null_print_client = cstate->null_print;		/* default */
 
 	/* We use fe_msgbuf as a per-row buffer regardless of copy_dest */
-	cstate->fe_msgbuf = makeLongStringInfo();
+	cstate->fe_msgbuf = makeStringInfo();
 
 	/* Get info about the columns we need to process. */
 	cstate->out_functions = (FmgrInfo *) palloc(num_phys_attrs * sizeof(FmgrInfo));
@@ -2909,8 +2909,8 @@ BeginCopyFrom(ParseState *pstate,
 	cstate->cur_attval = NULL;
 
 	/* Set up variables to avoid per-attribute overhead. */
-	initLongStringInfo(&cstate->attribute_buf);
-	initLongStringInfo(&cstate->line_buf);
+	initStringInfo(&cstate->attribute_buf);
+	initStringInfo(&cstate->line_buf);
 	cstate->line_buf_converted = false;
 	cstate->raw_buf = (char *) palloc(RAW_BUF_SIZE + 1);
 	cstate->raw_buf_index = cstate->raw_buf_len = 0;
diff --git a/src/backend/lib/stringinfo.c b/src/backend/lib/stringinfo.c
@@ -4,8 +4,7 @@
  *
  * StringInfo provides an indefinitely-extensible string data type.
  * It can be used to buffer either ordinary C strings (null-terminated text)
- * or arbitrary binary data.  All storage is allocated with palloc() and
- * friends.
+ * or arbitrary binary data.  All storage is allocated with palloc().
  *
  * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
@@ -37,29 +36,11 @@ makeStringInfo(void)
 	return res;
 }
 
-/*
- * makeLongStringInfo
- *
- * Same as makeStringInfo, for larger strings.
- */
-StringInfo
-makeLongStringInfo(void)
-{
-	StringInfo	res;
-
-	res = (StringInfo) palloc(sizeof(StringInfoData));
-
-	initLongStringInfo(res);
-
-	return res;
-}
-
-
 /*
  * initStringInfo
  *
  * Initialize a StringInfoData struct (with previously undefined contents)
- * to describe an empty string; don't enable long strings yet.
+ * to describe an empty string.
  */
 void
 initStringInfo(StringInfo str)
@@ -68,22 +49,9 @@ initStringInfo(StringInfo str)
 
 	str->data = (char *) palloc(size);
 	str->maxlen = size;
-	str->long_ok = false;
 	resetStringInfo(str);
 }
 
-/*
- * initLongStringInfo
- *
- * Same as initStringInfo, plus enable long strings.
- */
-void
-initLongStringInfo(StringInfo str)
-{
-	initStringInfo(str);
-	str->long_ok = true;
-}
-
 /*
  * resetStringInfo
  *
@@ -174,7 +142,7 @@ appendStringInfoVA(StringInfo str, const char *fmt, va_list args)
 	/*
 	 * Return pvsnprintf's estimate of the space needed.  (Although this is
 	 * given as a size_t, we know it will fit in int because it's not more
-	 * than either MaxAllocSize or half an int's width.)
+	 * than MaxAllocSize.)
 	 */
 	return (int) nprinted;
 }
@@ -276,25 +244,15 @@ appendBinaryStringInfo(StringInfo str, const char *data, int datalen)
 void
 enlargeStringInfo(StringInfo str, int needed)
 {
-	Size		newlen;
-	Size		limit;
-
-	/*
-	 * Determine the upper size limit.  Because of overflow concerns outside
-	 * of this module, we limit ourselves to 4-byte signed integer range,
-	 * even for "long_ok" strings.
-	 */
-	limit = str->long_ok ?
-		(((Size) 1) << (sizeof(int32) * 8 - 1)) - 1 :
-		MaxAllocSize;
+	int			newlen;
 
 	/*
 	 * Guard against out-of-range "needed" values.  Without this, we can get
 	 * an overflow or infinite loop in the following.
 	 */
 	if (needed < 0)				/* should not happen */
 		elog(ERROR, "invalid string enlargement request size: %d", needed);
-	if (((Size) needed) >= (limit - (Size) str->len))
+	if (((Size) needed) >= (MaxAllocSize - (Size) str->len))
 		ereport(ERROR,
 				(errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
 				 errmsg("out of memory"),
@@ -303,7 +261,7 @@ enlargeStringInfo(StringInfo str, int needed)
 
 	needed += str->len + 1;		/* total space required now */
 
-	/* Because of the above test, we now have needed <= limit */
+	/* Because of the above test, we now have needed <= MaxAllocSize */
 
 	if (needed <= str->maxlen)
 		return;					/* got enough space already */
@@ -313,20 +271,19 @@ enlargeStringInfo(StringInfo str, int needed)
 	 * for efficiency, double the buffer size each time it overflows.
 	 * Actually, we might need to more than double it if 'needed' is big...
 	 */
-	newlen = 2 * (Size) str->maxlen;
-	while ((Size) needed > newlen)
+	newlen = 2 * str->maxlen;
+	while (needed > newlen)
 		newlen = 2 * newlen;
 
 	/*
-	 * Clamp to the limit in case we went past it.  (We used to depend on
-	 * limit <= INT32_MAX/2, to avoid overflow in the loop above; we no longer
-	 * depend on that, but if "needed" and str->maxlen ever become wider, we
-	 * will need similar caution here.)  We will still have newlen >= needed.
+	 * Clamp to MaxAllocSize in case we went past it.  Note we are assuming
+	 * here that MaxAllocSize <= INT_MAX/2, else the above loop could
+	 * overflow.  We will still have newlen >= needed.
 	 */
-	if (newlen > limit)
-		newlen = limit;
+	if (newlen > (int) MaxAllocSize)
+		newlen = (int) MaxAllocSize;
 
-	str->data = (char *) repalloc_huge(str->data, newlen);
+	str->data = (char *) repalloc(str->data, newlen);
 
 	str->maxlen = newlen;
 }
diff --git a/src/include/lib/stringinfo.h b/src/include/lib/stringinfo.h
@@ -30,8 +30,6 @@
  *		cursor	is initialized to zero by makeStringInfo or initStringInfo,
  *				but is not otherwise touched by the stringinfo.c routines.
  *				Some routines use it to scan through a StringInfo.
- *		long_ok whether this StringInfo can allocate more than MaxAllocSize
- *				bytes (but still up to 2GB).
  *-------------------------
  */
 typedef struct StringInfoData
@@ -40,7 +38,6 @@ typedef struct StringInfoData
 	int			len;
 	int			maxlen;
 	int			cursor;
-	bool		long_ok;
 } StringInfoData;
 
 typedef StringInfoData *StringInfo;
@@ -49,11 +46,11 @@ typedef StringInfoData *StringInfo;
 /*------------------------
  * There are two ways to create a StringInfo object initially:
  *
- * StringInfo stringptr = makeStringInfo(); // or makeLongStringInfo();
+ * StringInfo stringptr = makeStringInfo();
  *		Both the StringInfoData and the data buffer are palloc'd.
  *
  * StringInfoData string;
- * initStringInfo(&string); // or initLongStringInfo();
+ * initStringInfo(&string);
  *		The data buffer is palloc'd but the StringInfoData is just local.
  *		This is the easiest approach for a StringInfo object that will
  *		only live as long as the current routine.
@@ -70,26 +67,21 @@ typedef StringInfoData *StringInfo;
 
 /*------------------------
  * makeStringInfo
- * makeLongStringInfo
- * Create an empty 'StringInfoData' & return a pointer to it.  The former
- * allows up to 1 GB in size, per palloc(); the latter allows up to 2 GB.
+ * Create an empty 'StringInfoData' & return a pointer to it.
  */
 extern StringInfo makeStringInfo(void);
-extern StringInfo makeLongStringInfo(void);
 
 /*------------------------
  * initStringInfo
- * initLongStringInfo
  * Initialize a StringInfoData struct (with previously undefined contents)
- * to describe an empty string.  Size limits as above.
+ * to describe an empty string.
  */
 extern void initStringInfo(StringInfo str);
-extern void initLongStringInfo(StringInfo str);
 
 /*------------------------
  * resetStringInfo
  * Clears the current content of the StringInfo, if any. The
- * StringInfo remains valid.  The long_ok flag is not reset.
+ * StringInfo remains valid.
  */
 extern void resetStringInfo(StringInfo str);
 
