Skip to content

Commit 38e444a

Browse files
peterepetere
committed
Make sure the pg_dump tar archiver can handle members larger than 2 GB, but
does not create members larger than allowed by the tar format. Also, fix the generation of the tar header to conform to POSIX.
1 parent 172f9a4 commit 38e444a

File tree

2 files changed

+57
-22
lines changed

2 files changed

+57
-22
lines changed

‎doc/src/sgml/ref/pg_dump.sgml

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
<!--
2-
$Header: /cvsroot/pgsql/doc/src/sgml/ref/pg_dump.sgml,v 1.49 2002/08/27 18:57:26 petere Exp $
2+
$Header: /cvsroot/pgsql/doc/src/sgml/ref/pg_dump.sgml,v 1.50 2002/09/06 21:58:36 petere Exp $
33
PostgreSQL documentation
44
-->
55

@@ -662,6 +662,15 @@ CREATE DATABASE foo WITH TEMPLATE = template0;
662662

663663
</itemizedlist>
664664
</para>
665+
666+
<para>
667+
Members of tar archives are limited to a size less than 8 GB.
668+
(This is an inherent limitation of the tar file format.) Therefore
669+
this format cannot be used if the textual representation of a table
670+
exceeds that size. The total size of a tar archive and any of the
671+
other output formats is not limited, except possibly by the
672+
operating system.
673+
</para>
665674
</refsect1>
666675

667676
<refsect1 id="pg-dump-examples">

‎src/bin/pg_dump/pg_backup_tar.c

Lines changed: 47 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
*
1717
*
1818
* IDENTIFICATION
19-
* $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_tar.c,v 1.28 2002/09/04 20:31:34 momjian Exp $
19+
* $Header: /cvsroot/pgsql/src/bin/pg_dump/pg_backup_tar.c,v 1.29 2002/09/06 21:58:36 petere Exp $
2020
*
2121
*-------------------------------------------------------------------------
2222
*/
@@ -73,6 +73,17 @@ typedef struct
7373
ArchiveHandle *AH;
7474
} TAR_MEMBER;
7575

76+
/*
77+
* Maximum file size for a tar member: The limit inherent in the
78+
* format is 2^33-1 bytes (nearly 8 GB). But we don't want to exceed
79+
* what we can represent by an off_t.
80+
*/
81+
#ifdef INT64_IS_BUSTED
82+
#define MAX_TAR_MEMBER_FILELEN INT_MAX
83+
#else
84+
#define MAX_TAR_MEMBER_FILELEN (((int64) 1 << Min(33, sizeof(off_t)*8 - 1)) - 1)
85+
#endif
86+
7687
typedef struct
7788
{
7889
int hasSeek;
@@ -1006,6 +1017,8 @@ _tarAddFile(ArchiveHandle *AH, TAR_MEMBER *th)
10061017
*/
10071018
fseeko(tmp, 0, SEEK_END);
10081019
th->fileLen = ftello(tmp);
1020+
if (th->fileLen > MAX_TAR_MEMBER_FILELEN)
1021+
die_horribly(AH, modulename, "archive member too large for tar format\n");
10091022
fseeko(tmp, 0, SEEK_SET);
10101023

10111024
_tarWriteHeader(th);
@@ -1219,6 +1232,23 @@ _tarGetHeader(ArchiveHandle *AH, TAR_MEMBER *th)
12191232
return 1;
12201233
}
12211234

1235+
1236+
/*
1237+
* Utility routine to print possibly larger than 32 bit integers in a
1238+
* portable fashion. Filled with zeros.
1239+
*/
1240+
static void print_val(char *s, uint64 val, unsigned int base, size_t len)
1241+
{
1242+
int i;
1243+
for (i = len; i > 0; i--)
1244+
{
1245+
int digit = val % base;
1246+
s[i - 1] = '0' + digit;
1247+
val = val / base;
1248+
}
1249+
}
1250+
1251+
12221252
static void
12231253
_tarWriteHeader(TAR_MEMBER *th)
12241254
{
@@ -1235,34 +1265,30 @@ _tarWriteHeader(TAR_MEMBER *th)
12351265
sprintf(&h[100], "100600 ");
12361266

12371267
/* User ID 8 */
1238-
sprintf(&h[108], " 04000 ");
1268+
sprintf(&h[108], "004000 ");
12391269

12401270
/* Group 8 */
1241-
sprintf(&h[116], " 02000 ");
1271+
sprintf(&h[116], "002000 ");
12421272

1243-
/* File size 12 */
1244-
/* FIXME: This goes only up to 2^30. -- What about larger files? */
1245-
sprintf(&h[124], "%10o ", (unsigned int) th->fileLen);
1273+
/* File size 12 - 11 digits, 1 space, no NUL */
1274+
print_val(&h[124], th->fileLen, 8, 11);
1275+
sprintf(&h[135], " ");
12461276

12471277
/* Mod Time 12 */
1248-
sprintf(&h[136], "%10o ", (int) time(NULL));
1278+
sprintf(&h[136], "%011o ", (int) time(NULL));
12491279

12501280
/* Checksum 8 */
1251-
sprintf(&h[148], "%6o ", lastSum);
1281+
sprintf(&h[148], "%06o ", lastSum);
12521282

1253-
/* Type 1 */
1254-
/* sprintf(&h[156], "%c", LF_NORMAL); */
1283+
/* Type - regular file */
12551284
sprintf(&h[156], "0");
12561285

12571286
/* Link tag 100 (NULL) */
12581287

1259-
/* Magic 8 */
1260-
sprintf(&h[257], "ustar ");
1261-
1262-
/*
1263-
* GNU Version... sprintf(&h[257], "ustar"); sprintf(&h[263], "00");
1264-
*/
1288+
/* Magic 6 + Version 2 */
1289+
sprintf(&h[257], "ustar00");
12651290

1291+
#if 0
12661292
/* User 32 */
12671293
sprintf(&h[265], "%.31s", ""); /* How do I get username reliably?
12681294
* Do I need to? */
@@ -1272,15 +1298,15 @@ _tarWriteHeader(TAR_MEMBER *th)
12721298
* I need to? */
12731299

12741300
/* Maj Dev 8 */
1275-
/* sprintf(&h[329], "%6o ", 0); */
1276-
1277-
/* Min Dev */
1278-
/* sprintf(&h[337], "%6o ", 0); */
1301+
sprintf(&h[329], "%6o ", 0);
12791302

1303+
/* Min Dev 8 */
1304+
sprintf(&h[337], "%6o ", 0);
1305+
#endif
12801306

12811307
while ((sum = _tarChecksum(h)) != lastSum)
12821308
{
1283-
sprintf(&h[148], "%6o ", sum);
1309+
sprintf(&h[148], "%06o ", sum);
12841310
lastSum = sum;
12851311
}
12861312

0 commit comments

Comments
 (0)