Skip to content

Commit 3c6248a

Browse files
tglsfdctglsfdc
committed
Remove the recently added USE_SEGMENTED_FILES option, and indeed remove all
support for a nonsegmented mode from md.c. Per recent discussions, there doesn't seem to be much value in a "never segment" option as opposed to segmenting with a suitably large segment size. So instead provide a configure-time switch to set the desired segment size in units of gigabytes. While at it, expose a configure switch for BLCKSZ as well. Zdenek Kotala
1 parent 94b0b54 commit 3c6248a

File tree

8 files changed

+223
-170
lines changed

8 files changed

+223
-170
lines changed

‎configure

Lines changed: 88 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1357,7 +1357,6 @@ Optional Features:
13571357
--enable-debug build with debugging symbols (-g)
13581358
--enable-profiling build with profiling enabled
13591359
--enable-dtrace build with DTrace support
1360-
--disable-segmented-files disable data file segmentation (requires largefile support)
13611360
--enable-depend turn on automatic dependency tracking
13621361
--enable-cassert enable assertion checks (for debugging)
13631362
--enable-thread-safety make client libraries thread-safe
@@ -1373,7 +1372,9 @@ Optional Packages:
13731372
--with-includes=DIRS look for additional header files in DIRS
13741373
--with-libraries=DIRS look for additional libraries in DIRS
13751374
--with-libs=DIRS alternative spelling of --with-libraries
1376-
--with-pgport=PORTNUM change default port number [5432]
1375+
--with-pgport=PORTNUM set default port number [5432]
1376+
--with-blocksize=BLOCKSIZE set block size in kB [8]
1377+
--with-segsize=SEGSIZE set segment size in GB [1]
13771378
--with-tcl build Tcl modules (PL/Tcl)
13781379
--with-tclconfig=DIR tclConfig.sh is in DIR
13791380
--with-perl build Perl modules (PL/Perl)
@@ -2549,34 +2550,102 @@ fi
25492550

25502551

25512552
#
2552-
# Data file segmentation
2553+
# Block size
25532554
#
2555+
{ echo "$as_me:$LINENO: checking for block size" >&5
2556+
echo $ECHO_N "checking for block size... $ECHO_C" >&6; }
25542557

2555-
pgac_args="$pgac_args enable_segmented_files"
2558+
pgac_args="$pgac_args with_blocksize"
25562559

2557-
# Check whether --enable-segmented-files was given.
2558-
if test "${enable_segmented_files+set}" = set; then
2559-
enableval=$enable_segmented_files;
2560-
case $enableval in
2560+
2561+
# Check whether --with-blocksize was given.
2562+
if test "${with_blocksize+set}" = set; then
2563+
withval=$with_blocksize;
2564+
case $withval in
25612565
yes)
2562-
:
2566+
{ { echo "$as_me:$LINENO: error: argument required for --with-blocksize option" >&5
2567+
echo "$as_me: error: argument required for --with-blocksize option" >&2;}
2568+
{ (exit 1); exit 1; }; }
25632569
;;
25642570
no)
2565-
:
2571+
{ { echo "$as_me:$LINENO: error: argument required for --with-blocksize option" >&5
2572+
echo "$as_me: error: argument required for --with-blocksize option" >&2;}
2573+
{ (exit 1); exit 1; }; }
25662574
;;
25672575
*)
2568-
{ { echo "$as_me:$LINENO: error: no argument expected for --enable-segmented-files option" >&5
2569-
echo "$as_me: error: no argument expected for --enable-segmented-files option" >&2;}
2570-
{ (exit 1); exit 1; }; }
2576+
blocksize=$withval
25712577
;;
25722578
esac
25732579

25742580
else
2575-
enable_segmented_files=yes
2581+
blocksize=8
2582+
fi
2583+
2584+
2585+
case ${blocksize} in
2586+
1) BLCKSZ=1024;;
2587+
2) BLCKSZ=2048;;
2588+
4) BLCKSZ=4096;;
2589+
8) BLCKSZ=8192;;
2590+
16) BLCKSZ=16384;;
2591+
32) BLCKSZ=32768;;
2592+
*) { { echo "$as_me:$LINENO: error: Invalid block size. Allowed values are 1,2,4,8,16,32." >&5
2593+
echo "$as_me: error: Invalid block size. Allowed values are 1,2,4,8,16,32." >&2;}
2594+
{ (exit 1); exit 1; }; }
2595+
esac
2596+
{ echo "$as_me:$LINENO: result: ${blocksize}kB" >&5
2597+
echo "${ECHO_T}${blocksize}kB" >&6; }
2598+
2599+
2600+
cat >>confdefs.h <<_ACEOF
2601+
#define BLCKSZ ${BLCKSZ}
2602+
_ACEOF
2603+
2604+
2605+
#
2606+
# File segment size
2607+
#
2608+
{ echo "$as_me:$LINENO: checking for segment size" >&5
2609+
echo $ECHO_N "checking for segment size... $ECHO_C" >&6; }
2610+
2611+
pgac_args="$pgac_args with_segsize"
2612+
25762613

2614+
# Check whether --with-segsize was given.
2615+
if test "${with_segsize+set}" = set; then
2616+
withval=$with_segsize;
2617+
case $withval in
2618+
yes)
2619+
{ { echo "$as_me:$LINENO: error: argument required for --with-segsize option" >&5
2620+
echo "$as_me: error: argument required for --with-segsize option" >&2;}
2621+
{ (exit 1); exit 1; }; }
2622+
;;
2623+
no)
2624+
{ { echo "$as_me:$LINENO: error: argument required for --with-segsize option" >&5
2625+
echo "$as_me: error: argument required for --with-segsize option" >&2;}
2626+
{ (exit 1); exit 1; }; }
2627+
;;
2628+
*)
2629+
segsize=$withval
2630+
;;
2631+
esac
2632+
2633+
else
2634+
segsize=1
25772635
fi
25782636

25792637

2638+
# this expression is set up to avoid unnecessary integer overflow
2639+
RELSEG_SIZE=`expr '(' 1024 '*' ${segsize} / ${blocksize} ')' '*' 1024`
2640+
test $? -eq 0 || exit 1
2641+
{ echo "$as_me:$LINENO: result: ${segsize}GB" >&5
2642+
echo "${ECHO_T}${segsize}GB" >&6; }
2643+
2644+
2645+
cat >>confdefs.h <<_ACEOF
2646+
#define RELSEG_SIZE ${RELSEG_SIZE}
2647+
_ACEOF
2648+
25802649

25812650
#
25822651
# C compiler
@@ -24287,12 +24356,11 @@ _ACEOF
2428724356

2428824357

2428924358

24290-
if test "$ac_cv_sizeof_off_t" -lt 8 -o "$enable_segmented_files" = "yes"; then
24291-
24292-
cat >>confdefs.h <<\_ACEOF
24293-
#define USE_SEGMENTED_FILES 1
24294-
_ACEOF
24295-
24359+
# If we don't have largefile support, can't handle segsize >= 2GB.
24360+
if test "$ac_cv_sizeof_off_t" -lt 8 -a "$segsize" != "1"; then
24361+
{ { echo "$as_me:$LINENO: error: Large file support is not enabled. Segment size cannot be larger than 1GB." >&5
24362+
echo "$as_me: error: Large file support is not enabled. Segment size cannot be larger than 1GB." >&2;}
24363+
{ (exit 1); exit 1; }; }
2429624364
fi
2429724365

2429824366
# SunOS doesn't handle negative byte comparisons properly with +/- return

‎configure.in

Lines changed: 66 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
dnl Process this file with autoconf to produce a configure script.
2-
dnl $PostgreSQL: pgsql/configure.in,v 1.557 2008/04/28 22:47:03 tgl Exp $
2+
dnl $PostgreSQL: pgsql/configure.in,v 1.558 2008/05/02 01:08:26 tgl Exp $
33
dnl
44
dnl Developers, please strive to achieve this order:
55
dnl
@@ -155,7 +155,7 @@ AC_SUBST(WANTED_LANGUAGES)
155155
# Default port number (--with-pgport), default 5432
156156
#
157157
AC_MSG_CHECKING([for default port number])
158-
PGAC_ARG_REQ(with, pgport, [ --with-pgport=PORTNUM change default port number [[5432]]],
158+
PGAC_ARG_REQ(with, pgport, [ --with-pgport=PORTNUM set default port number [[5432]]],
159159
[default_port=$withval],
160160
[default_port=5432])
161161
AC_MSG_RESULT([$default_port])
@@ -218,10 +218,67 @@ AC_SUBST(DTRACEFLAGS)])
218218
AC_SUBST(enable_dtrace)
219219

220220
#
221-
# Data file segmentation
222-
#
223-
PGAC_ARG_BOOL(enable, segmented-files, yes,
224-
[ --disable-segmented-files disable data file segmentation (requires largefile support)])
221+
# Block size
222+
#
223+
AC_MSG_CHECKING([for block size])
224+
PGAC_ARG_REQ(with, blocksize, [ --with-blocksize=BLOCKSIZE set block size in kB [[8]]],
225+
[blocksize=$withval],
226+
[blocksize=8])
227+
case ${blocksize} in
228+
1) BLCKSZ=1024;;
229+
2) BLCKSZ=2048;;
230+
4) BLCKSZ=4096;;
231+
8) BLCKSZ=8192;;
232+
16) BLCKSZ=16384;;
233+
32) BLCKSZ=32768;;
234+
*) AC_MSG_ERROR([Invalid block size. Allowed values are 1,2,4,8,16,32.])
235+
esac
236+
AC_MSG_RESULT([${blocksize}kB])
237+
238+
AC_DEFINE_UNQUOTED([BLCKSZ], ${BLCKSZ}, [
239+
Size of a disk block --- this also limits the size of a tuple. You
240+
can set it bigger if you need bigger tuples (although TOAST should
241+
reduce the need to have large tuples, since fields can be spread
242+
across multiple tuples).
243+
244+
BLCKSZ must be a power of 2. The maximum possible value of BLCKSZ
245+
is currently 2^15 (32768). This is determined by the 15-bit widths
246+
of the lp_off and lp_len fields in ItemIdData (see
247+
include/storage/itemid.h).
248+
249+
Changing BLCKSZ requires an initdb.
250+
])
251+
252+
#
253+
# File segment size
254+
#
255+
AC_MSG_CHECKING([for segment size])
256+
PGAC_ARG_REQ(with, segsize, [ --with-segsize=SEGSIZE set segment size in GB [[1]]],
257+
[segsize=$withval],
258+
[segsize=1])
259+
# this expression is set up to avoid unnecessary integer overflow
260+
RELSEG_SIZE=`expr '(' 1024 '*' ${segsize} / ${blocksize} ')' '*' 1024`
261+
test $? -eq 0 || exit 1
262+
AC_MSG_RESULT([${segsize}GB])
263+
264+
AC_DEFINE_UNQUOTED([RELSEG_SIZE], ${RELSEG_SIZE}, [
265+
RELSEG_SIZE is the maximum number of blocks allowed in one disk file.
266+
Thus, the maximum size of a single file is RELSEG_SIZE * BLCKSZ;
267+
relations bigger than that are divided into multiple files.
268+
269+
RELSEG_SIZE * BLCKSZ must be less than your OS' limit on file size.
270+
This is often 2 GB or 4GB in a 32-bit operating system, unless you
271+
have large file support enabled. By default, we make the limit 1 GB
272+
to avoid any possible integer-overflow problems within the OS.
273+
A limit smaller than necessary only means we divide a large
274+
relation into more chunks than necessary, so it seems best to err
275+
in the direction of a small limit.
276+
277+
A power-of-2 value is recommended to save a few cycles in md.c,
278+
but is not absolutely required.
279+
280+
Changing RELSEG_SIZE requires an initdb.
281+
])
225282

226283
#
227284
# C compiler
@@ -1469,8 +1526,9 @@ fi
14691526
# Check for largefile support (must be after AC_SYS_LARGEFILE)
14701527
AC_CHECK_SIZEOF([off_t])
14711528

1472-
if test "$ac_cv_sizeof_off_t" -lt 8 -o "$enable_segmented_files" = "yes"; then
1473-
AC_DEFINE([USE_SEGMENTED_FILES], 1, [Define to split data files into 1GB segments.])
1529+
# If we don't have largefile support, can't handle segsize >= 2GB.
1530+
if test "$ac_cv_sizeof_off_t" -lt 8 -a "$segsize" != "1"; then
1531+
AC_MSG_ERROR([Large file support is not enabled. Segment size cannot be larger than 1GB.])
14741532
fi
14751533

14761534
# SunOS doesn't handle negative byte comparisons properly with +/- return

‎doc/src/sgml/installation.sgml

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
<!-- $PostgreSQL: pgsql/doc/src/sgml/installation.sgml,v 1.307 2008/04/21 00:26:44 tgl Exp $ -->
1+
<!-- $PostgreSQL: pgsql/doc/src/sgml/installation.sgml,v 1.308 2008/05/02 01:08:26 tgl Exp $ -->
22

33
<chapter id="installation">
44
<title><![%standalone-include[<productname>PostgreSQL</>]]>
@@ -1067,15 +1067,38 @@ su - postgres
10671067
</varlistentry>
10681068

10691069
<varlistentry>
1070-
<term><option>--disable-segmented-files</option></term>
1070+
<term><option>--with-segsize=<replaceable>SEGSIZE</replaceable></option></term>
10711071
<listitem>
10721072
<para>
1073-
Store large tables as single operating-system files, rather than
1074-
dividing them into 1GB segments as is the default. This option
1075-
is ignored unless the operating system has <quote>largefile</>
1076-
support (which most do, nowadays). It can be helpful to reduce
1077-
the number of file descriptors consumed when working with very
1078-
large tables.
1073+
Set the <firstterm>segment size</>, in gigabytes. Large tables are
1074+
divided into multiple operating-system files, each of size equal
1075+
to the segment size. This avoids problems with file size limits
1076+
that exist on many platforms. The default segment size, 1 gigabyte,
1077+
is safe on all supported platforms. If your operating system has
1078+
<quote>largefile</> support (which most do, nowadays), you can use
1079+
a larger segment size. This can be helpful to reduce the number of
1080+
file descriptors consumed when working with very large tables.
1081+
But be careful not to select a value larger than is supported
1082+
by your platform and the filesystem(s) you intend to use. Other
1083+
tools you might wish to use, such as <application>tar</>, could
1084+
also set limits on the usable file size.
1085+
It is recommended, though not absolutely required, that this value
1086+
be a power of 2.
1087+
Note that changing this value requires an initdb.
1088+
</para>
1089+
</listitem>
1090+
</varlistentry>
1091+
1092+
<varlistentry>
1093+
<term><option>--with-blocksize=<replaceable>BLOCKSIZE</replaceable></option></term>
1094+
<listitem>
1095+
<para>
1096+
Set the <firstterm>block size</>, in kilobytes. This is the unit
1097+
of storage and I/O within tables. The default, 8 kilobytes,
1098+
is suitable for most situations; but other values may be useful
1099+
in special cases.
1100+
The value must be a power of 2 between 1 and 32 (kilobytes).
1101+
Note that changing this value requires an initdb.
10791102
</para>
10801103
</listitem>
10811104
</varlistentry>

‎doc/src/sgml/storage.sgml

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
<!-- $PostgreSQL: pgsql/doc/src/sgml/storage.sgml,v 1.22 2008/03/10 20:06:27 tgl Exp $ -->
1+
<!-- $PostgreSQL: pgsql/doc/src/sgml/storage.sgml,v 1.23 2008/05/02 01:08:26 tgl Exp $ -->
22

33
<chapter id="storage">
44

@@ -138,14 +138,13 @@ Avoid assuming that filenode and table OID are the same.
138138
</caution>
139139

140140
<para>
141-
When a table or index exceeds 1 GB, it is normally divided into gigabyte-sized
141+
When a table or index exceeds 1 GB, it is divided into gigabyte-sized
142142
<firstterm>segments</>. The first segment's file name is the same as the
143143
filenode; subsequent segments are named filenode.1, filenode.2, etc.
144144
This arrangement avoids problems on platforms that have file size limitations.
145-
(But if the platform does not have such a limitation, and
146-
<option>--disable-segmented-files</option> was specified when
147-
<productname>PostgreSQL</> was built, then each table or index is stored
148-
as a single file, without segmentation.)
145+
(Actually, 1 GB is just the default segment size. The segment size can be
146+
adjusted using the configuration option <option>--with-segsize</option>
147+
when building <productname>PostgreSQL</>.)
149148
The contents of tables and indexes are discussed further in
150149
<xref linkend="storage-page-layout">.
151150
</para>

‎src/backend/storage/file/buffile.c

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
* Portions Copyright (c) 1994, Regents of the University of California
88
*
99
* IDENTIFICATION
10-
* $PostgreSQL: pgsql/src/backend/storage/file/buffile.c,v 1.30 2008/03/10 20:06:27 tgl Exp $
10+
* $PostgreSQL: pgsql/src/backend/storage/file/buffile.c,v 1.31 2008/05/02 01:08:27 tgl Exp $
1111
*
1212
* NOTES:
1313
*
@@ -38,9 +38,9 @@
3838
#include "storage/buffile.h"
3939

4040
/*
41-
* We break BufFiles into gigabyte-sized segments, whether or not
42-
* USE_SEGMENTED_FILES is defined. The reason is that we'd like large
43-
* temporary BufFiles to be spread across multiple tablespaces when available.
41+
* We break BufFiles into gigabyte-sized segments, regardless of RELSEG_SIZE.
42+
* The reason is that we'd like large temporary BufFiles to be spread across
43+
* multiple tablespaces when available.
4444
*/
4545
#define MAX_PHYSICAL_FILESIZE 0x40000000
4646
#define BUFFILE_SEG_SIZE (MAX_PHYSICAL_FILESIZE / BLCKSZ)

0 commit comments

Comments
 (0)