diff --git a/contrib/test_decoding/expected/ddl.out b/contrib/test_decoding/expected/ddl.out index 2c999fd3eb781..9613558920bf5 100644 --- a/contrib/test_decoding/expected/ddl.out +++ b/contrib/test_decoding/expected/ddl.out @@ -438,12 +438,12 @@ CREATE TABLE replication_metadata ( WITH (user_catalog_table = true) ; \d+ replication_metadata - Table "public.replication_metadata" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description -----------+---------+-----------+----------+--------------------------------------------------+----------+--------------+------------- - id | integer | | not null | nextval('replication_metadata_id_seq'::regclass) | plain | | - relation | name | | not null | | plain | | - options | text[] | | | | extended | | + Table "public.replication_metadata" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +----------+---------+-----------+----------+--------------------------------------------------+----------+-------------+--------------+------------- + id | integer | | not null | nextval('replication_metadata_id_seq'::regclass) | plain | | | + relation | name | | not null | | plain | | | + options | text[] | | | | extended | pglz | | Indexes: "replication_metadata_pkey" PRIMARY KEY, btree (id) Options: user_catalog_table=true @@ -452,12 +452,12 @@ INSERT INTO replication_metadata(relation, options) VALUES ('foo', ARRAY['a', 'b']); ALTER TABLE replication_metadata RESET (user_catalog_table); \d+ replication_metadata - Table "public.replication_metadata" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description -----------+---------+-----------+----------+--------------------------------------------------+----------+--------------+------------- - id | integer | | not null | nextval('replication_metadata_id_seq'::regclass) | plain | | - relation | name | | not null | | plain | | - options | text[] | | | | extended | | + Table "public.replication_metadata" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +----------+---------+-----------+----------+--------------------------------------------------+----------+-------------+--------------+------------- + id | integer | | not null | nextval('replication_metadata_id_seq'::regclass) | plain | | | + relation | name | | not null | | plain | | | + options | text[] | | | | extended | pglz | | Indexes: "replication_metadata_pkey" PRIMARY KEY, btree (id) @@ -465,12 +465,12 @@ INSERT INTO replication_metadata(relation, options) VALUES ('bar', ARRAY['a', 'b']); ALTER TABLE replication_metadata SET (user_catalog_table = true); \d+ replication_metadata - Table "public.replication_metadata" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description -----------+---------+-----------+----------+--------------------------------------------------+----------+--------------+------------- - id | integer | | not null | nextval('replication_metadata_id_seq'::regclass) | plain | | - relation | name | | not null | | plain | | - options | text[] | | | | extended | | + Table "public.replication_metadata" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +----------+---------+-----------+----------+--------------------------------------------------+----------+-------------+--------------+------------- + id | integer | | not null | nextval('replication_metadata_id_seq'::regclass) | plain | | | + relation | name | | not null | | plain | | | + options | text[] | | | | extended | pglz | | Indexes: "replication_metadata_pkey" PRIMARY KEY, btree (id) Options: user_catalog_table=true @@ -483,13 +483,13 @@ ALTER TABLE replication_metadata ALTER COLUMN rewritemeornot TYPE text; ERROR: cannot rewrite table "replication_metadata" used as a catalog table ALTER TABLE replication_metadata SET (user_catalog_table = false); \d+ replication_metadata - Table "public.replication_metadata" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description -----------------+---------+-----------+----------+--------------------------------------------------+----------+--------------+------------- - id | integer | | not null | nextval('replication_metadata_id_seq'::regclass) | plain | | - relation | name | | not null | | plain | | - options | text[] | | | | extended | | - rewritemeornot | integer | | | | plain | | + Table "public.replication_metadata" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +----------------+---------+-----------+----------+--------------------------------------------------+----------+-------------+--------------+------------- + id | integer | | not null | nextval('replication_metadata_id_seq'::regclass) | plain | | | + relation | name | | not null | | plain | | | + options | text[] | | | | extended | pglz | | + rewritemeornot | integer | | | | plain | | | Indexes: "replication_metadata_pkey" PRIMARY KEY, btree (id) Options: user_catalog_table=false diff --git a/doc/src/sgml/catalogs.sgml b/doc/src/sgml/catalogs.sgml index 09690b6c76bdb..a3c9e544c4d41 100644 --- a/doc/src/sgml/catalogs.sgml +++ b/doc/src/sgml/catalogs.sgml @@ -70,6 +70,11 @@ access method support functions + + pg_attr_compression + table columns compression relationships and options + + pg_attrdef column default values @@ -592,9 +597,10 @@ The catalog pg_am stores information about relation access methods. There is one row for each access method supported by the system. - Currently, only tables and indexes have access methods. The requirements for table - and index access methods are discussed in detail in and - respectively. + Currently, there are tables, indexes and compression access methods. + The requirements for table, index and compression access methods + are discussed in detail in , + and respectively. @@ -898,6 +904,12 @@ + + <structname>pg_attr_compression</structname> + + pg_attr_compression + + <structname>pg_attrdef</structname> diff --git a/doc/src/sgml/compression-am.sgml b/doc/src/sgml/compression-am.sgml new file mode 100644 index 0000000000000..e23d817910132 --- /dev/null +++ b/doc/src/sgml/compression-am.sgml @@ -0,0 +1,178 @@ + + + + Compression Access Methods + + PostgreSQL supports two internal + built-in compression methods (pglz + and zlib), and also allows to add more custom compression + methods through compression access methods interface. + + + + Built-in Compression Access Methods + + These compression access methods are included in + PostgreSQL and don't need any external extensions. + +
+ Built-in Compression Access Methods + + + + Name + Options + + + + + pglz + + min_input_size (int), + max_input_size (int), + min_comp_rate (int), + first_success_by (int), + match_size_good (int), + match_size_drop (int) + + + + zlib + level (text), dict (text) + + + +
+ + Note that for zlib to work it should be installed in the + system and PostgreSQL should be compiled without + --without-zlib flag. + + + + + Basic API for compression methods + + + Each compression access method is described by a row in the + pg_am + system catalog. The pg_am entry + specifies a name and a handler function for the access + method. These entries can be created and deleted using the + and + SQL commands. + + + + A compression access method handler function must be declared to accept a + single argument of type internal and to return the + pseudo-type compression_am_handler. The argument is a dummy value that + simply serves to prevent handler functions from being called directly from + SQL commands. The result of the function must be a palloc'd struct of + type CompressionAmRoutine, which contains everything + that the core code needs to know to make use of the compression access method. + The CompressionAmRoutine struct, also called the access + method's API struct, contains pointers to support + functions for the access method. These support functions are plain C + functions and are not visible or callable at the SQL level. + The support functions are described in . + + + + The structure CompressionAmRoutine is defined thus: + +typedef struct CompressionAmRoutine +{ + NodeTag type; + + cmcheck_function cmcheck; /* can be NULL */ + cminitstate_function cminitstate; /* can be NULL */ + cmcompress_function cmcompress; + cmcompress_function cmdecompress; +} CompressionAmRoutine; + + + + + Compression Access Method Functions + + + The compression and auxiliary functions that an compression access + method must provide in CompressionAmRoutine are: + + + + +void +cmcheck (Form_pg_attribute att, List *options); + + Called when an attribute is linked with compression access method. Could + be used to check compatibility with the attribute and other additional + checks. + + + + Compression functions take special struct + CompressionAmOptions as first + parameter. This struct contains per backend cached state for each + attribute compression record. CompressionAmOptions is defined thus: + + +typedef struct CompressionAmOptions +{ + Oid acoid; /* Oid of attribute compression */ + Oid amoid; /* Oid of compression access method */ + List *acoptions; /* Parsed options, used for comparison */ + CompressionAmRoutine *amroutine; /* compression access method routine */ + + /* result of cminitstate function will be put here */ + void *acstate; +} CompressionAmOptions; + + + + + The acstate field is used to keep temporary state + between compression functions calls and stores the result of + cminitstate function. It could be useful to store + the parsed view of the compression options. + + + + Note that any invalidation of pg_attr_compression relation + will cause all the cached acstate options cleared. + They will be recreated on the next compression functions calls. + + + + +void * +cminitstate (Oid acoid, List *options); + + Called when CompressionAmOptions is being + initialized. Can return a pointer to memory that will be passed between + compression functions calls. + + + + +struct varlena * +cmcompress (CompressionAmOptions *cmoptions, + const struct varlena *value); + + Function is used to compress varlena. Could return NULL if data is + incompressible. If it returns varlena bigger than original the core will + not use it. + + + + +struct varlena * +cmdecompress (CompressionAmOptions *cmoptions, + const struct varlena *value); + + Function is used to decompress varlena. + + + + diff --git a/doc/src/sgml/filelist.sgml b/doc/src/sgml/filelist.sgml index 3da2365ea971c..3ec32dca4c5d4 100644 --- a/doc/src/sgml/filelist.sgml +++ b/doc/src/sgml/filelist.sgml @@ -91,6 +91,7 @@ + diff --git a/doc/src/sgml/indexam.sgml b/doc/src/sgml/indexam.sgml index dd54c6880241a..f49f16fdb8194 100644 --- a/doc/src/sgml/indexam.sgml +++ b/doc/src/sgml/indexam.sgml @@ -55,7 +55,7 @@ Basic API Structure for Indexes - Each index access method is described by a row in the + Each index access method is described by a row with INDEX type in the pg_am system catalog. The pg_am entry specifies a name and a handler function for the index diff --git a/doc/src/sgml/postgres.sgml b/doc/src/sgml/postgres.sgml index 3e115f1c76c8c..37141e955928f 100644 --- a/doc/src/sgml/postgres.sgml +++ b/doc/src/sgml/postgres.sgml @@ -252,6 +252,7 @@ &geqo; &tableam; &indexam; + &compression-am; &generic-wal; &btree; &gist; diff --git a/doc/src/sgml/ref/alter_table.sgml b/doc/src/sgml/ref/alter_table.sgml index 90bf19564c680..1484e429678ba 100644 --- a/doc/src/sgml/ref/alter_table.sgml +++ b/doc/src/sgml/ref/alter_table.sgml @@ -53,6 +53,7 @@ ALTER TABLE [ IF EXISTS ] name ALTER [ COLUMN ] column_name SET ( attribute_option = value [, ... ] ) ALTER [ COLUMN ] column_name RESET ( attribute_option [, ... ] ) ALTER [ COLUMN ] column_name SET STORAGE { PLAIN | EXTERNAL | EXTENDED | MAIN } + ALTER [ COLUMN ] column_name SET COMPRESSION compression_am [ WITH (compression_am_options) ] [ PRESERVE (compression_preserve_list) ] ADD table_constraint [ NOT VALID ] ADD table_constraint_using_index ALTER CONSTRAINT constraint_name [ DEFERRABLE | NOT DEFERRABLE ] [ INITIALLY DEFERRED | INITIALLY IMMEDIATE ] @@ -360,6 +361,23 @@ WITH ( MODULUS numeric_literal, REM + + + SET COMPRESSION compression_method_name [ WITH (compression_method_options) ] [ PRESERVE (compression_preserve_list) ] + + + + This form adds compression to a column. Compression access method should be + created with . If compression + method has options they could be specified with WITH + parameter. The PRESERVE list contains list of compression access methods + used on the column and determines which of them should be kept on the + column. Without PRESERVE or partial list of compression methods table + will be rewritten. + + + + ADD table_constraint [ NOT VALID ] diff --git a/doc/src/sgml/ref/create_access_method.sgml b/doc/src/sgml/ref/create_access_method.sgml index dae43dbaed588..79f1290a5801b 100644 --- a/doc/src/sgml/ref/create_access_method.sgml +++ b/doc/src/sgml/ref/create_access_method.sgml @@ -61,7 +61,7 @@ CREATE ACCESS METHOD name This clause specifies the type of access method to define. - Only TABLE and INDEX + TABLE, INDEX and COMPRESSION are supported at present. @@ -77,12 +77,15 @@ CREATE ACCESS METHOD name declared to take a single argument of type internal, and its return type depends on the type of access method; for TABLE access methods, it must - be table_am_handler and for INDEX - access methods, it must be index_am_handler. + be table_am_handler, for INDEX + access methods, it must be index_am_handler and + for COMPRESSION access methods, it must be + compression_am_handler. The C-level API that the handler function must implement varies depending on the type of access method. The table access method API - is described in and the index access method - API is described in . + is described in , the index access method + API is described in an the compression access + method API is described in . diff --git a/doc/src/sgml/ref/create_table.sgml b/doc/src/sgml/ref/create_table.sgml index 9009addb9c06f..d945b3eba9c42 100644 --- a/doc/src/sgml/ref/create_table.sgml +++ b/doc/src/sgml/ref/create_table.sgml @@ -69,6 +69,7 @@ CREATE [ [ GLOBAL | LOCAL ] { TEMPORARY | TEMP } | UNLOGGED ] TABLE [ IF NOT EXI GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY [ ( sequence_options ) ] | UNIQUE index_parameters | PRIMARY KEY index_parameters | + COMPRESSION compression_access_method [ WITH (compression_am_options) ] | REFERENCES reftable [ ( refcolumn ) ] [ MATCH FULL | MATCH PARTIAL | MATCH SIMPLE ] [ ON DELETE referential_action ] [ ON UPDATE referential_action ] } [ DEFERRABLE | NOT DEFERRABLE ] [ INITIALLY DEFERRED | INITIALLY IMMEDIATE ] @@ -953,6 +954,18 @@ WITH ( MODULUS numeric_literal, REM + + COMPRESSION compression_access_method [ WITH (compression_am_options) ] + + + This clause adds compression to a column. Compression method could be + created with . If compression + method has options they could be specified by WITH + parameter. + + + + EXCLUDE [ USING index_method ] ( exclude_element WITH operator [, ... ] ) index_parameters [ WHERE ( predicate ) ] diff --git a/doc/src/sgml/storage.sgml b/doc/src/sgml/storage.sgml index 1047c77a635c7..7709c7e2f6888 100644 --- a/doc/src/sgml/storage.sgml +++ b/doc/src/sgml/storage.sgml @@ -393,10 +393,12 @@ Further details appear in . -The compression technique used for either in-line or out-of-line compressed +The default compression technique used for either in-line or out-of-line compressed data is a fairly simple and very fast member of the LZ family of compression techniques. See -src/common/pg_lzcompress.c for the details. +src/common/pg_lzcompress.c for the details. Also custom +compressions could be used. Look at for +more information. diff --git a/src/backend/Makefile b/src/backend/Makefile index b03d5e510f64a..cafd178f16a78 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -45,7 +45,7 @@ OBJS = $(SUBDIROBJS) $(LOCALOBJS) $(top_builddir)/src/port/libpgport_srv.a \ LIBS := $(filter-out -lpgport -lpgcommon, $(LIBS)) $(LDAP_LIBS_BE) $(ICU_LIBS) # The backend doesn't need everything that's in LIBS, however -LIBS := $(filter-out -lz -lreadline -ledit -ltermcap -lncurses -lcurses, $(LIBS)) +LIBS := $(filter-out -lreadline -ledit -ltermcap -lncurses -lcurses, $(LIBS)) ifeq ($(with_systemd),yes) LIBS += -lsystemd diff --git a/src/backend/access/Makefile b/src/backend/access/Makefile index 0880e0a8bbb63..ca00737c53ef1 100644 --- a/src/backend/access/Makefile +++ b/src/backend/access/Makefile @@ -8,7 +8,7 @@ subdir = src/backend/access top_builddir = ../../.. include $(top_builddir)/src/Makefile.global -SUBDIRS = brin common gin gist hash heap index nbtree rmgrdesc spgist \ - table tablesample transam +SUBDIRS = brin common compression gin gist hash heap index nbtree \ + rmgrdesc spgist table tablesample transam include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/brin/brin_tuple.c b/src/backend/access/brin/brin_tuple.c index 2b3861710c3e1..4e3994533e6f1 100644 --- a/src/backend/access/brin/brin_tuple.c +++ b/src/backend/access/brin/brin_tuple.c @@ -96,6 +96,7 @@ brin_form_tuple(BrinDesc *brdesc, BlockNumber blkno, BrinMemTuple *tuple, int keyno; int idxattno; uint16 phony_infomask = 0; + uint16 phony_infomask2 = 0; bits8 *phony_nullbitmap; Size len, hoff, @@ -187,6 +188,7 @@ brin_form_tuple(BrinDesc *brdesc, BlockNumber blkno, BrinMemTuple *tuple, (char *) rettuple + hoff, data_len, &phony_infomask, + &phony_infomask2, phony_nullbitmap); /* done with these */ diff --git a/src/backend/access/common/heaptuple.c b/src/backend/access/common/heaptuple.c index a48a6cd757f44..35acc28481084 100644 --- a/src/backend/access/common/heaptuple.c +++ b/src/backend/access/common/heaptuple.c @@ -177,6 +177,7 @@ fill_val(Form_pg_attribute att, int *bitmask, char **dataP, uint16 *infomask, + uint16 *infomask2, Datum datum, bool isnull) { @@ -207,6 +208,9 @@ fill_val(Form_pg_attribute att, **bit |= *bitmask; } + if (OidIsValid(att->attcompression)) + *infomask2 |= HEAP_HASCUSTOMCOMPRESSED; + /* * XXX we use the att_align macros on the pointer value itself, not on an * offset. This is a bit of a hack. @@ -245,6 +249,15 @@ fill_val(Form_pg_attribute att, /* no alignment, since it's short by definition */ data_length = VARSIZE_EXTERNAL(val); memcpy(data, val, data_length); + + if (VARATT_IS_EXTERNAL_ONDISK(val)) + { + struct varatt_external toast_pointer; + + VARATT_EXTERNAL_GET_POINTER(toast_pointer, val); + if (VARATT_EXTERNAL_IS_CUSTOM_COMPRESSED(toast_pointer)) + *infomask2 |= HEAP_HASCUSTOMCOMPRESSED; + } } } else if (VARATT_IS_SHORT(val)) @@ -268,6 +281,9 @@ fill_val(Form_pg_attribute att, att->attalign); data_length = VARSIZE(val); memcpy(data, val, data_length); + + if (VARATT_IS_CUSTOM_COMPRESSED(val)) + *infomask2 |= HEAP_HASCUSTOMCOMPRESSED; } } else if (att->attlen == -2) @@ -304,7 +320,7 @@ void heap_fill_tuple(TupleDesc tupleDesc, Datum *values, bool *isnull, char *data, Size data_size, - uint16 *infomask, bits8 *bit) + uint16 *infomask, uint16 *infomask2, bits8 *bit) { bits8 *bitP; int bitmask; @@ -328,6 +344,7 @@ heap_fill_tuple(TupleDesc tupleDesc, } *infomask &= ~(HEAP_HASNULL | HEAP_HASVARWIDTH | HEAP_HASEXTERNAL); + *infomask2 &= ~HEAP_HASCUSTOMCOMPRESSED; for (i = 0; i < numberOfAttributes; i++) { @@ -338,6 +355,7 @@ heap_fill_tuple(TupleDesc tupleDesc, &bitmask, &data, infomask, + infomask2, values ? values[i] : PointerGetDatum(NULL), isnull ? isnull[i] : true); } @@ -752,6 +770,7 @@ expand_tuple(HeapTuple *targetHeapTuple, int bitMask = 0; char *targetData; uint16 *infoMask; + uint16 *infoMask2; Assert((targetHeapTuple && !targetMinimalTuple) || (!targetHeapTuple && targetMinimalTuple)); @@ -864,6 +883,7 @@ expand_tuple(HeapTuple *targetHeapTuple, + offsetof(HeapTupleHeaderData, t_bits)); targetData = (char *) (*targetHeapTuple)->t_data + hoff; infoMask = &(targetTHeader->t_infomask); + infoMask2 = &(targetTHeader->t_infomask2); } else { @@ -882,6 +902,7 @@ expand_tuple(HeapTuple *targetHeapTuple, + offsetof(MinimalTupleData, t_bits)); targetData = (char *) *targetMinimalTuple + hoff; infoMask = &((*targetMinimalTuple)->t_infomask); + infoMask2 = &((*targetMinimalTuple)->t_infomask2); } if (targetNullLen > 0) @@ -934,6 +955,7 @@ expand_tuple(HeapTuple *targetHeapTuple, &bitMask, &targetData, infoMask, + infoMask2, attrmiss[attnum].am_value, false); } @@ -944,6 +966,7 @@ expand_tuple(HeapTuple *targetHeapTuple, &bitMask, &targetData, infoMask, + infoMask2, (Datum) 0, true); } @@ -1093,6 +1116,7 @@ heap_form_tuple(TupleDesc tupleDescriptor, (char *) td + hoff, data_len, &td->t_infomask, + &td->t_infomask2, (hasnull ? td->t_bits : NULL)); return tuple; @@ -1415,6 +1439,7 @@ heap_form_minimal_tuple(TupleDesc tupleDescriptor, (char *) tuple + hoff, data_len, &tuple->t_infomask, + &tuple->t_infomask2, (hasnull ? tuple->t_bits : NULL)); return tuple; diff --git a/src/backend/access/common/indextuple.c b/src/backend/access/common/indextuple.c index cb23be859de58..dc399e0335574 100644 --- a/src/backend/access/common/indextuple.c +++ b/src/backend/access/common/indextuple.c @@ -50,6 +50,7 @@ index_form_tuple(TupleDesc tupleDescriptor, unsigned short infomask = 0; bool hasnull = false; uint16 tupmask = 0; + uint16 tupmask2 = 0; int numberOfAttributes = tupleDescriptor->natts; #ifdef TOAST_INDEX_HACK @@ -77,13 +78,30 @@ index_form_tuple(TupleDesc tupleDescriptor, /* * If value is stored EXTERNAL, must fetch it so we are not depending - * on outside storage. This should be improved someday. + * on outside storage. This should be improved someday. If value also + * was compressed by custom compression method then we should + * decompress it too. */ if (VARATT_IS_EXTERNAL(DatumGetPointer(values[i]))) + { + struct varatt_external toast_pointer; + + VARATT_EXTERNAL_GET_POINTER(toast_pointer, DatumGetPointer(values[i])); + if (VARATT_EXTERNAL_IS_CUSTOM_COMPRESSED(toast_pointer)) + untoasted_values[i] = + PointerGetDatum(heap_tuple_untoast_attr((struct varlena *) + DatumGetPointer(values[i]))); + else + untoasted_values[i] = + PointerGetDatum(heap_tuple_fetch_attr((struct varlena *) + DatumGetPointer(values[i]))); + untoasted_free[i] = true; + } + else if (VARATT_IS_CUSTOM_COMPRESSED(DatumGetPointer(values[i]))) { untoasted_values[i] = - PointerGetDatum(heap_tuple_fetch_attr((struct varlena *) - DatumGetPointer(values[i]))); + PointerGetDatum(heap_tuple_untoast_attr((struct varlena *) + DatumGetPointer(values[i]))); untoasted_free[i] = true; } @@ -95,7 +113,7 @@ index_form_tuple(TupleDesc tupleDescriptor, VARSIZE(DatumGetPointer(untoasted_values[i])) > TOAST_INDEX_TARGET && (att->attstorage == 'x' || att->attstorage == 'm')) { - Datum cvalue = toast_compress_datum(untoasted_values[i]); + Datum cvalue = toast_compress_datum(untoasted_values[i], InvalidOid); if (DatumGetPointer(cvalue) != NULL) { @@ -145,6 +163,7 @@ index_form_tuple(TupleDesc tupleDescriptor, (char *) tp + hoff, data_size, &tupmask, + &tupmask2, (hasnull ? (bits8 *) tp + sizeof(IndexTupleData) : NULL)); #ifdef TOAST_INDEX_HACK diff --git a/src/backend/access/common/reloptions.c b/src/backend/access/common/reloptions.c index 577302149931b..643558751ae30 100644 --- a/src/backend/access/common/reloptions.c +++ b/src/backend/access/common/reloptions.c @@ -988,11 +988,100 @@ untransformRelOptions(Datum options) val = (Node *) makeString(pstrdup(p)); } result = lappend(result, makeDefElem(pstrdup(s), val, -1)); + pfree(s); } return result; } +/* + * helper function for qsort to compare DefElem + */ +static int +compare_options(const void *a, const void *b) +{ + DefElem *da = (DefElem *) lfirst(*(ListCell **) a); + DefElem *db = (DefElem *) lfirst(*(ListCell **) b); + + return strcmp(da->defname, db->defname); +} + +/* + * Convert a DefElem list to the text array format that is used in + * pg_foreign_data_wrapper, pg_foreign_server, pg_user_mapping, + * pg_foreign_table and pg_attr_compression + * + * Returns the array in the form of a Datum, or PointerGetDatum(NULL) + * if the list is empty. + * + * Note: The array is usually stored to database without further + * processing, hence any validation should be done before this + * conversion. + */ +Datum +optionListToArray(List *options, bool sorted) +{ + ArrayBuildState *astate = NULL; + ListCell *cell; + List *resoptions = NIL; + int len = list_length(options); + + /* sort by option name if needed */ + if (sorted && len > 1) + resoptions = list_qsort(options, compare_options); + else + resoptions = options; + + foreach(cell, resoptions) + { + DefElem *def = lfirst(cell); + const char *value; + Size len; + text *t; + + value = defGetString(def); + len = VARHDRSZ + strlen(def->defname) + 1 + strlen(value); + t = palloc(len + 1); + SET_VARSIZE(t, len); + sprintf(VARDATA(t), "%s=%s", def->defname, value); + + astate = accumArrayResult(astate, PointerGetDatum(t), + false, TEXTOID, + CurrentMemoryContext); + } + + /* free if the the modified version of list was used */ + if (resoptions != options) + list_free(resoptions); + + if (astate) + return makeArrayResult(astate, CurrentMemoryContext); + + return PointerGetDatum(NULL); +} + +/* + * Return human readable list of reloptions + */ +char * +formatRelOptions(List *options) +{ + StringInfoData buf; + ListCell *cell; + + initStringInfo(&buf); + + foreach(cell, options) + { + DefElem *def = (DefElem *) lfirst(cell); + + appendStringInfo(&buf, "%s%s=%s", buf.len > 0 ? ", " : "", + def->defname, defGetString(def)); + } + + return buf.data; +} + /* * Extract and parse reloptions from a pg_class tuple. * diff --git a/src/backend/access/common/tupdesc.c b/src/backend/access/common/tupdesc.c index 6bc4e4c036097..14d7715a7fd09 100644 --- a/src/backend/access/common/tupdesc.c +++ b/src/backend/access/common/tupdesc.c @@ -20,6 +20,7 @@ #include "postgres.h" #include "access/htup_details.h" +#include "access/reloptions.h" #include "access/tupdesc_details.h" #include "catalog/pg_collation.h" #include "catalog/pg_type.h" @@ -72,6 +73,7 @@ CreateTemplateTupleDesc(int natts) desc->constr = NULL; desc->tdtypeid = RECORDOID; desc->tdtypmod = -1; + desc->tdflags = 0; desc->tdrefcount = -1; /* assume not reference-counted */ return desc; @@ -94,8 +96,17 @@ CreateTupleDesc(int natts, Form_pg_attribute *attrs) desc = CreateTemplateTupleDesc(natts); for (i = 0; i < natts; ++i) + { memcpy(TupleDescAttr(desc, i), attrs[i], ATTRIBUTE_FIXED_PART_SIZE); + /* + * If even one of attributes is compressed we save information about + * it to TupleDesc flags + */ + if (OidIsValid(attrs[i]->attcompression)) + desc->tdflags |= TD_ATTR_CUSTOM_COMPRESSED; + } + return desc; } @@ -137,6 +148,7 @@ CreateTupleDescCopy(TupleDesc tupdesc) /* We can copy the tuple type identification, too */ desc->tdtypeid = tupdesc->tdtypeid; desc->tdtypmod = tupdesc->tdtypmod; + desc->tdflags = tupdesc->tdflags; return desc; } @@ -217,6 +229,7 @@ CreateTupleDescCopyConstr(TupleDesc tupdesc) /* We can copy the tuple type identification, too */ desc->tdtypeid = tupdesc->tdtypeid; desc->tdtypmod = tupdesc->tdtypmod; + desc->tdflags = tupdesc->tdflags; return desc; } @@ -304,6 +317,7 @@ TupleDescCopyEntry(TupleDesc dst, AttrNumber dstAttno, dstAtt->atthasmissing = false; dstAtt->attidentity = '\0'; dstAtt->attgenerated = '\0'; + dstAtt->attcompression = InvalidOid; } /* @@ -418,6 +432,8 @@ equalTupleDescs(TupleDesc tupdesc1, TupleDesc tupdesc2) return false; if (tupdesc1->tdtypeid != tupdesc2->tdtypeid) return false; + if (tupdesc1->tdflags != tupdesc2->tdflags) + return false; for (i = 0; i < tupdesc1->natts; i++) { @@ -470,6 +486,8 @@ equalTupleDescs(TupleDesc tupdesc1, TupleDesc tupdesc2) return false; if (attr1->attcollation != attr2->attcollation) return false; + if (attr1->attcompression != attr2->attcompression) + return false; /* attacl, attoptions and attfdwoptions are not even present... */ } @@ -557,6 +575,7 @@ equalTupleDescs(TupleDesc tupdesc1, TupleDesc tupdesc2) } else if (tupdesc2->constr != NULL) return false; + return true; } @@ -663,6 +682,7 @@ TupleDescInitEntry(TupleDesc desc, att->attalign = typeForm->typalign; att->attstorage = typeForm->typstorage; att->attcollation = typeForm->typcollation; + att->attcompression = InvalidOid; ReleaseSysCache(tuple); } diff --git a/src/backend/access/compression/Makefile b/src/backend/access/compression/Makefile new file mode 100644 index 0000000000000..7ea5ee2e43ddd --- /dev/null +++ b/src/backend/access/compression/Makefile @@ -0,0 +1,17 @@ +#------------------------------------------------------------------------- +# +# Makefile-- +# Makefile for access/compression +# +# IDENTIFICATION +# src/backend/access/compression/Makefile +# +#------------------------------------------------------------------------- + +subdir = src/backend/access/compression +top_builddir = ../../../.. +include $(top_builddir)/src/Makefile.global + +OBJS = cm_pglz.o cm_zlib.o cmapi.o + +include $(top_srcdir)/src/backend/common.mk diff --git a/src/backend/access/compression/cm_pglz.c b/src/backend/access/compression/cm_pglz.c new file mode 100644 index 0000000000000..03c323f2c27ae --- /dev/null +++ b/src/backend/access/compression/cm_pglz.c @@ -0,0 +1,190 @@ +/*------------------------------------------------------------------------- + * + * cm_pglz.c + * pglz compression method + * + * Copyright (c) 2015-2018, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/access/compression/cm_pglz.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "access/cmapi.h" +#include "commands/defrem.h" +#include "common/pg_lzcompress.h" +#include "nodes/parsenodes.h" +#include "utils/builtins.h" + +#define PGLZ_OPTIONS_COUNT 6 + +static char *PGLZ_options[PGLZ_OPTIONS_COUNT] = { + "min_input_size", + "max_input_size", + "min_comp_rate", + "first_success_by", + "match_size_good", + "match_size_drop" +}; + +/* + * Convert value from reloptions to int32, and report if it is not correct. + * Also checks parameter names + */ +static int32 +parse_option(char *name, char *value) +{ + int i; + + for (i = 0; i < PGLZ_OPTIONS_COUNT; i++) + { + if (strcmp(PGLZ_options[i], name) == 0) + return pg_atoi(value, 4, 0); + } + + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_PARAMETER), + errmsg("unexpected parameter for pglz: \"%s\"", name))); +} + +/* + * Check PGLZ options if specified + */ +static void +pglz_cmcheck(Form_pg_attribute att, List *options) +{ + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + parse_option(def->defname, defGetString(def)); + } +} + +/* + * Configure PGLZ_Strategy struct for compression function + */ +static void * +pglz_cminitstate(Oid acoid, List *options) +{ + ListCell *lc; + PGLZ_Strategy *strategy = palloc(sizeof(PGLZ_Strategy)); + + /* initialize with default strategy values */ + memcpy(strategy, PGLZ_strategy_default, sizeof(PGLZ_Strategy)); + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + int32 val = parse_option(def->defname, defGetString(def)); + + /* fill the strategy */ + if (strcmp(def->defname, "min_input_size") == 0) + strategy->min_input_size = val; + else if (strcmp(def->defname, "max_input_size") == 0) + strategy->max_input_size = val; + else if (strcmp(def->defname, "min_comp_rate") == 0) + strategy->min_comp_rate = val; + else if (strcmp(def->defname, "first_success_by") == 0) + strategy->first_success_by = val; + else if (strcmp(def->defname, "match_size_good") == 0) + strategy->match_size_good = val; + else if (strcmp(def->defname, "match_size_drop") == 0) + strategy->match_size_drop = val; + } + return (void *) strategy; +} + +static struct varlena * +pglz_cmcompress(CompressionAmOptions *cmoptions, const struct varlena *value) +{ + int32 valsize, + len; + struct varlena *tmp = NULL; + PGLZ_Strategy *strategy; + + valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); + strategy = (PGLZ_Strategy *) cmoptions->acstate; + + Assert(strategy != NULL); + if (valsize < strategy->min_input_size || + valsize > strategy->max_input_size) + return NULL; + + tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) + + VARHDRSZ_CUSTOM_COMPRESSED); + len = pglz_compress(VARDATA_ANY(value), + valsize, + (char *) tmp + VARHDRSZ_CUSTOM_COMPRESSED, + strategy); + + if (len >= 0) + { + SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_CUSTOM_COMPRESSED); + return tmp; + } + + pfree(tmp); + return NULL; +} + +static struct varlena * +pglz_cmdecompress(CompressionAmOptions *cmoptions, const struct varlena *value) +{ + struct varlena *result; + int32 rawsize; + + Assert(VARATT_IS_CUSTOM_COMPRESSED(value)); + result = (struct varlena *) palloc(VARRAWSIZE_4B_C(value) + VARHDRSZ); + + rawsize = pglz_decompress((char *) value + VARHDRSZ_CUSTOM_COMPRESSED, + VARSIZE(value) - VARHDRSZ_CUSTOM_COMPRESSED, + VARDATA(result), + VARRAWSIZE_4B_C(value), true); + + if (rawsize < 0) + elog(ERROR, "pglz: compressed data is corrupted"); + + SET_VARSIZE(result, rawsize + VARHDRSZ); + return result; +} + +static struct varlena * +pglz_cmdecompress_slice(CompressionAmOptions *cmoptions, const struct varlena *value, + int32 slicelength) +{ + struct varlena *result; + int32 rawsize; + + Assert(VARATT_IS_CUSTOM_COMPRESSED(value)); + result = (struct varlena *) palloc(VARRAWSIZE_4B_C(value) + VARHDRSZ); + + rawsize = pglz_decompress((char *) value + VARHDRSZ_CUSTOM_COMPRESSED, + VARSIZE(value) - VARHDRSZ_CUSTOM_COMPRESSED, + VARDATA(result), + slicelength, false); + + if (rawsize < 0) + elog(ERROR, "pglz: compressed data is corrupted"); + + SET_VARSIZE(result, rawsize + VARHDRSZ); + return result; +} + +/* pglz is the default compression method */ +Datum +pglzhandler(PG_FUNCTION_ARGS) +{ + CompressionAmRoutine *routine = makeNode(CompressionAmRoutine); + + routine->cmcheck = pglz_cmcheck; + routine->cminitstate = pglz_cminitstate; + routine->cmcompress = pglz_cmcompress; + routine->cmdecompress = pglz_cmdecompress; + routine->cmdecompress_slice = pglz_cmdecompress_slice; + + PG_RETURN_POINTER(routine); +} diff --git a/src/backend/access/compression/cm_zlib.c b/src/backend/access/compression/cm_zlib.c new file mode 100644 index 0000000000000..e9bf2772e2fc6 --- /dev/null +++ b/src/backend/access/compression/cm_zlib.c @@ -0,0 +1,250 @@ +/*------------------------------------------------------------------------- + * + * cm_zlib.c + * zlib compression method + * + * Copyright (c) 2015-2018, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/access/compression/cm_zlib.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "access/cmapi.h" +#include "commands/defrem.h" +#include "nodes/parsenodes.h" +#include "utils/builtins.h" + +#ifdef HAVE_LIBZ +#include + +#define ZLIB_MAX_DICTIONARY_LENGTH 32768 +#define ZLIB_DICTIONARY_DELIM (" ,") + +typedef struct +{ + int level; + Bytef dict[ZLIB_MAX_DICTIONARY_LENGTH]; + unsigned int dictlen; +} zlib_state; + +/* + * Check options if specified. All validation is located here so + * we don't need do it again in cminitstate function. + */ +static void +zlib_cmcheck(Form_pg_attribute att, List *options) +{ + ListCell *lc; + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + if (strcmp(def->defname, "level") == 0) + { + int8 level = pg_atoi(defGetString(def), sizeof(int8), 0); + if (level < 0 || level > 9) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("unexpected value for zlib compression level: \"%s\"", + defGetString(def)), + errhint("expected value between 0 and 9") + )); + } + else if (strcmp(def->defname, "dict") == 0) + { + int ntokens = 0; + char *val, + *tok; + + val = pstrdup(defGetString(def)); + if (strlen(val) > (ZLIB_MAX_DICTIONARY_LENGTH - 1)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + (errmsg("zlib dictionary length should be less than %d", ZLIB_MAX_DICTIONARY_LENGTH)))); + + while ((tok = strtok(val, ZLIB_DICTIONARY_DELIM)) != NULL) + { + ntokens++; + val = NULL; + } + + if (ntokens < 2) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + (errmsg("zlib dictionary is too small")))); + } + else + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_PARAMETER), + errmsg("unexpected parameter for zlib: \"%s\"", def->defname))); + } +} + +static void * +zlib_cminitstate(Oid acoid, List *options) +{ + zlib_state *state = NULL; + + state = palloc0(sizeof(zlib_state)); + state->level = Z_DEFAULT_COMPRESSION; + + if (list_length(options) > 0) + { + ListCell *lc; + + foreach(lc, options) + { + DefElem *def = (DefElem *) lfirst(lc); + + if (strcmp(def->defname, "level") == 0) + state->level = pg_atoi(defGetString(def), sizeof(int), 0); + else if (strcmp(def->defname, "dict") == 0) + { + char *val, + *tok; + + val = pstrdup(defGetString(def)); + + /* Fill the zlib dictionary */ + while ((tok = strtok(val, ZLIB_DICTIONARY_DELIM)) != NULL) + { + int len = strlen(tok); + memcpy((void *) (state->dict + state->dictlen), tok, len); + state->dictlen += len + 1; + Assert(state->dictlen <= ZLIB_MAX_DICTIONARY_LENGTH); + + /* add space as dictionary delimiter */ + state->dict[state->dictlen - 1] = ' '; + val = NULL; + } + } + } + } + + return state; +} + +static struct varlena * +zlib_cmcompress(CompressionAmOptions *cmoptions, const struct varlena *value) +{ + int32 valsize, + len; + struct varlena *tmp = NULL; + z_streamp zp; + int res; + zlib_state *state = (zlib_state *) cmoptions->acstate; + + zp = (z_streamp) palloc(sizeof(z_stream)); + zp->zalloc = Z_NULL; + zp->zfree = Z_NULL; + zp->opaque = Z_NULL; + + if (deflateInit(zp, state->level) != Z_OK) + elog(ERROR, "could not initialize compression library: %s", zp->msg); + + if (state->dictlen > 0) + { + res = deflateSetDictionary(zp, state->dict, state->dictlen); + if (res != Z_OK) + elog(ERROR, "could not set dictionary for zlib: %s", zp->msg); + } + + valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); + tmp = (struct varlena *) palloc(valsize + VARHDRSZ_CUSTOM_COMPRESSED); + zp->next_in = (void *) VARDATA_ANY(value); + zp->avail_in = valsize; + zp->avail_out = valsize; + zp->next_out = (void *)((char *) tmp + VARHDRSZ_CUSTOM_COMPRESSED); + + do { + res = deflate(zp, Z_FINISH); + if (res == Z_STREAM_ERROR) + elog(ERROR, "could not compress data: %s", zp->msg); + } while (zp->avail_in != 0); + + Assert(res == Z_STREAM_END); + + len = valsize - zp->avail_out; + if (deflateEnd(zp) != Z_OK) + elog(ERROR, "could not close compression stream: %s", zp->msg); + pfree(zp); + + if (len > 0) + { + SET_VARSIZE_COMPRESSED(tmp, len + VARHDRSZ_CUSTOM_COMPRESSED); + return tmp; + } + + pfree(tmp); + return NULL; +} + +static struct varlena * +zlib_cmdecompress(CompressionAmOptions *cmoptions, const struct varlena *value) +{ + struct varlena *result; + z_streamp zp; + int res = Z_OK; + zlib_state *state = (zlib_state *) cmoptions->acstate; + + zp = (z_streamp) palloc(sizeof(z_stream)); + zp->zalloc = Z_NULL; + zp->zfree = Z_NULL; + zp->opaque = Z_NULL; + + if (inflateInit(zp) != Z_OK) + elog(ERROR, "could not initialize compression library: %s", zp->msg); + + Assert(VARATT_IS_CUSTOM_COMPRESSED(value)); + zp->next_in = (void *) ((char *) value + VARHDRSZ_CUSTOM_COMPRESSED); + zp->avail_in = VARSIZE(value) - VARHDRSZ_CUSTOM_COMPRESSED; + zp->avail_out = VARRAWSIZE_4B_C(value); + + result = (struct varlena *) palloc(zp->avail_out + VARHDRSZ); + SET_VARSIZE(result, zp->avail_out + VARHDRSZ); + zp->next_out = (void *) VARDATA(result); + + while (zp->avail_in > 0) + { + res = inflate(zp, 0); + if (res == Z_NEED_DICT && state->dictlen > 0) + { + res = inflateSetDictionary(zp, state->dict, state->dictlen); + if (res != Z_OK) + elog(ERROR, "could not set dictionary for zlib"); + continue; + } + if (!(res == Z_OK || res == Z_STREAM_END)) + elog(ERROR, "could not uncompress data: %s", zp->msg); + } + + if (inflateEnd(zp) != Z_OK) + elog(ERROR, "could not close compression library: %s", zp->msg); + + pfree(zp); + return result; +} +#endif + +Datum +zlibhandler(PG_FUNCTION_ARGS) +{ +#ifndef HAVE_LIBZ + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("not built with zlib support"))); +#else + CompressionAmRoutine *routine = makeNode(CompressionAmRoutine); + + routine->cmcheck = zlib_cmcheck; + routine->cminitstate = zlib_cminitstate; + routine->cmcompress = zlib_cmcompress; + routine->cmdecompress = zlib_cmdecompress; + routine->cmdecompress_slice = NULL; + + PG_RETURN_POINTER(routine); +#endif +} diff --git a/src/backend/access/compression/cmapi.c b/src/backend/access/compression/cmapi.c new file mode 100644 index 0000000000000..504da0638ff00 --- /dev/null +++ b/src/backend/access/compression/cmapi.c @@ -0,0 +1,96 @@ +/*------------------------------------------------------------------------- + * + * compression/cmapi.c + * Functions for compression access methods + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/access/compression/cmapi.c + *------------------------------------------------------------------------- + */ + +#include "postgres.h" +#include "fmgr.h" +#include "access/cmapi.h" +#include "access/htup_details.h" +#include "access/reloptions.h" +#include "catalog/pg_am.h" +#include "catalog/pg_attr_compression.h" +#include "commands/defrem.h" +#include "utils/syscache.h" + +/* + * InvokeCompressionAmHandler - call the specified access method handler routine to get + * its CompressionAmRoutine struct, which will be palloc'd in the caller's context. + */ +CompressionAmRoutine * +InvokeCompressionAmHandler(Oid amhandler) +{ + Datum datum; + CompressionAmRoutine *routine; + + datum = OidFunctionCall0(amhandler); + routine = (CompressionAmRoutine *) DatumGetPointer(datum); + + if (routine == NULL || !IsA(routine, CompressionAmRoutine)) + elog(ERROR, "compression method handler function %u " + "did not return an CompressionAmRoutine struct", + amhandler); + + return routine; +} + +/* + * GetAttrCompressionOptions + * + * Parse array of attribute compression options and return it as a list. + */ +List * +GetAttrCompressionOptions(Oid acoid) +{ + HeapTuple tuple; + List *result = NIL; + bool isnull; + Datum acoptions; + + tuple = SearchSysCache1(ATTCOMPRESSIONOID, ObjectIdGetDatum(acoid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for an attribute compression %u", acoid); + + /* options could be NULL, so we can't use form struct */ + acoptions = SysCacheGetAttr(ATTCOMPRESSIONOID, tuple, + Anum_pg_attr_compression_acoptions, &isnull); + + if (!isnull) + result = untransformRelOptions(acoptions); + + ReleaseSysCache(tuple); + return result; +} + +/* + * GetAttrCompressionAmOid + * + * Return access method Oid by attribute compression Oid + */ +Oid +GetAttrCompressionAmOid(Oid acoid) +{ + Oid result; + HeapTuple tuple; + Form_pg_attr_compression acform; + + /* extract access method Oid */ + tuple = SearchSysCache1(ATTCOMPRESSIONOID, ObjectIdGetDatum(acoid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for attribute compression %u", acoid); + + acform = (Form_pg_attr_compression) GETSTRUCT(tuple); + result = get_compression_am_oid(NameStr(acform->acname), false); + ReleaseSysCache(tuple); + + return result; +} diff --git a/src/backend/access/heap/heapam.c b/src/backend/access/heap/heapam.c index d768b9b061cd9..e10536363b182 100644 --- a/src/backend/access/heap/heapam.c +++ b/src/backend/access/heap/heapam.c @@ -71,7 +71,7 @@ static HeapTuple heap_prepare_insert(Relation relation, HeapTuple tup, - TransactionId xid, CommandId cid, int options); + TransactionId xid, CommandId cid, int options, BulkInsertState bistate); static XLogRecPtr log_heap_update(Relation reln, Buffer oldbuf, Buffer newbuf, HeapTuple oldtup, HeapTuple newtup, HeapTuple old_key_tup, @@ -1817,13 +1817,14 @@ UpdateXmaxHintBits(HeapTupleHeader tuple, Buffer buffer, TransactionId xid) * GetBulkInsertState - prepare status object for a bulk insert */ BulkInsertState -GetBulkInsertState(void) +GetBulkInsertState(HTAB *preserved_am_info) { BulkInsertState bistate; bistate = (BulkInsertState) palloc(sizeof(BulkInsertStateData)); bistate->strategy = GetAccessStrategy(BAS_BULKWRITE); bistate->current_buf = InvalidBuffer; + bistate->preserved_am_info = preserved_am_info; return bistate; } @@ -1885,7 +1886,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, * Note: below this point, heaptup is the data we actually intend to store * into the relation; tup is the caller's original untoasted data. */ - heaptup = heap_prepare_insert(relation, tup, xid, cid, options); + heaptup = heap_prepare_insert(relation, tup, xid, cid, options, bistate); /* * Find buffer to insert this tuple into. If the page is all visible, @@ -2052,7 +2053,7 @@ heap_insert(Relation relation, HeapTuple tup, CommandId cid, */ static HeapTuple heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid, - CommandId cid, int options) + CommandId cid, int options, BulkInsertState bistate) { /* * Parallel operations are required to be strictly read-only in a parallel @@ -2089,8 +2090,12 @@ heap_prepare_insert(Relation relation, HeapTuple tup, TransactionId xid, Assert(!HeapTupleHasExternal(tup)); return tup; } - else if (HeapTupleHasExternal(tup) || tup->t_len > TOAST_TUPLE_THRESHOLD) - return toast_insert_or_update(relation, tup, NULL, options); + else if (HeapTupleHasExternal(tup) + || RelationGetDescr(relation)->tdflags & TD_ATTR_CUSTOM_COMPRESSED + || HeapTupleHasCustomCompressed(tup) + || tup->t_len > TOAST_TUPLE_THRESHOLD) + return toast_insert_or_update(relation, tup, NULL, options, + bistate ? bistate->preserved_am_info : NULL); else return tup; } @@ -2138,7 +2143,7 @@ heap_multi_insert(Relation relation, TupleTableSlot **slots, int ntuples, slots[i]->tts_tableOid = RelationGetRelid(relation); tuple->t_tableOid = slots[i]->tts_tableOid; heaptuples[i] = heap_prepare_insert(relation, tuple, xid, cid, - options); + options, bistate); } /* @@ -3406,6 +3411,8 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, else need_toast = (HeapTupleHasExternal(&oldtup) || HeapTupleHasExternal(newtup) || + RelationGetDescr(relation)->tdflags & TD_ATTR_CUSTOM_COMPRESSED || + HeapTupleHasCustomCompressed(newtup) || newtup->t_len > TOAST_TUPLE_THRESHOLD); pagefree = PageGetHeapFreeSpace(page); @@ -3508,7 +3515,7 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup, if (need_toast) { /* Note we always use WAL and FSM during updates */ - heaptup = toast_insert_or_update(relation, newtup, &oldtup, 0); + heaptup = toast_insert_or_update(relation, newtup, &oldtup, 0, NULL); newtupsize = MAXALIGN(heaptup->t_len); } else diff --git a/src/backend/access/heap/rewriteheap.c b/src/backend/access/heap/rewriteheap.c index 369694fa2e9be..de690b0d9fd79 100644 --- a/src/backend/access/heap/rewriteheap.c +++ b/src/backend/access/heap/rewriteheap.c @@ -665,7 +665,7 @@ raw_heap_insert(RewriteState state, HeapTuple tup) options |= HEAP_INSERT_NO_LOGICAL; heaptup = toast_insert_or_update(state->rs_new_rel, tup, NULL, - options); + options, NULL); } else heaptup = tup; diff --git a/src/backend/access/heap/tuptoaster.c b/src/backend/access/heap/tuptoaster.c index 55d6e91d1c388..513e74f5d7e73 100644 --- a/src/backend/access/heap/tuptoaster.c +++ b/src/backend/access/heap/tuptoaster.c @@ -30,17 +30,25 @@ #include #include +#include "access/cmapi.h" #include "access/genam.h" #include "access/heapam.h" +#include "access/reloptions.h" #include "access/tuptoaster.h" #include "access/xact.h" #include "catalog/catalog.h" +#include "catalog/pg_am.h" +#include "commands/defrem.h" #include "common/pg_lzcompress.h" #include "miscadmin.h" #include "utils/expandeddatum.h" #include "utils/fmgroids.h" +#include "utils/hsearch.h" +#include "utils/inval.h" +#include "utils/memutils.h" #include "utils/rel.h" #include "utils/snapmgr.h" +#include "utils/syscache.h" #include "utils/typcache.h" @@ -52,19 +60,51 @@ typedef struct toast_compress_header { int32 vl_len_; /* varlena header (do not touch directly!) */ - int32 rawsize; + uint32 info; /* flags (2 bits) and rawsize */ } toast_compress_header; +/* + * If the compression method were used, then data also contains + * Oid of compression options + */ +typedef struct toast_compress_header_custom +{ + int32 vl_len_; /* varlena header (do not touch directly!) */ + uint32 info; /* flags (2 high bits) and rawsize */ + Oid cmid; /* Oid from pg_attr_compression */ +} toast_compress_header_custom; + +static HTAB *amoptions_cache = NULL; +static MemoryContext amoptions_cache_mcxt = NULL; + +#define RAWSIZEMASK (0x3FFFFFFFU) + /* * Utilities for manipulation of header information for compressed * toast entries. + * + * Since version 11 TOAST_COMPRESS_SET_RAWSIZE also marks compressed + * varlenas as custom compressed. Such varlenas will contain 0x02 (0b10) in + * two highest bits. */ -#define TOAST_COMPRESS_HDRSZ ((int32) sizeof(toast_compress_header)) -#define TOAST_COMPRESS_RAWSIZE(ptr) (((toast_compress_header *) (ptr))->rawsize) +#define TOAST_COMPRESS_HDRSZ ((int32) sizeof(toast_compress_header)) +#define TOAST_COMPRESS_HDRSZ_CUSTOM ((int32) sizeof(toast_compress_header_custom)) +#define TOAST_COMPRESS_RAWSIZE(ptr) (((toast_compress_header *) (ptr))->info & RAWSIZEMASK) #define TOAST_COMPRESS_RAWDATA(ptr) \ (((char *) (ptr)) + TOAST_COMPRESS_HDRSZ) #define TOAST_COMPRESS_SET_RAWSIZE(ptr, len) \ - (((toast_compress_header *) (ptr))->rawsize = (len)) +do { \ + Assert(len > 0 && len <= RAWSIZEMASK); \ + ((toast_compress_header *) (ptr))->info = (len) | (0x02 << 30); \ +} while (0) +#define TOAST_COMPRESS_SET_CMID(ptr, oid) \ + (((toast_compress_header_custom *) (ptr))->cmid = (oid)) + +#define VARATT_EXTERNAL_SET_CUSTOM(toast_pointer) \ +do { \ + ((toast_pointer).va_extinfo |= (1 << 31)); \ + ((toast_pointer).va_extinfo &= ~(1 << 30)); \ +} while (0) static void toast_delete_datum(Relation rel, Datum value, bool is_speculative); static Datum toast_save_datum(Relation rel, Datum value, @@ -83,7 +123,8 @@ static int toast_open_indexes(Relation toastrel, static void toast_close_indexes(Relation *toastidxs, int num_indexes, LOCKMODE lock); static void init_toast_snapshot(Snapshot toast_snapshot); - +static void init_amoptions_cache(void); +static bool attr_compression_options_are_equal(Oid acoid1, Oid acoid2); /* ---------- * heap_tuple_fetch_attr - @@ -425,7 +466,7 @@ toast_datum_size(Datum value) struct varatt_external toast_pointer; VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); - result = toast_pointer.va_extsize; + result = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer); } else if (VARATT_IS_EXTERNAL_INDIRECT(attr)) { @@ -515,6 +556,98 @@ toast_delete(Relation rel, HeapTuple oldtup, bool is_speculative) } } +/* ---------- + * toast_prepare_varlena + * + * Untoast or fetch varlena if needed. This function could return + * compressed varlena. + */ +static struct varlena * +toast_prepare_varlena(Form_pg_attribute att, struct varlena *value, + List *preserved_amoids, bool *modified) +{ + struct varlena *tmpval = NULL; + + *modified = false; + + /* + * We took care of UPDATE, so any external value we find still in the + * tuple must be someone else's that we cannot reuse (this includes the + * case of an out-of-line in-memory datum). Fetch it back (without + * decompression, unless we are forcing PLAIN storage or it has a custom + * compression). If necessary, we'll push it out as a new external value + * below. + */ + if (VARATT_IS_EXTERNAL(value)) + { + *modified = true; + if (att->attstorage == 'p') + { + value = heap_tuple_untoast_attr(value); + + /* untoasted value does not need any further work */ + return value; + } + else + value = heap_tuple_fetch_attr(value); + } + + /* + * Process custom compressed datum. + * + * 1) If destination column has identical compression move the data as is + * and only change attribute compression Oid. 2) If it's rewrite from + * ALTER command check list of preserved compression access methods. 3) In + * other cases just untoast the datum. + */ + if (VARATT_IS_CUSTOM_COMPRESSED(value)) + { + bool storage_ok = (att->attstorage == 'm' || att->attstorage == 'x'); + toast_compress_header_custom *hdr; + + hdr = (toast_compress_header_custom *) value; + + /* nothing to do */ + if (storage_ok && hdr->cmid == att->attcompression) + return value; + + if (storage_ok && + OidIsValid(att->attcompression) && + attr_compression_options_are_equal(att->attcompression, hdr->cmid)) + { + /* identical compression, just change Oid to new one */ + tmpval = palloc(VARSIZE(value)); + memcpy(tmpval, value, VARSIZE(value)); + TOAST_COMPRESS_SET_CMID(tmpval, att->attcompression); + } + else if (preserved_amoids != NULL) + { + Oid amoid = GetAttrCompressionAmOid(hdr->cmid); + + /* decompress the value if it's not in preserved list */ + if (!list_member_oid(preserved_amoids, amoid)) + tmpval = heap_tuple_untoast_attr(value); + } + else + /* just decompress the value */ + tmpval = heap_tuple_untoast_attr(value); + + } + else if (OidIsValid(att->attcompression)) + tmpval = heap_tuple_untoast_attr(value); + + if (tmpval && tmpval != value) + { + /* if value was external we need to free fetched data */ + if (*modified) + pfree(value); + + value = tmpval; + *modified = true; + } + + return value; +} /* ---------- * toast_insert_or_update - @@ -526,6 +659,8 @@ toast_delete(Relation rel, HeapTuple oldtup, bool is_speculative) * newtup: the candidate new tuple to be inserted * oldtup: the old row version for UPDATE, or NULL for INSERT * options: options to be passed to heap_insert() for toast rows + * preserved_am_info: hash table with attnum as key, for lists of compression + * methods which should be preserved on decompression. * Result: * either newtup if no toasting is needed, or a palloc'd modified tuple * that is what should actually get stored @@ -536,7 +671,7 @@ toast_delete(Relation rel, HeapTuple oldtup, bool is_speculative) */ HeapTuple toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, - int options) + int options, HTAB *preserved_am_info) { HeapTuple result_tuple; TupleDesc tupleDesc; @@ -671,27 +806,37 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, */ if (att->attlen == -1) { + bool modified = false; + List *preserved_amoids = NIL; + /* * If the table's attribute says PLAIN always, force it so. */ if (att->attstorage == 'p') toast_action[i] = 'p'; + if (VARATT_IS_EXTERNAL(new_value)) + toast_oldexternal[i] = new_value; + /* - * We took care of UPDATE above, so any external value we find - * still in the tuple must be someone else's that we cannot reuse - * (this includes the case of an out-of-line in-memory datum). - * Fetch it back (without decompression, unless we are forcing - * PLAIN storage). If necessary, we'll push it out as a new - * external value below. + * If it's ALTER SET COMPRESSION command we should check that + * access method of compression in preserved list. */ - if (VARATT_IS_EXTERNAL(new_value)) + if (preserved_am_info != NULL) + { + bool found; + AttrCmPreservedInfo *pinfo; + + pinfo = hash_search(preserved_am_info, &att->attnum, + HASH_FIND, &found); + if (found) + preserved_amoids = pinfo->preserved_amoids; + } + + new_value = toast_prepare_varlena(att, + new_value, preserved_amoids, &modified); + if (modified) { - toast_oldexternal[i] = new_value; - if (att->attstorage == 'p') - new_value = heap_tuple_untoast_attr(new_value); - else - new_value = heap_tuple_fetch_attr(new_value); toast_values[i] = PointerGetDatum(new_value); toast_free[i] = true; need_change = true; @@ -743,12 +888,14 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, Datum old_value; Datum new_value; + Form_pg_attribute att; + /* * Search for the biggest yet unprocessed internal attribute */ for (i = 0; i < numAttrs; i++) { - Form_pg_attribute att = TupleDescAttr(tupleDesc, i); + char attstorage; if (toast_action[i] != ' ') continue; @@ -756,7 +903,9 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, continue; /* can't happen, toast_action would be 'p' */ if (VARATT_IS_COMPRESSED(DatumGetPointer(toast_values[i]))) continue; - if (att->attstorage != 'x' && att->attstorage != 'e') + + attstorage = (TupleDescAttr(tupleDesc, i))->attstorage; + if (attstorage != 'x' && attstorage != 'e') continue; if (toast_sizes[i] > biggest_size) { @@ -772,10 +921,11 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, * Attempt to compress it inline, if it has attstorage 'x' */ i = biggest_attno; - if (TupleDescAttr(tupleDesc, i)->attstorage == 'x') + att = TupleDescAttr(tupleDesc, i); + if (att->attstorage == 'x') { old_value = toast_values[i]; - new_value = toast_compress_datum(old_value); + new_value = toast_compress_datum(old_value, att->attcompression); if (DatumGetPointer(new_value) != NULL) { @@ -916,7 +1066,8 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, */ i = biggest_attno; old_value = toast_values[i]; - new_value = toast_compress_datum(old_value); + new_value = toast_compress_datum(old_value, + TupleDescAttr(tupleDesc, i)->attcompression); if (DatumGetPointer(new_value) != NULL) { @@ -1044,6 +1195,7 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, (char *) new_data + new_header_len, new_data_len, &(new_data->t_infomask), + &(new_data->t_infomask2), has_nulls ? new_data->t_bits : NULL); } else @@ -1265,6 +1417,7 @@ toast_flatten_tuple_to_datum(HeapTupleHeader tup, (char *) new_data + new_header_len, new_data_len, &(new_data->t_infomask), + &(new_data->t_infomask2), has_nulls ? new_data->t_bits : NULL); /* @@ -1344,6 +1497,18 @@ toast_build_flattened_tuple(TupleDesc tupleDesc, return new_tuple; } +/* --------- + * toast_set_compressed_datum_info + * + * Save metadata in compressed datum. This information will required + * for decompression. + */ +void +toast_set_compressed_datum_info(struct varlena *val, Oid cmid, int32 rawsize) +{ + TOAST_COMPRESS_SET_RAWSIZE(val, rawsize); + TOAST_COMPRESS_SET_CMID(val, cmid); +} /* ---------- * toast_compress_datum - @@ -1359,54 +1524,47 @@ toast_build_flattened_tuple(TupleDesc tupleDesc, * ---------- */ Datum -toast_compress_datum(Datum value) +toast_compress_datum(Datum value, Oid acoid) { - struct varlena *tmp; - int32 valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); - int32 len; + struct varlena *tmp = NULL; + int32 valsize; + CompressionAmOptions *cmoptions = NULL; Assert(!VARATT_IS_EXTERNAL(DatumGetPointer(value))); Assert(!VARATT_IS_COMPRESSED(DatumGetPointer(value))); - /* - * No point in wasting a palloc cycle if value size is out of the allowed - * range for compression - */ - if (valsize < PGLZ_strategy_default->min_input_size || - valsize > PGLZ_strategy_default->max_input_size) - return PointerGetDatum(NULL); + /* Fallback to default compression if not specified */ + if (!OidIsValid(acoid)) + acoid = DefaultCompressionOid; - tmp = (struct varlena *) palloc(PGLZ_MAX_OUTPUT(valsize) + - TOAST_COMPRESS_HDRSZ); + cmoptions = lookup_compression_am_options(acoid); + tmp = cmoptions->amroutine->cmcompress(cmoptions, (const struct varlena *) value); + if (!tmp) + return PointerGetDatum(NULL); /* - * We recheck the actual size even if pglz_compress() reports success, - * because it might be satisfied with having saved as little as one byte - * in the compressed data --- which could turn into a net loss once you - * consider header and alignment padding. Worst case, the compressed - * format might require three padding bytes (plus header, which is - * included in VARSIZE(tmp)), whereas the uncompressed format would take - * only one header byte and no padding if the value is short enough. So - * we insist on a savings of more than 2 bytes to ensure we have a gain. + * We recheck the actual size even if compression function reports + * success, because it might be satisfied with having saved as little as + * one byte in the compressed data --- which could turn into a net loss + * once you consider header and alignment padding. Worst case, the + * compressed format might require three padding bytes (plus header, which + * is included in VARSIZE(tmp)), whereas the uncompressed format would + * take only one header byte and no padding if the value is short enough. + * So we insist on a savings of more than 2 bytes to ensure we have a + * gain. */ - len = pglz_compress(VARDATA_ANY(DatumGetPointer(value)), - valsize, - TOAST_COMPRESS_RAWDATA(tmp), - PGLZ_strategy_default); - if (len >= 0 && - len + TOAST_COMPRESS_HDRSZ < valsize - 2) + valsize = VARSIZE_ANY_EXHDR(DatumGetPointer(value)); + + if (VARSIZE(tmp) < valsize - 2) { - TOAST_COMPRESS_SET_RAWSIZE(tmp, valsize); - SET_VARSIZE_COMPRESSED(tmp, len + TOAST_COMPRESS_HDRSZ); /* successful compression */ + toast_set_compressed_datum_info(tmp, cmoptions->acoid, valsize); return PointerGetDatum(tmp); } - else - { - /* incompressible data */ - pfree(tmp); - return PointerGetDatum(NULL); - } + + /* incompressible data */ + pfree(tmp); + return PointerGetDatum(NULL); } @@ -1501,19 +1659,20 @@ toast_save_datum(Relation rel, Datum value, &num_indexes); /* - * Get the data pointer and length, and compute va_rawsize and va_extsize. + * Get the data pointer and length, and compute va_rawsize and va_extinfo. * * va_rawsize is the size of the equivalent fully uncompressed datum, so * we have to adjust for short headers. * - * va_extsize is the actual size of the data payload in the toast records. + * va_extinfo contains the actual size of the data payload in the toast + * records. */ if (VARATT_IS_SHORT(dval)) { data_p = VARDATA_SHORT(dval); data_todo = VARSIZE_SHORT(dval) - VARHDRSZ_SHORT; toast_pointer.va_rawsize = data_todo + VARHDRSZ; /* as if not short */ - toast_pointer.va_extsize = data_todo; + toast_pointer.va_extinfo = data_todo; /* no flags */ } else if (VARATT_IS_COMPRESSED(dval)) { @@ -1521,7 +1680,10 @@ toast_save_datum(Relation rel, Datum value, data_todo = VARSIZE(dval) - VARHDRSZ; /* rawsize in a compressed datum is just the size of the payload */ toast_pointer.va_rawsize = VARRAWSIZE_4B_C(dval) + VARHDRSZ; - toast_pointer.va_extsize = data_todo; + toast_pointer.va_extinfo = data_todo; + if (VARATT_IS_CUSTOM_COMPRESSED(dval)) + VARATT_EXTERNAL_SET_CUSTOM(toast_pointer); + /* Assert that the numbers look like it's compressed */ Assert(VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)); } @@ -1530,7 +1692,7 @@ toast_save_datum(Relation rel, Datum value, data_p = VARDATA(dval); data_todo = VARSIZE(dval) - VARHDRSZ; toast_pointer.va_rawsize = VARSIZE(dval); - toast_pointer.va_extsize = data_todo; + toast_pointer.va_extinfo = data_todo; /* no flags */ } /* @@ -1890,7 +2052,7 @@ toast_fetch_datum(struct varlena *attr) /* Must copy to access aligned fields */ VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr); - ressize = toast_pointer.va_extsize; + ressize = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer); numchunks = ((ressize - 1) / TOAST_MAX_CHUNK_SIZE) + 1; result = (struct varlena *) palloc(ressize + VARHDRSZ); @@ -2077,7 +2239,7 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, int32 length) */ Assert(!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)); - attrsize = toast_pointer.va_extsize; + attrsize = VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer); totalchunks = ((attrsize - 1) / TOAST_MAX_CHUNK_SIZE) + 1; if (sliceoffset >= attrsize) @@ -2270,20 +2432,36 @@ toast_decompress_datum(struct varlena *attr) Assert(VARATT_IS_COMPRESSED(attr)); - result = (struct varlena *) - palloc(TOAST_COMPRESS_RAWSIZE(attr) + VARHDRSZ); - SET_VARSIZE(result, TOAST_COMPRESS_RAWSIZE(attr) + VARHDRSZ); + if (VARATT_IS_CUSTOM_COMPRESSED(attr)) + { + CompressionAmOptions *cmoptions; + toast_compress_header_custom *hdr; - if (pglz_decompress(TOAST_COMPRESS_RAWDATA(attr), - VARSIZE(attr) - TOAST_COMPRESS_HDRSZ, - VARDATA(result), - TOAST_COMPRESS_RAWSIZE(attr), true) < 0) - elog(ERROR, "compressed data is corrupted"); + hdr = (toast_compress_header_custom *) attr; + cmoptions = lookup_compression_am_options(hdr->cmid); + result = cmoptions->amroutine->cmdecompress(cmoptions, attr); + } + else + { + int rawsize; + + result = (struct varlena *) + palloc(TOAST_COMPRESS_RAWSIZE(attr) + VARHDRSZ); + SET_VARSIZE(result, TOAST_COMPRESS_RAWSIZE(attr) + VARHDRSZ); + + rawsize = pglz_decompress(TOAST_COMPRESS_RAWDATA(attr), + VARSIZE(attr) - TOAST_COMPRESS_HDRSZ, + VARDATA(result), + TOAST_COMPRESS_RAWSIZE(attr), true); + if (rawsize < 0) + elog(ERROR, "compressed data is corrupted"); + + SET_VARSIZE(result, rawsize + VARHDRSZ); + } return result; } - /* ---------- * toast_decompress_datum_slice - * @@ -2299,16 +2477,32 @@ toast_decompress_datum_slice(struct varlena *attr, int32 slicelength) Assert(VARATT_IS_COMPRESSED(attr)); - result = (struct varlena *) palloc(slicelength + VARHDRSZ); + if (VARATT_IS_CUSTOM_COMPRESSED(attr)) + { + CompressionAmOptions *cmoptions; + toast_compress_header_custom *hdr; - rawsize = pglz_decompress(TOAST_COMPRESS_RAWDATA(attr), - VARSIZE(attr) - TOAST_COMPRESS_HDRSZ, - VARDATA(result), - slicelength, false); - if (rawsize < 0) - elog(ERROR, "compressed data is corrupted"); + hdr = (toast_compress_header_custom *) attr; + cmoptions = lookup_compression_am_options(hdr->cmid); + if (cmoptions->amroutine->cmdecompress_slice) + result = cmoptions->amroutine->cmdecompress_slice(cmoptions, attr, slicelength); + else + result = cmoptions->amroutine->cmdecompress(cmoptions, attr); + } + else + { + result = (struct varlena *) palloc(slicelength + VARHDRSZ); + + rawsize = pglz_decompress(TOAST_COMPRESS_RAWDATA(attr), + VARSIZE(attr) - TOAST_COMPRESS_HDRSZ, + VARDATA(result), + slicelength, false); + if (rawsize < 0) + elog(ERROR, "compressed data is corrupted"); + + SET_VARSIZE(result, rawsize + VARHDRSZ); + } - SET_VARSIZE(result, rawsize + VARHDRSZ); return result; } @@ -2409,3 +2603,159 @@ init_toast_snapshot(Snapshot toast_snapshot) InitToastSnapshot(*toast_snapshot, snapshot->lsn, snapshot->whenTaken); } + +/* ---------- + * remove_cached_amoptions + * + * Remove specified compression options from cache + */ +static void +remove_cached_amoptions(CompressionAmOptions *entry) +{ + MemoryContextDelete(entry->mcxt); + + if (hash_search(amoptions_cache, (void *) &entry->acoid, + HASH_REMOVE, NULL) == NULL) + elog(ERROR, "hash table corrupted"); +} + +/* ---------- + * invalidate_amoptions_cache + * + * Flush cache entries when pg_attr_compression is updated. + */ +static void +invalidate_amoptions_cache(Datum arg, int cacheid, uint32 hashvalue) +{ + HASH_SEQ_STATUS status; + CompressionAmOptions *entry; + + hash_seq_init(&status, amoptions_cache); + while ((entry = (CompressionAmOptions *) hash_seq_search(&status)) != NULL) + remove_cached_amoptions(entry); +} + +/* ---------- + * init_amoptions_cache + * + * Initialize a local cache for attribute compression options. + */ +static void +init_amoptions_cache(void) +{ + HASHCTL ctl; + + amoptions_cache_mcxt = AllocSetContextCreate(TopMemoryContext, + "attr compression cache", + ALLOCSET_DEFAULT_SIZES); + MemSet(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(Oid); + ctl.entrysize = sizeof(CompressionAmOptions); + ctl.hcxt = amoptions_cache_mcxt; + amoptions_cache = hash_create("attr compression cache", 100, &ctl, + HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); + + /* Set up invalidation callback on pg_attr_compression. */ + CacheRegisterSyscacheCallback(ATTCOMPRESSIONOID, + invalidate_amoptions_cache, + (Datum) 0); +} + +/* ---------- + * lookup_compression_am_options + * + * Return or generate cache record for attribute compression. + */ +CompressionAmOptions * +lookup_compression_am_options(Oid acoid) +{ + bool found, + optisnull; + CompressionAmOptions *result; + Datum acoptions; + Form_pg_attr_compression acform; + HeapTuple tuple; + Oid amoid; + regproc amhandler; + + /* + * This call could invalidate system cache so we need to call it before + * we're putting something to our cache. + */ + tuple = SearchSysCache1(ATTCOMPRESSIONOID, ObjectIdGetDatum(acoid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for attribute compression %u", acoid); + + acform = (Form_pg_attr_compression) GETSTRUCT(tuple); + amoid = get_compression_am_oid(NameStr(acform->acname), false); + amhandler = get_am_handler_oid(amoid, AMTYPE_COMPRESSION, false); + acoptions = SysCacheGetAttr(ATTCOMPRESSIONOID, tuple, + Anum_pg_attr_compression_acoptions, &optisnull); + + if (!amoptions_cache) + init_amoptions_cache(); + + result = hash_search(amoptions_cache, &acoid, HASH_ENTER, &found); + if (!found) + { + MemoryContext oldcxt = CurrentMemoryContext; + + result->mcxt = AllocSetContextCreate(amoptions_cache_mcxt, + "compression am options", + ALLOCSET_DEFAULT_SIZES); + + PG_TRY(); + { + result->acoid = acoid; + result->amoid = amoid; + + MemoryContextSwitchTo(result->mcxt); + result->amroutine = InvokeCompressionAmHandler(amhandler); + if (optisnull) + result->acoptions = NIL; + else + result->acoptions = untransformRelOptions(acoptions); + result->acstate = result->amroutine->cminitstate ? + result->amroutine->cminitstate(acoid, result->acoptions) : NULL; + } + PG_CATCH(); + { + MemoryContextSwitchTo(oldcxt); + remove_cached_amoptions(result); + PG_RE_THROW(); + } + PG_END_TRY(); + MemoryContextSwitchTo(oldcxt); + } + + ReleaseSysCache(tuple); + + if (!result->amroutine) + /* should not happen but need to check if something goes wrong */ + elog(ERROR, "compression access method routine is NULL"); + + return result; +} + +/* ---------- + * attr_compression_options_are_equal + * + * Compare two attribute compression records + */ +static bool +attr_compression_options_are_equal(Oid acoid1, Oid acoid2) +{ + CompressionAmOptions *a; + CompressionAmOptions *b; + + a = lookup_compression_am_options(acoid1); + b = lookup_compression_am_options(acoid2); + + if (a->amoid != b->amoid) + return false; + + if (list_length(a->acoptions) != list_length(b->acoptions)) + return false; + + return equal(a->acoptions, b->acoptions); +} diff --git a/src/backend/bootstrap/bootstrap.c b/src/backend/bootstrap/bootstrap.c index 43627ab8f4e7a..7f93ddff41d54 100644 --- a/src/backend/bootstrap/bootstrap.c +++ b/src/backend/bootstrap/bootstrap.c @@ -763,6 +763,7 @@ DefineAttr(char *name, char *type, int attnum, int nullness) attrtypes[attnum]->attcacheoff = -1; attrtypes[attnum]->atttypmod = -1; attrtypes[attnum]->attislocal = true; + attrtypes[attnum]->attcompression = InvalidOid; if (nullness == BOOTCOL_NULL_FORCE_NOT_NULL) { diff --git a/src/backend/catalog/Makefile b/src/backend/catalog/Makefile index 8bece078dd19b..20edc2aa42b87 100644 --- a/src/backend/catalog/Makefile +++ b/src/backend/catalog/Makefile @@ -34,7 +34,7 @@ CATALOG_HEADERS := \ pg_attrdef.h pg_constraint.h pg_inherits.h pg_index.h pg_operator.h \ pg_opfamily.h pg_opclass.h pg_am.h pg_amop.h pg_amproc.h \ pg_language.h pg_largeobject_metadata.h pg_largeobject.h pg_aggregate.h \ - pg_statistic_ext.h pg_statistic_ext_data.h \ + pg_statistic_ext.h pg_statistic_ext_data.h pg_attr_compression.h \ pg_statistic.h pg_rewrite.h pg_trigger.h pg_event_trigger.h pg_description.h \ pg_cast.h pg_enum.h pg_namespace.h pg_conversion.h pg_depend.h \ pg_database.h pg_db_role_setting.h pg_tablespace.h pg_pltemplate.h \ diff --git a/src/backend/catalog/dependency.c b/src/backend/catalog/dependency.c index 6315fc4b2fd3c..0a5f8c7946d2c 100644 --- a/src/backend/catalog/dependency.c +++ b/src/backend/catalog/dependency.c @@ -25,6 +25,7 @@ #include "catalog/pg_am.h" #include "catalog/pg_amop.h" #include "catalog/pg_amproc.h" +#include "catalog/pg_attr_compression.h" #include "catalog/pg_attrdef.h" #include "catalog/pg_authid.h" #include "catalog/pg_cast.h" @@ -181,7 +182,8 @@ static const Oid object_classes[] = { PublicationRelationId, /* OCLASS_PUBLICATION */ PublicationRelRelationId, /* OCLASS_PUBLICATION_REL */ SubscriptionRelationId, /* OCLASS_SUBSCRIPTION */ - TransformRelationId /* OCLASS_TRANSFORM */ + TransformRelationId, /* OCLASS_TRANSFORM */ + AttrCompressionRelationId /* OCLASS_ATTR_COMPRESSION */ }; @@ -1480,6 +1482,10 @@ doDeletion(const ObjectAddress *object, int flags) DropTransformById(object->objectId); break; + case OCLASS_ATTR_COMPRESSION: + RemoveAttributeCompression(object->objectId); + break; + /* * These global object types are not supported here. */ @@ -2775,6 +2781,9 @@ getObjectClass(const ObjectAddress *object) case TransformRelationId: return OCLASS_TRANSFORM; + + case AttrCompressionRelationId: + return OCLASS_ATTR_COMPRESSION; } /* shouldn't get here */ diff --git a/src/backend/catalog/heap.c b/src/backend/catalog/heap.c index 3b8c8b193a7f8..7a7100d662b88 100644 --- a/src/backend/catalog/heap.c +++ b/src/backend/catalog/heap.c @@ -725,6 +725,7 @@ InsertPgAttributeTuple(Relation pg_attribute_rel, values[Anum_pg_attribute_attislocal - 1] = BoolGetDatum(new_attribute->attislocal); values[Anum_pg_attribute_attinhcount - 1] = Int32GetDatum(new_attribute->attinhcount); values[Anum_pg_attribute_attcollation - 1] = ObjectIdGetDatum(new_attribute->attcollation); + values[Anum_pg_attribute_attcompression - 1] = ObjectIdGetDatum(new_attribute->attcompression); /* start out with empty permissions and empty options */ nulls[Anum_pg_attribute_attacl - 1] = true; @@ -1636,6 +1637,9 @@ RemoveAttributeById(Oid relid, AttrNumber attnum) /* Unset this so no one tries to look up the generation expression */ attStruct->attgenerated = '\0'; + /* Unset attribute compression Oid */ + attStruct->attcompression = InvalidOid; + /* * Change the column name to something that isn't likely to conflict */ diff --git a/src/backend/catalog/index.c b/src/backend/catalog/index.c index bb60b23093e3b..0e185f6d00a10 100644 --- a/src/backend/catalog/index.c +++ b/src/backend/catalog/index.c @@ -367,6 +367,7 @@ ConstructTupleDescriptor(Relation heapRelation, to->attstorage = typeTup->typstorage; to->attalign = typeTup->typalign; to->atttypmod = exprTypmod(indexkey); + to->attcompression = InvalidOid; ReleaseSysCache(tuple); @@ -453,6 +454,7 @@ ConstructTupleDescriptor(Relation heapRelation, to->attbyval = typeTup->typbyval; to->attalign = typeTup->typalign; to->attstorage = typeTup->typstorage; + to->attcompression = InvalidOid; ReleaseSysCache(tuple); } diff --git a/src/backend/catalog/objectaddress.c b/src/backend/catalog/objectaddress.c index caf48cefa9820..e709a61ac3f24 100644 --- a/src/backend/catalog/objectaddress.c +++ b/src/backend/catalog/objectaddress.c @@ -26,6 +26,7 @@ #include "catalog/pg_am.h" #include "catalog/pg_amop.h" #include "catalog/pg_amproc.h" +#include "catalog/pg_attr_compression.h" #include "catalog/pg_attrdef.h" #include "catalog/pg_authid.h" #include "catalog/pg_cast.h" @@ -525,6 +526,18 @@ static const ObjectPropertyType ObjectProperty[] = InvalidAttrNumber, /* no ACL (same as relation) */ OBJECT_STATISTIC_EXT, true + }, + { + AttrCompressionRelationId, + AttrCompressionIndexId, + ATTCOMPRESSIONOID, + -1, + InvalidAttrNumber, + InvalidAttrNumber, + InvalidAttrNumber, + InvalidAttrNumber, + -1, + false } }; @@ -3618,6 +3631,37 @@ getObjectDescription(const ObjectAddress *object) break; } + case OCLASS_ATTR_COMPRESSION: + { + char *attname; + HeapTuple tup; + Form_pg_attr_compression acform; + + tup = SearchSysCache1(ATTCOMPRESSIONOID, ObjectIdGetDatum(object->objectId)); + if (!HeapTupleIsValid(tup)) + elog(ERROR, "cache lookup failed for attribute compression %u", object->objectId); + + acform = (Form_pg_attr_compression) GETSTRUCT(tup); + if (OidIsValid(acform->acrelid)) + { + appendStringInfo(&buffer, _("compression on ")); + getRelationDescription(&buffer, acform->acrelid); + + attname = get_attname(acform->acrelid, acform->acattnum, true); + if (attname) + { + appendStringInfo(&buffer, _(" column %s"), attname); + pfree(attname); + } + } + else + appendStringInfo(&buffer, _("attribute compression %u"), object->objectId); + + ReleaseSysCache(tup); + + break; + } + case OCLASS_TRANSFORM: { HeapTuple trfTup; @@ -4147,6 +4191,10 @@ getObjectTypeDescription(const ObjectAddress *object) appendStringInfoString(&buffer, "transform"); break; + case OCLASS_ATTR_COMPRESSION: + appendStringInfoString(&buffer, "attribute compression"); + break; + /* * There's intentionally no default: case here; we want the * compiler to warn if a new OCLASS hasn't been handled above. @@ -4693,6 +4741,15 @@ getObjectIdentityParts(const ObjectAddress *object, break; } + case OCLASS_ATTR_COMPRESSION: + { + appendStringInfo(&buffer, "%u", + object->objectId); + if (objname) + *objname = list_make1(psprintf("%u", object->objectId)); + break; + } + case OCLASS_REWRITE: { Relation ruleDesc; diff --git a/src/backend/commands/Makefile b/src/backend/commands/Makefile index d64628566d361..ce7ef52e7af7d 100644 --- a/src/backend/commands/Makefile +++ b/src/backend/commands/Makefile @@ -13,8 +13,8 @@ top_builddir = ../../.. include $(top_builddir)/src/Makefile.global OBJS = amcmds.o aggregatecmds.o alter.o analyze.o async.o cluster.o comment.o \ - collationcmds.o constraint.o conversioncmds.o copy.o createas.o \ - dbcommands.o define.o discard.o dropcmds.o \ + collationcmds.o compressioncmds.o constraint.o conversioncmds.o copy.o \ + createas.o dbcommands.o define.o discard.o dropcmds.o \ event_trigger.o explain.o extension.o foreigncmds.o functioncmds.o \ indexcmds.o lockcmds.o matview.o operatorcmds.o opclasscmds.o \ policy.o portalcmds.o prepare.o proclang.o publicationcmds.o \ diff --git a/src/backend/commands/alter.c b/src/backend/commands/alter.c index 70dbcb0756c3a..e537361f26879 100644 --- a/src/backend/commands/alter.c +++ b/src/backend/commands/alter.c @@ -638,6 +638,7 @@ AlterObjectNamespace_oid(Oid classId, Oid objid, Oid nspOid, case OCLASS_PUBLICATION_REL: case OCLASS_SUBSCRIPTION: case OCLASS_TRANSFORM: + case OCLASS_ATTR_COMPRESSION: /* ignore object types that don't have schema-qualified names */ break; diff --git a/src/backend/commands/amcmds.c b/src/backend/commands/amcmds.c index c0e40980d5fce..fe898b7289ac5 100644 --- a/src/backend/commands/amcmds.c +++ b/src/backend/commands/amcmds.c @@ -179,6 +179,70 @@ get_am_type_oid(const char *amname, char amtype, bool missing_ok) return oid; } +/* + * get_am_handler_oid - return access method handler Oid with additional + * type checking. + * + * If noerror is false, throw an error if access method not found or its + * type not equal to specified type, or its handler is not valid. + * + * If amtype is not '\0', an error is raised if the AM found is not of the + * given type. + */ +regproc +get_am_handler_oid(Oid amOid, char amtype, bool noerror) +{ + HeapTuple tup; + Form_pg_am amform; + regproc amhandler; + + /* Get handler function OID for the access method */ + tup = SearchSysCache1(AMOID, ObjectIdGetDatum(amOid)); + if (!HeapTupleIsValid(tup)) + { + if (noerror) + return InvalidOid; + + elog(ERROR, "cache lookup failed for access method %u", amOid); + } + + amform = (Form_pg_am) GETSTRUCT(tup); + amhandler = amform->amhandler; + + if (amtype != '\0' && amform->amtype != amtype) + { + if (noerror) + { + ReleaseSysCache(tup); + return InvalidOid; + } + + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("access method \"%s\" is not of type %s", + NameStr(amform->amname), + get_am_type_string(amtype)))); + } + + /* Complain if handler OID is invalid */ + if (!RegProcedureIsValid(amhandler)) + { + if (noerror) + { + ReleaseSysCache(tup); + return InvalidOid; + } + + ereport(ERROR, + (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), + errmsg("access method \"%s\" does not have a handler", + NameStr(amform->amname)))); + } + + ReleaseSysCache(tup); + return amhandler; +} + /* * get_index_am_oid - given an access method name, look up its OID * and verify it corresponds to an index AM. @@ -199,6 +263,16 @@ get_table_am_oid(const char *amname, bool missing_ok) return get_am_type_oid(amname, AMTYPE_TABLE, missing_ok); } +/* + * get_compression_am_oid - given an access method name, look up its OID + * and verify it corresponds to an compression AM. + */ +Oid +get_compression_am_oid(const char *amname, bool missing_ok) +{ + return get_am_type_oid(amname, AMTYPE_COMPRESSION, missing_ok); +} + /* * get_am_oid - given an access method name, look up its OID. * The type is not checked. @@ -241,6 +315,8 @@ get_am_type_string(char amtype) return "INDEX"; case AMTYPE_TABLE: return "TABLE"; + case AMTYPE_COMPRESSION: + return "COMPRESSION"; default: /* shouldn't happen */ elog(ERROR, "invalid access method type '%c'", amtype); @@ -278,6 +354,9 @@ lookup_am_handler_func(List *handler_name, char amtype) case AMTYPE_TABLE: expectedType = TABLE_AM_HANDLEROID; break; + case AMTYPE_COMPRESSION: + expectedType = COMPRESSION_AM_HANDLEROID; + break; default: elog(ERROR, "unrecognized access method type \"%c\"", amtype); } diff --git a/src/backend/commands/compressioncmds.c b/src/backend/commands/compressioncmds.c new file mode 100644 index 0000000000000..39ee3cf38e266 --- /dev/null +++ b/src/backend/commands/compressioncmds.c @@ -0,0 +1,672 @@ +/*------------------------------------------------------------------------- + * + * compressioncmds.c + * Routines for SQL commands for compression access methods + * + * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/backend/commands/compressioncmds.c + *------------------------------------------------------------------------- + */ +#include "postgres.h" +#include "miscadmin.h" + +#include "access/cmapi.h" +#include "access/heapam.h" +#include "access/htup_details.h" +#include "access/reloptions.h" +#include "access/xact.h" +#include "catalog/catalog.h" +#include "catalog/dependency.h" +#include "catalog/indexing.h" +#include "catalog/pg_attr_compression_d.h" +#include "catalog/pg_am_d.h" +#include "catalog/pg_collation_d.h" +#include "catalog/pg_depend.h" +#include "catalog/pg_proc_d.h" +#include "catalog/pg_type_d.h" +#include "commands/defrem.h" +#include "parser/parse_func.h" +#include "utils/builtins.h" +#include "utils/fmgroids.h" +#include "utils/lsyscache.h" +#include "utils/rel.h" +#include "utils/syscache.h" +#include "utils/snapmgr.h" + +/* Set by pg_upgrade_support functions */ +Oid binary_upgrade_next_attr_compression_oid = InvalidOid; + +/* + * When conditions of compression satisfies one if builtin attribute + * compresssion tuples the compressed attribute will be linked to + * builtin compression without new record in pg_attr_compression. + * So the fact that the column has a builtin compression we only can find out + * by its dependency. + */ +static void +lookup_builtin_dependencies(Oid attrelid, AttrNumber attnum, + List **amoids) +{ + LOCKMODE lock = AccessShareLock; + Oid amoid = InvalidOid; + HeapTuple tup; + Relation rel; + SysScanDesc scan; + ScanKeyData key[3]; + + rel = heap_open(DependRelationId, lock); + + ScanKeyInit(&key[0], + Anum_pg_depend_classid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(RelationRelationId)); + ScanKeyInit(&key[1], + Anum_pg_depend_objid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(attrelid)); + ScanKeyInit(&key[2], + Anum_pg_depend_objsubid, + BTEqualStrategyNumber, F_INT4EQ, + Int32GetDatum((int32) attnum)); + + scan = systable_beginscan(rel, DependDependerIndexId, true, + NULL, 3, key); + + while (HeapTupleIsValid(tup = systable_getnext(scan))) + { + Form_pg_depend depform = (Form_pg_depend) GETSTRUCT(tup); + + if (depform->refclassid == AttrCompressionRelationId) + { + Assert(IsBuiltinCompression(depform->refobjid)); + amoid = GetAttrCompressionAmOid(depform->refobjid); + *amoids = list_append_unique_oid(*amoids, amoid); + } + } + + systable_endscan(scan); + heap_close(rel, lock); +} + +/* + * Find identical attribute compression for reuse and fill the list with + * used compression access methods. + */ +static Oid +lookup_attribute_compression(Oid attrelid, AttrNumber attnum, + Oid amoid, Datum acoptions, List **previous_amoids) +{ + Relation rel; + HeapTuple tuple; + SysScanDesc scan; + FmgrInfo arrayeq_info; + Oid result = InvalidOid; + ScanKeyData key[2]; + + /* fill FmgrInfo for array_eq function */ + fmgr_info(F_ARRAY_EQ, &arrayeq_info); + + Assert((attrelid > 0 && attnum > 0) || (attrelid == 0 && attnum == 0)); + if (previous_amoids) + lookup_builtin_dependencies(attrelid, attnum, previous_amoids); + + rel = heap_open(AttrCompressionRelationId, AccessShareLock); + ScanKeyInit(&key[0], + Anum_pg_attr_compression_acrelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(attrelid)); + + ScanKeyInit(&key[1], + Anum_pg_attr_compression_acattnum, + BTEqualStrategyNumber, F_INT2EQ, + Int16GetDatum(attnum)); + + scan = systable_beginscan(rel, AttrCompressionRelidAttnumIndexId, + true, NULL, 2, key); + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + Oid acoid, + tup_amoid; + Datum values[Natts_pg_attr_compression]; + bool nulls[Natts_pg_attr_compression]; + char *amname; + + heap_deform_tuple(tuple, RelationGetDescr(rel), values, nulls); + acoid = DatumGetObjectId(values[Anum_pg_attr_compression_acoid - 1]); + amname = NameStr(*DatumGetName(values[Anum_pg_attr_compression_acname - 1])); + tup_amoid = get_am_oid(amname, false); + + if (previous_amoids) + *previous_amoids = list_append_unique_oid(*previous_amoids, tup_amoid); + + if (tup_amoid != amoid) + continue; + + /* + * even if we found the match, we still need to acquire all previous + * access methods so don't break cycle if previous_amoids is not NULL + */ + if (nulls[Anum_pg_attr_compression_acoptions - 1]) + { + if (DatumGetPointer(acoptions) == NULL) + result = acoid; + } + else if (DatumGetPointer(acoptions) != NULL) + { + bool equal; + + /* check if arrays for WITH options are equal */ + equal = DatumGetBool(CallerFInfoFunctionCall2( + array_eq, &arrayeq_info, DEFAULT_COLLATION_OID, acoptions, + values[Anum_pg_attr_compression_acoptions - 1])); + + if (equal) + result = acoid; + } + + if (previous_amoids == NULL && OidIsValid(result)) + break; + } + + systable_endscan(scan); + heap_close(rel, AccessShareLock); + return result; +} + +/* + * Link compression with an attribute. Creates a row in pg_attr_compression + * if needed. + * + * When compression is not specified returns default attribute compression. + * It is possible case for CREATE TABLE and ADD COLUMN commands + * where COMPRESSION syntax is optional. + * + * If any of builtin attribute compression tuples satisfies conditions + * returns it. + * + * For ALTER command check for previous attribute compression record with + * identical compression options and reuse it if found any. + * + * Note we create attribute compression for EXTERNAL storage too, so when + * storage is changed we can start compression on future tuples right away. + */ +Oid +CreateAttributeCompression(Form_pg_attribute att, + ColumnCompression *compression, + bool *need_rewrite, List **preserved_amoids) +{ + Relation rel; + HeapTuple newtup; + Oid acoid = InvalidOid, + amoid; + regproc amhandler; + Datum arropt; + Datum values[Natts_pg_attr_compression]; + bool nulls[Natts_pg_attr_compression]; + + ObjectAddress myself, + amref; + + CompressionAmRoutine *routine; + + /* No compression for PLAIN storage. */ + if (att->attstorage == 'p') + return InvalidOid; + + /* Fallback to default compression if it's not specified */ + if (compression == NULL) + return DefaultCompressionOid; + + amoid = get_compression_am_oid(compression->amname, false); + if (compression->options) + arropt = optionListToArray(compression->options, true); + else + arropt = PointerGetDatum(NULL); + + /* Try to find builtin compression first */ + acoid = lookup_attribute_compression(0, 0, amoid, arropt, NULL); + + /* no rewrite by default */ + if (need_rewrite != NULL) + *need_rewrite = false; + + if (IsBinaryUpgrade) + { + /* Skip the rewrite checks and searching of identical compression */ + goto add_tuple; + } + + /* + * attrelid will be invalid on CREATE TABLE, no need for table rewrite + * check. + */ + if (OidIsValid(att->attrelid)) + { + Oid attacoid; + List *previous_amoids = NIL; + + /* + * Try to find identical compression from previous tuples, and fill + * the list of previous compresssion methods + */ + attacoid = lookup_attribute_compression(att->attrelid, att->attnum, + amoid, arropt, &previous_amoids); + if (!OidIsValid(acoid)) + acoid = attacoid; + + /* + * Determine if the column needs rewrite or not. Rewrite conditions: - + * SET COMPRESSION without PRESERVE - SET COMPRESSION with PRESERVE + * but not with full list of previous access methods. + */ + if (need_rewrite != NULL) + { + Assert(preserved_amoids != NULL); + + if (compression->preserve == NIL) + *need_rewrite = true; + else + { + ListCell *cell; + + foreach(cell, compression->preserve) + { + char *amname_p = strVal(lfirst(cell)); + Oid amoid_p = get_am_oid(amname_p, false); + + if (!list_member_oid(previous_amoids, amoid_p)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("\"%s\" compression access method cannot be preserved", amname_p), + errhint("use \"pg_column_compression\" function for list of compression methods") + )); + + *preserved_amoids = list_append_unique_oid(*preserved_amoids, amoid_p); + + /* + * Remove from previous list, also protect from multiple + * mentions of one access method in PRESERVE list + */ + previous_amoids = list_delete_oid(previous_amoids, amoid_p); + } + + /* + * If the list of previous Oids is not empty after deletions + * then we need to rewrite tuples in the table. + * + * In binary upgrade list will not be free since it contains + * Oid of builtin compression access method. + */ + if (list_length(previous_amoids) != 0) + *need_rewrite = true; + } + } + + /* Cleanup */ + list_free(previous_amoids); + } + + /* Return Oid if we already found identical compression on this column */ + if (OidIsValid(acoid)) + { + if (DatumGetPointer(arropt) != NULL) + pfree(DatumGetPointer(arropt)); + + return acoid; + } + +add_tuple: + /* Initialize buffers for new tuple values */ + memset(values, 0, sizeof(values)); + memset(nulls, false, sizeof(nulls)); + + amhandler = get_am_handler_oid(amoid, AMTYPE_COMPRESSION, false); + + rel = heap_open(AttrCompressionRelationId, RowExclusiveLock); + + if (IsBinaryUpgrade) + { + /* acoid should be found in some cases */ + if (binary_upgrade_next_attr_compression_oid < FirstNormalObjectId && + (!OidIsValid(acoid) || binary_upgrade_next_attr_compression_oid != acoid)) + elog(ERROR, "could not link to built-in attribute compression"); + + acoid = binary_upgrade_next_attr_compression_oid; + } + else + { + acoid = GetNewOidWithIndex(rel, AttrCompressionIndexId, + Anum_pg_attr_compression_acoid); + + } + + if (acoid < FirstNormalObjectId) + { + /* this is built-in attribute compression */ + heap_close(rel, RowExclusiveLock); + return acoid; + } + + /* we need routine only to call cmcheck function */ + routine = InvokeCompressionAmHandler(amhandler); + if (routine->cmcheck != NULL) + routine->cmcheck(att, compression->options); + pfree(routine); + + values[Anum_pg_attr_compression_acoid - 1] = ObjectIdGetDatum(acoid); + values[Anum_pg_attr_compression_acname - 1] = CStringGetDatum(compression->amname); + values[Anum_pg_attr_compression_acrelid - 1] = ObjectIdGetDatum(att->attrelid); + values[Anum_pg_attr_compression_acattnum - 1] = Int32GetDatum(att->attnum); + + if (compression->options) + values[Anum_pg_attr_compression_acoptions - 1] = arropt; + else + nulls[Anum_pg_attr_compression_acoptions - 1] = true; + + newtup = heap_form_tuple(RelationGetDescr(rel), values, nulls); + CatalogTupleInsert(rel, newtup); + heap_freetuple(newtup); + heap_close(rel, RowExclusiveLock); + + ObjectAddressSet(myself, AttrCompressionRelationId, acoid); + ObjectAddressSet(amref, AccessMethodRelationId, amoid); + + recordDependencyOn(&myself, &amref, DEPENDENCY_NORMAL); + recordDependencyOnCurrentExtension(&myself, false); + + /* Make the changes visible */ + CommandCounterIncrement(); + return acoid; +} + +/* + * Remove the attribute compression record from pg_attr_compression. + */ +void +RemoveAttributeCompression(Oid acoid) +{ + Relation relation; + HeapTuple tup; + + tup = SearchSysCache1(ATTCOMPRESSIONOID, ObjectIdGetDatum(acoid)); + if (!HeapTupleIsValid(tup)) + elog(ERROR, "cache lookup failed for attribute compression %u", acoid); + + /* check we're not trying to remove builtin attribute compression */ + Assert(((Form_pg_attr_compression) GETSTRUCT(tup))->acrelid != 0); + + /* delete the record from catalogs */ + relation = heap_open(AttrCompressionRelationId, RowExclusiveLock); + CatalogTupleDelete(relation, &tup->t_self); + heap_close(relation, RowExclusiveLock); + ReleaseSysCache(tup); +} + +/* + * CleanupAttributeCompression + * + * Remove entries in pg_attr_compression of the column except current + * attribute compression and related with specified list of access methods. + */ +void +CleanupAttributeCompression(Oid relid, AttrNumber attnum, List *keepAmOids) +{ + Oid acoid, + amoid; + Relation rel; + SysScanDesc scan; + ScanKeyData key[3]; + HeapTuple tuple, + attrtuple; + Form_pg_attribute attform; + List *removed = NIL; + ListCell *lc; + + attrtuple = SearchSysCache2(ATTNUM, + ObjectIdGetDatum(relid), + Int16GetDatum(attnum)); + + if (!HeapTupleIsValid(attrtuple)) + elog(ERROR, "cache lookup failed for attribute %d of relation %u", + attnum, relid); + attform = (Form_pg_attribute) GETSTRUCT(attrtuple); + acoid = attform->attcompression; + ReleaseSysCache(attrtuple); + + Assert(relid > 0 && attnum > 0); + Assert(!IsBinaryUpgrade); + + rel = heap_open(AttrCompressionRelationId, RowExclusiveLock); + + ScanKeyInit(&key[0], + Anum_pg_attr_compression_acrelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relid)); + + ScanKeyInit(&key[1], + Anum_pg_attr_compression_acattnum, + BTEqualStrategyNumber, F_INT2EQ, + Int16GetDatum(attnum)); + + scan = systable_beginscan(rel, AttrCompressionRelidAttnumIndexId, + true, NULL, 2, key); + + /* + * Remove attribute compression tuples and collect removed Oids + * to list. + */ + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + Form_pg_attr_compression acform; + + acform = (Form_pg_attr_compression) GETSTRUCT(tuple); + amoid = get_am_oid(NameStr(acform->acname), false); + + /* skip current compression */ + if (acform->acoid == acoid) + continue; + + if (!list_member_oid(keepAmOids, amoid)) + { + removed = lappend_oid(removed, acform->acoid); + CatalogTupleDelete(rel, &tuple->t_self); + } + } + + systable_endscan(scan); + heap_close(rel, RowExclusiveLock); + + /* + * Now remove dependencies between attribute compression (dependent) + * and column. + */ + rel = heap_open(DependRelationId, RowExclusiveLock); + foreach(lc, removed) + { + Oid tup_acoid = lfirst_oid(lc); + + ScanKeyInit(&key[0], + Anum_pg_depend_classid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(AttrCompressionRelationId)); + ScanKeyInit(&key[1], + Anum_pg_depend_objid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(tup_acoid)); + + scan = systable_beginscan(rel, DependDependerIndexId, true, + NULL, 2, key); + + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + CatalogTupleDelete(rel, &tuple->t_self); + + systable_endscan(scan); + } + heap_close(rel, RowExclusiveLock); + + /* Now remove dependencies with builtin compressions */ + rel = heap_open(DependRelationId, RowExclusiveLock); + ScanKeyInit(&key[0], + Anum_pg_depend_classid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(RelationRelationId)); + ScanKeyInit(&key[1], + Anum_pg_depend_objid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relid)); + ScanKeyInit(&key[2], + Anum_pg_depend_objsubid, + BTEqualStrategyNumber, F_INT4EQ, + Int32GetDatum((int32) attnum)); + + scan = systable_beginscan(rel, DependDependerIndexId, true, + NULL, 3, key); + + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + Form_pg_depend depform = (Form_pg_depend) GETSTRUCT(tuple); + + if (depform->refclassid != AttrCompressionRelationId) + continue; + + /* skip current compression */ + if (depform->refobjid == acoid) + continue; + + amoid = GetAttrCompressionAmOid(depform->refobjid); + if (!list_member_oid(keepAmOids, amoid)) + CatalogTupleDelete(rel, &tuple->t_self); + } + + systable_endscan(scan); + heap_close(rel, RowExclusiveLock); +} + +/* + * Construct ColumnCompression node by attribute compression Oid. + */ +ColumnCompression * +MakeColumnCompression(Oid acoid) +{ + HeapTuple tuple; + Form_pg_attr_compression acform; + ColumnCompression *node; + + if (!OidIsValid(acoid)) + return NULL; + + tuple = SearchSysCache1(ATTCOMPRESSIONOID, ObjectIdGetDatum(acoid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for attribute compression %u", acoid); + + acform = (Form_pg_attr_compression) GETSTRUCT(tuple); + node = makeNode(ColumnCompression); + node->amname = pstrdup(NameStr(acform->acname)); + ReleaseSysCache(tuple); + + /* + * fill attribute compression options too. We could've do it above but + * it's easier to call this helper. + */ + node->options = GetAttrCompressionOptions(acoid); + + return node; +} + +/* + * Compare compression options for two columns. + */ +void +CheckCompressionMismatch(ColumnCompression *c1, ColumnCompression *c2, + const char *attributeName) +{ + if (strcmp(c1->amname, c2->amname)) + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("column \"%s\" has a compression method conflict", + attributeName), + errdetail("%s versus %s", c1->amname, c2->amname))); + + if (!equal(c1->options, c2->options)) + ereport(ERROR, + (errcode(ERRCODE_DATATYPE_MISMATCH), + errmsg("column \"%s\" has a compression options conflict", + attributeName), + errdetail("(%s) versus (%s)", + formatRelOptions(c1->options), + formatRelOptions(c2->options)))); +} + +/* + * Return list of compression methods used in specified column. + */ +Datum +pg_column_compression(PG_FUNCTION_ARGS) +{ + Oid relOid = PG_GETARG_OID(0); + char *attname = TextDatumGetCString(PG_GETARG_TEXT_P(1)); + Relation rel; + HeapTuple tuple; + AttrNumber attnum; + List *amoids = NIL; + Oid amoid; + ListCell *lc; + + ScanKeyData key[2]; + SysScanDesc scan; + StringInfoData result; + + attnum = get_attnum(relOid, attname); + if (attnum == InvalidAttrNumber) + PG_RETURN_NULL(); + + /* Collect related builtin compression access methods */ + lookup_builtin_dependencies(relOid, attnum, &amoids); + + /* Collect other related access methods */ + rel = heap_open(AttrCompressionRelationId, AccessShareLock); + + ScanKeyInit(&key[0], + Anum_pg_attr_compression_acrelid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relOid)); + ScanKeyInit(&key[1], + Anum_pg_attr_compression_acattnum, + BTEqualStrategyNumber, F_INT2EQ, + Int16GetDatum(attnum)); + + scan = systable_beginscan(rel, AttrCompressionRelidAttnumIndexId, + true, NULL, 2, key); + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + Form_pg_attr_compression acform; + + acform = (Form_pg_attr_compression) GETSTRUCT(tuple); + amoid = get_am_oid(NameStr(acform->acname), false); + amoids = list_append_unique_oid(amoids, amoid); + } + + systable_endscan(scan); + heap_close(rel, AccessShareLock); + + if (!list_length(amoids)) + PG_RETURN_NULL(); + + /* Construct the list separated by comma */ + amoid = InvalidOid; + initStringInfo(&result); + foreach(lc, amoids) + { + if (OidIsValid(amoid)) + appendStringInfoString(&result, ", "); + + amoid = lfirst_oid(lc); + appendStringInfoString(&result, get_am_name(amoid)); + } + + PG_RETURN_TEXT_P(CStringGetTextDatum(result.data)); +} diff --git a/src/backend/commands/copy.c b/src/backend/commands/copy.c index f1161f0fee16d..af88740946bbc 100644 --- a/src/backend/commands/copy.c +++ b/src/backend/commands/copy.c @@ -2354,7 +2354,7 @@ CopyMultiInsertBufferInit(ResultRelInfo *rri) buffer = (CopyMultiInsertBuffer *) palloc(sizeof(CopyMultiInsertBuffer)); memset(buffer->slots, 0, sizeof(TupleTableSlot *) * MAX_BUFFERED_TUPLES); buffer->resultRelInfo = rri; - buffer->bistate = GetBulkInsertState(); + buffer->bistate = GetBulkInsertState(NULL); buffer->nused = 0; return buffer; @@ -2975,7 +2975,7 @@ CopyFrom(CopyState cstate) { singleslot = table_slot_create(resultRelInfo->ri_RelationDesc, &estate->es_tupleTable); - bistate = GetBulkInsertState(); + bistate = GetBulkInsertState(NULL); } has_before_insert_row_trig = (resultRelInfo->ri_TrigDesc && diff --git a/src/backend/commands/createas.c b/src/backend/commands/createas.c index 4c1d909d380ce..134c1c125c07c 100644 --- a/src/backend/commands/createas.c +++ b/src/backend/commands/createas.c @@ -560,7 +560,7 @@ intorel_startup(DestReceiver *self, int operation, TupleDesc typeinfo) */ myState->ti_options = TABLE_INSERT_SKIP_FSM | (XLogIsNeeded() ? 0 : TABLE_INSERT_SKIP_WAL); - myState->bistate = GetBulkInsertState(); + myState->bistate = GetBulkInsertState(NULL); /* Not using WAL requires smgr_targblock be initially invalid */ Assert(RelationGetTargetBlock(intoRelationDesc) == InvalidBlockNumber); diff --git a/src/backend/commands/event_trigger.c b/src/backend/commands/event_trigger.c index efef120c038fd..a99e018b0b010 100644 --- a/src/backend/commands/event_trigger.c +++ b/src/backend/commands/event_trigger.c @@ -1216,6 +1216,7 @@ EventTriggerSupportsObjectClass(ObjectClass objclass) case OCLASS_PUBLICATION_REL: case OCLASS_SUBSCRIPTION: case OCLASS_TRANSFORM: + case OCLASS_ATTR_COMPRESSION: return true; /* diff --git a/src/backend/commands/foreigncmds.c b/src/backend/commands/foreigncmds.c index d7bc6e35f029c..0c4c71909eca0 100644 --- a/src/backend/commands/foreigncmds.c +++ b/src/backend/commands/foreigncmds.c @@ -49,50 +49,6 @@ typedef struct /* Internal functions */ static void import_error_callback(void *arg); - -/* - * Convert a DefElem list to the text array format that is used in - * pg_foreign_data_wrapper, pg_foreign_server, pg_user_mapping, and - * pg_foreign_table. - * - * Returns the array in the form of a Datum, or PointerGetDatum(NULL) - * if the list is empty. - * - * Note: The array is usually stored to database without further - * processing, hence any validation should be done before this - * conversion. - */ -static Datum -optionListToArray(List *options) -{ - ArrayBuildState *astate = NULL; - ListCell *cell; - - foreach(cell, options) - { - DefElem *def = lfirst(cell); - const char *value; - Size len; - text *t; - - value = defGetString(def); - len = VARHDRSZ + strlen(def->defname) + 1 + strlen(value); - t = palloc(len + 1); - SET_VARSIZE(t, len); - sprintf(VARDATA(t), "%s=%s", def->defname, value); - - astate = accumArrayResult(astate, PointerGetDatum(t), - false, TEXTOID, - CurrentMemoryContext); - } - - if (astate) - return makeArrayResult(astate, CurrentMemoryContext); - - return PointerGetDatum(NULL); -} - - /* * Transform a list of DefElem into text array format. This is substantially * the same thing as optionListToArray(), except we recognize SET/ADD/DROP @@ -179,7 +135,7 @@ transformGenericOptions(Oid catalogId, } } - result = optionListToArray(resultOptions); + result = optionListToArray(resultOptions, false); if (OidIsValid(fdwvalidator)) { diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index 537d0e8ceff65..ee63c09afef23 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -465,7 +465,7 @@ transientrel_startup(DestReceiver *self, int operation, TupleDesc typeinfo) myState->ti_options = TABLE_INSERT_SKIP_FSM | TABLE_INSERT_FROZEN; if (!XLogIsNeeded()) myState->ti_options |= TABLE_INSERT_SKIP_WAL; - myState->bistate = GetBulkInsertState(); + myState->bistate = GetBulkInsertState(NULL); /* Not using WAL requires smgr_targblock be initially invalid */ Assert(RelationGetTargetBlock(transientrel) == InvalidBlockNumber); diff --git a/src/backend/commands/tablecmds.c b/src/backend/commands/tablecmds.c index 3aee2d82ce50b..d55f574c4d6f3 100644 --- a/src/backend/commands/tablecmds.c +++ b/src/backend/commands/tablecmds.c @@ -24,6 +24,7 @@ #include "access/sysattr.h" #include "access/tableam.h" #include "access/tupconvert.h" +#include "access/tuptoaster.h" #include "access/xact.h" #include "access/xlog.h" #include "catalog/catalog.h" @@ -35,6 +36,7 @@ #include "catalog/objectaccess.h" #include "catalog/partition.h" #include "catalog/pg_am.h" +#include "catalog/pg_attr_compression.h" #include "catalog/pg_collation.h" #include "catalog/pg_constraint.h" #include "catalog/pg_depend.h" @@ -42,6 +44,7 @@ #include "catalog/pg_inherits.h" #include "catalog/pg_namespace.h" #include "catalog/pg_opclass.h" +#include "catalog/pg_proc.h" #include "catalog/pg_tablespace.h" #include "catalog/pg_trigger.h" #include "catalog/pg_type.h" @@ -88,6 +91,7 @@ #include "storage/smgr.h" #include "utils/acl.h" #include "utils/builtins.h" +#include "utils/datum.h" #include "utils/fmgroids.h" #include "utils/inval.h" #include "utils/lsyscache.h" @@ -162,6 +166,9 @@ typedef struct AlteredTableInfo List *constraints; /* List of NewConstraint */ List *newvals; /* List of NewColumnValue */ bool verify_new_notnull; /* T if we should recheck NOT NULL */ + bool new_notnull; /* T if we added new NOT NULL constraints */ + HTAB *preservedAmInfo; /* Hash table for preserved compression + * methods */ int rewrite; /* Reason for forced rewrite, if any */ Oid newTableSpace; /* new tablespace; 0 means no change */ bool chgPersistence; /* T if SET LOGGED/UNLOGGED is used */ @@ -368,6 +375,8 @@ static bool check_for_column_name_collision(Relation rel, const char *colname, bool if_not_exists); static void add_column_datatype_dependency(Oid relid, int32 attnum, Oid typid); static void add_column_collation_dependency(Oid relid, int32 attnum, Oid collid); +static void add_column_compression_dependency(Oid relid, int32 attnum, Oid acoid); +static void set_column_compression_relid(Oid acoid, Oid relid); static void ATPrepDropNotNull(Relation rel, bool recurse, bool recursing); static ObjectAddress ATExecDropNotNull(Relation rel, const char *colName, LOCKMODE lockmode); static void ATPrepSetNotNull(List **wqueue, Relation rel, @@ -501,6 +510,8 @@ static void ATExecGenericOptions(Relation rel, List *options); static void ATExecEnableRowSecurity(Relation rel); static void ATExecDisableRowSecurity(Relation rel); static void ATExecForceNoForceRowSecurity(Relation rel, bool force_rls); +static ObjectAddress ATExecSetCompression(AlteredTableInfo *tab, Relation rel, + const char *column, ColumnCompression *compression, LOCKMODE lockmode); static void index_copy_data(Relation rel, RelFileNode newrnode); static const char *storage_name(char c); @@ -554,6 +565,7 @@ ObjectAddress DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, ObjectAddress *typaddress, const char *queryString) { + int i; char relname[NAMEDATALEN]; Oid namespaceId; Oid relationId; @@ -766,6 +778,8 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, * We can set the atthasdef flags now in the tuple descriptor; this just * saves StoreAttrDefault from having to do an immediate update of the * pg_attribute rows. + * + * Also create attribute compression records if needed. */ rawDefaults = NIL; cookedDefaults = NIL; @@ -816,6 +830,13 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, if (colDef->generated) attr->attgenerated = colDef->generated; + + if (!IsBinaryUpgrade && + (relkind == RELKIND_RELATION || relkind == RELKIND_PARTITIONED_TABLE)) + attr->attcompression = CreateAttributeCompression(attr, + colDef->compression, NULL, NULL); + else + attr->attcompression = InvalidOid; } /* @@ -883,6 +904,23 @@ DefineRelation(CreateStmt *stmt, char relkind, Oid ownerId, */ rel = relation_open(relationId, AccessExclusiveLock); + /* + * Specify relation Oid in attribute compression tuples, and create + * dependencies. + */ + for (i = 0; i < (RelationGetDescr(rel))->natts; i++) + { + Form_pg_attribute attr; + + attr = TupleDescAttr(RelationGetDescr(rel), i); + if (OidIsValid(attr->attcompression)) + { + set_column_compression_relid(attr->attcompression, relationId); + add_column_compression_dependency(attr->attrelid, attr->attnum, + attr->attcompression); + } + } + /* * Now add any newly specified column default and generation expressions * to the new relation. These are passed to us in the form of raw @@ -2307,6 +2345,19 @@ MergeAttributes(List *schema, List *supers, char relpersistence, storage_name(def->storage), storage_name(attribute->attstorage)))); + if (OidIsValid(attribute->attcompression)) + { + ColumnCompression *compression = + MakeColumnCompression(attribute->attcompression); + + if (!def->compression) + def->compression = compression; + else + CheckCompressionMismatch(def->compression, + compression, + attributeName); + } + def->inhcount++; /* Merge of NOT NULL constraints = OR 'em together */ def->is_not_null |= attribute->attnotnull; @@ -2341,6 +2392,7 @@ MergeAttributes(List *schema, List *supers, char relpersistence, def->collOid = attribute->attcollation; def->constraints = NIL; def->location = -1; + def->compression = MakeColumnCompression(attribute->attcompression); inhSchema = lappend(inhSchema, def); newattno[parent_attno - 1] = ++child_attno; } @@ -2550,6 +2602,13 @@ MergeAttributes(List *schema, List *supers, char relpersistence, storage_name(def->storage), storage_name(newdef->storage)))); + if (!def->compression) + def->compression = newdef->compression; + else if (newdef->compression) + CheckCompressionMismatch(def->compression, + newdef->compression, + attributeName); + /* Mark the column as locally defined */ def->is_local = true; /* Merge of NOT NULL constraints = OR 'em together */ @@ -3635,6 +3694,7 @@ AlterTableGetLockLevel(List *cmds) */ case AT_GenericOptions: case AT_AlterColumnGenericOptions: + case AT_SetCompression: cmd_lockmode = AccessExclusiveLock; break; @@ -4110,6 +4170,7 @@ ATPrepCmd(List **wqueue, Relation rel, AlterTableCmd *cmd, case AT_DisableRowSecurity: case AT_ForceRowSecurity: case AT_NoForceRowSecurity: + case AT_SetCompression: ATSimplePermissions(rel, ATT_TABLE); /* These commands never recurse */ /* No command-specific prep needed */ @@ -4473,6 +4534,11 @@ ATExecCmd(List **wqueue, AlteredTableInfo *tab, Relation rel, Assert(rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE); ATExecDetachPartition(rel, ((PartitionCmd *) cmd->def)->name); break; + case AT_SetCompression: + address = ATExecSetCompression(tab, rel, cmd->name, + (ColumnCompression *) cmd->def, + lockmode); + break; default: /* oops */ elog(ERROR, "unrecognized alter table type: %d", (int) cmd->subtype); @@ -4533,8 +4599,9 @@ ATRewriteTables(AlterTableStmt *parsetree, List **wqueue, LOCKMODE lockmode) /* * We only need to rewrite the table if at least one column needs to - * be recomputed, we are adding/removing the OID column, or we are - * changing its persistence. + * be recomputed, we are adding/removing the OID column, we are + * changing its persistence, or we are changing compression type of a + * column without preserving old ones. * * There are two reasons for requiring a rewrite when changing * persistence: on one hand, we need to ensure that the buffers @@ -4768,7 +4835,7 @@ ATRewriteTable(AlteredTableInfo *tab, Oid OIDNewHeap, LOCKMODE lockmode) if (newrel) { mycid = GetCurrentCommandId(true); - bistate = GetBulkInsertState(); + bistate = GetBulkInsertState(tab->preservedAmInfo); ti_options = TABLE_INSERT_SKIP_FSM; if (!XLogIsNeeded()) @@ -5052,6 +5119,24 @@ ATRewriteTable(AlteredTableInfo *tab, Oid OIDNewHeap, LOCKMODE lockmode) FreeExecutorState(estate); + /* Remove old compression options */ + if (tab->rewrite & AT_REWRITE_ALTER_COMPRESSION) + { + AttrCmPreservedInfo *pinfo; + HASH_SEQ_STATUS status; + + Assert(tab->preservedAmInfo); + hash_seq_init(&status, tab->preservedAmInfo); + while ((pinfo = (AttrCmPreservedInfo *) hash_seq_search(&status)) != NULL) + { + CleanupAttributeCompression(tab->relid, pinfo->attnum, + pinfo->preserved_amoids); + list_free(pinfo->preserved_amoids); + } + + hash_destroy(tab->preservedAmInfo); + } + table_close(oldrel, NoLock); if (newrel) { @@ -5702,6 +5787,16 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel, attribute.attislocal = colDef->is_local; attribute.attinhcount = colDef->inhcount; attribute.attcollation = collOid; + + /* create attribute compresssion record */ + if (rel->rd_rel->relkind == RELKIND_RELATION || + rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + attribute.attcompression = CreateAttributeCompression(&attribute, + colDef->compression, + NULL, NULL); + else + attribute.attcompression = InvalidOid; + /* attribute.attacl is handled by InsertPgAttributeTuple */ ReleaseSysCache(typeTuple); @@ -5866,6 +5961,7 @@ ATExecAddColumn(List **wqueue, AlteredTableInfo *tab, Relation rel, */ add_column_datatype_dependency(myrelid, newattnum, attribute.atttypid); add_column_collation_dependency(myrelid, newattnum, attribute.attcollation); + add_column_compression_dependency(myrelid, newattnum, attribute.attcompression); /* * Propagate to children as appropriate. Unlike most other ALTER @@ -5988,6 +6084,30 @@ add_column_datatype_dependency(Oid relid, int32 attnum, Oid typid) recordDependencyOn(&myself, &referenced, DEPENDENCY_NORMAL); } +/* Update acrelid in pg_attr_compression */ +static void +set_column_compression_relid(Oid acoid, Oid relid) +{ + Relation rel; + HeapTuple tuple; + Form_pg_attr_compression acform; + + Assert(OidIsValid(relid)); + if (IsBuiltinCompression(acoid)) + return; + + rel = heap_open(AttrCompressionRelationId, RowExclusiveLock); + tuple = SearchSysCache1(ATTCOMPRESSIONOID, ObjectIdGetDatum(acoid)); + if (!HeapTupleIsValid(tuple)) + elog(ERROR, "cache lookup failed for attribute compression %u", acoid); + + acform = (Form_pg_attr_compression) GETSTRUCT(tuple); + acform->acrelid = relid; + CatalogTupleUpdate(rel, &tuple->t_self, tuple); + ReleaseSysCache(tuple); + heap_close(rel, RowExclusiveLock); +} + /* * Install a column's dependency on its collation. */ @@ -6010,6 +6130,106 @@ add_column_collation_dependency(Oid relid, int32 attnum, Oid collid) } } +/* + * Install a dependency for compression options on its column. + * + * If it is builtin attribute compression then column depends on it. This + * is used to determine connection between column and builtin attribute + * compression in ALTER SET COMPRESSION command. + * + * If dependency is already there the whole thing is skipped. + */ +static void +add_column_compression_dependency(Oid relid, int32 attnum, Oid acoid) +{ + bool found = false; + Relation depRel; + ObjectAddress acref, + attref; + HeapTuple depTup; + ScanKeyData key[3]; + SysScanDesc scan; + + if (!OidIsValid(acoid)) + return; + + Assert(relid > 0 && attnum > 0); + ObjectAddressSet(acref, AttrCompressionRelationId, acoid); + ObjectAddressSubSet(attref, RelationRelationId, relid, attnum); + + depRel = heap_open(DependRelationId, AccessShareLock); + + if (IsBuiltinCompression(acoid)) + { + ScanKeyInit(&key[0], + Anum_pg_depend_classid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(RelationRelationId)); + ScanKeyInit(&key[1], + Anum_pg_depend_objid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relid)); + ScanKeyInit(&key[2], + Anum_pg_depend_objsubid, + BTEqualStrategyNumber, F_INT4EQ, + Int32GetDatum((int32) attnum)); + + scan = systable_beginscan(depRel, DependDependerIndexId, true, + NULL, 3, key); + + while (HeapTupleIsValid(depTup = systable_getnext(scan))) + { + Form_pg_depend foundDep = (Form_pg_depend) GETSTRUCT(depTup); + + if (foundDep->refclassid == AttrCompressionRelationId && + foundDep->refobjid == acoid) + { + found = true; + break; + } + } + systable_endscan(scan); + + if (!found) + recordDependencyOn(&attref, &acref, DEPENDENCY_NORMAL); + } + else + { + ScanKeyInit(&key[0], + Anum_pg_depend_refclassid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(RelationRelationId)); + ScanKeyInit(&key[1], + Anum_pg_depend_refobjid, + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(relid)); + ScanKeyInit(&key[2], + Anum_pg_depend_refobjsubid, + BTEqualStrategyNumber, F_INT4EQ, + Int32GetDatum((int32) attnum)); + + scan = systable_beginscan(depRel, DependReferenceIndexId, true, + NULL, 3, key); + + while (HeapTupleIsValid(depTup = systable_getnext(scan))) + { + Form_pg_depend foundDep = (Form_pg_depend) GETSTRUCT(depTup); + + if (foundDep->classid == AttrCompressionRelationId && + foundDep->objid == acoid) + { + found = true; + break; + } + } + systable_endscan(scan); + + if (!found) + recordDependencyOn(&acref, &attref, DEPENDENCY_INTERNAL); + } + heap_close(depRel, AccessShareLock); +} + /* * ALTER TABLE ALTER COLUMN DROP NOT NULL */ @@ -6930,6 +7150,10 @@ ATExecSetStorage(Relation rel, const char *colName, Node *newValue, LOCKMODE loc errmsg("column data type %s can only have storage PLAIN", format_type_be(attrtuple->atttypid)))); + /* use default compression if storage is not PLAIN */ + if (!OidIsValid(attrtuple->attcompression) && (newstorage != 'p')) + attrtuple->attcompression = DefaultCompressionOid; + CatalogTupleUpdate(attrelation, &tuple->t_self, tuple); InvokeObjectPostAlterHook(RelationRelationId, @@ -9959,6 +10183,45 @@ createForeignKeyCheckTriggers(Oid myRelOid, Oid refRelOid, indexOid, false); } +/* + * Initialize hash table used to keep rewrite rules for + * compression changes in ALTER commands. + */ +static void +setupCompressionRewriteRules(AlteredTableInfo *tab, const char *column, + AttrNumber attnum, List *preserved_amoids) +{ + bool found; + AttrCmPreservedInfo *pinfo; + + Assert(!IsBinaryUpgrade); + tab->rewrite |= AT_REWRITE_ALTER_COMPRESSION; + + /* initialize hash for oids */ + if (tab->preservedAmInfo == NULL) + { + HASHCTL ctl; + + MemSet(&ctl, 0, sizeof(ctl)); + ctl.keysize = sizeof(AttrNumber); + ctl.entrysize = sizeof(AttrCmPreservedInfo); + tab->preservedAmInfo = + hash_create("preserved access methods cache", 10, &ctl, + HASH_ELEM | HASH_BLOBS); + } + pinfo = (AttrCmPreservedInfo *) hash_search(tab->preservedAmInfo, + &attnum, HASH_ENTER, &found); + + if (found) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot alter compression of column \"%s\" twice", column), + errhint("Remove one of statements from the command."))); + + pinfo->attnum = attnum; + pinfo->preserved_amoids = preserved_amoids; +} + /* * ALTER TABLE DROP CONSTRAINT * @@ -10746,6 +11009,12 @@ ATExecAlterColumnType(AlteredTableInfo *tab, Relation rel, colName))); break; + case OCLASS_ATTR_COMPRESSION: + + /* Just check that dependency is the right type */ + Assert(foundDep->deptype == DEPENDENCY_INTERNAL); + break; + case OCLASS_DEFAULT: /* @@ -10856,7 +11125,8 @@ ATExecAlterColumnType(AlteredTableInfo *tab, Relation rel, foundDep->refobjid == attTup->attcollation) && !(foundDep->refclassid == RelationRelationId && foundDep->refobjid == RelationGetRelid(rel) && - foundDep->refobjsubid != 0) + foundDep->refobjsubid != 0) && + foundDep->refclassid != AttrCompressionRelationId ) elog(ERROR, "found unexpected dependency for column: %s", getObjectDescription(&foundObject)); @@ -10948,6 +11218,34 @@ ATExecAlterColumnType(AlteredTableInfo *tab, Relation rel, ReleaseSysCache(typeTuple); + if (rel->rd_rel->relkind == RELKIND_RELATION || + rel->rd_rel->relkind == RELKIND_PARTITIONED_TABLE) + { + /* Setup attribute compression for new attribute */ + if (OidIsValid(attTup->attcompression) && attTup->attstorage == 'p') + { + /* Set up rewrite of table */ + attTup->attcompression = InvalidOid; + setupCompressionRewriteRules(tab, colName, attOldTup->attnum, NIL); + } + else if (OidIsValid(attTup->attcompression)) + { + /* create new record */ + ColumnCompression *compression = MakeColumnCompression(attTup->attcompression); + + if (!IsBuiltinCompression(attTup->attcompression)) + setupCompressionRewriteRules(tab, colName, attOldTup->attnum, NIL); + + attTup->attcompression = CreateAttributeCompression(attTup, compression, NULL, NULL); + } + else if (attTup->attstorage != 'p') + attTup->attcompression = DefaultCompressionOid; + else + attTup->attcompression = InvalidOid; + } + else + attTup->attcompression = InvalidOid; + CatalogTupleUpdate(attrelation, &heapTup->t_self, heapTup); table_close(attrelation, RowExclusiveLock); @@ -10956,6 +11254,11 @@ ATExecAlterColumnType(AlteredTableInfo *tab, Relation rel, add_column_datatype_dependency(RelationGetRelid(rel), attnum, targettype); add_column_collation_dependency(RelationGetRelid(rel), attnum, targetcollid); + /* Create dependency for new attribute compression */ + if (OidIsValid(attTup->attcompression)) + add_column_compression_dependency(RelationGetRelid(rel), attnum, + attTup->attcompression); + /* * Drop any pg_statistic entry for the column, since it's now wrong type */ @@ -14023,6 +14326,86 @@ ATExecGenericOptions(Relation rel, List *options) heap_freetuple(tuple); } +static ObjectAddress +ATExecSetCompression(AlteredTableInfo *tab, + Relation rel, + const char *column, + ColumnCompression *compression, + LOCKMODE lockmode) +{ + Oid acoid; + Relation attrel; + HeapTuple atttuple; + Form_pg_attribute atttableform; + AttrNumber attnum; + bool need_rewrite; + Datum values[Natts_pg_attribute]; + bool nulls[Natts_pg_attribute]; + bool replace[Natts_pg_attribute]; + List *preserved_amoids = NIL; + ObjectAddress address; + + attrel = heap_open(AttributeRelationId, RowExclusiveLock); + + atttuple = SearchSysCacheAttName(RelationGetRelid(rel), column); + if (!HeapTupleIsValid(atttuple)) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("column \"%s\" of relation \"%s\" does not exist", + column, RelationGetRelationName(rel)))); + + /* Prevent them from altering a system attribute */ + atttableform = (Form_pg_attribute) GETSTRUCT(atttuple); + attnum = atttableform->attnum; + if (attnum <= 0) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("cannot alter system column \"%s\"", column))); + + /* Prevent from altering untoastable attributes with PLAIN storage */ + if (atttableform->attstorage == 'p' && !TypeIsToastable(atttableform->atttypid)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("column data type %s does not support compression", + format_type_be(atttableform->atttypid)))); + + /* Initialize buffers for new tuple values */ + memset(values, 0, sizeof(values)); + memset(nulls, false, sizeof(nulls)); + memset(replace, false, sizeof(replace)); + + /* create pg_attr_compression record and its dependencies */ + acoid = CreateAttributeCompression(atttableform, compression, + &need_rewrite, &preserved_amoids); + add_column_compression_dependency(atttableform->attrelid, + atttableform->attnum, acoid); + + /* + * Save list of preserved access methods to cache which will be passed to + * toast_insert_or_update + */ + if (need_rewrite) + setupCompressionRewriteRules(tab, column, atttableform->attnum, + preserved_amoids); + + atttableform->attcompression = acoid; + CatalogTupleUpdate(attrel, &atttuple->t_self, atttuple); + + InvokeObjectPostAlterHook(RelationRelationId, + RelationGetRelid(rel), + atttableform->attnum); + + ReleaseSysCache(atttuple); + heap_close(attrel, RowExclusiveLock); + + /* make changes visible */ + CommandCounterIncrement(); + + ObjectAddressSet(address, AttrCompressionRelationId, acoid); + return address; +} + + /* * Preparation phase for SET LOGGED/UNLOGGED * diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c index 78deade89b4d3..313d812b538ab 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -2877,6 +2877,7 @@ _copyColumnDef(const ColumnDef *from) COPY_STRING_FIELD(colname); COPY_NODE_FIELD(typeName); + COPY_NODE_FIELD(compression); COPY_SCALAR_FIELD(inhcount); COPY_SCALAR_FIELD(is_local); COPY_SCALAR_FIELD(is_not_null); @@ -2896,6 +2897,18 @@ _copyColumnDef(const ColumnDef *from) return newnode; } +static ColumnCompression * +_copyColumnCompression(const ColumnCompression *from) +{ + ColumnCompression *newnode = makeNode(ColumnCompression); + + COPY_STRING_FIELD(amname); + COPY_NODE_FIELD(options); + COPY_NODE_FIELD(preserve); + + return newnode; +} + static Constraint * _copyConstraint(const Constraint *from) { @@ -5582,6 +5595,9 @@ copyObjectImpl(const void *from) case T_ColumnDef: retval = _copyColumnDef(from); break; + case T_ColumnCompression: + retval = _copyColumnCompression(from); + break; case T_Constraint: retval = _copyConstraint(from); break; diff --git a/src/backend/nodes/equalfuncs.c b/src/backend/nodes/equalfuncs.c index 4f2ebe5118e97..bfaecaac69837 100644 --- a/src/backend/nodes/equalfuncs.c +++ b/src/backend/nodes/equalfuncs.c @@ -2557,6 +2557,7 @@ _equalColumnDef(const ColumnDef *a, const ColumnDef *b) { COMPARE_STRING_FIELD(colname); COMPARE_NODE_FIELD(typeName); + COMPARE_NODE_FIELD(compression); COMPARE_SCALAR_FIELD(inhcount); COMPARE_SCALAR_FIELD(is_local); COMPARE_SCALAR_FIELD(is_not_null); @@ -2576,6 +2577,16 @@ _equalColumnDef(const ColumnDef *a, const ColumnDef *b) return true; } +static bool +_equalColumnCompression(const ColumnCompression *a, const ColumnCompression *b) +{ + COMPARE_STRING_FIELD(amname); + COMPARE_NODE_FIELD(options); + COMPARE_NODE_FIELD(preserve); + + return true; +} + static bool _equalConstraint(const Constraint *a, const Constraint *b) { @@ -3642,6 +3653,9 @@ equal(const void *a, const void *b) case T_ColumnDef: retval = _equalColumnDef(a, b); break; + case T_ColumnCompression: + retval = _equalColumnCompression(a, b); + break; case T_Constraint: retval = _equalConstraint(a, b); break; diff --git a/src/backend/nodes/nodeFuncs.c b/src/backend/nodes/nodeFuncs.c index 05ae73f7db8fd..9555c6bbeb0c8 100644 --- a/src/backend/nodes/nodeFuncs.c +++ b/src/backend/nodes/nodeFuncs.c @@ -3688,6 +3688,8 @@ raw_expression_tree_walker(Node *node, if (walker(coldef->typeName, context)) return true; + if (walker(coldef->compression, context)) + return true; if (walker(coldef->raw_default, context)) return true; if (walker(coldef->collClause, context)) @@ -3695,6 +3697,14 @@ raw_expression_tree_walker(Node *node, /* for now, constraints are ignored */ } break; + case T_ColumnCompression: + { + ColumnCompression *colcmp = (ColumnCompression *) node; + + if (walker(colcmp->options, context)) + return true; + } + break; case T_IndexElem: { IndexElem *indelem = (IndexElem *) node; diff --git a/src/backend/nodes/outfuncs.c b/src/backend/nodes/outfuncs.c index 8400dd319e23f..aa97a20bd462b 100644 --- a/src/backend/nodes/outfuncs.c +++ b/src/backend/nodes/outfuncs.c @@ -2782,6 +2782,7 @@ _outColumnDef(StringInfo str, const ColumnDef *node) WRITE_STRING_FIELD(colname); WRITE_NODE_FIELD(typeName); + WRITE_NODE_FIELD(compression); WRITE_INT_FIELD(inhcount); WRITE_BOOL_FIELD(is_local); WRITE_BOOL_FIELD(is_not_null); @@ -2799,6 +2800,16 @@ _outColumnDef(StringInfo str, const ColumnDef *node) WRITE_LOCATION_FIELD(location); } +static void +_outColumnCompression(StringInfo str, const ColumnCompression *node) +{ + WRITE_NODE_TYPE("COLUMNCOMPRESSION"); + + WRITE_STRING_FIELD(amname); + WRITE_NODE_FIELD(options); + WRITE_NODE_FIELD(preserve); +} + static void _outTypeName(StringInfo str, const TypeName *node) { @@ -4136,6 +4147,9 @@ outNode(StringInfo str, const void *obj) case T_ColumnDef: _outColumnDef(str, obj); break; + case T_ColumnCompression: + _outColumnCompression(str, obj); + break; case T_TypeName: _outTypeName(str, obj); break; diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 208b4a1f28a75..71899247578d5 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -405,6 +405,7 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); transform_element_list transform_type_list TriggerTransitions TriggerReferencing publication_name_list + optCompressionParameters vacuum_relation_list opt_vacuum_relation_list %type group_by_list @@ -590,6 +591,10 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); %type hash_partbound %type hash_partbound_elem +%type optColumnCompression alterColumnCompression +%type compressionClause +%type optCompressionPreserve + /* * Non-keyword token types. These are hard-wired into the "flex" lexer. * They must be listed first so that their numeric codes do not depend on @@ -622,9 +627,9 @@ static Node *makeRecursiveViewSelect(char *relname, List *aliases, Node *query); CACHE CALL CALLED CASCADE CASCADED CASE CAST CATALOG_P CHAIN CHAR_P CHARACTER CHARACTERISTICS CHECK CHECKPOINT CLASS CLOSE CLUSTER COALESCE COLLATE COLLATION COLUMN COLUMNS COMMENT COMMENTS COMMIT - COMMITTED CONCURRENTLY CONFIGURATION CONFLICT CONNECTION CONSTRAINT - CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COPY COST CREATE - CROSS CSV CUBE CURRENT_P + COMMITTED COMPRESSION CONCURRENTLY CONFIGURATION CONFLICT + CONNECTION CONSTRAINT CONSTRAINTS CONTENT_P CONTINUE_P CONVERSION_P COPY + COST CREATE CROSS CSV CUBE CURRENT_P CURRENT_CATALOG CURRENT_DATE CURRENT_ROLE CURRENT_SCHEMA CURRENT_TIME CURRENT_TIMESTAMP CURRENT_USER CURSOR CYCLE @@ -2219,6 +2224,15 @@ alter_table_cmd: n->missing_ok = true; $$ = (Node *)n; } + /* ALTER TABLE ALTER [COLUMN] SET (COMPRESSION [WITH ()]) */ + | ALTER opt_column ColId SET alterColumnCompression + { + AlterTableCmd *n = makeNode(AlterTableCmd); + n->subtype = AT_SetCompression; + n->name = $3; + n->def = $5; + $$ = (Node *)n; + } /* ALTER TABLE DROP [COLUMN] IF EXISTS [RESTRICT|CASCADE] */ | DROP opt_column IF_P EXISTS ColId opt_drop_behavior { @@ -3326,11 +3340,12 @@ TypedTableElement: | TableConstraint { $$ = $1; } ; -columnDef: ColId Typename create_generic_options ColQualList +columnDef: ColId Typename optColumnCompression create_generic_options ColQualList { ColumnDef *n = makeNode(ColumnDef); n->colname = $1; n->typeName = $2; + n->compression = (ColumnCompression *) $3; n->inhcount = 0; n->is_local = true; n->is_not_null = false; @@ -3339,8 +3354,8 @@ columnDef: ColId Typename create_generic_options ColQualList n->raw_default = NULL; n->cooked_default = NULL; n->collOid = InvalidOid; - n->fdwoptions = $3; - SplitColQualList($4, &n->constraints, &n->collClause, + n->fdwoptions = $4; + SplitColQualList($5, &n->constraints, &n->collClause, yyscanner); n->location = @1; $$ = (Node *)n; @@ -3385,6 +3400,43 @@ columnOptions: ColId ColQualList } ; +optCompressionPreserve: + PRESERVE '(' name_list ')' { $$ = $3; } + | /*EMPTY*/ { $$ = NULL; } + ; + +optCompressionParameters: + WITH '(' generic_option_list ')' { $$ = $3; } + | /*EMPTY*/ { $$ = NULL; } + ; + +compressionClause: + COMPRESSION name { $$ = pstrdup($2); } + ; + +optColumnCompression: + compressionClause optCompressionParameters + { + ColumnCompression *n = makeNode(ColumnCompression); + n->amname = $1; + n->options = (List *) $2; + n->preserve = NIL; + $$ = (Node *) n; + } + | /*EMPTY*/ { $$ = NULL; } + ; + +alterColumnCompression: + compressionClause optCompressionParameters optCompressionPreserve + { + ColumnCompression *n = makeNode(ColumnCompression); + n->amname = $1; + n->options = (List *) $2; + n->preserve = (List *) $3; + $$ = (Node *) n; + } + ; + ColQualList: ColQualList ColConstraint { $$ = lappend($1, $2); } | /*EMPTY*/ { $$ = NIL; } @@ -3614,6 +3666,7 @@ TableLikeOption: | INDEXES { $$ = CREATE_TABLE_LIKE_INDEXES; } | STATISTICS { $$ = CREATE_TABLE_LIKE_STATISTICS; } | STORAGE { $$ = CREATE_TABLE_LIKE_STORAGE; } + | COMPRESSION { $$ = CREATE_TABLE_LIKE_COMPRESSION; } | ALL { $$ = CREATE_TABLE_LIKE_ALL; } ; @@ -5312,6 +5365,7 @@ CreateAmStmt: CREATE ACCESS METHOD name TYPE_P am_type HANDLER handler_name am_type: INDEX { $$ = AMTYPE_INDEX; } | TABLE { $$ = AMTYPE_TABLE; } + | COMPRESSION { $$ = AMTYPE_COMPRESSION; } ; /***************************************************************************** @@ -15055,6 +15109,7 @@ unreserved_keyword: | COMMENTS | COMMIT | COMMITTED + | COMPRESSION | CONFIGURATION | CONFLICT | CONNECTION diff --git a/src/backend/parser/parse_utilcmd.c b/src/backend/parser/parse_utilcmd.c index 7450d74b7acb4..f7f27c025f2a6 100644 --- a/src/backend/parser/parse_utilcmd.c +++ b/src/backend/parser/parse_utilcmd.c @@ -1097,6 +1097,13 @@ transformTableLikeClause(CreateStmtContext *cxt, TableLikeClause *table_like_cla else def->storage = 0; + /* Likewise, copy compression if requested */ + if (table_like_clause->options & CREATE_TABLE_LIKE_COMPRESSION + && OidIsValid(attribute->attcompression)) + def->compression = MakeColumnCompression(attribute->attcompression); + else + def->compression = NULL; + /* Likewise, copy comment if requested */ if ((table_like_clause->options & CREATE_TABLE_LIKE_COMMENTS) && (comment = GetComment(attribute->attrelid, diff --git a/src/backend/replication/logical/reorderbuffer.c b/src/backend/replication/logical/reorderbuffer.c index e7c32f2a13235..659cb6e3fcf5e 100644 --- a/src/backend/replication/logical/reorderbuffer.c +++ b/src/backend/replication/logical/reorderbuffer.c @@ -3122,7 +3122,7 @@ ReorderBufferToastReplace(ReorderBuffer *rb, ReorderBufferTXN *txn, VARSIZE(chunk) - VARHDRSZ); data_done += VARSIZE(chunk) - VARHDRSZ; } - Assert(data_done == toast_pointer.va_extsize); + Assert(data_done == VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer)); /* make sure its marked as compressed or not */ if (VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer)) diff --git a/src/backend/utils/adt/expandedrecord.c b/src/backend/utils/adt/expandedrecord.c index 166c8630267bc..600a8700e3b42 100644 --- a/src/backend/utils/adt/expandedrecord.c +++ b/src/backend/utils/adt/expandedrecord.c @@ -808,6 +808,7 @@ ER_flatten_into(ExpandedObjectHeader *eohptr, (char *) tuphdr + erh->hoff, erh->data_len, &tuphdr->t_infomask, + &tuphdr->t_infomask2, (erh->hasnull ? tuphdr->t_bits : NULL)); } diff --git a/src/backend/utils/adt/pg_upgrade_support.c b/src/backend/utils/adt/pg_upgrade_support.c index 99db5ba3894e1..0e81e70e095b4 100644 --- a/src/backend/utils/adt/pg_upgrade_support.c +++ b/src/backend/utils/adt/pg_upgrade_support.c @@ -116,6 +116,16 @@ binary_upgrade_set_next_pg_authid_oid(PG_FUNCTION_ARGS) PG_RETURN_VOID(); } +Datum +binary_upgrade_set_next_attr_compression_oid(PG_FUNCTION_ARGS) +{ + Oid acoid = PG_GETARG_OID(0); + + CHECK_IS_BINARY_UPGRADE; + binary_upgrade_next_attr_compression_oid = acoid; + PG_RETURN_VOID(); +} + Datum binary_upgrade_create_empty_extension(PG_FUNCTION_ARGS) { diff --git a/src/backend/utils/adt/pseudotypes.c b/src/backend/utils/adt/pseudotypes.c index 5c886cfe9630f..db18724625311 100644 --- a/src/backend/utils/adt/pseudotypes.c +++ b/src/backend/utils/adt/pseudotypes.c @@ -419,3 +419,4 @@ PSEUDOTYPE_DUMMY_IO_FUNCS(opaque); PSEUDOTYPE_DUMMY_IO_FUNCS(anyelement); PSEUDOTYPE_DUMMY_IO_FUNCS(anynonarray); PSEUDOTYPE_DUMMY_IO_FUNCS(table_am_handler); +PSEUDOTYPE_DUMMY_IO_FUNCS(compression_am_handler); diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c index 2b992d783273a..d7f9b8be72015 100644 --- a/src/backend/utils/cache/relcache.c +++ b/src/backend/utils/cache/relcache.c @@ -585,6 +585,10 @@ RelationBuildTupleDesc(Relation relation) ndef++; } + /* mark tupledesc as it contains attributes with custom compression */ + if (attp->attcompression) + relation->rd_att->tdflags |= TD_ATTR_CUSTOM_COMPRESSED; + /* Likewise for a missing value */ if (attp->atthasmissing) { diff --git a/src/backend/utils/cache/syscache.c b/src/backend/utils/cache/syscache.c index 16297a52a1906..fbb5a9b0b167b 100644 --- a/src/backend/utils/cache/syscache.c +++ b/src/backend/utils/cache/syscache.c @@ -27,6 +27,7 @@ #include "catalog/pg_am.h" #include "catalog/pg_amop.h" #include "catalog/pg_amproc.h" +#include "catalog/pg_attr_compression.h" #include "catalog/pg_auth_members.h" #include "catalog/pg_authid.h" #include "catalog/pg_cast.h" @@ -188,6 +189,17 @@ static const struct cachedesc cacheinfo[] = { }, 16 }, + {AttrCompressionRelationId, /* ATTCOMPRESSIONOID */ + AttrCompressionIndexId, + 1, + { + Anum_pg_attr_compression_acoid, + 0, + 0, + 0 + }, + 8, + }, {AttributeRelationId, /* ATTNAME */ AttributeRelidNameIndexId, 2, diff --git a/src/bin/pg_dump/pg_backup.h b/src/bin/pg_dump/pg_backup.h index db30b54a92f13..6153a4e6fb626 100644 --- a/src/bin/pg_dump/pg_backup.h +++ b/src/bin/pg_dump/pg_backup.h @@ -78,6 +78,7 @@ typedef struct _restoreOptions int no_publications; /* Skip publication entries */ int no_security_labels; /* Skip security label entries */ int no_subscriptions; /* Skip subscription entries */ + int no_compression_methods; /* Skip compression methods */ int strict_names; const char *filename; @@ -150,6 +151,7 @@ typedef struct _dumpOptions int no_security_labels; int no_publications; int no_subscriptions; + int no_compression_methods; int no_synchronized_snapshots; int no_unlogged_table_data; int serializable_deferrable; diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index 806fc78f04848..763e768c36f65 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -41,11 +41,13 @@ #include "getopt_long.h" #include "access/attnum.h" +#include "access/cmapi.h" #include "access/sysattr.h" #include "access/transam.h" #include "catalog/pg_aggregate_d.h" #include "catalog/pg_am_d.h" #include "catalog/pg_attribute_d.h" +#include "catalog/pg_attr_compression_d.h" #include "catalog/pg_cast_d.h" #include "catalog/pg_class_d.h" #include "catalog/pg_default_acl_d.h" @@ -388,6 +390,7 @@ main(int argc, char **argv) {"no-synchronized-snapshots", no_argument, &dopt.no_synchronized_snapshots, 1}, {"no-unlogged-table-data", no_argument, &dopt.no_unlogged_table_data, 1}, {"no-subscriptions", no_argument, &dopt.no_subscriptions, 1}, + {"no-compression-methods", no_argument, &dopt.no_compression_methods, 1}, {"no-sync", no_argument, NULL, 7}, {"on-conflict-do-nothing", no_argument, &dopt.do_nothing, 1}, {"rows-per-insert", required_argument, NULL, 10}, @@ -882,6 +885,8 @@ main(int argc, char **argv) * We rely on dependency information to help us determine a safe order, so * the initial sort is mostly for cosmetic purposes: we sort by name to * ensure that logically identical schemas will dump identically. + * + * If we do a parallel dump, we want the largest tables to go first. */ sortDumpableObjectsByTypeName(dobjs, numObjs); @@ -8227,9 +8232,12 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) int i_attcollation; int i_attfdwoptions; int i_attmissingval; + int i_attcmoptions; + int i_attcmname; PGresult *res; int ntups; bool hasdefaults; + bool createWithCompression; for (i = 0; i < numTables; i++) { @@ -8278,6 +8286,23 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) appendPQExpBufferStr(q, "'' AS attgenerated,\n"); + createWithCompression = (!dopt->binary_upgrade && fout->remoteVersion >= 120000); + + if (createWithCompression) + appendPQExpBuffer(q, + "pg_catalog.array_to_string(ARRAY(" + "SELECT pg_catalog.quote_ident(option_name) || " + "' ' || pg_catalog.quote_literal(option_value) " + "FROM pg_catalog.pg_options_to_table(c.acoptions) " + "ORDER BY option_name" + "), E',\n ') AS attcmoptions,\n" + "c.acname AS attcmname,\n"); + else + appendPQExpBuffer(q, + "NULL AS attcmoptions,\n" + "NULL AS attcmname,\n"); + + if (fout->remoteVersion >= 110000) appendPQExpBufferStr(q, "CASE WHEN a.atthasmissing AND NOT a.attisdropped " @@ -8330,7 +8355,13 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) /* need left join here to not fail on dropped columns ... */ appendPQExpBuffer(q, "FROM pg_catalog.pg_attribute a LEFT JOIN pg_catalog.pg_type t " - "ON a.atttypid = t.oid\n" + "ON a.atttypid = t.oid\n"); + + if (createWithCompression) + appendPQExpBuffer(q, "LEFT JOIN pg_catalog.pg_attr_compression c " + "ON a.attcompression = c.acoid\n"); + + appendPQExpBuffer(q, "WHERE a.attrelid = '%u'::pg_catalog.oid " "AND a.attnum > 0::pg_catalog.int2\n" "ORDER BY a.attnum", @@ -8359,6 +8390,8 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) i_attcollation = PQfnumber(res, "attcollation"); i_attfdwoptions = PQfnumber(res, "attfdwoptions"); i_attmissingval = PQfnumber(res, "attmissingval"); + i_attcmname = PQfnumber(res, "attcmname"); + i_attcmoptions = PQfnumber(res, "attcmoptions"); tbinfo->numatts = ntups; tbinfo->attnames = (char **) pg_malloc(ntups * sizeof(char *)); @@ -8377,9 +8410,12 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) tbinfo->attcollation = (Oid *) pg_malloc(ntups * sizeof(Oid)); tbinfo->attfdwoptions = (char **) pg_malloc(ntups * sizeof(char *)); tbinfo->attmissingval = (char **) pg_malloc(ntups * sizeof(char *)); + tbinfo->attcmoptions = (char **) pg_malloc(ntups * sizeof(char *)); + tbinfo->attcmnames = (char **) pg_malloc(ntups * sizeof(char *)); tbinfo->notnull = (bool *) pg_malloc(ntups * sizeof(bool)); tbinfo->inhNotNull = (bool *) pg_malloc(ntups * sizeof(bool)); tbinfo->attrdefs = (AttrDefInfo **) pg_malloc(ntups * sizeof(AttrDefInfo *)); + tbinfo->attcompression = NULL; hasdefaults = false; for (j = 0; j < ntups; j++) @@ -8405,6 +8441,8 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) tbinfo->attcollation[j] = atooid(PQgetvalue(res, j, i_attcollation)); tbinfo->attfdwoptions[j] = pg_strdup(PQgetvalue(res, j, i_attfdwoptions)); tbinfo->attmissingval[j] = pg_strdup(PQgetvalue(res, j, i_attmissingval)); + tbinfo->attcmoptions[j] = pg_strdup(PQgetvalue(res, j, i_attcmoptions)); + tbinfo->attcmnames[j] = pg_strdup(PQgetvalue(res, j, i_attcmname)); tbinfo->attrdefs[j] = NULL; /* fix below */ if (PQgetvalue(res, j, i_atthasdef)[0] == 't') hasdefaults = true; @@ -8619,6 +8657,103 @@ getTableAttrs(Archive *fout, TableInfo *tblinfo, int numTables) } PQclear(res); } + + /* + * Get compression info + */ + if (fout->remoteVersion >= 120000 && dopt->binary_upgrade) + { + int i_acname; + int i_acoid; + int i_parsedoptions; + int i_curattnum; + int start; + + pg_log_info("finding compression info for table \"%s.%s\"", + tbinfo->dobj.namespace->dobj.name, + tbinfo->dobj.name); + + tbinfo->attcompression = pg_malloc0(tbinfo->numatts * sizeof(AttrCompressionInfo *)); + + resetPQExpBuffer(q); + appendPQExpBuffer(q, + "SELECT attrelid::pg_catalog.regclass AS relname, attname," + " (CASE WHEN deptype = 'i' THEN refobjsubid ELSE objsubid END) AS curattnum," + " (CASE WHEN deptype = 'n' THEN attcompression = refobjid" + " ELSE attcompression = objid END) AS iscurrent," + " acname, acoid," + " (CASE WHEN acoptions IS NOT NULL" + " THEN pg_catalog.array_to_string(ARRAY(" + " SELECT pg_catalog.quote_ident(option_name) || " + " ' ' || pg_catalog.quote_literal(option_value) " + " FROM pg_catalog.pg_options_to_table(acoptions) " + " ORDER BY option_name" + " ), E',\n ')" + " ELSE NULL END) AS parsedoptions " + " FROM pg_depend d" + " JOIN pg_attribute a ON" + " (classid = 'pg_class'::pg_catalog.regclass::pg_catalog.oid AND a.attrelid = d.objid" + " AND a.attnum = d.objsubid AND d.deptype = 'n'" + " AND d.refclassid = 'pg_attr_compression'::pg_catalog.regclass::pg_catalog.oid)" + " OR (d.refclassid = 'pg_class'::pg_catalog.regclass::pg_catalog.oid" + " AND d.refobjid = a.attrelid" + " AND d.refobjsubid = a.attnum AND d.deptype = 'i'" + " AND d.classid = 'pg_attr_compression'::pg_catalog.regclass::pg_catalog.oid)" + " JOIN pg_attr_compression c ON" + " (d.deptype = 'i' AND d.objid = c.acoid AND a.attnum = c.acattnum" + " AND a.attrelid = c.acrelid) OR" + " (d.deptype = 'n' AND d.refobjid = c.acoid AND c.acattnum = 0" + " AND c.acrelid = 0)" + " WHERE (deptype = 'n' AND d.objid = %d) OR (deptype = 'i' AND d.refobjid = %d)" + " ORDER BY curattnum, iscurrent;", + tbinfo->dobj.catId.oid, tbinfo->dobj.catId.oid); + + res = ExecuteSqlQuery(fout, q->data, PGRES_TUPLES_OK); + ntups = PQntuples(res); + + if (ntups > 0) + { + int k; + + i_acname = PQfnumber(res, "acname"); + i_acoid = PQfnumber(res, "acoid"); + i_parsedoptions = PQfnumber(res, "parsedoptions"); + i_curattnum = PQfnumber(res, "curattnum"); + + start = 0; + + for (j = 0; j < ntups; j++) + { + int attnum = atoi(PQgetvalue(res, j, i_curattnum)); + + if ((j == ntups - 1) || atoi(PQgetvalue(res, j + 1, i_curattnum)) != attnum) + { + AttrCompressionInfo *cminfo = pg_malloc(sizeof(AttrCompressionInfo)); + + cminfo->nitems = j - start + 1; + cminfo->items = pg_malloc(sizeof(AttrCompressionItem *) * cminfo->nitems); + + for (k = start; k < start + cminfo->nitems; k++) + { + AttrCompressionItem *cmitem = pg_malloc0(sizeof(AttrCompressionItem)); + + cmitem->acname = pg_strdup(PQgetvalue(res, k, i_acname)); + cmitem->acoid = atooid(PQgetvalue(res, k, i_acoid)); + + if (!PQgetisnull(res, k, i_parsedoptions)) + cmitem->parsedoptions = pg_strdup(PQgetvalue(res, k, i_parsedoptions)); + + cminfo->items[k - start] = cmitem; + } + + tbinfo->attcompression[attnum - 1] = cminfo; + start = j + 1; /* start from next */ + } + } + } + + PQclear(res); + } } destroyPQExpBuffer(q); @@ -12728,6 +12863,9 @@ dumpAccessMethod(Archive *fout, AccessMethodInfo *aminfo) break; case AMTYPE_TABLE: appendPQExpBufferStr(q, "TYPE TABLE "); + break; + case AMTYPE_COMPRESSION: + appendPQExpBufferStr(q, "TYPE COMPRESSION "); break; default: pg_log_warning("invalid type \"%c\" of access method \"%s\"", @@ -15615,6 +15753,7 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo) { bool print_default; bool print_notnull; + bool has_custom_compression; /* * Default value --- suppress if to be printed separately. @@ -15639,6 +15778,14 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo) !dopt->binary_upgrade) continue; + /* + * Compression will require a record in + * pg_attr_compression + */ + has_custom_compression = (tbinfo->attcmnames[j] && + ((strcmp(tbinfo->attcmnames[j], "pglz") != 0) || + nonemptyReloptions(tbinfo->attcmoptions[j]))); + /* Format properly if not first attr */ if (actual_atts == 0) appendPQExpBufferStr(q, " ("); @@ -15674,6 +15821,25 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo) tbinfo->atttypnames[j]); } + /* + * Compression + * + * In binary-upgrade mode, compression is assigned by + * ALTER. Even if we're skipping compression the attribute + * will get default compression. It's the task for ALTER + * command to restore compression info. + */ + if (!dopt->no_compression_methods && !dopt->binary_upgrade && + tbinfo->attcmnames[j] && strlen(tbinfo->attcmnames[j]) && + has_custom_compression) + { + appendPQExpBuffer(q, " COMPRESSION %s", + tbinfo->attcmnames[j]); + if (nonemptyReloptions(tbinfo->attcmoptions[j])) + appendPQExpBuffer(q, " WITH (%s)", + tbinfo->attcmoptions[j]); + } + if (print_default) { if (tbinfo->attgenerated[j] == ATTRIBUTE_GENERATED_STORED) @@ -16121,6 +16287,34 @@ dumpTableSchema(Archive *fout, TableInfo *tbinfo) appendPQExpBuffer(q, "OPTIONS (\n %s\n);\n", tbinfo->attfdwoptions[j]); } + + /* + * Dump per-column compression options + */ + if (tbinfo->attcompression && tbinfo->attcompression[j]) + { + AttrCompressionInfo *cminfo = tbinfo->attcompression[j]; + + if (cminfo->nitems) + appendPQExpBuffer(q, "\n-- For binary upgrade, recreate compression metadata on column %s\n", + fmtId(tbinfo->attnames[j])); + + for (int i = 0; i < cminfo->nitems; i++) + { + AttrCompressionItem *item = cminfo->items[i]; + + appendPQExpBuffer(q, + "SELECT binary_upgrade_set_next_attr_compression_oid('%d'::pg_catalog.oid);\n", + item->acoid); + appendPQExpBuffer(q, "ALTER TABLE %s ALTER COLUMN %s\nSET COMPRESSION %s", + qualrelname, fmtId(tbinfo->attnames[j]), item->acname); + + if (item->parsedoptions) + appendPQExpBuffer(q, "\nWITH (%s);\n", item->parsedoptions); + else + appendPQExpBuffer(q, ";\n"); + } + } } if (ftoptions) diff --git a/src/bin/pg_dump/pg_dump.h b/src/bin/pg_dump/pg_dump.h index c3c2ea147369b..e273ee30f3117 100644 --- a/src/bin/pg_dump/pg_dump.h +++ b/src/bin/pg_dump/pg_dump.h @@ -327,6 +327,10 @@ typedef struct _tableInfo bool needs_override; /* has GENERATED ALWAYS AS IDENTITY */ char *amname; /* relation access method */ + char **attcmoptions; /* per-attribute current compression options */ + char **attcmnames; /* per-attribute current compression method names */ + struct _attrCompressionInfo **attcompression; /* per-attribute all compression data */ + /* * Stuff computed only for dumpable tables. */ @@ -348,6 +352,19 @@ typedef struct _attrDefInfo bool separate; /* true if must dump as separate item */ } AttrDefInfo; +typedef struct _attrCompressionItem +{ + Oid acoid; /* attribute compression oid */ + char *acname; /* compression access method name */ + char *parsedoptions; /* WITH options */ +} AttrCompressionItem; + +typedef struct _attrCompressionInfo +{ + int nitems; + AttrCompressionItem **items; +} AttrCompressionInfo; + typedef struct _tableDataInfo { DumpableObject dobj; diff --git a/src/bin/pg_dump/pg_dumpall.c b/src/bin/pg_dump/pg_dumpall.c index 158c0c74b2040..0822d549c0ee9 100644 --- a/src/bin/pg_dump/pg_dumpall.c +++ b/src/bin/pg_dump/pg_dumpall.c @@ -77,6 +77,7 @@ static int no_comments = 0; static int no_publications = 0; static int no_security_labels = 0; static int no_subscriptions = 0; +static int no_compression_methods = 0; static int no_unlogged_table_data = 0; static int no_role_passwords = 0; static int server_version; @@ -143,6 +144,7 @@ main(int argc, char *argv[]) {"no-role-passwords", no_argument, &no_role_passwords, 1}, {"no-security-labels", no_argument, &no_security_labels, 1}, {"no-subscriptions", no_argument, &no_subscriptions, 1}, + {"no-compression-methods", no_argument, &no_compression_methods, 1}, {"no-sync", no_argument, NULL, 4}, {"no-unlogged-table-data", no_argument, &no_unlogged_table_data, 1}, {"on-conflict-do-nothing", no_argument, &on_conflict_do_nothing, 1}, @@ -428,6 +430,8 @@ main(int argc, char *argv[]) appendPQExpBufferStr(pgdumpopts, " --no-security-labels"); if (no_subscriptions) appendPQExpBufferStr(pgdumpopts, " --no-subscriptions"); + if (no_compression_methods) + appendPQExpBufferStr(pgdumpopts, " --no-compression-methods"); if (no_unlogged_table_data) appendPQExpBufferStr(pgdumpopts, " --no-unlogged-table-data"); if (on_conflict_do_nothing) @@ -649,6 +653,7 @@ help(void) printf(_(" --no-role-passwords do not dump passwords for roles\n")); printf(_(" --no-security-labels do not dump security label assignments\n")); printf(_(" --no-subscriptions do not dump subscriptions\n")); + printf(_(" --no-compression-methods do not dump compression methods\n")); printf(_(" --no-sync do not wait for changes to be written safely to disk\n")); printf(_(" --no-tablespaces do not dump tablespace assignments\n")); printf(_(" --no-unlogged-table-data do not dump unlogged table data\n")); diff --git a/src/bin/pg_dump/pg_restore.c b/src/bin/pg_dump/pg_restore.c index f9b1ae6809d5c..4b6d9302192b2 100644 --- a/src/bin/pg_dump/pg_restore.c +++ b/src/bin/pg_dump/pg_restore.c @@ -75,6 +75,7 @@ main(int argc, char **argv) static int no_publications = 0; static int no_security_labels = 0; static int no_subscriptions = 0; + static int no_compression_methods = 0; static int strict_names = 0; struct option cmdopts[] = { @@ -124,6 +125,7 @@ main(int argc, char **argv) {"no-publications", no_argument, &no_publications, 1}, {"no-security-labels", no_argument, &no_security_labels, 1}, {"no-subscriptions", no_argument, &no_subscriptions, 1}, + {"no-compression-methods", no_argument, &no_compression_methods, 1}, {NULL, 0, NULL, 0} }; @@ -380,6 +382,7 @@ main(int argc, char **argv) opts->no_publications = no_publications; opts->no_security_labels = no_security_labels; opts->no_subscriptions = no_subscriptions; + opts->no_compression_methods = no_compression_methods; if (if_exists && !opts->dropSchema) { diff --git a/src/bin/pg_dump/t/002_pg_dump.pl b/src/bin/pg_dump/t/002_pg_dump.pl index c56bf00e4b40e..fdde896477c61 100644 --- a/src/bin/pg_dump/t/002_pg_dump.pl +++ b/src/bin/pg_dump/t/002_pg_dump.pl @@ -651,6 +651,43 @@ }, }, + # compression data in binary upgrade mode + 'ALTER TABLE test_table_compression ALTER COLUMN ... SET COMPRESSION' => { + all_runs => 1, + catch_all => 'ALTER TABLE ... commands', + regexp => qr/^ + \QCREATE TABLE dump_test.test_table_compression (\E\n + \s+\Qcol1 text,\E\n + \s+\Qcol2 text,\E\n + \s+\Qcol3 text,\E\n + \s+\Qcol4 text\E\n + \); + .* + \QSELECT binary_upgrade_set_next_attr_compression_oid('\E\d+\Q'::pg_catalog.oid);\E\n + \QALTER TABLE dump_test.test_table_compression ALTER COLUMN col1\E\n + \QSET COMPRESSION pglz;\E\n + .* + \QSELECT binary_upgrade_set_next_attr_compression_oid('\E\d+\Q'::pg_catalog.oid);\E\n + \QALTER TABLE dump_test.test_table_compression ALTER COLUMN col2\E\n + \QSET COMPRESSION pglz2;\E\n + .* + \QSELECT binary_upgrade_set_next_attr_compression_oid('\E\d+\Q'::pg_catalog.oid);\E\n + \QALTER TABLE dump_test.test_table_compression ALTER COLUMN col3\E\n + \QSET COMPRESSION pglz\E\n + \QWITH (min_input_size '1000');\E\n + .* + \QSELECT binary_upgrade_set_next_attr_compression_oid('\E\d+\Q'::pg_catalog.oid);\E\n + \QALTER TABLE dump_test.test_table_compression ALTER COLUMN col4\E\n + \QSET COMPRESSION pglz2\E\n + \QWITH (min_input_size '1000');\E\n + \QSELECT binary_upgrade_set_next_attr_compression_oid('\E\d+\Q'::pg_catalog.oid);\E\n + \QALTER TABLE dump_test.test_table_compression ALTER COLUMN col4\E\n + \QSET COMPRESSION pglz2\E\n + \QWITH (min_input_size '2000');\E\n + /xms, + like => { binary_upgrade => 1, }, + }, + 'ALTER TABLE ONLY test_table ALTER COLUMN col1 SET STATISTICS 90' => { create_order => 93, create_sql => @@ -1373,6 +1410,17 @@ like => { %full_runs, section_pre_data => 1, }, }, + 'CREATE ACCESS METHOD pglz2' => { + all_runs => 1, + catch_all => 'CREATE ... commands', + create_order => 52, + create_sql => + 'CREATE ACCESS METHOD pglz2 TYPE COMPRESSION HANDLER pglzhandler;', + regexp => + qr/CREATE ACCESS METHOD pglz2 TYPE COMPRESSION HANDLER pglzhandler;/m, + like => { %full_runs, section_pre_data => 1, }, + }, + 'CREATE COLLATION test0 FROM "C"' => { create_order => 76, create_sql => 'CREATE COLLATION test0 FROM "C";', @@ -2480,6 +2528,53 @@ }, }, + 'CREATE TABLE test_table_compression' => { + create_order => 55, + create_sql => 'CREATE TABLE dump_test.test_table_compression ( + col1 text, + col2 text COMPRESSION pglz2, + col3 text COMPRESSION pglz WITH (min_input_size \'1000\'), + col4 text COMPRESSION pglz2 WITH (min_input_size \'1000\') + );', + regexp => qr/^ + \QCREATE TABLE dump_test.test_table_compression (\E\n + \s+\Qcol1 text,\E\n + \s+\Qcol2 text COMPRESSION pglz2,\E\n + \s+\Qcol3 text COMPRESSION pglz WITH (min_input_size '1000'),\E\n + \s+\Qcol4 text COMPRESSION pglz2 WITH (min_input_size '2000')\E\n + \); + /xm, + like => + { %full_runs, %dump_test_schema_runs, section_pre_data => 1, }, + unlike => { + binary_upgrade => 1, + exclude_dump_test_schema => 1, + }, + }, + + 'ALTER TABLE test_table_compression' => { + create_order => 56, + create_sql => 'ALTER TABLE dump_test.test_table_compression + ALTER COLUMN col4 + SET COMPRESSION pglz2 + WITH (min_input_size \'2000\') + PRESERVE (pglz2);', + regexp => qr/^ + \QCREATE TABLE dump_test.test_table_compression (\E\n + \s+\Qcol1 text,\E\n + \s+\Qcol2 text COMPRESSION pglz2,\E\n + \s+\Qcol3 text COMPRESSION pglz WITH (min_input_size '1000'),\E\n + \s+\Qcol4 text COMPRESSION pglz2 WITH (min_input_size '2000')\E\n + \); + /xm, + like => + { %full_runs, %dump_test_schema_runs, section_pre_data => 1, }, + unlike => { + binary_upgrade => 1, + exclude_dump_test_schema => 1, + }, + }, + 'CREATE STATISTICS extended_stats_no_options' => { create_order => 97, create_sql => 'CREATE STATISTICS dump_test.test_ext_stats_no_options diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c index 8b4cd53631c0f..1a64927b97f2e 100644 --- a/src/bin/psql/describe.c +++ b/src/bin/psql/describe.c @@ -1472,6 +1472,7 @@ describeOneTableDetails(const char *schemaname, fdwopts_col = -1, attstorage_col = -1, attstattarget_col = -1, + attcompression_col = -1, attdescr_col = -1; int numrows; struct @@ -1889,6 +1890,24 @@ describeOneTableDetails(const char *schemaname, appendPQExpBufferStr(&buf, ",\n a.attstorage"); attstorage_col = cols++; + /* compresssion info */ + if (pset.sversion >= 120000 && + (tableinfo.relkind == RELKIND_RELATION || + tableinfo.relkind == RELKIND_PARTITIONED_TABLE)) + { + appendPQExpBufferStr(&buf, ",\n CASE WHEN attcompression = 0 THEN NULL ELSE " + " (SELECT c.acname || " + " (CASE WHEN acoptions IS NULL " + " THEN '' " + " ELSE '(' || array_to_string(ARRAY(SELECT quote_ident(option_name) || ' ' || quote_literal(option_value)" + " FROM pg_options_to_table(acoptions)), ', ') || ')'" + " END) " + " FROM pg_catalog.pg_attr_compression c " + " WHERE c.acoid = a.attcompression) " + " END AS attcmname"); + attcompression_col = cols++; + } + /* stats target, if relevant to relkind */ if (tableinfo.relkind == RELKIND_RELATION || tableinfo.relkind == RELKIND_INDEX || @@ -2015,6 +2034,8 @@ describeOneTableDetails(const char *schemaname, headers[cols++] = gettext_noop("FDW options"); if (attstorage_col >= 0) headers[cols++] = gettext_noop("Storage"); + if (attcompression_col >= 0) + headers[cols++] = gettext_noop("Compression"); if (attstattarget_col >= 0) headers[cols++] = gettext_noop("Stats target"); if (attdescr_col >= 0) @@ -2090,6 +2111,27 @@ describeOneTableDetails(const char *schemaname, false, false); } + /* Column compression. */ + if (attcompression_col >= 0) + { + bool mustfree = false; + const int trunclen = 100; + char *val = PQgetvalue(res, i, attcompression_col); + + /* truncate the options if they're too long */ + if (strlen(val) > trunclen + 3) + { + char *trunc = pg_malloc0(trunclen + 4); + strncpy(trunc, val, trunclen); + strncpy(trunc + trunclen, "...", 4); + + val = trunc; + mustfree = true; + } + + printTableAddCell(&cont, val, false, mustfree); + } + /* Statistics target, if the relkind supports this feature */ if (attstattarget_col >= 0) printTableAddCell(&cont, PQgetvalue(res, i, attstattarget_col), diff --git a/src/bin/psql/tab-complete.c b/src/bin/psql/tab-complete.c index 12355348c953f..11ec11cd4badc 100644 --- a/src/bin/psql/tab-complete.c +++ b/src/bin/psql/tab-complete.c @@ -1958,11 +1958,14 @@ psql_completion(const char *text, int start, int end) /* ALTER TABLE ALTER [COLUMN] SET */ else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET") || Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET")) - COMPLETE_WITH("(", "DEFAULT", "NOT NULL", "STATISTICS", "STORAGE"); + COMPLETE_WITH("(", "COMPRESSION", "DEFAULT", "NOT NULL", "STATISTICS", "STORAGE"); /* ALTER TABLE ALTER [COLUMN] SET ( */ else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "(") || Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "(")) COMPLETE_WITH("n_distinct", "n_distinct_inherited"); + else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "COMPRESSION", MatchAny) || + Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "COMPRESSION", MatchAny)) + COMPLETE_WITH("WITH (", "PRESERVE ("); /* ALTER TABLE ALTER [COLUMN] SET STORAGE */ else if (Matches("ALTER", "TABLE", MatchAny, "ALTER", "COLUMN", MatchAny, "SET", "STORAGE") || Matches("ALTER", "TABLE", MatchAny, "ALTER", MatchAny, "SET", "STORAGE")) diff --git a/src/include/access/cmapi.h b/src/include/access/cmapi.h new file mode 100644 index 0000000000000..86c5e7ba64381 --- /dev/null +++ b/src/include/access/cmapi.h @@ -0,0 +1,84 @@ +/*------------------------------------------------------------------------- + * + * cmapi.h + * API for Postgres compression AM. + * + * Copyright (c) 2015-2017, PostgreSQL Global Development Group + * + * src/include/access/cmapi.h + * + *------------------------------------------------------------------------- + */ + +#ifndef CMAPI_H +#define CMAPI_H + +#include "postgres.h" +#include "catalog/pg_attr_compression.h" +#include "catalog/pg_attribute.h" +#include "nodes/pg_list.h" + +#define IsBuiltinCompression(cmid) ((cmid) < FirstBootstrapObjectId) +#define DefaultCompressionOid (PGLZ_AC_OID) + +typedef struct CompressionAmRoutine CompressionAmRoutine; + +/* + * CompressionAmOptions contains all information needed to compress varlena. + * + * For optimization purposes it will be created once for each attribute + * compression and stored in cache, until its renewal on global cache reset, + * or until deletion of related attribute compression. + */ +typedef struct CompressionAmOptions +{ + Oid acoid; /* Oid of attribute compression, + should go always first */ + Oid amoid; /* Oid of compression access method */ + List *acoptions; /* Parsed options, used for comparison */ + CompressionAmRoutine *amroutine; /* compression access method routine */ + MemoryContext mcxt; + + /* result of cminitstate function will be put here */ + void *acstate; +} CompressionAmOptions; + +typedef void (*cmcheck_function) (Form_pg_attribute att, List *options); +typedef struct varlena *(*cmcompress_function) + (CompressionAmOptions *cmoptions, const struct varlena *value); +typedef struct varlena *(*cmdecompress_slice_function) + (CompressionAmOptions *cmoptions, const struct varlena *value, + int32 slicelength); +typedef void *(*cminitstate_function) (Oid acoid, List *options); + +/* + * API struct for a compression AM. + * + * 'cmcheck' - called when attribute is linking with compression method. + * This function should check compability of compression method with + * the attribute and its options. + * + * 'cminitstate' - called when CompressionAmOptions instance is created. + * Should return pointer to a memory in a caller memory context, or NULL. + * Could be used to pass some internal state between compression function + * calls, like internal structure for parsed compression options. + * + * 'cmcompress' and 'cmdecompress' - varlena compression functions. + */ +struct CompressionAmRoutine +{ + NodeTag type; + + cmcheck_function cmcheck; /* can be NULL */ + cminitstate_function cminitstate; /* can be NULL */ + cmcompress_function cmcompress; + cmcompress_function cmdecompress; + cmdecompress_slice_function cmdecompress_slice; +}; + +/* access/compression/cmapi.c */ +extern CompressionAmRoutine *InvokeCompressionAmHandler(Oid amhandler); +extern List *GetAttrCompressionOptions(Oid acoid); +extern Oid GetAttrCompressionAmOid(Oid acoid); + +#endif /* CMAPI_H */ diff --git a/src/include/access/heapam.h b/src/include/access/heapam.h index dffb57bf11a40..47263084c5382 100644 --- a/src/include/access/heapam.h +++ b/src/include/access/heapam.h @@ -26,6 +26,7 @@ #include "storage/lockdefs.h" #include "utils/relcache.h" #include "utils/snapshot.h" +#include "utils/hsearch.h" /* "options" flag bits for heap_insert */ @@ -130,7 +131,7 @@ extern bool heap_hot_search_buffer(ItemPointer tid, Relation relation, extern void heap_get_latest_tid(TableScanDesc scan, ItemPointer tid); extern void setLastTid(const ItemPointer tid); -extern BulkInsertState GetBulkInsertState(void); +extern BulkInsertState GetBulkInsertState(HTAB *); extern void FreeBulkInsertState(BulkInsertState); extern void ReleaseBulkInsertStatePin(BulkInsertState bistate); diff --git a/src/include/access/hio.h b/src/include/access/hio.h index dbaabcc073d94..7383a63f9d5e9 100644 --- a/src/include/access/hio.h +++ b/src/include/access/hio.h @@ -31,6 +31,8 @@ typedef struct BulkInsertStateData { BufferAccessStrategy strategy; /* our BULKWRITE strategy object */ Buffer current_buf; /* current insertion target page */ + HTAB *preserved_am_info; /* hash table with preserved compression + * methods for attributes */ } BulkInsertStateData; diff --git a/src/include/access/htup_details.h b/src/include/access/htup_details.h index 27f963e9e8dea..97dd38f111041 100644 --- a/src/include/access/htup_details.h +++ b/src/include/access/htup_details.h @@ -274,7 +274,9 @@ struct HeapTupleHeaderData * information stored in t_infomask2: */ #define HEAP_NATTS_MASK 0x07FF /* 11 bits for number of attributes */ -/* bits 0x1800 are available */ +/* bit 0x800 is available */ +#define HEAP_HASCUSTOMCOMPRESSED 0x1000 /* tuple contains custom compressed + * varlena(s) */ #define HEAP_KEYS_UPDATED 0x2000 /* tuple was updated and key cols * modified, or tuple deleted */ #define HEAP_HOT_UPDATED 0x4000 /* tuple was HOT-updated */ @@ -673,6 +675,9 @@ struct MinimalTupleData #define HeapTupleHasExternal(tuple) \ (((tuple)->t_data->t_infomask & HEAP_HASEXTERNAL) != 0) +#define HeapTupleHasCustomCompressed(tuple) \ + (((tuple)->t_data->t_infomask2 & HEAP_HASCUSTOMCOMPRESSED) != 0) + #define HeapTupleIsHotUpdated(tuple) \ HeapTupleHeaderIsHotUpdated((tuple)->t_data) @@ -779,7 +784,7 @@ extern Size heap_compute_data_size(TupleDesc tupleDesc, extern void heap_fill_tuple(TupleDesc tupleDesc, Datum *values, bool *isnull, char *data, Size data_size, - uint16 *infomask, bits8 *bit); + uint16 *infomask, uint16 *infomask2, bits8 *bit); extern bool heap_attisnull(HeapTuple tup, int attnum, TupleDesc tupleDesc); extern Datum nocachegetattr(HeapTuple tup, int attnum, TupleDesc att); diff --git a/src/include/access/reloptions.h b/src/include/access/reloptions.h index a1912f41e631d..f06dc16fd192f 100644 --- a/src/include/access/reloptions.h +++ b/src/include/access/reloptions.h @@ -269,6 +269,8 @@ extern void fillRelOptions(void *rdopts, Size basesize, relopt_value *options, int numoptions, bool validate, const relopt_parse_elt *elems, int nelems); +extern char *formatRelOptions(List *options); +extern List *untransformRelOptions(Datum options); extern bytea *default_reloptions(Datum reloptions, bool validate, relopt_kind kind); diff --git a/src/include/access/tupdesc.h b/src/include/access/tupdesc.h index a06800555c82b..f357272b0a982 100644 --- a/src/include/access/tupdesc.h +++ b/src/include/access/tupdesc.h @@ -14,7 +14,9 @@ #ifndef TUPDESC_H #define TUPDESC_H +#include "postgres.h" #include "access/attnum.h" +#include "access/cmapi.h" #include "catalog/pg_attribute.h" #include "nodes/pg_list.h" @@ -45,6 +47,10 @@ typedef struct TupleConstr bool has_generated_stored; } TupleConstr; +/* tupledesc flags */ +#define TD_ATTR_CUSTOM_COMPRESSED 0x01 /* is TupleDesc contain attributes + * with custom compression? */ + /* * This struct is passed around within the backend to describe the structure * of tuples. For tuples coming from on-disk relations, the information is @@ -81,6 +87,7 @@ typedef struct TupleDescData int natts; /* number of attributes in the tuple */ Oid tdtypeid; /* composite type ID for tuple type */ int32 tdtypmod; /* typmod for tuple type */ + char tdflags; /* tuple additional flags */ int tdrefcount; /* reference count, or -1 if not counting */ TupleConstr *constr; /* constraints, or NULL if none */ /* attrs[N] is the description of Attribute Number N+1 */ diff --git a/src/include/access/tuptoaster.h b/src/include/access/tuptoaster.h index f0aea2496bf73..113bc693c67e8 100644 --- a/src/include/access/tuptoaster.h +++ b/src/include/access/tuptoaster.h @@ -16,6 +16,7 @@ #include "access/htup_details.h" #include "storage/lockdefs.h" #include "utils/relcache.h" +#include "utils/hsearch.h" /* * This enables de-toasting of index entries. Needed until VACUUM is @@ -101,6 +102,15 @@ /* Size of an EXTERNAL datum that contains an indirection pointer */ #define INDIRECT_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_indirect)) +/* + * va_extinfo in varatt_external contains actual length of the external data + * and optional flags + */ +#define VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) \ + ((toast_pointer).va_extinfo & 0x3FFFFFFF) +#define VARATT_EXTERNAL_IS_CUSTOM_COMPRESSED(toast_pointer) \ + (((toast_pointer).va_extinfo >> 30) == 0x02) + /* * Testing whether an externally-stored value is compressed now requires * comparing extsize (the actual length of the external data) to rawsize @@ -109,7 +119,7 @@ * saves space, so we expect either equality or less-than. */ #define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \ - ((toast_pointer).va_extsize < (toast_pointer).va_rawsize - VARHDRSZ) + (VARATT_EXTERNAL_GET_EXTSIZE(toast_pointer) < (toast_pointer).va_rawsize - VARHDRSZ) /* * Macro to fetch the possibly-unaligned contents of an EXTERNAL datum @@ -126,6 +136,16 @@ do { \ memcpy(&(toast_pointer), VARDATA_EXTERNAL(attre), sizeof(toast_pointer)); \ } while (0) +/* + * Used to pass to preserved compression access methods from + * ALTER TABLE SET COMPRESSION into toast_insert_or_update. + */ +typedef struct AttrCmPreservedInfo +{ + AttrNumber attnum; + List *preserved_amoids; +} AttrCmPreservedInfo; + /* ---------- * toast_insert_or_update - * @@ -134,7 +154,7 @@ do { \ */ extern HeapTuple toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup, - int options); + int options, HTAB *preserved_cmlist); /* ---------- * toast_delete - @@ -210,7 +230,7 @@ extern HeapTuple toast_build_flattened_tuple(TupleDesc tupleDesc, * Create a compressed version of a varlena datum, if possible * ---------- */ -extern Datum toast_compress_datum(Datum value); +extern Datum toast_compress_datum(Datum value, Oid cmoptoid); /* ---------- * toast_raw_datum_size - @@ -236,4 +256,19 @@ extern Size toast_datum_size(Datum value); */ extern Oid toast_get_valid_index(Oid toastoid, LOCKMODE lock); +/* + * lookup_compression_am_options - + * + * Return cached CompressionAmOptions for specified attribute compression. + */ +extern CompressionAmOptions *lookup_compression_am_options(Oid acoid); + +/* + * toast_set_compressed_datum_info - + * + * Save metadata in compressed datum + */ +extern void toast_set_compressed_datum_info(struct varlena *val, Oid cmid, + int32 rawsize); + #endif /* TUPTOASTER_H */ diff --git a/src/include/catalog/binary_upgrade.h b/src/include/catalog/binary_upgrade.h index 2927b7a4d3cb7..00f91e4b90489 100644 --- a/src/include/catalog/binary_upgrade.h +++ b/src/include/catalog/binary_upgrade.h @@ -25,6 +25,8 @@ extern PGDLLIMPORT Oid binary_upgrade_next_toast_pg_class_oid; extern PGDLLIMPORT Oid binary_upgrade_next_pg_enum_oid; extern PGDLLIMPORT Oid binary_upgrade_next_pg_authid_oid; +extern PGDLLIMPORT Oid binary_upgrade_next_attr_compression_oid; + extern PGDLLIMPORT bool binary_upgrade_record_init_privs; #endif /* BINARY_UPGRADE_H */ diff --git a/src/include/catalog/dependency.h b/src/include/catalog/dependency.h index ef9c86864cd60..3a5c03ff6a0e7 100644 --- a/src/include/catalog/dependency.h +++ b/src/include/catalog/dependency.h @@ -125,10 +125,11 @@ typedef enum ObjectClass OCLASS_PUBLICATION, /* pg_publication */ OCLASS_PUBLICATION_REL, /* pg_publication_rel */ OCLASS_SUBSCRIPTION, /* pg_subscription */ - OCLASS_TRANSFORM /* pg_transform */ + OCLASS_TRANSFORM, /* pg_transform */ + OCLASS_ATTR_COMPRESSION /* pg_attr_compression */ } ObjectClass; -#define LAST_OCLASS OCLASS_TRANSFORM +#define LAST_OCLASS OCLASS_ATTR_COMPRESSION /* flag bits for performDeletion/performMultipleDeletions: */ #define PERFORM_DELETION_INTERNAL 0x0001 /* internal action */ diff --git a/src/include/catalog/indexing.h b/src/include/catalog/indexing.h index ef4445b017a10..bf98919d6fe12 100644 --- a/src/include/catalog/indexing.h +++ b/src/include/catalog/indexing.h @@ -90,6 +90,11 @@ DECLARE_UNIQUE_INDEX(pg_attrdef_adrelid_adnum_index, 2656, on pg_attrdef using b DECLARE_UNIQUE_INDEX(pg_attrdef_oid_index, 2657, on pg_attrdef using btree(oid oid_ops)); #define AttrDefaultOidIndexId 2657 +DECLARE_UNIQUE_INDEX(pg_attr_compression_index, 2030, on pg_attr_compression using btree(acoid oid_ops)); +#define AttrCompressionIndexId 2030 +DECLARE_INDEX(pg_attr_compression_relid_attnum_index, 2121, on pg_attr_compression using btree(acrelid oid_ops, acattnum int2_ops)); +#define AttrCompressionRelidAttnumIndexId 2121 + DECLARE_UNIQUE_INDEX(pg_attribute_relid_attnam_index, 2658, on pg_attribute using btree(attrelid oid_ops, attname name_ops)); #define AttributeRelidNameIndexId 2658 DECLARE_UNIQUE_INDEX(pg_attribute_relid_attnum_index, 2659, on pg_attribute using btree(attrelid oid_ops, attnum int2_ops)); diff --git a/src/include/catalog/pg_am.dat b/src/include/catalog/pg_am.dat index 393b41dd684b8..773b8597e1714 100644 --- a/src/include/catalog/pg_am.dat +++ b/src/include/catalog/pg_am.dat @@ -33,5 +33,11 @@ { oid => '3580', oid_symbol => 'BRIN_AM_OID', descr => 'block range index (BRIN) access method', amname => 'brin', amhandler => 'brinhandler', amtype => 'i' }, +{ oid => '4191', oid_symbol => 'PGLZ_COMPRESSION_AM_OID', + descr => 'pglz compression access method', + amname => 'pglz', amhandler => 'pglzhandler', amtype => 'c' }, +{ oid => '4192', oid_symbol => 'ZLIB_COMPRESSION_AM_OID', + descr => 'zlib compression access method', + amname => 'zlib', amhandler => 'zlibhandler', amtype => 'c' }, ] diff --git a/src/include/catalog/pg_am.h b/src/include/catalog/pg_am.h index 706b5e81cba6b..8199f5c05d5e2 100644 --- a/src/include/catalog/pg_am.h +++ b/src/include/catalog/pg_am.h @@ -54,6 +54,7 @@ typedef FormData_pg_am *Form_pg_am; */ #define AMTYPE_INDEX 'i' /* index access method */ #define AMTYPE_TABLE 't' /* table access method */ +#define AMTYPE_COMPRESSION 'c' /* compression access method */ #endif /* EXPOSE_TO_CLIENT_CODE */ diff --git a/src/include/catalog/pg_attr_compression.dat b/src/include/catalog/pg_attr_compression.dat new file mode 100644 index 0000000000000..cd716b4a5bc15 --- /dev/null +++ b/src/include/catalog/pg_attr_compression.dat @@ -0,0 +1,24 @@ +#---------------------------------------------------------------------- +# +# pg_attr_compression.dat +# Initial contents of the pg_attr_compression system relation. +# +# Predefined compression options for builtin compression access methods. +# It is safe to use Oids that equal to Oids of access methods, since +# these are just numbers and not system Oids. +# +# Note that predefined options should have 0 in acrelid and acattnum. +# +# Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# +# src/include/catalog/pg_attr_compression.dat +# +#---------------------------------------------------------------------- + +[ + +{ acoid => '4191', acname => 'pglz' }, +{ acoid => '4192', acname => 'zlib' }, + +] diff --git a/src/include/catalog/pg_attr_compression.h b/src/include/catalog/pg_attr_compression.h new file mode 100644 index 0000000000000..d6550a0eec505 --- /dev/null +++ b/src/include/catalog/pg_attr_compression.h @@ -0,0 +1,53 @@ +/*------------------------------------------------------------------------- + * + * pg_attr_compression.h + * + * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * src/include/catalog/pg_attr_compression.h + * + * NOTES + * The Catalog.pm module reads this file and derives schema + * information. + * + *------------------------------------------------------------------------- + */ +#ifndef PG_ATTR_COMPRESSION_H +#define PG_ATTR_COMPRESSION_H + +#include "catalog/genbki.h" +#include "catalog/pg_am_d.h" +#include "catalog/pg_attr_compression_d.h" + +/* ---------------- + * pg_attr_compression definition. cpp turns this into + * typedef struct FormData_pg_attr_compression + * ---------------- + */ +CATALOG(pg_attr_compression,5555,AttrCompressionRelationId) +{ + Oid acoid; /* attribute compression oid */ + NameData acname; /* name of compression AM */ + Oid acrelid BKI_DEFAULT(0); /* attribute relation */ + int16 acattnum BKI_DEFAULT(0); /* attribute number in the relation */ + +#ifdef CATALOG_VARLEN /* variable-length fields start here */ + text acoptions[1] BKI_DEFAULT(_null_); /* specific options from WITH */ +#endif +} FormData_pg_attr_compression; + +/* ---------------- + * Form_pg_attr_compresssion corresponds to a pointer to a tuple with + * the format of pg_attr_compression relation. + * ---------------- + */ +typedef FormData_pg_attr_compression *Form_pg_attr_compression; + +#ifdef EXPOSE_TO_CLIENT_CODE +/* builtin attribute compression Oids */ +#define PGLZ_AC_OID (PGLZ_COMPRESSION_AM_OID) +#define ZLIB_AC_OID (ZLIB_COMPRESSION_AM_OID) +#endif + +#endif /* PG_ATTR_COMPRESSION_H */ diff --git a/src/include/catalog/pg_attribute.h b/src/include/catalog/pg_attribute.h index 04004b5703ee4..e326df218052d 100644 --- a/src/include/catalog/pg_attribute.h +++ b/src/include/catalog/pg_attribute.h @@ -163,6 +163,9 @@ CATALOG(pg_attribute,1249,AttributeRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(75, /* attribute's collation */ Oid attcollation; + /* attribute's compression options or InvalidOid */ + Oid attcompression BKI_DEFAULT(0); + #ifdef CATALOG_VARLEN /* variable-length fields start here */ /* NOTE: The following fields are not present in tuple descriptors. */ @@ -187,10 +190,10 @@ CATALOG(pg_attribute,1249,AttributeRelationId) BKI_BOOTSTRAP BKI_ROWTYPE_OID(75, * ATTRIBUTE_FIXED_PART_SIZE is the size of the fixed-layout, * guaranteed-not-null part of a pg_attribute row. This is in fact as much * of the row as gets copied into tuple descriptors, so don't expect you - * can access fields beyond attcollation except in a real tuple! + * can access fields beyond attcompression except in a real tuple! */ #define ATTRIBUTE_FIXED_PART_SIZE \ - (offsetof(FormData_pg_attribute,attcollation) + sizeof(Oid)) + (offsetof(FormData_pg_attribute,attcompression) + sizeof(Oid)) /* ---------------- * Form_pg_attribute corresponds to a pointer to a tuple with diff --git a/src/include/catalog/pg_class.dat b/src/include/catalog/pg_class.dat index 9bcf28676da04..82e75d70f9ce8 100644 --- a/src/include/catalog/pg_class.dat +++ b/src/include/catalog/pg_class.dat @@ -34,7 +34,7 @@ relname => 'pg_attribute', reltype => 'pg_attribute', relam => 'heap', relfilenode => '0', relpages => '0', reltuples => '0', relallvisible => '0', reltoastrelid => '0', relhasindex => 'f', relisshared => 'f', - relpersistence => 'p', relkind => 'r', relnatts => '25', relchecks => '0', + relpersistence => 'p', relkind => 'r', relnatts => '26', relchecks => '0', relhasrules => 'f', relhastriggers => 'f', relhassubclass => 'f', relrowsecurity => 'f', relforcerowsecurity => 'f', relispopulated => 't', relreplident => 'n', relispartition => 'f', relfrozenxid => '3', diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 29f0944774fcb..c44b53b777e8b 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -909,6 +909,16 @@ prorettype => 'void', proargtypes => 'regclass int8', prosrc => 'brin_desummarize_range' }, +# Compression access method handlers +{ oid => '4193', descr => 'pglz compression access method handler', + proname => 'pglzhandler', provolatile => 'v', + prorettype => 'compression_am_handler', proargtypes => 'internal', + prosrc => 'pglzhandler' }, +{ oid => '4194', descr => 'zlib compression access method handler', + proname => 'zlibhandler', provolatile => 'v', + prorettype => 'compression_am_handler', proargtypes => 'internal', + prosrc => 'zlibhandler' }, + { oid => '338', descr => 'validate an operator class', proname => 'amvalidate', provolatile => 'v', prorettype => 'bool', proargtypes => 'oid', prosrc => 'amvalidate' }, @@ -6872,6 +6882,9 @@ descr => 'bytes required to store the value, perhaps with compression', proname => 'pg_column_size', provolatile => 's', prorettype => 'int4', proargtypes => 'any', prosrc => 'pg_column_size' }, +{ oid => '2023', descr => 'list of compression methods used by the column', + proname => 'pg_column_compression', provolatile => 's', prorettype => 'text', + proargtypes => 'regclass text', prosrc => 'pg_column_compression' }, { oid => '2322', descr => 'total disk space usage for the specified tablespace', proname => 'pg_tablespace_size', provolatile => 'v', prorettype => 'int8', @@ -7048,6 +7061,13 @@ { oid => '268', descr => 'I/O', proname => 'table_am_handler_out', prorettype => 'cstring', proargtypes => 'table_am_handler', prosrc => 'table_am_handler_out' }, +{ oid => '270', descr => 'I/O', + proname => 'compression_am_handler_in', proisstrict => 'f', + prorettype => 'compression_am_handler', proargtypes => 'cstring', + prosrc => 'compression_am_handler_in' }, +{ oid => '271', descr => 'I/O', + proname => 'compression_am_handler_out', prorettype => 'cstring', + proargtypes => 'compression_am_handler', prosrc => 'compression_am_handler_out' }, # tablesample method handlers { oid => '3313', descr => 'BERNOULLI tablesample method handler', @@ -10080,6 +10100,10 @@ proname => 'binary_upgrade_set_missing_value', provolatile => 'v', proparallel => 'u', prorettype => 'void', proargtypes => 'oid text text', prosrc => 'binary_upgrade_set_missing_value' }, +{ oid => '4035', descr => 'for use by pg_upgrade', + proname => 'binary_upgrade_set_next_attr_compression_oid', provolatile => 'v', + proparallel => 'r', prorettype => 'void', proargtypes => 'oid', + prosrc => 'binary_upgrade_set_next_attr_compression_oid' }, # conversion functions { oid => '4300', diff --git a/src/include/catalog/pg_type.dat b/src/include/catalog/pg_type.dat index be49e0011445d..0be6ba0306c95 100644 --- a/src/include/catalog/pg_type.dat +++ b/src/include/catalog/pg_type.dat @@ -578,6 +578,11 @@ typcategory => 'P', typinput => 'index_am_handler_in', typoutput => 'index_am_handler_out', typreceive => '-', typsend => '-', typalign => 'i' }, +{ oid => '5559', + typname => 'compression_am_handler', typlen => '4', typbyval => 't', typtype => 'p', + typcategory => 'P', typinput => 'compression_am_handler_in', + typoutput => 'compression_am_handler_out', typreceive => '-', typsend => '-', + typalign => 'i' }, { oid => '3310', descr => 'pseudo-type for the result of a tablesample method function', typname => 'tsm_handler', typlen => '4', typbyval => 't', typtype => 'p', diff --git a/src/include/catalog/toasting.h b/src/include/catalog/toasting.h index cc5dfed0bf6b9..22f1b18ceec90 100644 --- a/src/include/catalog/toasting.h +++ b/src/include/catalog/toasting.h @@ -75,6 +75,7 @@ DECLARE_TOAST(pg_trigger, 2336, 2337); DECLARE_TOAST(pg_ts_dict, 4169, 4170); DECLARE_TOAST(pg_type, 4171, 4172); DECLARE_TOAST(pg_user_mapping, 4173, 4174); +DECLARE_TOAST(pg_attr_compression, 5556, 5558); /* shared catalogs */ DECLARE_TOAST(pg_authid, 4175, 4176); diff --git a/src/include/commands/defrem.h b/src/include/commands/defrem.h index b4e7db67c3fa1..7101af08c07aa 100644 --- a/src/include/commands/defrem.h +++ b/src/include/commands/defrem.h @@ -147,6 +147,7 @@ extern Oid RemoveUserMapping(DropUserMappingStmt *stmt); extern void RemoveUserMappingById(Oid umId); extern void CreateForeignTable(CreateForeignTableStmt *stmt, Oid relid); extern void ImportForeignSchema(ImportForeignSchemaStmt *stmt); +extern Datum optionListToArray(List *options, bool sorted); extern Datum transformGenericOptions(Oid catalogId, Datum oldOptions, List *options, @@ -157,8 +158,21 @@ extern ObjectAddress CreateAccessMethod(CreateAmStmt *stmt); extern void RemoveAccessMethodById(Oid amOid); extern Oid get_index_am_oid(const char *amname, bool missing_ok); extern Oid get_table_am_oid(const char *amname, bool missing_ok); +extern Oid get_compression_am_oid(const char *amname, bool missing_ok); extern Oid get_am_oid(const char *amname, bool missing_ok); extern char *get_am_name(Oid amOid); +extern regproc get_am_handler_oid(Oid amOid, char amtype, bool noerror); + +/* commands/compressioncmds.c */ +extern ColumnCompression *MakeColumnCompression(Oid acoid); +extern Oid CreateAttributeCompression(Form_pg_attribute attr, + ColumnCompression *compression, + bool *need_rewrite, + List **preserved_amoids); +extern void RemoveAttributeCompression(Oid acoid); +extern void CheckCompressionMismatch(ColumnCompression *c1, + ColumnCompression *c2, const char *attributeName); +void CleanupAttributeCompression(Oid relid, AttrNumber attnum, List *keepAmOids); /* support routines in commands/define.c */ diff --git a/src/include/commands/event_trigger.h b/src/include/commands/event_trigger.h index 575e9243e517e..3b70907d0b55f 100644 --- a/src/include/commands/event_trigger.h +++ b/src/include/commands/event_trigger.h @@ -31,6 +31,7 @@ typedef struct EventTriggerData #define AT_REWRITE_ALTER_PERSISTENCE 0x01 #define AT_REWRITE_DEFAULT_VAL 0x02 #define AT_REWRITE_COLUMN_REWRITE 0x04 +#define AT_REWRITE_ALTER_COMPRESSION 0x08 /* * EventTriggerData is the node type that is passed as fmgr "context" info diff --git a/src/include/nodes/nodes.h b/src/include/nodes/nodes.h index 4e2fb39105b2e..3b50a2e1f51e7 100644 --- a/src/include/nodes/nodes.h +++ b/src/include/nodes/nodes.h @@ -475,6 +475,7 @@ typedef enum NodeTag T_PartitionBoundSpec, T_PartitionRangeDatum, T_PartitionCmd, + T_ColumnCompression, T_VacuumRelation, /* @@ -505,6 +506,7 @@ typedef enum NodeTag T_FdwRoutine, /* in foreign/fdwapi.h */ T_IndexAmRoutine, /* in access/amapi.h */ T_TableAmRoutine, /* in access/tableam.h */ + T_CompressionAmRoutine, /* in access/cmapi.h */ T_TsmRoutine, /* in access/tsmapi.h */ T_ForeignKeyCacheInfo, /* in utils/rel.h */ T_CallContext, /* in nodes/parsenodes.h */ diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index 94ded3c135ed8..e0675e507e7a5 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -622,6 +622,20 @@ typedef struct RangeTableSample int location; /* method name location, or -1 if unknown */ } RangeTableSample; +/* + * ColumnCompression - compression parameters for some attribute + * + * This represents compression information defined using clause: + * .. COMPRESSION WITH () PRESERVE + */ +typedef struct ColumnCompression +{ + NodeTag type; + char *amname; + List *options; + List *preserve; +} ColumnCompression; + /* * ColumnDef - column definition (used in various creates) * @@ -645,6 +659,7 @@ typedef struct ColumnDef NodeTag type; char *colname; /* name of column */ TypeName *typeName; /* type of column */ + ColumnCompression *compression; int inhcount; /* number of times column is inherited */ bool is_local; /* column has local (non-inherited) def'n */ bool is_not_null; /* NOT NULL constraint specified? */ @@ -683,6 +698,7 @@ typedef enum TableLikeOption CREATE_TABLE_LIKE_INDEXES = 1 << 5, CREATE_TABLE_LIKE_STATISTICS = 1 << 6, CREATE_TABLE_LIKE_STORAGE = 1 << 7, + CREATE_TABLE_LIKE_COMPRESSION = 1 << 8, CREATE_TABLE_LIKE_ALL = PG_INT32_MAX } TableLikeOption; @@ -1824,7 +1840,8 @@ typedef enum AlterTableType AT_DetachPartition, /* DETACH PARTITION */ AT_AddIdentity, /* ADD IDENTITY */ AT_SetIdentity, /* SET identity column options */ - AT_DropIdentity /* DROP IDENTITY */ + AT_DropIdentity, /* DROP IDENTITY */ + AT_SetCompression /* SET COMPRESSION */ } AlterTableType; typedef struct ReplicaIdentityStmt diff --git a/src/include/parser/kwlist.h b/src/include/parser/kwlist.h index 00ace8425e21c..f274434fee627 100644 --- a/src/include/parser/kwlist.h +++ b/src/include/parser/kwlist.h @@ -87,6 +87,7 @@ PG_KEYWORD("comment", COMMENT, UNRESERVED_KEYWORD) PG_KEYWORD("comments", COMMENTS, UNRESERVED_KEYWORD) PG_KEYWORD("commit", COMMIT, UNRESERVED_KEYWORD) PG_KEYWORD("committed", COMMITTED, UNRESERVED_KEYWORD) +PG_KEYWORD("compression", COMPRESSION, UNRESERVED_KEYWORD) PG_KEYWORD("concurrently", CONCURRENTLY, TYPE_FUNC_NAME_KEYWORD) PG_KEYWORD("configuration", CONFIGURATION, UNRESERVED_KEYWORD) PG_KEYWORD("conflict", CONFLICT, UNRESERVED_KEYWORD) diff --git a/src/include/postgres.h b/src/include/postgres.h index 057a3413acd8a..a8e818347b30a 100644 --- a/src/include/postgres.h +++ b/src/include/postgres.h @@ -55,7 +55,7 @@ /* * struct varatt_external is a traditional "TOAST pointer", that is, the * information needed to fetch a Datum stored out-of-line in a TOAST table. - * The data is compressed if and only if va_extsize < va_rawsize - VARHDRSZ. + * The data is compressed if and only if size in va_extinfo < va_rawsize - VARHDRSZ. * This struct must not contain any padding, because we sometimes compare * these pointers using memcmp. * @@ -67,7 +67,8 @@ typedef struct varatt_external { int32 va_rawsize; /* Original data size (includes header) */ - int32 va_extsize; /* External saved size (doesn't) */ + uint32 va_extinfo; /* External saved size (without header) and + * flags */ Oid va_valueid; /* Unique ID of value within TOAST table */ Oid va_toastrelid; /* RelID of TOAST table containing it */ } varatt_external; @@ -145,9 +146,18 @@ typedef union struct /* Compressed-in-line format */ { uint32 va_header; - uint32 va_rawsize; /* Original data size (excludes header) */ + uint32 va_info; /* Original data size (excludes header) and + * flags */ char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */ } va_compressed; + struct /* Compressed-in-line format */ + { + uint32 va_header; + uint32 va_info; /* Original data size (excludes header) and + * flags */ + Oid va_cmid; /* Oid of compression options */ + char va_data[FLEXIBLE_ARRAY_MEMBER]; /* Compressed data */ + } va_custom_compressed; } varattrib_4b; typedef struct @@ -280,8 +290,14 @@ typedef struct #define VARDATA_1B(PTR) (((varattrib_1b *) (PTR))->va_data) #define VARDATA_1B_E(PTR) (((varattrib_1b_e *) (PTR))->va_data) +/* va_info in va_compress contains raw size of datum and optional flags */ #define VARRAWSIZE_4B_C(PTR) \ - (((varattrib_4b *) (PTR))->va_compressed.va_rawsize) + (((varattrib_4b *) (PTR))->va_compressed.va_info & 0x3FFFFFFF) +#define VARFLAGS_4B_C(PTR) \ + (((varattrib_4b *) (PTR))->va_compressed.va_info >> 30) + +#define VARHDRSZ_CUSTOM_COMPRESSED \ + (offsetof(varattrib_4b, va_custom_compressed.va_data)) /* Externally visible macros */ @@ -310,6 +326,8 @@ typedef struct #define VARDATA_EXTERNAL(PTR) VARDATA_1B_E(PTR) #define VARATT_IS_COMPRESSED(PTR) VARATT_IS_4B_C(PTR) +#define VARATT_IS_CUSTOM_COMPRESSED(PTR) (VARATT_IS_4B_C(PTR) && \ + (VARFLAGS_4B_C(PTR) == 0x02)) #define VARATT_IS_EXTERNAL(PTR) VARATT_IS_1B_E(PTR) #define VARATT_IS_EXTERNAL_ONDISK(PTR) \ (VARATT_IS_EXTERNAL(PTR) && VARTAG_EXTERNAL(PTR) == VARTAG_ONDISK) diff --git a/src/include/utils/syscache.h b/src/include/utils/syscache.h index 918765cc993f6..20b8788a9013b 100644 --- a/src/include/utils/syscache.h +++ b/src/include/utils/syscache.h @@ -37,6 +37,7 @@ enum SysCacheIdentifier AMOPOPID, AMOPSTRATEGY, AMPROCNUM, + ATTCOMPRESSIONOID, ATTNAME, ATTNUM, AUTHMEMMEMROLE, diff --git a/src/test/regress/expected/copy2.out b/src/test/regress/expected/copy2.out index c53ed3ebf592b..1753a9744fddf 100644 --- a/src/test/regress/expected/copy2.out +++ b/src/test/regress/expected/copy2.out @@ -464,10 +464,10 @@ begin end $$ language plpgsql immutable; alter table check_con_tbl add check (check_con_function(check_con_tbl.*)); \d+ check_con_tbl - Table "public.check_con_tbl" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - f1 | integer | | | | plain | | + Table "public.check_con_tbl" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + f1 | integer | | | | plain | | | Check constraints: "check_con_tbl_check" CHECK (check_con_function(check_con_tbl.*)) diff --git a/src/test/regress/expected/create_cm.out b/src/test/regress/expected/create_cm.out new file mode 100644 index 0000000000000..d3b4095fea42f --- /dev/null +++ b/src/test/regress/expected/create_cm.out @@ -0,0 +1,404 @@ +-- test drop +DROP ACCESS METHOD pglz; --fail +ERROR: cannot drop access method pglz because it is required by the database system +CREATE ACCESS METHOD pglz1 TYPE COMPRESSION HANDLER pglzhandler; +CREATE TABLE droptest(d1 TEXT COMPRESSION pglz1); +DROP ACCESS METHOD pglz1; +ERROR: cannot drop access method pglz1 because other objects depend on it +DETAIL: column d1 of table droptest depends on access method pglz1 +HINT: Use DROP ... CASCADE to drop the dependent objects too. +DROP ACCESS METHOD pglz1 CASCADE; +NOTICE: drop cascades to column d1 of table droptest +\d+ droptest + Table "public.droptest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+---------+-------------+--------------+------------- + +DROP TABLE droptest; +CREATE ACCESS METHOD pglz1 TYPE COMPRESSION HANDLER pglzhandler; +-- test auto drop of related attribute compression +CREATE ACCESS METHOD pglz2 TYPE COMPRESSION HANDLER pglzhandler; +CREATE TABLE cmaltertest (f1 TEXT COMPRESSION pglz2); +SELECT acname, acattnum, acoptions FROM pg_attr_compression + WHERE acrelid = 'cmaltertest'::REGCLASS; + acname | acattnum | acoptions +--------+----------+----------- + pglz2 | 1 | +(1 row) + +ALTER TABLE cmaltertest DROP COLUMN f1; +SELECT acname, acattnum, acoptions FROM pg_attr_compression + WHERE acrelid = 'cmaltertest'::REGCLASS; + acname | acattnum | acoptions +--------+----------+----------- +(0 rows) + +DROP TABLE cmaltertest; +-- test drop +DROP ACCESS METHOD pglz2; +-- test alter data type +CREATE TABLE cmaltertest(at1 TEXT); +ALTER TABLE cmaltertest ALTER COLUMN at1 SET COMPRESSION pglz1 PRESERVE (pglz); +SELECT pg_column_compression('cmaltertest', 'at1'); + pg_column_compression +----------------------- + pglz, pglz1 +(1 row) + +ALTER TABLE cmaltertest ALTER COLUMN at1 SET DATA TYPE INT USING at1::INTEGER; +\d+ cmaltertest + Table "public.cmaltertest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + at1 | integer | | | | plain | | | + +SELECT pg_column_compression('cmaltertest', 'at1'); + pg_column_compression +----------------------- + +(1 row) + +SELECT acname, acattnum, acoptions FROM pg_attr_compression + WHERE acrelid = 'cmaltertest'::REGCLASS; + acname | acattnum | acoptions +--------+----------+----------- +(0 rows) + +ALTER TABLE cmaltertest ALTER COLUMN at1 SET DATA TYPE TEXT; +SELECT pg_column_compression('cmaltertest', 'at1'); + pg_column_compression +----------------------- + pglz +(1 row) + +DROP TABLE cmaltertest; +-- test storages +CREATE TABLE cmstoragetest(st1 TEXT, st2 INT); +ALTER TABLE cmstoragetest ALTER COLUMN st2 + SET COMPRESSION pglz WITH (min_input_size '100'); -- fail +ERROR: column data type integer does not support compression +ALTER TABLE cmstoragetest ALTER COLUMN st1 + SET COMPRESSION pglz WITH (min_input_size '100', min_comp_rate '50'); +ALTER TABLE cmstoragetest ALTER COLUMN st1 SET STORAGE EXTERNAL; +\d+ cmstoragetest + Table "public.cmstoragetest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+------------------------------------------------+--------------+------------- + st1 | text | | | | external | pglz(min_comp_rate '50', min_input_size '100') | | + st2 | integer | | | | plain | | | + +ALTER TABLE cmstoragetest ALTER COLUMN st1 SET STORAGE MAIN; +\d+ cmstoragetest + Table "public.cmstoragetest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+------------------------------------------------+--------------+------------- + st1 | text | | | | main | pglz(min_comp_rate '50', min_input_size '100') | | + st2 | integer | | | | plain | | | + +ALTER TABLE cmstoragetest ALTER COLUMN st1 SET STORAGE PLAIN; +\d+ cmstoragetest + Table "public.cmstoragetest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+------------------------------------------------+--------------+------------- + st1 | text | | | | plain | pglz(min_comp_rate '50', min_input_size '100') | | + st2 | integer | | | | plain | | | + +ALTER TABLE cmstoragetest ALTER COLUMN st1 SET STORAGE EXTENDED; +\d+ cmstoragetest + Table "public.cmstoragetest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+------------------------------------------------+--------------+------------- + st1 | text | | | | extended | pglz(min_comp_rate '50', min_input_size '100') | | + st2 | integer | | | | plain | | | + +DROP TABLE cmstoragetest; +CREATE ACCESS METHOD pglz2 TYPE COMPRESSION HANDLER pglzhandler; +-- test PRESERVE +CREATE TABLE cmtest(f1 TEXT); +\d+ cmtest + Table "public.cmtest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | pglz | | + +-- view to check dependencies +CREATE VIEW cmtest_deps AS + SELECT classid, objsubid, refclassid, refobjsubid, deptype + FROM pg_depend + WHERE (refclassid = 'pg_catalog.pg_attr_compression'::REGCLASS OR + classid = 'pg_catalog.pg_attr_compression'::REGCLASS) AND + (objid = 'cmtest'::REGCLASS OR refobjid = 'cmtest'::REGCLASS) + ORDER by objid, refobjid; +INSERT INTO cmtest VALUES(repeat('1234567890',1001)); +-- one normal dependency +SELECT * FROM cmtest_deps; + classid | objsubid | refclassid | refobjsubid | deptype +---------+----------+------------+-------------+--------- + 1259 | 1 | 5555 | 0 | n +(1 row) + +-- check decompression +SELECT length(f1) FROM cmtest; + length +-------- + 10010 +(1 row) + +SELECT length(f1) FROM cmtest; + length +-------- + 10010 +(1 row) + +CREATE FUNCTION on_cmtest_rewrite() +RETURNS event_trigger AS $$ +BEGIN + RAISE NOTICE 'cmtest rewrite'; +END; +$$ LANGUAGE plpgsql; +CREATE EVENT TRIGGER notice_on_cmtest_rewrite ON table_rewrite + EXECUTE PROCEDURE on_cmtest_rewrite(); +-- no rewrite +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz1 PRESERVE (pglz); +INSERT INTO cmtest VALUES(repeat('1234567890',1002)); +SELECT length(f1) FROM cmtest; + length +-------- + 10010 + 10020 +(2 rows) + +SELECT pg_column_compression('cmtest', 'f1'); + pg_column_compression +----------------------- + pglz, pglz1 +(1 row) + +-- one normal and one internal dependency +SELECT * FROM cmtest_deps; + classid | objsubid | refclassid | refobjsubid | deptype +---------+----------+------------+-------------+--------- + 1259 | 1 | 5555 | 0 | n + 5555 | 0 | 1259 | 1 | i +(2 rows) + +-- rewrite +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz2 PRESERVE (pglz1); +NOTICE: cmtest rewrite +INSERT INTO cmtest VALUES(repeat('1234567890',1003)); +SELECT length(f1) FROM cmtest; + length +-------- + 10010 + 10020 + 10030 +(3 rows) + +SELECT pg_column_compression('cmtest', 'f1'); + pg_column_compression +----------------------- + pglz1, pglz2 +(1 row) + +-- two internal dependencies +SELECT * FROM cmtest_deps; + classid | objsubid | refclassid | refobjsubid | deptype +---------+----------+------------+-------------+--------- + 5555 | 0 | 1259 | 1 | i + 5555 | 0 | 1259 | 1 | i +(2 rows) + +-- rewrite +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz; +NOTICE: cmtest rewrite +INSERT INTO cmtest VALUES(repeat('1234567890',1004)); +SELECT length(f1) FROM cmtest; + length +-------- + 10010 + 10020 + 10030 + 10040 +(4 rows) + +SELECT pg_column_compression('cmtest', 'f1'); + pg_column_compression +----------------------- + pglz +(1 row) + +-- one nornal dependency +SELECT * FROM cmtest_deps; + classid | objsubid | refclassid | refobjsubid | deptype +---------+----------+------------+-------------+--------- + 1259 | 1 | 5555 | 0 | n +(1 row) + +-- no rewrites +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz1 PRESERVE (pglz); +INSERT INTO cmtest VALUES(repeat('1234567890',1005)); +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz + WITH (min_input_size '1000') PRESERVE (pglz, pglz1); +INSERT INTO cmtest VALUES(repeat('1234567890',1006)); +-- one nornal dependency and two internal dependencies +SELECT * FROM cmtest_deps; + classid | objsubid | refclassid | refobjsubid | deptype +---------+----------+------------+-------------+--------- + 1259 | 1 | 5555 | 0 | n + 5555 | 0 | 1259 | 1 | i + 5555 | 0 | 1259 | 1 | i +(3 rows) + +-- remove function and related event trigger +DROP FUNCTION on_cmtest_rewrite CASCADE; +NOTICE: drop cascades to event trigger notice_on_cmtest_rewrite +-- test moving +CREATE TABLE cmdata(f1 text COMPRESSION pglz WITH (first_success_by '10000')); +INSERT INTO cmdata VALUES(repeat('1234567890',1000)); +INSERT INTO cmdata VALUES(repeat('1234567890',1001)); +-- copy with table creation +SELECT * INTO cmmove1 FROM cmdata; +-- we update using datum from different table +CREATE TABLE cmmove2(f1 text COMPRESSION pglz WITH (min_input_size '100')); +INSERT INTO cmmove2 VALUES (repeat('1234567890',1004)); +UPDATE cmmove2 SET f1 = cmdata.f1 FROM cmdata; +-- copy to existing table +CREATE TABLE cmmove3(f1 text COMPRESSION pglz WITH (min_input_size '100')); +INSERT INTO cmmove3 SELECT * FROM cmdata; +-- drop original compression information +DROP TABLE cmdata; +-- check data is ok +SELECT length(f1) FROM cmmove1; + length +-------- + 10000 + 10010 +(2 rows) + +SELECT length(f1) FROM cmmove2; + length +-------- + 10000 +(1 row) + +SELECT length(f1) FROM cmmove3; + length +-------- + 10000 + 10010 +(2 rows) + +-- create different types of tables +CREATE TABLE cmexample (f1 TEXT COMPRESSION pglz1); +CREATE TABLE cmtestlike1 (LIKE cmexample INCLUDING COMPRESSION); +CREATE TABLE cmtestlike2 (f2 INT) INHERITS (cmexample); +\d+ cmtestlike1 + Table "public.cmtestlike1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | pglz1 | | + +\d+ cmtestlike2 + Table "public.cmtestlike2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | pglz1 | | + f2 | integer | | | | plain | | | +Inherits: cmexample + +-- test two columns +CREATE TABLE cmaltertest(f1 TEXT, f2 TEXT COMPRESSION pglz1); +\d+ cmaltertest; + Table "public.cmaltertest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | pglz | | + f2 | text | | | | extended | pglz1 | | + +-- fail, changing one column twice +ALTER TABLE cmaltertest ALTER COLUMN f1 SET COMPRESSION pglz, + ALTER COLUMN f1 SET COMPRESSION pglz; +ERROR: cannot alter compression of column "f1" twice +HINT: Remove one of statements from the command. +-- with rewrite +ALTER TABLE cmaltertest ALTER COLUMN f1 SET COMPRESSION pglz1, + ALTER COLUMN f2 SET COMPRESSION pglz PRESERVE (pglz1); +SELECT pg_column_compression('cmaltertest', 'f1'); + pg_column_compression +----------------------- + pglz1 +(1 row) + +SELECT pg_column_compression('cmaltertest', 'f2'); + pg_column_compression +----------------------- + pglz, pglz1 +(1 row) + +-- no rewrite +ALTER TABLE cmaltertest ALTER COLUMN f1 SET COMPRESSION pglz PRESERVE (pglz1), + ALTER COLUMN f2 SET COMPRESSION pglz2 PRESERVE (pglz1, pglz); +SELECT pg_column_compression('cmaltertest', 'f1'); + pg_column_compression +----------------------- + pglz, pglz1 +(1 row) + +SELECT pg_column_compression('cmaltertest', 'f2'); + pg_column_compression +----------------------- + pglz, pglz1, pglz2 +(1 row) + +-- make pglz2 droppable +ALTER TABLE cmaltertest ALTER COLUMN f2 SET COMPRESSION pglz1; +SELECT pg_column_compression('cmaltertest', 'f1'); + pg_column_compression +----------------------- + pglz, pglz1 +(1 row) + +SELECT pg_column_compression('cmaltertest', 'f2'); + pg_column_compression +----------------------- + pglz1 +(1 row) + +SELECT acname, acattnum, acoptions FROM pg_attr_compression + WHERE acrelid = 'cmaltertest'::REGCLASS OR acrelid = 'cmtest'::REGCLASS; + acname | acattnum | acoptions +--------+----------+----------------------- + pglz1 | 1 | + pglz | 1 | {min_input_size=1000} + pglz1 | 2 | + pglz1 | 1 | +(4 rows) + +-- zlib compression +CREATE TABLE zlibtest(f1 TEXT COMPRESSION zlib WITH (invalid 'param')); +ERROR: unexpected parameter for zlib: "invalid" +CREATE TABLE zlibtest(f1 TEXT COMPRESSION zlib WITH (level 'best')); +ERROR: invalid input syntax for type integer: "best" +CREATE TABLE zlibtest(f1 TEXT COMPRESSION zlib); +ALTER TABLE zlibtest + ALTER COLUMN f1 SET COMPRESSION zlib WITH (level '9'); +ALTER TABLE zlibtest + ALTER COLUMN f1 SET COMPRESSION zlib WITH (level '1'); +ALTER TABLE zlibtest + ALTER COLUMN f1 SET COMPRESSION zlib WITH (dict 'one'); +ERROR: zlib dictionary is too small +INSERT INTO zlibtest VALUES(repeat('1234567890',1004)); +ALTER TABLE zlibtest + ALTER COLUMN f1 SET COMPRESSION zlib WITH (dict 'one two') PRESERVE (zlib); +INSERT INTO zlibtest VALUES(repeat('1234567890 one two three',1004)); +SELECT length(f1) FROM zlibtest; + length +-------- + 10040 + 24096 +(2 rows) + +DROP ACCESS METHOD pglz2; +DROP VIEW cmtest_deps; +DROP TABLE cmmove1, cmmove2, cmmove3; +DROP TABLE cmexample, cmtestlike1, cmtestlike2, zlibtest; diff --git a/src/test/regress/expected/create_cm_1.out b/src/test/regress/expected/create_cm_1.out new file mode 100644 index 0000000000000..e38ae0cecffc7 --- /dev/null +++ b/src/test/regress/expected/create_cm_1.out @@ -0,0 +1,407 @@ +-- test drop +DROP ACCESS METHOD pglz; --fail +ERROR: cannot drop access method pglz because it is required by the database system +CREATE ACCESS METHOD pglz1 TYPE COMPRESSION HANDLER pglzhandler; +CREATE TABLE droptest(d1 TEXT COMPRESSION pglz1); +DROP ACCESS METHOD pglz1; +ERROR: cannot drop access method pglz1 because other objects depend on it +DETAIL: column d1 of table droptest depends on access method pglz1 +HINT: Use DROP ... CASCADE to drop the dependent objects too. +DROP ACCESS METHOD pglz1 CASCADE; +NOTICE: drop cascades to column d1 of table droptest +\d+ droptest + Table "public.droptest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+---------+-------------+--------------+------------- + +DROP TABLE droptest; +CREATE ACCESS METHOD pglz1 TYPE COMPRESSION HANDLER pglzhandler; +-- test auto drop of related attribute compression +CREATE ACCESS METHOD pglz2 TYPE COMPRESSION HANDLER pglzhandler; +CREATE TABLE cmaltertest (f1 TEXT COMPRESSION pglz2); +SELECT acname, acattnum, acoptions FROM pg_attr_compression + WHERE acrelid = 'cmaltertest'::REGCLASS; + acname | acattnum | acoptions +--------+----------+----------- + pglz2 | 1 | +(1 row) + +ALTER TABLE cmaltertest DROP COLUMN f1; +SELECT acname, acattnum, acoptions FROM pg_attr_compression + WHERE acrelid = 'cmaltertest'::REGCLASS; + acname | acattnum | acoptions +--------+----------+----------- +(0 rows) + +DROP TABLE cmaltertest; +-- test drop +DROP ACCESS METHOD pglz2; +-- test alter data type +CREATE TABLE cmaltertest(at1 TEXT); +ALTER TABLE cmaltertest ALTER COLUMN at1 SET COMPRESSION pglz1 PRESERVE (pglz); +SELECT pg_column_compression('cmaltertest', 'at1'); + pg_column_compression +----------------------- + pglz, pglz1 +(1 row) + +ALTER TABLE cmaltertest ALTER COLUMN at1 SET DATA TYPE INT USING at1::INTEGER; +\d+ cmaltertest + Table "public.cmaltertest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + at1 | integer | | | | plain | | | + +SELECT pg_column_compression('cmaltertest', 'at1'); + pg_column_compression +----------------------- + +(1 row) + +SELECT acname, acattnum, acoptions FROM pg_attr_compression + WHERE acrelid = 'cmaltertest'::REGCLASS; + acname | acattnum | acoptions +--------+----------+----------- +(0 rows) + +ALTER TABLE cmaltertest ALTER COLUMN at1 SET DATA TYPE TEXT; +SELECT pg_column_compression('cmaltertest', 'at1'); + pg_column_compression +----------------------- + pglz +(1 row) + +DROP TABLE cmaltertest; +-- test storages +CREATE TABLE cmstoragetest(st1 TEXT, st2 INT); +ALTER TABLE cmstoragetest ALTER COLUMN st2 + SET COMPRESSION pglz WITH (min_input_size '100'); -- fail +ERROR: column data type integer does not support compression +ALTER TABLE cmstoragetest ALTER COLUMN st1 + SET COMPRESSION pglz WITH (min_input_size '100', min_comp_rate '50'); +ALTER TABLE cmstoragetest ALTER COLUMN st1 SET STORAGE EXTERNAL; +\d+ cmstoragetest + Table "public.cmstoragetest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+------------------------------------------------+--------------+------------- + st1 | text | | | | external | pglz(min_comp_rate '50', min_input_size '100') | | + st2 | integer | | | | plain | | | + +ALTER TABLE cmstoragetest ALTER COLUMN st1 SET STORAGE MAIN; +\d+ cmstoragetest + Table "public.cmstoragetest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+------------------------------------------------+--------------+------------- + st1 | text | | | | main | pglz(min_comp_rate '50', min_input_size '100') | | + st2 | integer | | | | plain | | | + +ALTER TABLE cmstoragetest ALTER COLUMN st1 SET STORAGE PLAIN; +\d+ cmstoragetest + Table "public.cmstoragetest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+------------------------------------------------+--------------+------------- + st1 | text | | | | plain | pglz(min_comp_rate '50', min_input_size '100') | | + st2 | integer | | | | plain | | | + +ALTER TABLE cmstoragetest ALTER COLUMN st1 SET STORAGE EXTENDED; +\d+ cmstoragetest + Table "public.cmstoragetest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+------------------------------------------------+--------------+------------- + st1 | text | | | | extended | pglz(min_comp_rate '50', min_input_size '100') | | + st2 | integer | | | | plain | | | + +DROP TABLE cmstoragetest; +CREATE ACCESS METHOD pglz2 TYPE COMPRESSION HANDLER pglzhandler; +-- test PRESERVE +CREATE TABLE cmtest(f1 TEXT); +\d+ cmtest + Table "public.cmtest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | pglz | | + +-- view to check dependencies +CREATE VIEW cmtest_deps AS + SELECT classid, objsubid, refclassid, refobjsubid, deptype + FROM pg_depend + WHERE (refclassid = 'pg_catalog.pg_attr_compression'::REGCLASS OR + classid = 'pg_catalog.pg_attr_compression'::REGCLASS) AND + (objid = 'cmtest'::REGCLASS OR refobjid = 'cmtest'::REGCLASS) + ORDER by objid, refobjid; +INSERT INTO cmtest VALUES(repeat('1234567890',1001)); +-- one normal dependency +SELECT * FROM cmtest_deps; + classid | objsubid | refclassid | refobjsubid | deptype +---------+----------+------------+-------------+--------- + 1259 | 1 | 5555 | 0 | n +(1 row) + +-- check decompression +SELECT length(f1) FROM cmtest; + length +-------- + 10010 +(1 row) + +SELECT length(f1) FROM cmtest; + length +-------- + 10010 +(1 row) + +CREATE FUNCTION on_cmtest_rewrite() +RETURNS event_trigger AS $$ +BEGIN + RAISE NOTICE 'cmtest rewrite'; +END; +$$ LANGUAGE plpgsql; +CREATE EVENT TRIGGER notice_on_cmtest_rewrite ON table_rewrite + EXECUTE PROCEDURE on_cmtest_rewrite(); +-- no rewrite +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz1 PRESERVE (pglz); +INSERT INTO cmtest VALUES(repeat('1234567890',1002)); +SELECT length(f1) FROM cmtest; + length +-------- + 10010 + 10020 +(2 rows) + +SELECT pg_column_compression('cmtest', 'f1'); + pg_column_compression +----------------------- + pglz, pglz1 +(1 row) + +-- one normal and one internal dependency +SELECT * FROM cmtest_deps; + classid | objsubid | refclassid | refobjsubid | deptype +---------+----------+------------+-------------+--------- + 1259 | 1 | 5555 | 0 | n + 5555 | 0 | 1259 | 1 | i +(2 rows) + +-- rewrite +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz2 PRESERVE (pglz1); +NOTICE: cmtest rewrite +INSERT INTO cmtest VALUES(repeat('1234567890',1003)); +SELECT length(f1) FROM cmtest; + length +-------- + 10010 + 10020 + 10030 +(3 rows) + +SELECT pg_column_compression('cmtest', 'f1'); + pg_column_compression +----------------------- + pglz1, pglz2 +(1 row) + +-- two internal dependencies +SELECT * FROM cmtest_deps; + classid | objsubid | refclassid | refobjsubid | deptype +---------+----------+------------+-------------+--------- + 5555 | 0 | 1259 | 1 | i + 5555 | 0 | 1259 | 1 | i +(2 rows) + +-- rewrite +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz; +NOTICE: cmtest rewrite +INSERT INTO cmtest VALUES(repeat('1234567890',1004)); +SELECT length(f1) FROM cmtest; + length +-------- + 10010 + 10020 + 10030 + 10040 +(4 rows) + +SELECT pg_column_compression('cmtest', 'f1'); + pg_column_compression +----------------------- + pglz +(1 row) + +-- one nornal dependency +SELECT * FROM cmtest_deps; + classid | objsubid | refclassid | refobjsubid | deptype +---------+----------+------------+-------------+--------- + 1259 | 1 | 5555 | 0 | n +(1 row) + +-- no rewrites +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz1 PRESERVE (pglz); +INSERT INTO cmtest VALUES(repeat('1234567890',1005)); +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz + WITH (min_input_size '1000') PRESERVE (pglz, pglz1); +INSERT INTO cmtest VALUES(repeat('1234567890',1006)); +-- one nornal dependency and two internal dependencies +SELECT * FROM cmtest_deps; + classid | objsubid | refclassid | refobjsubid | deptype +---------+----------+------------+-------------+--------- + 1259 | 1 | 5555 | 0 | n + 5555 | 0 | 1259 | 1 | i + 5555 | 0 | 1259 | 1 | i +(3 rows) + +-- remove function and related event trigger +DROP FUNCTION on_cmtest_rewrite CASCADE; +NOTICE: drop cascades to event trigger notice_on_cmtest_rewrite +-- test moving +CREATE TABLE cmdata(f1 text COMPRESSION pglz WITH (first_success_by '10000')); +INSERT INTO cmdata VALUES(repeat('1234567890',1000)); +INSERT INTO cmdata VALUES(repeat('1234567890',1001)); +-- copy with table creation +SELECT * INTO cmmove1 FROM cmdata; +-- we update using datum from different table +CREATE TABLE cmmove2(f1 text COMPRESSION pglz WITH (min_input_size '100')); +INSERT INTO cmmove2 VALUES (repeat('1234567890',1004)); +UPDATE cmmove2 SET f1 = cmdata.f1 FROM cmdata; +-- copy to existing table +CREATE TABLE cmmove3(f1 text COMPRESSION pglz WITH (min_input_size '100')); +INSERT INTO cmmove3 SELECT * FROM cmdata; +-- drop original compression information +DROP TABLE cmdata; +-- check data is ok +SELECT length(f1) FROM cmmove1; + length +-------- + 10000 + 10010 +(2 rows) + +SELECT length(f1) FROM cmmove2; + length +-------- + 10000 +(1 row) + +SELECT length(f1) FROM cmmove3; + length +-------- + 10000 + 10010 +(2 rows) + +-- create different types of tables +CREATE TABLE cmexample (f1 TEXT COMPRESSION pglz1); +CREATE TABLE cmtestlike1 (LIKE cmexample INCLUDING COMPRESSION); +CREATE TABLE cmtestlike2 (f2 INT) INHERITS (cmexample); +\d+ cmtestlike1 + Table "public.cmtestlike1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | pglz1 | | + +\d+ cmtestlike2 + Table "public.cmtestlike2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | pglz1 | | + f2 | integer | | | | plain | | | +Inherits: cmexample + +-- test two columns +CREATE TABLE cmaltertest(f1 TEXT, f2 TEXT COMPRESSION pglz1); +\d+ cmaltertest; + Table "public.cmaltertest" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | text | | | | extended | pglz | | + f2 | text | | | | extended | pglz1 | | + +-- fail, changing one column twice +ALTER TABLE cmaltertest ALTER COLUMN f1 SET COMPRESSION pglz, + ALTER COLUMN f1 SET COMPRESSION pglz; +ERROR: cannot alter compression of column "f1" twice +HINT: Remove one of statements from the command. +-- with rewrite +ALTER TABLE cmaltertest ALTER COLUMN f1 SET COMPRESSION pglz1, + ALTER COLUMN f2 SET COMPRESSION pglz PRESERVE (pglz1); +SELECT pg_column_compression('cmaltertest', 'f1'); + pg_column_compression +----------------------- + pglz1 +(1 row) + +SELECT pg_column_compression('cmaltertest', 'f2'); + pg_column_compression +----------------------- + pglz, pglz1 +(1 row) + +-- no rewrite +ALTER TABLE cmaltertest ALTER COLUMN f1 SET COMPRESSION pglz PRESERVE (pglz1), + ALTER COLUMN f2 SET COMPRESSION pglz2 PRESERVE (pglz1, pglz); +SELECT pg_column_compression('cmaltertest', 'f1'); + pg_column_compression +----------------------- + pglz, pglz1 +(1 row) + +SELECT pg_column_compression('cmaltertest', 'f2'); + pg_column_compression +----------------------- + pglz, pglz1, pglz2 +(1 row) + +-- make pglz2 droppable +ALTER TABLE cmaltertest ALTER COLUMN f2 SET COMPRESSION pglz1; +SELECT pg_column_compression('cmaltertest', 'f1'); + pg_column_compression +----------------------- + pglz, pglz1 +(1 row) + +SELECT pg_column_compression('cmaltertest', 'f2'); + pg_column_compression +----------------------- + pglz1 +(1 row) + +SELECT acname, acattnum, acoptions FROM pg_attr_compression + WHERE acrelid = 'cmaltertest'::REGCLASS OR acrelid = 'cmtest'::REGCLASS; + acname | acattnum | acoptions +--------+----------+----------------------- + pglz1 | 1 | + pglz | 1 | {min_input_size=1000} + pglz1 | 2 | + pglz1 | 1 | +(4 rows) + +-- zlib compression +CREATE TABLE zlibtest(f1 TEXT COMPRESSION zlib WITH (invalid 'param')); +ERROR: not built with zlib support +CREATE TABLE zlibtest(f1 TEXT COMPRESSION zlib WITH (level 'best')); +ERROR: not built with zlib support +CREATE TABLE zlibtest(f1 TEXT COMPRESSION zlib); +ALTER TABLE zlibtest + ALTER COLUMN f1 SET COMPRESSION zlib WITH (level '9'); +ERROR: not built with zlib support +ALTER TABLE zlibtest + ALTER COLUMN f1 SET COMPRESSION zlib WITH (level '1'); +ERROR: not built with zlib support +ALTER TABLE zlibtest + ALTER COLUMN f1 SET COMPRESSION zlib WITH (dict 'one'); +ERROR: not built with zlib support +INSERT INTO zlibtest VALUES(repeat('1234567890',1004)); +ERROR: not built with zlib support +ALTER TABLE zlibtest + ALTER COLUMN f1 SET COMPRESSION zlib WITH (dict 'one two') PRESERVE (zlib); +ERROR: not built with zlib support +INSERT INTO zlibtest VALUES(repeat('1234567890 one two three',1004)); +ERROR: not built with zlib support +SELECT length(f1) FROM zlibtest; + length +-------- +(0 rows) + +DROP ACCESS METHOD pglz2; +DROP VIEW cmtest_deps; +DROP TABLE cmmove1, cmmove2, cmmove3; +DROP TABLE cmexample, cmtestlike1, cmtestlike2, zlibtest; diff --git a/src/test/regress/expected/create_table.out b/src/test/regress/expected/create_table.out index 262abf2bfb81b..7cfb8e3e4ebb2 100644 --- a/src/test/regress/expected/create_table.out +++ b/src/test/regress/expected/create_table.out @@ -479,11 +479,11 @@ Partition key: RANGE (a oid_ops, plusone(b), c, d COLLATE "C") Number of partitions: 0 \d+ partitioned2 - Partitioned table "public.partitioned2" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | integer | | | | plain | | - b | text | | | | extended | | + Partitioned table "public.partitioned2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | integer | | | | plain | | | + b | text | | | | extended | pglz | | Partition key: RANGE (((a + 1)), substr(b, 1, 5)) Number of partitions: 0 @@ -492,11 +492,11 @@ ERROR: no partition of relation "partitioned2" found for row DETAIL: Partition key of the failing row contains ((a + 1), substr(b, 1, 5)) = (2, hello). CREATE TABLE part2_1 PARTITION OF partitioned2 FOR VALUES FROM (-1, 'aaaaa') TO (100, 'ccccc'); \d+ part2_1 - Table "public.part2_1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | integer | | | | plain | | - b | text | | | | extended | | + Table "public.part2_1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | integer | | | | plain | | | + b | text | | | | extended | pglz | | Partition of: partitioned2 FOR VALUES FROM ('-1', 'aaaaa') TO (100, 'ccccc') Partition constraint: (((a + 1) IS NOT NULL) AND (substr(b, 1, 5) IS NOT NULL) AND (((a + 1) > '-1'::integer) OR (((a + 1) = '-1'::integer) AND (substr(b, 1, 5) >= 'aaaaa'::text))) AND (((a + 1) < 100) OR (((a + 1) = 100) AND (substr(b, 1, 5) < 'ccccc'::text)))) @@ -513,10 +513,10 @@ CREATE TABLE part_p2 PARTITION OF list_parted FOR VALUES IN (2); CREATE TABLE part_p3 PARTITION OF list_parted FOR VALUES IN ((2+1)); CREATE TABLE part_null PARTITION OF list_parted FOR VALUES IN (null); \d+ list_parted - Partitioned table "public.list_parted" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - a | integer | | | | plain | | + Partitioned table "public.list_parted" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + a | integer | | | | plain | | | Partition key: LIST (a) Partitions: part_null FOR VALUES IN (NULL), part_p1 FOR VALUES IN (1), @@ -919,21 +919,21 @@ create table test_part_coll_cast2 partition of test_part_coll_posix for values f drop table test_part_coll_posix; -- Partition bound in describe output \d+ part_b - Table "public.part_b" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | not null | 1 | plain | | + Table "public.part_b" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | not null | 1 | plain | | | Partition of: parted FOR VALUES IN ('b') Partition constraint: ((a IS NOT NULL) AND (a = 'b'::text)) -- Both partition bound and partition key in describe output \d+ part_c - Partitioned table "public.part_c" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | not null | 0 | plain | | + Partitioned table "public.part_c" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | not null | 0 | plain | | | Partition of: parted FOR VALUES IN ('c') Partition constraint: ((a IS NOT NULL) AND (a = 'c'::text)) Partition key: RANGE (b) @@ -941,11 +941,11 @@ Partitions: part_c_1_10 FOR VALUES FROM (1) TO (10) -- a level-2 partition's constraint will include the parent's expressions \d+ part_c_1_10 - Table "public.part_c_1_10" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | not null | 0 | plain | | + Table "public.part_c_1_10" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | not null | 0 | plain | | | Partition of: part_c FOR VALUES FROM (1) TO (10) Partition constraint: ((a IS NOT NULL) AND (a = 'c'::text) AND (b IS NOT NULL) AND (b >= 1) AND (b < 10)) @@ -974,46 +974,46 @@ Number of partitions: 3 (Use \d+ to list them.) CREATE TABLE range_parted4 (a int, b int, c int) PARTITION BY RANGE (abs(a), abs(b), c); CREATE TABLE unbounded_range_part PARTITION OF range_parted4 FOR VALUES FROM (MINVALUE, MINVALUE, MINVALUE) TO (MAXVALUE, MAXVALUE, MAXVALUE); \d+ unbounded_range_part - Table "public.unbounded_range_part" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - a | integer | | | | plain | | - b | integer | | | | plain | | - c | integer | | | | plain | | + Table "public.unbounded_range_part" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + a | integer | | | | plain | | | + b | integer | | | | plain | | | + c | integer | | | | plain | | | Partition of: range_parted4 FOR VALUES FROM (MINVALUE, MINVALUE, MINVALUE) TO (MAXVALUE, MAXVALUE, MAXVALUE) Partition constraint: ((abs(a) IS NOT NULL) AND (abs(b) IS NOT NULL) AND (c IS NOT NULL)) DROP TABLE unbounded_range_part; CREATE TABLE range_parted4_1 PARTITION OF range_parted4 FOR VALUES FROM (MINVALUE, MINVALUE, MINVALUE) TO (1, MAXVALUE, MAXVALUE); \d+ range_parted4_1 - Table "public.range_parted4_1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - a | integer | | | | plain | | - b | integer | | | | plain | | - c | integer | | | | plain | | + Table "public.range_parted4_1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + a | integer | | | | plain | | | + b | integer | | | | plain | | | + c | integer | | | | plain | | | Partition of: range_parted4 FOR VALUES FROM (MINVALUE, MINVALUE, MINVALUE) TO (1, MAXVALUE, MAXVALUE) Partition constraint: ((abs(a) IS NOT NULL) AND (abs(b) IS NOT NULL) AND (c IS NOT NULL) AND (abs(a) <= 1)) CREATE TABLE range_parted4_2 PARTITION OF range_parted4 FOR VALUES FROM (3, 4, 5) TO (6, 7, MAXVALUE); \d+ range_parted4_2 - Table "public.range_parted4_2" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - a | integer | | | | plain | | - b | integer | | | | plain | | - c | integer | | | | plain | | + Table "public.range_parted4_2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + a | integer | | | | plain | | | + b | integer | | | | plain | | | + c | integer | | | | plain | | | Partition of: range_parted4 FOR VALUES FROM (3, 4, 5) TO (6, 7, MAXVALUE) Partition constraint: ((abs(a) IS NOT NULL) AND (abs(b) IS NOT NULL) AND (c IS NOT NULL) AND ((abs(a) > 3) OR ((abs(a) = 3) AND (abs(b) > 4)) OR ((abs(a) = 3) AND (abs(b) = 4) AND (c >= 5))) AND ((abs(a) < 6) OR ((abs(a) = 6) AND (abs(b) <= 7)))) CREATE TABLE range_parted4_3 PARTITION OF range_parted4 FOR VALUES FROM (6, 8, MINVALUE) TO (9, MAXVALUE, MAXVALUE); \d+ range_parted4_3 - Table "public.range_parted4_3" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - a | integer | | | | plain | | - b | integer | | | | plain | | - c | integer | | | | plain | | + Table "public.range_parted4_3" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + a | integer | | | | plain | | | + b | integer | | | | plain | | | + c | integer | | | | plain | | | Partition of: range_parted4 FOR VALUES FROM (6, 8, MINVALUE) TO (9, MAXVALUE, MAXVALUE) Partition constraint: ((abs(a) IS NOT NULL) AND (abs(b) IS NOT NULL) AND (c IS NOT NULL) AND ((abs(a) > 6) OR ((abs(a) = 6) AND (abs(b) >= 8))) AND (abs(a) <= 9)) @@ -1045,11 +1045,11 @@ SELECT obj_description('parted_col_comment'::regclass); (1 row) \d+ parted_col_comment - Partitioned table "public.parted_col_comment" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+--------------- - a | integer | | | | plain | | Partition key - b | text | | | | extended | | + Partitioned table "public.parted_col_comment" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+--------------- + a | integer | | | | plain | | | Partition key + b | text | | | | extended | pglz | | Partition key: LIST (a) Number of partitions: 0 @@ -1058,10 +1058,10 @@ DROP TABLE parted_col_comment; CREATE TABLE arrlp (a int[]) PARTITION BY LIST (a); CREATE TABLE arrlp12 PARTITION OF arrlp FOR VALUES IN ('{1}', '{2}'); \d+ arrlp12 - Table "public.arrlp12" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+-----------+-----------+----------+---------+----------+--------------+------------- - a | integer[] | | | | extended | | + Table "public.arrlp12" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+-----------+-----------+----------+---------+----------+-------------+--------------+------------- + a | integer[] | | | | extended | pglz | | Partition of: arrlp FOR VALUES IN ('{1}', '{2}') Partition constraint: ((a IS NOT NULL) AND ((a = '{1}'::integer[]) OR (a = '{2}'::integer[]))) @@ -1071,10 +1071,10 @@ create table boolspart (a bool) partition by list (a); create table boolspart_t partition of boolspart for values in (true); create table boolspart_f partition of boolspart for values in (false); \d+ boolspart - Partitioned table "public.boolspart" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - a | boolean | | | | plain | | + Partitioned table "public.boolspart" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + a | boolean | | | | plain | | | Partition key: LIST (a) Partitions: boolspart_f FOR VALUES IN (false), boolspart_t FOR VALUES IN (true) diff --git a/src/test/regress/expected/create_table_like.out b/src/test/regress/expected/create_table_like.out index b153d6adb1db0..7ec153f5ed887 100644 --- a/src/test/regress/expected/create_table_like.out +++ b/src/test/regress/expected/create_table_like.out @@ -204,32 +204,32 @@ CREATE TABLE ctlt4 (a text, c text); ALTER TABLE ctlt4 ALTER COLUMN c SET STORAGE EXTERNAL; CREATE TABLE ctlt12_storage (LIKE ctlt1 INCLUDING STORAGE, LIKE ctlt2 INCLUDING STORAGE); \d+ ctlt12_storage - Table "public.ctlt12_storage" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+------+-----------+----------+---------+----------+--------------+------------- - a | text | | not null | | main | | - b | text | | | | extended | | - c | text | | | | external | | + Table "public.ctlt12_storage" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | not null | | main | pglz | | + b | text | | | | extended | pglz | | + c | text | | | | external | pglz | | CREATE TABLE ctlt12_comments (LIKE ctlt1 INCLUDING COMMENTS, LIKE ctlt2 INCLUDING COMMENTS); \d+ ctlt12_comments - Table "public.ctlt12_comments" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+------+-----------+----------+---------+----------+--------------+------------- - a | text | | not null | | extended | | A - b | text | | | | extended | | B - c | text | | | | extended | | C + Table "public.ctlt12_comments" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | not null | | extended | pglz | | A + b | text | | | | extended | pglz | | B + c | text | | | | extended | pglz | | C CREATE TABLE ctlt1_inh (LIKE ctlt1 INCLUDING CONSTRAINTS INCLUDING COMMENTS) INHERITS (ctlt1); NOTICE: merging column "a" with inherited definition NOTICE: merging column "b" with inherited definition NOTICE: merging constraint "ctlt1_a_check" with inherited definition \d+ ctlt1_inh - Table "public.ctlt1_inh" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+------+-----------+----------+---------+----------+--------------+------------- - a | text | | not null | | main | | A - b | text | | | | extended | | B + Table "public.ctlt1_inh" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | not null | | main | pglz | | A + b | text | | | | extended | pglz | | B Check constraints: "ctlt1_a_check" CHECK (length(a) > 2) Inherits: ctlt1 @@ -243,12 +243,12 @@ SELECT description FROM pg_description, pg_constraint c WHERE classoid = 'pg_con CREATE TABLE ctlt13_inh () INHERITS (ctlt1, ctlt3); NOTICE: merging multiple inherited definitions of column "a" \d+ ctlt13_inh - Table "public.ctlt13_inh" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+------+-----------+----------+---------+----------+--------------+------------- - a | text | | not null | | main | | - b | text | | | | extended | | - c | text | | | | external | | + Table "public.ctlt13_inh" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | not null | | main | pglz | | + b | text | | | | extended | pglz | | + c | text | | | | external | pglz | | Check constraints: "ctlt1_a_check" CHECK (length(a) > 2) "ctlt3_a_check" CHECK (length(a) < 5) @@ -258,12 +258,12 @@ Inherits: ctlt1, CREATE TABLE ctlt13_like (LIKE ctlt3 INCLUDING CONSTRAINTS INCLUDING COMMENTS INCLUDING STORAGE) INHERITS (ctlt1); NOTICE: merging column "a" with inherited definition \d+ ctlt13_like - Table "public.ctlt13_like" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+------+-----------+----------+---------+----------+--------------+------------- - a | text | | not null | | main | | A3 - b | text | | | | extended | | - c | text | | | | external | | C + Table "public.ctlt13_like" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | not null | | main | pglz | | A3 + b | text | | | | extended | pglz | | + c | text | | | | external | pglz | | C Check constraints: "ctlt1_a_check" CHECK (length(a) > 2) "ctlt3_a_check" CHECK (length(a) < 5) @@ -277,11 +277,11 @@ SELECT description FROM pg_description, pg_constraint c WHERE classoid = 'pg_con CREATE TABLE ctlt_all (LIKE ctlt1 INCLUDING ALL); \d+ ctlt_all - Table "public.ctlt_all" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+------+-----------+----------+---------+----------+--------------+------------- - a | text | | not null | | main | | A - b | text | | | | extended | | B + Table "public.ctlt_all" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | not null | | main | pglz | | A + b | text | | | | extended | pglz | | B Indexes: "ctlt_all_pkey" PRIMARY KEY, btree (a) "ctlt_all_b_idx" btree (b) diff --git a/src/test/regress/expected/domain.out b/src/test/regress/expected/domain.out index 4ff1b4af418b8..5255951060d62 100644 --- a/src/test/regress/expected/domain.out +++ b/src/test/regress/expected/domain.out @@ -272,10 +272,10 @@ explain (verbose, costs off) create rule silly as on delete to dcomptable do instead update dcomptable set d1.r = (d1).r - 1, d1.i = (d1).i + 1 where (d1).i > 0; \d+ dcomptable - Table "public.dcomptable" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+-----------+-----------+----------+---------+----------+--------------+------------- - d1 | dcomptype | | | | extended | | + Table "public.dcomptable" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+-----------+-----------+----------+---------+----------+-------------+--------------+------------- + d1 | dcomptype | | | | extended | pglz | | Indexes: "dcomptable_d1_key" UNIQUE CONSTRAINT, btree (d1) Rules: @@ -409,10 +409,10 @@ create rule silly as on delete to dcomptable do instead update dcomptable set d1[1].r = d1[1].r - 1, d1[1].i = d1[1].i + 1 where d1[1].i > 0; \d+ dcomptable - Table "public.dcomptable" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+------------+-----------+----------+---------+----------+--------------+------------- - d1 | dcomptypea | | | | extended | | + Table "public.dcomptable" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+------------+-----------+----------+---------+----------+-------------+--------------+------------- + d1 | dcomptypea | | | | extended | pglz | | Indexes: "dcomptable_d1_key" UNIQUE CONSTRAINT, btree (d1) Rules: diff --git a/src/test/regress/expected/foreign_data.out b/src/test/regress/expected/foreign_data.out index b9e25820bc02f..89466da6037a0 100644 --- a/src/test/regress/expected/foreign_data.out +++ b/src/test/regress/expected/foreign_data.out @@ -1383,12 +1383,12 @@ CREATE TABLE fd_pt1 ( CREATE FOREIGN TABLE ft2 () INHERITS (fd_pt1) SERVER s0 OPTIONS (delimiter ',', quote '"', "be quoted" 'value'); \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Child tables: ft2 \d+ ft2 @@ -1404,12 +1404,12 @@ Inherits: fd_pt1 DROP FOREIGN TABLE ft2; \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | CREATE FOREIGN TABLE ft2 ( c1 integer NOT NULL, @@ -1428,12 +1428,12 @@ FDW options: (delimiter ',', quote '"', "be quoted" 'value') ALTER FOREIGN TABLE ft2 INHERIT fd_pt1; \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Child tables: ft2 \d+ ft2 @@ -1471,12 +1471,12 @@ Child tables: ct3, ft3 \d+ ct3 - Table "public.ct3" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.ct3" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Inherits: ft2 \d+ ft3 @@ -1496,17 +1496,17 @@ ALTER TABLE fd_pt1 ADD COLUMN c6 integer; ALTER TABLE fd_pt1 ADD COLUMN c7 integer NOT NULL; ALTER TABLE fd_pt1 ADD COLUMN c8 integer; \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | | | extended | | - c3 | date | | | | plain | | - c4 | integer | | | | plain | | - c5 | integer | | | 0 | plain | | - c6 | integer | | | | plain | | - c7 | integer | | not null | | plain | | - c8 | integer | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | + c4 | integer | | | | plain | | | + c5 | integer | | | 0 | plain | | | + c6 | integer | | | | plain | | | + c7 | integer | | not null | | plain | | | + c8 | integer | | | | plain | | | Child tables: ft2 \d+ ft2 @@ -1528,17 +1528,17 @@ Child tables: ct3, ft3 \d+ ct3 - Table "public.ct3" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | | | extended | | - c3 | date | | | | plain | | - c4 | integer | | | | plain | | - c5 | integer | | | 0 | plain | | - c6 | integer | | | | plain | | - c7 | integer | | not null | | plain | | - c8 | integer | | | | plain | | + Table "public.ct3" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | + c4 | integer | | | | plain | | | + c5 | integer | | | 0 | plain | | | + c6 | integer | | | | plain | | | + c7 | integer | | not null | | plain | | | + c8 | integer | | | | plain | | | Inherits: ft2 \d+ ft3 @@ -1570,17 +1570,17 @@ ALTER TABLE fd_pt1 ALTER COLUMN c1 SET (n_distinct = 100); ALTER TABLE fd_pt1 ALTER COLUMN c8 SET STATISTICS -1; ALTER TABLE fd_pt1 ALTER COLUMN c8 SET STORAGE EXTERNAL; \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | 10000 | - c2 | text | | | | extended | | - c3 | date | | | | plain | | - c4 | integer | | | 0 | plain | | - c5 | integer | | | | plain | | - c6 | integer | | not null | | plain | | - c7 | integer | | | | plain | | - c8 | text | | | | external | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | 10000 | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | + c4 | integer | | | 0 | plain | | | + c5 | integer | | | | plain | | | + c6 | integer | | not null | | plain | | | + c7 | integer | | | | plain | | | + c8 | text | | | | external | pglz | | Child tables: ft2 \d+ ft2 @@ -1608,12 +1608,12 @@ ALTER TABLE fd_pt1 DROP COLUMN c6; ALTER TABLE fd_pt1 DROP COLUMN c7; ALTER TABLE fd_pt1 DROP COLUMN c8; \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | 10000 | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | 10000 | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Child tables: ft2 \d+ ft2 @@ -1645,12 +1645,12 @@ SELECT relname, conname, contype, conislocal, coninhcount, connoinherit -- child does not inherit NO INHERIT constraints \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | 10000 | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | 10000 | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Check constraints: "fd_pt1chk1" CHECK (c1 > 0) NO INHERIT "fd_pt1chk2" CHECK (c2 <> ''::text) @@ -1692,12 +1692,12 @@ ALTER FOREIGN TABLE ft2 ADD CONSTRAINT fd_pt1chk2 CHECK (c2 <> ''); ALTER FOREIGN TABLE ft2 INHERIT fd_pt1; -- child does not inherit NO INHERIT constraints \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | 10000 | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | 10000 | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Check constraints: "fd_pt1chk1" CHECK (c1 > 0) NO INHERIT "fd_pt1chk2" CHECK (c2 <> ''::text) @@ -1723,12 +1723,12 @@ ALTER TABLE fd_pt1 DROP CONSTRAINT fd_pt1chk2 CASCADE; INSERT INTO fd_pt1 VALUES (1, 'fd_pt1'::text, '1994-01-01'::date); ALTER TABLE fd_pt1 ADD CONSTRAINT fd_pt1chk3 CHECK (c2 <> '') NOT VALID; \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | 10000 | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | 10000 | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Check constraints: "fd_pt1chk3" CHECK (c2 <> ''::text) NOT VALID Child tables: ft2 @@ -1750,12 +1750,12 @@ Inherits: fd_pt1 -- VALIDATE CONSTRAINT need do nothing on foreign tables ALTER TABLE fd_pt1 VALIDATE CONSTRAINT fd_pt1chk3; \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | 10000 | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | 10000 | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Check constraints: "fd_pt1chk3" CHECK (c2 <> ''::text) Child tables: ft2 @@ -1781,12 +1781,12 @@ ALTER TABLE fd_pt1 RENAME COLUMN c3 TO f3; -- changes name of a constraint recursively ALTER TABLE fd_pt1 RENAME CONSTRAINT fd_pt1chk3 TO f2_check; \d+ fd_pt1 - Table "public.fd_pt1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - f1 | integer | | not null | | plain | 10000 | - f2 | text | | | | extended | | - f3 | date | | | | plain | | + Table "public.fd_pt1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | integer | | not null | | plain | | 10000 | + f2 | text | | | | extended | pglz | | + f3 | date | | | | plain | | | Check constraints: "f2_check" CHECK (f2 <> ''::text) Child tables: ft2 @@ -1845,12 +1845,12 @@ CREATE TABLE fd_pt2 ( CREATE FOREIGN TABLE fd_pt2_1 PARTITION OF fd_pt2 FOR VALUES IN (1) SERVER s0 OPTIONS (delimiter ',', quote '"', "be quoted" 'value'); \d+ fd_pt2 - Partitioned table "public.fd_pt2" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Partitioned table "public.fd_pt2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Partition key: LIST (c1) Partitions: fd_pt2_1 FOR VALUES IN (1) @@ -1890,12 +1890,12 @@ ERROR: table "fd_pt2_1" contains column "c4" not found in parent "fd_pt2" DETAIL: The new partition may contain only the columns present in parent. DROP FOREIGN TABLE fd_pt2_1; \d+ fd_pt2 - Partitioned table "public.fd_pt2" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Partitioned table "public.fd_pt2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Partition key: LIST (c1) Number of partitions: 0 @@ -1917,12 +1917,12 @@ FDW options: (delimiter ',', quote '"', "be quoted" 'value') -- no attach partition validation occurs for foreign tables ALTER TABLE fd_pt2 ATTACH PARTITION fd_pt2_1 FOR VALUES IN (1); \d+ fd_pt2 - Partitioned table "public.fd_pt2" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Partitioned table "public.fd_pt2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Partition key: LIST (c1) Partitions: fd_pt2_1 FOR VALUES IN (1) @@ -1945,12 +1945,12 @@ ERROR: cannot add column to a partition ALTER TABLE fd_pt2_1 ALTER c3 SET NOT NULL; ALTER TABLE fd_pt2_1 ADD CONSTRAINT p21chk CHECK (c2 <> ''); \d+ fd_pt2 - Partitioned table "public.fd_pt2" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | | | extended | | - c3 | date | | | | plain | | + Partitioned table "public.fd_pt2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | | | extended | pglz | | + c3 | date | | | | plain | | | Partition key: LIST (c1) Partitions: fd_pt2_1 FOR VALUES IN (1) @@ -1975,12 +1975,12 @@ ERROR: column "c1" is marked NOT NULL in parent table ALTER TABLE fd_pt2 DETACH PARTITION fd_pt2_1; ALTER TABLE fd_pt2 ALTER c2 SET NOT NULL; \d+ fd_pt2 - Partitioned table "public.fd_pt2" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | not null | | extended | | - c3 | date | | | | plain | | + Partitioned table "public.fd_pt2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | not null | | extended | pglz | | + c3 | date | | | | plain | | | Partition key: LIST (c1) Number of partitions: 0 @@ -2003,12 +2003,12 @@ ALTER TABLE fd_pt2 ATTACH PARTITION fd_pt2_1 FOR VALUES IN (1); ALTER TABLE fd_pt2 DETACH PARTITION fd_pt2_1; ALTER TABLE fd_pt2 ADD CONSTRAINT fd_pt2chk1 CHECK (c1 > 0); \d+ fd_pt2 - Partitioned table "public.fd_pt2" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - c1 | integer | | not null | | plain | | - c2 | text | | not null | | extended | | - c3 | date | | | | plain | | + Partitioned table "public.fd_pt2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + c1 | integer | | not null | | plain | | | + c2 | text | | not null | | extended | pglz | | + c3 | date | | | | plain | | | Partition key: LIST (c1) Check constraints: "fd_pt2chk1" CHECK (c1 > 0) diff --git a/src/test/regress/expected/inherit.out b/src/test/regress/expected/inherit.out index c6abcfc3cb106..01a16a532ccea 100644 --- a/src/test/regress/expected/inherit.out +++ b/src/test/regress/expected/inherit.out @@ -1042,13 +1042,13 @@ ALTER TABLE inhts RENAME aa TO aaa; -- to be failed ERROR: cannot rename inherited column "aa" ALTER TABLE inhts RENAME d TO dd; \d+ inhts - Table "public.inhts" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - aa | integer | | | | plain | | - b | integer | | | | plain | | - c | integer | | | | plain | | - dd | integer | | | | plain | | + Table "public.inhts" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + aa | integer | | | | plain | | | + b | integer | | | | plain | | | + c | integer | | | | plain | | | + dd | integer | | | | plain | | | Inherits: inht1, inhs1 @@ -1061,14 +1061,14 @@ NOTICE: merging multiple inherited definitions of column "aa" NOTICE: merging multiple inherited definitions of column "b" ALTER TABLE inht1 RENAME aa TO aaa; \d+ inht4 - Table "public.inht4" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - aaa | integer | | | | plain | | - b | integer | | | | plain | | - x | integer | | | | plain | | - y | integer | | | | plain | | - z | integer | | | | plain | | + Table "public.inht4" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + aaa | integer | | | | plain | | | + b | integer | | | | plain | | | + x | integer | | | | plain | | | + y | integer | | | | plain | | | + z | integer | | | | plain | | | Inherits: inht2, inht3 @@ -1078,14 +1078,14 @@ ALTER TABLE inht1 RENAME aaa TO aaaa; ALTER TABLE inht1 RENAME b TO bb; -- to be failed ERROR: cannot rename inherited column "b" \d+ inhts - Table "public.inhts" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - aaaa | integer | | | | plain | | - b | integer | | | | plain | | - x | integer | | | | plain | | - c | integer | | | | plain | | - d | integer | | | | plain | | + Table "public.inhts" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + aaaa | integer | | | | plain | | | + b | integer | | | | plain | | | + x | integer | | | | plain | | | + c | integer | | | | plain | | | + d | integer | | | | plain | | | Inherits: inht2, inhs1 @@ -1125,33 +1125,33 @@ drop cascades to table inht4 CREATE TABLE test_constraints (id int, val1 varchar, val2 int, UNIQUE(val1, val2)); CREATE TABLE test_constraints_inh () INHERITS (test_constraints); \d+ test_constraints - Table "public.test_constraints" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+-------------------+-----------+----------+---------+----------+--------------+------------- - id | integer | | | | plain | | - val1 | character varying | | | | extended | | - val2 | integer | | | | plain | | + Table "public.test_constraints" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+-------------------+-----------+----------+---------+----------+-------------+--------------+------------- + id | integer | | | | plain | | | + val1 | character varying | | | | extended | pglz | | + val2 | integer | | | | plain | | | Indexes: "test_constraints_val1_val2_key" UNIQUE CONSTRAINT, btree (val1, val2) Child tables: test_constraints_inh ALTER TABLE ONLY test_constraints DROP CONSTRAINT test_constraints_val1_val2_key; \d+ test_constraints - Table "public.test_constraints" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+-------------------+-----------+----------+---------+----------+--------------+------------- - id | integer | | | | plain | | - val1 | character varying | | | | extended | | - val2 | integer | | | | plain | | + Table "public.test_constraints" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+-------------------+-----------+----------+---------+----------+-------------+--------------+------------- + id | integer | | | | plain | | | + val1 | character varying | | | | extended | pglz | | + val2 | integer | | | | plain | | | Child tables: test_constraints_inh \d+ test_constraints_inh - Table "public.test_constraints_inh" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+-------------------+-----------+----------+---------+----------+--------------+------------- - id | integer | | | | plain | | - val1 | character varying | | | | extended | | - val2 | integer | | | | plain | | + Table "public.test_constraints_inh" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+-------------------+-----------+----------+---------+----------+-------------+--------------+------------- + id | integer | | | | plain | | | + val1 | character varying | | | | extended | pglz | | + val2 | integer | | | | plain | | | Inherits: test_constraints DROP TABLE test_constraints_inh; @@ -1162,27 +1162,27 @@ CREATE TABLE test_ex_constraints ( ); CREATE TABLE test_ex_constraints_inh () INHERITS (test_ex_constraints); \d+ test_ex_constraints - Table "public.test_ex_constraints" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+--------+-----------+----------+---------+---------+--------------+------------- - c | circle | | | | plain | | + Table "public.test_ex_constraints" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+--------+-----------+----------+---------+---------+-------------+--------------+------------- + c | circle | | | | plain | | | Indexes: "test_ex_constraints_c_excl" EXCLUDE USING gist (c WITH &&) Child tables: test_ex_constraints_inh ALTER TABLE test_ex_constraints DROP CONSTRAINT test_ex_constraints_c_excl; \d+ test_ex_constraints - Table "public.test_ex_constraints" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+--------+-----------+----------+---------+---------+--------------+------------- - c | circle | | | | plain | | + Table "public.test_ex_constraints" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+--------+-----------+----------+---------+---------+-------------+--------------+------------- + c | circle | | | | plain | | | Child tables: test_ex_constraints_inh \d+ test_ex_constraints_inh - Table "public.test_ex_constraints_inh" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+--------+-----------+----------+---------+---------+--------------+------------- - c | circle | | | | plain | | + Table "public.test_ex_constraints_inh" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+--------+-----------+----------+---------+---------+-------------+--------------+------------- + c | circle | | | | plain | | | Inherits: test_ex_constraints DROP TABLE test_ex_constraints_inh; @@ -1192,37 +1192,37 @@ CREATE TABLE test_primary_constraints(id int PRIMARY KEY); CREATE TABLE test_foreign_constraints(id1 int REFERENCES test_primary_constraints(id)); CREATE TABLE test_foreign_constraints_inh () INHERITS (test_foreign_constraints); \d+ test_primary_constraints - Table "public.test_primary_constraints" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - id | integer | | not null | | plain | | + Table "public.test_primary_constraints" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + id | integer | | not null | | plain | | | Indexes: "test_primary_constraints_pkey" PRIMARY KEY, btree (id) Referenced by: TABLE "test_foreign_constraints" CONSTRAINT "test_foreign_constraints_id1_fkey" FOREIGN KEY (id1) REFERENCES test_primary_constraints(id) \d+ test_foreign_constraints - Table "public.test_foreign_constraints" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - id1 | integer | | | | plain | | + Table "public.test_foreign_constraints" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + id1 | integer | | | | plain | | | Foreign-key constraints: "test_foreign_constraints_id1_fkey" FOREIGN KEY (id1) REFERENCES test_primary_constraints(id) Child tables: test_foreign_constraints_inh ALTER TABLE test_foreign_constraints DROP CONSTRAINT test_foreign_constraints_id1_fkey; \d+ test_foreign_constraints - Table "public.test_foreign_constraints" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - id1 | integer | | | | plain | | + Table "public.test_foreign_constraints" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + id1 | integer | | | | plain | | | Child tables: test_foreign_constraints_inh \d+ test_foreign_constraints_inh - Table "public.test_foreign_constraints_inh" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - id1 | integer | | | | plain | | + Table "public.test_foreign_constraints_inh" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + id1 | integer | | | | plain | | | Inherits: test_foreign_constraints DROP TABLE test_foreign_constraints_inh; diff --git a/src/test/regress/expected/insert.out b/src/test/regress/expected/insert.out index 75e25cdf48477..1829626e37a84 100644 --- a/src/test/regress/expected/insert.out +++ b/src/test/regress/expected/insert.out @@ -142,11 +142,11 @@ create rule irule3 as on insert to inserttest2 do also insert into inserttest (f4[1].if1, f4[1].if2[2]) select new.f1, new.f2; \d+ inserttest2 - Table "public.inserttest2" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+--------+-----------+----------+---------+----------+--------------+------------- - f1 | bigint | | | | plain | | - f2 | text | | | | extended | | + Table "public.inserttest2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+--------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | bigint | | | | plain | | | + f2 | text | | | | extended | pglz | | Rules: irule1 AS ON INSERT TO inserttest2 DO INSERT INTO inserttest (f3.if2[1], f3.if2[2]) @@ -448,11 +448,11 @@ from hash_parted order by part; -- test \d+ output on a table which has both partitioned and unpartitioned -- partitions \d+ list_parted - Partitioned table "public.list_parted" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | | | plain | | + Partitioned table "public.list_parted" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | | | plain | | | Partition key: LIST (lower(a)) Partitions: part_aa_bb FOR VALUES IN ('aa', 'bb'), part_cc_dd FOR VALUES IN ('cc', 'dd'), @@ -470,10 +470,10 @@ drop table hash_parted; create table list_parted (a int) partition by list (a); create table part_default partition of list_parted default; \d+ part_default - Table "public.part_default" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - a | integer | | | | plain | | + Table "public.part_default" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + a | integer | | | | plain | | | Partition of: list_parted DEFAULT No partition constraint @@ -855,11 +855,11 @@ create table mcrparted6_common_ge_10 partition of mcrparted for values from ('co create table mcrparted7_gt_common_lt_d partition of mcrparted for values from ('common', maxvalue) to ('d', minvalue); create table mcrparted8_ge_d partition of mcrparted for values from ('d', minvalue) to (maxvalue, maxvalue); \d+ mcrparted - Partitioned table "public.mcrparted" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | | | plain | | + Partitioned table "public.mcrparted" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | | | plain | | | Partition key: RANGE (a, b) Partitions: mcrparted1_lt_b FOR VALUES FROM (MINVALUE, MINVALUE) TO ('b', MINVALUE), mcrparted2_b FOR VALUES FROM ('b', MINVALUE) TO ('c', MINVALUE), @@ -871,74 +871,74 @@ Partitions: mcrparted1_lt_b FOR VALUES FROM (MINVALUE, MINVALUE) TO ('b', MINVAL mcrparted8_ge_d FOR VALUES FROM ('d', MINVALUE) TO (MAXVALUE, MAXVALUE) \d+ mcrparted1_lt_b - Table "public.mcrparted1_lt_b" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | | | plain | | + Table "public.mcrparted1_lt_b" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | | | plain | | | Partition of: mcrparted FOR VALUES FROM (MINVALUE, MINVALUE) TO ('b', MINVALUE) Partition constraint: ((a IS NOT NULL) AND (b IS NOT NULL) AND (a < 'b'::text)) \d+ mcrparted2_b - Table "public.mcrparted2_b" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | | | plain | | + Table "public.mcrparted2_b" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | | | plain | | | Partition of: mcrparted FOR VALUES FROM ('b', MINVALUE) TO ('c', MINVALUE) Partition constraint: ((a IS NOT NULL) AND (b IS NOT NULL) AND (a >= 'b'::text) AND (a < 'c'::text)) \d+ mcrparted3_c_to_common - Table "public.mcrparted3_c_to_common" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | | | plain | | + Table "public.mcrparted3_c_to_common" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | | | plain | | | Partition of: mcrparted FOR VALUES FROM ('c', MINVALUE) TO ('common', MINVALUE) Partition constraint: ((a IS NOT NULL) AND (b IS NOT NULL) AND (a >= 'c'::text) AND (a < 'common'::text)) \d+ mcrparted4_common_lt_0 - Table "public.mcrparted4_common_lt_0" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | | | plain | | + Table "public.mcrparted4_common_lt_0" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | | | plain | | | Partition of: mcrparted FOR VALUES FROM ('common', MINVALUE) TO ('common', 0) Partition constraint: ((a IS NOT NULL) AND (b IS NOT NULL) AND (a = 'common'::text) AND (b < 0)) \d+ mcrparted5_common_0_to_10 - Table "public.mcrparted5_common_0_to_10" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | | | plain | | + Table "public.mcrparted5_common_0_to_10" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | | | plain | | | Partition of: mcrparted FOR VALUES FROM ('common', 0) TO ('common', 10) Partition constraint: ((a IS NOT NULL) AND (b IS NOT NULL) AND (a = 'common'::text) AND (b >= 0) AND (b < 10)) \d+ mcrparted6_common_ge_10 - Table "public.mcrparted6_common_ge_10" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | | | plain | | + Table "public.mcrparted6_common_ge_10" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | | | plain | | | Partition of: mcrparted FOR VALUES FROM ('common', 10) TO ('common', MAXVALUE) Partition constraint: ((a IS NOT NULL) AND (b IS NOT NULL) AND (a = 'common'::text) AND (b >= 10)) \d+ mcrparted7_gt_common_lt_d - Table "public.mcrparted7_gt_common_lt_d" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | | | plain | | + Table "public.mcrparted7_gt_common_lt_d" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | | | plain | | | Partition of: mcrparted FOR VALUES FROM ('common', MAXVALUE) TO ('d', MINVALUE) Partition constraint: ((a IS NOT NULL) AND (b IS NOT NULL) AND (a > 'common'::text) AND (a < 'd'::text)) \d+ mcrparted8_ge_d - Table "public.mcrparted8_ge_d" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | integer | | | | plain | | + Table "public.mcrparted8_ge_d" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | integer | | | | plain | | | Partition of: mcrparted FOR VALUES FROM ('d', MINVALUE) TO (MAXVALUE, MAXVALUE) Partition constraint: ((a IS NOT NULL) AND (b IS NOT NULL) AND (a >= 'd'::text)) diff --git a/src/test/regress/expected/opr_sanity.out b/src/test/regress/expected/opr_sanity.out index 85af36ee5bca2..87588e0f128ff 100644 --- a/src/test/regress/expected/opr_sanity.out +++ b/src/test/regress/expected/opr_sanity.out @@ -1827,6 +1827,18 @@ WHERE p2.oid = p1.amhandler AND p1.amtype = 's' AND -----+--------+-----+--------- (0 rows) +-- Check for compression amhandler functions with the wrong signature +SELECT p1.oid, p1.amname, p2.oid, p2.proname +FROM pg_am AS p1, pg_proc AS p2 +WHERE p2.oid = p1.amhandler AND p1.amtype = 'c' AND + (p2.prorettype != 'compression_am_handler'::regtype + OR p2.proretset + OR p2.pronargs != 1 + OR p2.proargtypes[0] != 'internal'::regtype); + oid | amname | oid | proname +-----+--------+-----+--------- +(0 rows) + -- **************** pg_amop **************** -- Look for illegal values in pg_amop fields SELECT p1.amopfamily, p1.amopstrategy diff --git a/src/test/regress/expected/psql.out b/src/test/regress/expected/psql.out index 4bcf0cc5dfd57..a3e88f1702de1 100644 --- a/src/test/regress/expected/psql.out +++ b/src/test/regress/expected/psql.out @@ -2779,34 +2779,34 @@ CREATE ACCESS METHOD heap_psql TYPE TABLE HANDLER heap_tableam_handler; CREATE TABLE tbl_heap_psql(f1 int, f2 char(100)) using heap_psql; CREATE TABLE tbl_heap(f1 int, f2 char(100)) using heap; \d+ tbl_heap_psql - Table "public.tbl_heap_psql" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+----------------+-----------+----------+---------+----------+--------------+------------- - f1 | integer | | | | plain | | - f2 | character(100) | | | | extended | | + Table "public.tbl_heap_psql" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+----------------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | integer | | | | plain | | | + f2 | character(100) | | | | extended | pglz | | \d+ tbl_heap - Table "public.tbl_heap" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+----------------+-----------+----------+---------+----------+--------------+------------- - f1 | integer | | | | plain | | - f2 | character(100) | | | | extended | | + Table "public.tbl_heap" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+----------------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | integer | | | | plain | | | + f2 | character(100) | | | | extended | pglz | | \set HIDE_TABLEAM off \d+ tbl_heap_psql - Table "public.tbl_heap_psql" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+----------------+-----------+----------+---------+----------+--------------+------------- - f1 | integer | | | | plain | | - f2 | character(100) | | | | extended | | + Table "public.tbl_heap_psql" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+----------------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | integer | | | | plain | | | + f2 | character(100) | | | | extended | pglz | | Access method: heap_psql \d+ tbl_heap - Table "public.tbl_heap" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+----------------+-----------+----------+---------+----------+--------------+------------- - f1 | integer | | | | plain | | - f2 | character(100) | | | | extended | | + Table "public.tbl_heap" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+----------------+-----------+----------+---------+----------+-------------+--------------+------------- + f1 | integer | | | | plain | | | + f2 | character(100) | | | | extended | pglz | | Access method: heap \set HIDE_TABLEAM on diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out index 0e5e8f2b92933..d8f4233d453db 100644 --- a/src/test/regress/expected/publication.out +++ b/src/test/regress/expected/publication.out @@ -65,11 +65,11 @@ SELECT pubname, puballtables FROM pg_publication WHERE pubname = 'testpub_forall (1 row) \d+ testpub_tbl2 - Table "public.testpub_tbl2" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+------------------------------------------+----------+--------------+------------- - id | integer | | not null | nextval('testpub_tbl2_id_seq'::regclass) | plain | | - data | text | | | | extended | | + Table "public.testpub_tbl2" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+------------------------------------------+----------+-------------+--------------+------------- + id | integer | | not null | nextval('testpub_tbl2_id_seq'::regclass) | plain | | | + data | text | | | | extended | pglz | | Indexes: "testpub_tbl2_pkey" PRIMARY KEY, btree (id) Publications: @@ -141,22 +141,22 @@ ALTER PUBLICATION testpub_default SET TABLE testpub_tbl1; ALTER PUBLICATION testpub_default ADD TABLE pub_test.testpub_nopk; ALTER PUBLICATION testpib_ins_trunct ADD TABLE pub_test.testpub_nopk, testpub_tbl1; \d+ pub_test.testpub_nopk - Table "pub_test.testpub_nopk" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - foo | integer | | | | plain | | - bar | integer | | | | plain | | + Table "pub_test.testpub_nopk" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + foo | integer | | | | plain | | | + bar | integer | | | | plain | | | Publications: "testpib_ins_trunct" "testpub_default" "testpub_fortbl" \d+ testpub_tbl1 - Table "public.testpub_tbl1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+------------------------------------------+----------+--------------+------------- - id | integer | | not null | nextval('testpub_tbl1_id_seq'::regclass) | plain | | - data | text | | | | extended | | + Table "public.testpub_tbl1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+------------------------------------------+----------+-------------+--------------+------------- + id | integer | | not null | nextval('testpub_tbl1_id_seq'::regclass) | plain | | | + data | text | | | | extended | pglz | | Indexes: "testpub_tbl1_pkey" PRIMARY KEY, btree (id) Publications: @@ -178,11 +178,11 @@ ALTER PUBLICATION testpub_default DROP TABLE testpub_tbl1, pub_test.testpub_nopk ALTER PUBLICATION testpub_default DROP TABLE pub_test.testpub_nopk; ERROR: relation "testpub_nopk" is not part of the publication \d+ testpub_tbl1 - Table "public.testpub_tbl1" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+------------------------------------------+----------+--------------+------------- - id | integer | | not null | nextval('testpub_tbl1_id_seq'::regclass) | plain | | - data | text | | | | extended | | + Table "public.testpub_tbl1" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+------------------------------------------+----------+-------------+--------------+------------- + id | integer | | not null | nextval('testpub_tbl1_id_seq'::regclass) | plain | | | + data | text | | | | extended | pglz | | Indexes: "testpub_tbl1_pkey" PRIMARY KEY, btree (id) Publications: diff --git a/src/test/regress/expected/replica_identity.out b/src/test/regress/expected/replica_identity.out index 739608aab4c73..7e38d015e4bc8 100644 --- a/src/test/regress/expected/replica_identity.out +++ b/src/test/regress/expected/replica_identity.out @@ -153,13 +153,13 @@ SELECT relreplident FROM pg_class WHERE oid = 'test_replica_identity'::regclass; (1 row) \d+ test_replica_identity - Table "public.test_replica_identity" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------------------------------------------------+----------+--------------+------------- - id | integer | | not null | nextval('test_replica_identity_id_seq'::regclass) | plain | | - keya | text | | not null | | extended | | - keyb | text | | not null | | extended | | - nonkey | text | | | | extended | | + Table "public.test_replica_identity" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------------------------------------------------+----------+-------------+--------------+------------- + id | integer | | not null | nextval('test_replica_identity_id_seq'::regclass) | plain | | | + keya | text | | not null | | extended | pglz | | + keyb | text | | not null | | extended | pglz | | + nonkey | text | | | | extended | pglz | | Indexes: "test_replica_identity_pkey" PRIMARY KEY, btree (id) "test_replica_identity_expr" UNIQUE, btree (keya, keyb, (3)) diff --git a/src/test/regress/expected/rowsecurity.out b/src/test/regress/expected/rowsecurity.out index aa3b083ee3aa6..15585f3c0e5dc 100644 --- a/src/test/regress/expected/rowsecurity.out +++ b/src/test/regress/expected/rowsecurity.out @@ -938,14 +938,14 @@ CREATE POLICY pp1 ON part_document AS PERMISSIVE CREATE POLICY pp1r ON part_document AS RESTRICTIVE TO regress_rls_dave USING (cid < 55); \d+ part_document - Partitioned table "regress_rls_schema.part_document" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ----------+---------+-----------+----------+---------+----------+--------------+------------- - did | integer | | | | plain | | - cid | integer | | | | plain | | - dlevel | integer | | not null | | plain | | - dauthor | name | | | | plain | | - dtitle | text | | | | extended | | + Partitioned table "regress_rls_schema.part_document" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +---------+---------+-----------+----------+---------+----------+-------------+--------------+------------- + did | integer | | | | plain | | | + cid | integer | | | | plain | | | + dlevel | integer | | not null | | plain | | | + dauthor | name | | | | plain | | | + dtitle | text | | | | extended | pglz | | Partition key: RANGE (cid) Policies: POLICY "pp1" diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 210e9cd146cf3..7d8c0b32abee5 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -2927,11 +2927,11 @@ select * from rules_log; create rule r3 as on delete to rules_src do notify rules_src_deletion; \d+ rules_src - Table "public.rules_src" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - f1 | integer | | | | plain | | - f2 | integer | | | | plain | | + Table "public.rules_src" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + f1 | integer | | | | plain | | | + f2 | integer | | | | plain | | | Rules: r1 AS ON UPDATE TO rules_src DO INSERT INTO rules_log (f1, f2, tag) VALUES (old.f1,old.f2,'old'::text), (new.f1,new.f2,'new'::text) @@ -2947,11 +2947,11 @@ Rules: create rule r4 as on insert to rules_src do instead insert into rules_log AS trgt SELECT NEW.* RETURNING trgt.f1, trgt.f2; create rule r5 as on update to rules_src do instead UPDATE rules_log AS trgt SET tag = 'updated' WHERE trgt.f1 = new.f1; \d+ rules_src - Table "public.rules_src" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------+---------+--------------+------------- - f1 | integer | | | | plain | | - f2 | integer | | | | plain | | + Table "public.rules_src" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+---------+-----------+----------+---------+---------+-------------+--------------+------------- + f1 | integer | | | | plain | | | + f2 | integer | | | | plain | | | Rules: r1 AS ON UPDATE TO rules_src DO INSERT INTO rules_log (f1, f2, tag) VALUES (old.f1,old.f2,'old'::text), (new.f1,new.f2,'new'::text) diff --git a/src/test/regress/expected/sanity_check.out b/src/test/regress/expected/sanity_check.out index 8ff0da185e3d6..1eec5f84cc5a1 100644 --- a/src/test/regress/expected/sanity_check.out +++ b/src/test/regress/expected/sanity_check.out @@ -32,6 +32,8 @@ check2_tbl|f check_tbl|f circle_tbl|t city|f +cmaltertest|f +cmtest|f copy_tbl|f d|f d_star|f @@ -103,6 +105,7 @@ pg_aggregate|t pg_am|t pg_amop|t pg_amproc|t +pg_attr_compression|t pg_attrdef|t pg_attribute|t pg_auth_members|t diff --git a/src/test/regress/expected/update.out b/src/test/regress/expected/update.out index 2083345c8eea9..b245059e37855 100644 --- a/src/test/regress/expected/update.out +++ b/src/test/regress/expected/update.out @@ -690,14 +690,14 @@ DROP TRIGGER d15_insert_trig ON part_d_15_20; :init_range_parted; create table part_def partition of range_parted default; \d+ part_def - Table "public.part_def" - Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+-------------------+-----------+----------+---------+----------+--------------+------------- - a | text | | | | extended | | - b | bigint | | | | plain | | - c | numeric | | | | main | | - d | integer | | | | plain | | - e | character varying | | | | extended | | + Table "public.part_def" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +--------+-------------------+-----------+----------+---------+----------+-------------+--------------+------------- + a | text | | | | extended | pglz | | + b | bigint | | | | plain | | | + c | numeric | | | | main | pglz | | + d | integer | | | | plain | | | + e | character varying | | | | extended | pglz | | Partition of: range_parted DEFAULT Partition constraint: (NOT ((a IS NOT NULL) AND (b IS NOT NULL) AND (((a = 'a'::text) AND (b >= '1'::bigint) AND (b < '10'::bigint)) OR ((a = 'a'::text) AND (b >= '10'::bigint) AND (b < '20'::bigint)) OR ((a = 'b'::text) AND (b >= '1'::bigint) AND (b < '10'::bigint)) OR ((a = 'b'::text) AND (b >= '10'::bigint) AND (b < '20'::bigint)) OR ((a = 'b'::text) AND (b >= '20'::bigint) AND (b < '30'::bigint))))) diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 8fb55f045e67e..4b766826f1dff 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -36,6 +36,7 @@ test: create_function_1 test: create_type test: create_table test: create_function_2 +test: create_cm # ---------- # Load huge amounts of data diff --git a/src/test/regress/serial_schedule b/src/test/regress/serial_schedule index a39ca1012a313..c03affb3344e8 100644 --- a/src/test/regress/serial_schedule +++ b/src/test/regress/serial_schedule @@ -79,6 +79,7 @@ test: drop_if_exists test: updatable_views test: roleattributes test: create_am +test: create_cm test: hash_func test: errors test: sanity_check diff --git a/src/test/regress/sql/create_cm.sql b/src/test/regress/sql/create_cm.sql new file mode 100644 index 0000000000000..a598484253e54 --- /dev/null +++ b/src/test/regress/sql/create_cm.sql @@ -0,0 +1,202 @@ +-- test drop +DROP ACCESS METHOD pglz; --fail + +CREATE ACCESS METHOD pglz1 TYPE COMPRESSION HANDLER pglzhandler; +CREATE TABLE droptest(d1 TEXT COMPRESSION pglz1); +DROP ACCESS METHOD pglz1; +DROP ACCESS METHOD pglz1 CASCADE; +\d+ droptest +DROP TABLE droptest; + +CREATE ACCESS METHOD pglz1 TYPE COMPRESSION HANDLER pglzhandler; + +-- test auto drop of related attribute compression +CREATE ACCESS METHOD pglz2 TYPE COMPRESSION HANDLER pglzhandler; +CREATE TABLE cmaltertest (f1 TEXT COMPRESSION pglz2); +SELECT acname, acattnum, acoptions FROM pg_attr_compression + WHERE acrelid = 'cmaltertest'::REGCLASS; +ALTER TABLE cmaltertest DROP COLUMN f1; +SELECT acname, acattnum, acoptions FROM pg_attr_compression + WHERE acrelid = 'cmaltertest'::REGCLASS; +DROP TABLE cmaltertest; + +-- test drop +DROP ACCESS METHOD pglz2; + +-- test alter data type +CREATE TABLE cmaltertest(at1 TEXT); +ALTER TABLE cmaltertest ALTER COLUMN at1 SET COMPRESSION pglz1 PRESERVE (pglz); +SELECT pg_column_compression('cmaltertest', 'at1'); +ALTER TABLE cmaltertest ALTER COLUMN at1 SET DATA TYPE INT USING at1::INTEGER; +\d+ cmaltertest +SELECT pg_column_compression('cmaltertest', 'at1'); +SELECT acname, acattnum, acoptions FROM pg_attr_compression + WHERE acrelid = 'cmaltertest'::REGCLASS; +ALTER TABLE cmaltertest ALTER COLUMN at1 SET DATA TYPE TEXT; +SELECT pg_column_compression('cmaltertest', 'at1'); +DROP TABLE cmaltertest; + +-- test storages +CREATE TABLE cmstoragetest(st1 TEXT, st2 INT); +ALTER TABLE cmstoragetest ALTER COLUMN st2 + SET COMPRESSION pglz WITH (min_input_size '100'); -- fail +ALTER TABLE cmstoragetest ALTER COLUMN st1 + SET COMPRESSION pglz WITH (min_input_size '100', min_comp_rate '50'); +ALTER TABLE cmstoragetest ALTER COLUMN st1 SET STORAGE EXTERNAL; +\d+ cmstoragetest +ALTER TABLE cmstoragetest ALTER COLUMN st1 SET STORAGE MAIN; +\d+ cmstoragetest +ALTER TABLE cmstoragetest ALTER COLUMN st1 SET STORAGE PLAIN; +\d+ cmstoragetest +ALTER TABLE cmstoragetest ALTER COLUMN st1 SET STORAGE EXTENDED; +\d+ cmstoragetest +DROP TABLE cmstoragetest; + +CREATE ACCESS METHOD pglz2 TYPE COMPRESSION HANDLER pglzhandler; + +-- test PRESERVE +CREATE TABLE cmtest(f1 TEXT); +\d+ cmtest +-- view to check dependencies +CREATE VIEW cmtest_deps AS + SELECT classid, objsubid, refclassid, refobjsubid, deptype + FROM pg_depend + WHERE (refclassid = 'pg_catalog.pg_attr_compression'::REGCLASS OR + classid = 'pg_catalog.pg_attr_compression'::REGCLASS) AND + (objid = 'cmtest'::REGCLASS OR refobjid = 'cmtest'::REGCLASS) + ORDER by objid, refobjid; +INSERT INTO cmtest VALUES(repeat('1234567890',1001)); + +-- one normal dependency +SELECT * FROM cmtest_deps; + +-- check decompression +SELECT length(f1) FROM cmtest; +SELECT length(f1) FROM cmtest; + +CREATE FUNCTION on_cmtest_rewrite() +RETURNS event_trigger AS $$ +BEGIN + RAISE NOTICE 'cmtest rewrite'; +END; +$$ LANGUAGE plpgsql; + +CREATE EVENT TRIGGER notice_on_cmtest_rewrite ON table_rewrite + EXECUTE PROCEDURE on_cmtest_rewrite(); + +-- no rewrite +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz1 PRESERVE (pglz); +INSERT INTO cmtest VALUES(repeat('1234567890',1002)); +SELECT length(f1) FROM cmtest; +SELECT pg_column_compression('cmtest', 'f1'); +-- one normal and one internal dependency +SELECT * FROM cmtest_deps; + +-- rewrite +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz2 PRESERVE (pglz1); +INSERT INTO cmtest VALUES(repeat('1234567890',1003)); +SELECT length(f1) FROM cmtest; +SELECT pg_column_compression('cmtest', 'f1'); +-- two internal dependencies +SELECT * FROM cmtest_deps; + +-- rewrite +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz; +INSERT INTO cmtest VALUES(repeat('1234567890',1004)); +SELECT length(f1) FROM cmtest; +SELECT pg_column_compression('cmtest', 'f1'); +-- one nornal dependency +SELECT * FROM cmtest_deps; + +-- no rewrites +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz1 PRESERVE (pglz); +INSERT INTO cmtest VALUES(repeat('1234567890',1005)); +ALTER TABLE cmtest ALTER COLUMN f1 SET COMPRESSION pglz + WITH (min_input_size '1000') PRESERVE (pglz, pglz1); +INSERT INTO cmtest VALUES(repeat('1234567890',1006)); +-- one nornal dependency and two internal dependencies +SELECT * FROM cmtest_deps; + +-- remove function and related event trigger +DROP FUNCTION on_cmtest_rewrite CASCADE; + +-- test moving +CREATE TABLE cmdata(f1 text COMPRESSION pglz WITH (first_success_by '10000')); +INSERT INTO cmdata VALUES(repeat('1234567890',1000)); +INSERT INTO cmdata VALUES(repeat('1234567890',1001)); + +-- copy with table creation +SELECT * INTO cmmove1 FROM cmdata; + +-- we update using datum from different table +CREATE TABLE cmmove2(f1 text COMPRESSION pglz WITH (min_input_size '100')); +INSERT INTO cmmove2 VALUES (repeat('1234567890',1004)); +UPDATE cmmove2 SET f1 = cmdata.f1 FROM cmdata; + +-- copy to existing table +CREATE TABLE cmmove3(f1 text COMPRESSION pglz WITH (min_input_size '100')); +INSERT INTO cmmove3 SELECT * FROM cmdata; + +-- drop original compression information +DROP TABLE cmdata; + +-- check data is ok +SELECT length(f1) FROM cmmove1; +SELECT length(f1) FROM cmmove2; +SELECT length(f1) FROM cmmove3; + +-- create different types of tables +CREATE TABLE cmexample (f1 TEXT COMPRESSION pglz1); +CREATE TABLE cmtestlike1 (LIKE cmexample INCLUDING COMPRESSION); +CREATE TABLE cmtestlike2 (f2 INT) INHERITS (cmexample); + +\d+ cmtestlike1 +\d+ cmtestlike2 + +-- test two columns +CREATE TABLE cmaltertest(f1 TEXT, f2 TEXT COMPRESSION pglz1); +\d+ cmaltertest; +-- fail, changing one column twice +ALTER TABLE cmaltertest ALTER COLUMN f1 SET COMPRESSION pglz, + ALTER COLUMN f1 SET COMPRESSION pglz; + +-- with rewrite +ALTER TABLE cmaltertest ALTER COLUMN f1 SET COMPRESSION pglz1, + ALTER COLUMN f2 SET COMPRESSION pglz PRESERVE (pglz1); +SELECT pg_column_compression('cmaltertest', 'f1'); +SELECT pg_column_compression('cmaltertest', 'f2'); + +-- no rewrite +ALTER TABLE cmaltertest ALTER COLUMN f1 SET COMPRESSION pglz PRESERVE (pglz1), + ALTER COLUMN f2 SET COMPRESSION pglz2 PRESERVE (pglz1, pglz); +SELECT pg_column_compression('cmaltertest', 'f1'); +SELECT pg_column_compression('cmaltertest', 'f2'); + +-- make pglz2 droppable +ALTER TABLE cmaltertest ALTER COLUMN f2 SET COMPRESSION pglz1; +SELECT pg_column_compression('cmaltertest', 'f1'); +SELECT pg_column_compression('cmaltertest', 'f2'); + +SELECT acname, acattnum, acoptions FROM pg_attr_compression + WHERE acrelid = 'cmaltertest'::REGCLASS OR acrelid = 'cmtest'::REGCLASS; + +-- zlib compression +CREATE TABLE zlibtest(f1 TEXT COMPRESSION zlib WITH (invalid 'param')); +CREATE TABLE zlibtest(f1 TEXT COMPRESSION zlib WITH (level 'best')); +CREATE TABLE zlibtest(f1 TEXT COMPRESSION zlib); +ALTER TABLE zlibtest + ALTER COLUMN f1 SET COMPRESSION zlib WITH (level '9'); +ALTER TABLE zlibtest + ALTER COLUMN f1 SET COMPRESSION zlib WITH (level '1'); +ALTER TABLE zlibtest + ALTER COLUMN f1 SET COMPRESSION zlib WITH (dict 'one'); +INSERT INTO zlibtest VALUES(repeat('1234567890',1004)); +ALTER TABLE zlibtest + ALTER COLUMN f1 SET COMPRESSION zlib WITH (dict 'one two') PRESERVE (zlib); +INSERT INTO zlibtest VALUES(repeat('1234567890 one two three',1004)); +SELECT length(f1) FROM zlibtest; + +DROP ACCESS METHOD pglz2; +DROP VIEW cmtest_deps; +DROP TABLE cmmove1, cmmove2, cmmove3; +DROP TABLE cmexample, cmtestlike1, cmtestlike2, zlibtest; diff --git a/src/test/regress/sql/opr_sanity.sql b/src/test/regress/sql/opr_sanity.sql index 1227ef79f0c8a..fe65d44041afd 100644 --- a/src/test/regress/sql/opr_sanity.sql +++ b/src/test/regress/sql/opr_sanity.sql @@ -1221,6 +1221,16 @@ WHERE p2.oid = p1.amhandler AND p1.amtype = 's' AND OR p2.pronargs != 1 OR p2.proargtypes[0] != 'internal'::regtype); +-- Check for compression amhandler functions with the wrong signature + +SELECT p1.oid, p1.amname, p2.oid, p2.proname +FROM pg_am AS p1, pg_proc AS p2 +WHERE p2.oid = p1.amhandler AND p1.amtype = 'c' AND + (p2.prorettype != 'compression_am_handler'::regtype + OR p2.proretset + OR p2.pronargs != 1 + OR p2.proargtypes[0] != 'internal'::regtype); + -- **************** pg_amop **************** -- Look for illegal values in pg_amop fields diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 432d2d812e517..d4ef37e701234 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -358,6 +358,7 @@ CollectedCommand CollectedCommandType ColorTrgm ColorTrgmInfo +ColumnCompression ColumnCompareData ColumnDef ColumnIOData @@ -383,6 +384,8 @@ CompositeIOData CompositeTypeStmt CompoundAffixFlag CompressionAlgorithm +CompressionAmOptions +CompressionAmRoutine CompressorState ConditionVariable ConditionalStack