Skip to content

Commit 63110c6

Browse files
michaelpqmichaelpq
committed
Use multi-inserts for pg_depend
This is a follow-up of the work done in e3931d0. This case is a bit different than pg_attribute and pg_shdepend: the maximum number of items to insert is known in advance, but there is no need to handle pinned dependencies. Hence, the base allocation for slots is done based on the number of items and the maximum allowed with a cap at 64kB. Slots are initialized once used to minimize the overhead of the operation. The insertions can be done for dependencies of the same type. More could be done by grouping the insertion of multiple dependency types in a single batch. This is left as future work. Some of the multi-insert logic is also simplified for pg_shdepend, as per the feedback discussed for this specific patch. This also moves to indexing.h the variable capping the maximum amount of data that can be used at once for a multi-insert, instead of having separate definitions for pg_attribute, pg_depend and pg_shdepend. Author: Daniel Gustafsson, Michael Paquier Reviewed-by: Andres Freund, Álvaro Herrera Discussion: https://postgr.es/m/20200807061619.GA23955@paquier.xyz
1 parent 4d41823 commit 63110c6

File tree

4 files changed

+105
-65
lines changed

4 files changed

+105
-65
lines changed

‎src/backend/catalog/heap.c

+1-7
Original file line numberDiff line numberDiff line change
@@ -709,12 +709,6 @@ CheckAttributeType(const char *attname,
709709
}
710710
}
711711

712-
/*
713-
* Cap the maximum amount of bytes allocated for InsertPgAttributeTuples()
714-
* slots.
715-
*/
716-
#define MAX_PGATTRIBUTE_INSERT_BYTES 65535
717-
718712
/*
719713
* InsertPgAttributeTuples
720714
* Construct and insert a set of tuples in pg_attribute.
@@ -750,7 +744,7 @@ InsertPgAttributeTuples(Relation pg_attribute_rel,
750744

751745
/* Initialize the number of slots to use */
752746
nslots = Min(tupdesc->natts,
753-
(MAX_PGATTRIBUTE_INSERT_BYTES / sizeof(FormData_pg_attribute)));
747+
(MAX_CATALOG_MULTI_INSERT_BYTES / sizeof(FormData_pg_attribute)));
754748
slot = palloc(sizeof(TupleTableSlot *) * nslots);
755749
for (int i = 0; i < nslots; i++)
756750
slot[i] = MakeSingleTupleTableSlot(td, &TTSOpsHeapTuple);

‎src/backend/catalog/pg_depend.c

+66-24
Original file line numberDiff line numberDiff line change
@@ -59,10 +59,11 @@ recordMultipleDependencies(const ObjectAddress *depender,
5959
{
6060
Relation dependDesc;
6161
CatalogIndexState indstate;
62-
HeapTuple tup;
63-
int i;
64-
bool nulls[Natts_pg_depend];
65-
Datum values[Natts_pg_depend];
62+
TupleTableSlot **slot;
63+
int i,
64+
max_slots,
65+
slot_init_count,
66+
slot_stored_count;
6667

6768
if (nreferenced <= 0)
6869
return; /* nothing to do */
@@ -76,50 +77,91 @@ recordMultipleDependencies(const ObjectAddress *depender,
7677

7778
dependDesc = table_open(DependRelationId, RowExclusiveLock);
7879

80+
/*
81+
* Allocate the slots to use, but delay costly initialization until we
82+
* know that they will be used.
83+
*/
84+
max_slots = Min(nreferenced,
85+
MAX_CATALOG_MULTI_INSERT_BYTES / sizeof(FormData_pg_depend));
86+
slot = palloc(sizeof(TupleTableSlot *) * max_slots);
87+
7988
/* Don't open indexes unless we need to make an update */
8089
indstate = NULL;
8190

82-
memset(nulls, false, sizeof(nulls));
83-
91+
/* number of slots currently storing tuples */
92+
slot_stored_count = 0;
93+
/* number of slots currently initialized */
94+
slot_init_count = 0;
8495
for (i = 0; i < nreferenced; i++, referenced++)
8596
{
8697
/*
8798
* If the referenced object is pinned by the system, there's no real
8899
* need to record dependencies on it. This saves lots of space in
89100
* pg_depend, so it's worth the time taken to check.
90101
*/
91-
if (!isObjectPinned(referenced, dependDesc))
92-
{
93-
/*
94-
* Record the Dependency. Note we don't bother to check for
95-
* duplicate dependencies; there's no harm in them.
96-
*/
97-
values[Anum_pg_depend_classid - 1] = ObjectIdGetDatum(depender->classId);
98-
values[Anum_pg_depend_objid - 1] = ObjectIdGetDatum(depender->objectId);
99-
values[Anum_pg_depend_objsubid - 1] = Int32GetDatum(depender->objectSubId);
100-
101-
values[Anum_pg_depend_refclassid - 1] = ObjectIdGetDatum(referenced->classId);
102-
values[Anum_pg_depend_refobjid - 1] = ObjectIdGetDatum(referenced->objectId);
103-
values[Anum_pg_depend_refobjsubid - 1] = Int32GetDatum(referenced->objectSubId);
102+
if (isObjectPinned(referenced, dependDesc))
103+
continue;
104104

105-
values[Anum_pg_depend_deptype - 1] = CharGetDatum((char) behavior);
105+
if (slot_init_count < max_slots)
106+
{
107+
slot[slot_stored_count] = MakeSingleTupleTableSlot(RelationGetDescr(dependDesc),
108+
&TTSOpsHeapTuple);
109+
slot_init_count++;
110+
}
106111

107-
tup = heap_form_tuple(dependDesc->rd_att, values, nulls);
112+
ExecClearTuple(slot[slot_stored_count]);
108113

114+
/*
115+
* Record the dependency. Note we don't bother to check for duplicate
116+
* dependencies; there's no harm in them.
117+
*/
118+
slot[slot_stored_count]->tts_values[Anum_pg_depend_refclassid - 1] = ObjectIdGetDatum(referenced->classId);
119+
slot[slot_stored_count]->tts_values[Anum_pg_depend_refobjid - 1] = ObjectIdGetDatum(referenced->objectId);
120+
slot[slot_stored_count]->tts_values[Anum_pg_depend_refobjsubid - 1] = Int32GetDatum(referenced->objectSubId);
121+
slot[slot_stored_count]->tts_values[Anum_pg_depend_deptype - 1] = CharGetDatum((char) behavior);
122+
slot[slot_stored_count]->tts_values[Anum_pg_depend_classid - 1] = ObjectIdGetDatum(depender->classId);
123+
slot[slot_stored_count]->tts_values[Anum_pg_depend_objid - 1] = ObjectIdGetDatum(depender->objectId);
124+
slot[slot_stored_count]->tts_values[Anum_pg_depend_objsubid - 1] = Int32GetDatum(depender->objectSubId);
125+
126+
memset(slot[slot_stored_count]->tts_isnull, false,
127+
slot[slot_stored_count]->tts_tupleDescriptor->natts * sizeof(bool));
128+
129+
ExecStoreVirtualTuple(slot[slot_stored_count]);
130+
slot_stored_count++;
131+
132+
/* If slots are full, insert a batch of tuples */
133+
if (slot_stored_count == max_slots)
134+
{
109135
/* fetch index info only when we know we need it */
110136
if (indstate == NULL)
111137
indstate = CatalogOpenIndexes(dependDesc);
112138

113-
CatalogTupleInsertWithInfo(dependDesc, tup, indstate);
114-
115-
heap_freetuple(tup);
139+
CatalogTuplesMultiInsertWithInfo(dependDesc, slot, slot_stored_count,
140+
indstate);
141+
slot_stored_count = 0;
116142
}
117143
}
118144

145+
/* Insert any tuples left in the buffer */
146+
if (slot_stored_count > 0)
147+
{
148+
/* fetch index info only when we know we need it */
149+
if (indstate == NULL)
150+
indstate = CatalogOpenIndexes(dependDesc);
151+
152+
CatalogTuplesMultiInsertWithInfo(dependDesc, slot, slot_stored_count,
153+
indstate);
154+
}
155+
119156
if (indstate != NULL)
120157
CatalogCloseIndexes(indstate);
121158

122159
table_close(dependDesc, RowExclusiveLock);
160+
161+
/* Drop only the number of slots used */
162+
for (i = 0; i < slot_init_count; i++)
163+
ExecDropSingleTupleTableSlot(slot[i]);
164+
pfree(slot);
123165
}
124166

125167
/*

‎src/backend/catalog/pg_shdepend.c

+32-34
Original file line numberDiff line numberDiff line change
@@ -786,12 +786,6 @@ checkSharedDependencies(Oid classId, Oid objectId,
786786
}
787787

788788

789-
/*
790-
* Cap the maximum amount of bytes allocated for copyTemplateDependencies()
791-
* slots.
792-
*/
793-
#define MAX_PGSHDEPEND_INSERT_BYTES 65535
794-
795789
/*
796790
* copyTemplateDependencies
797791
*
@@ -806,21 +800,20 @@ copyTemplateDependencies(Oid templateDbId, Oid newDbId)
806800
ScanKeyData key[1];
807801
SysScanDesc scan;
808802
HeapTuple tup;
809-
int slotCount;
810803
CatalogIndexState indstate;
811804
TupleTableSlot **slot;
812-
int nslots,
813-
max_slots;
814-
bool slot_init = true;
805+
int max_slots,
806+
slot_init_count,
807+
slot_stored_count;
815808

816809
sdepRel = table_open(SharedDependRelationId, RowExclusiveLock);
817810
sdepDesc = RelationGetDescr(sdepRel);
818811

819812
/*
820-
* Allocate the slots to use, but delay initialization until we know that
821-
* they will be used.
813+
* Allocate the slots to use, but delay costly initialization until we
814+
* know that they will be used.
822815
*/
823-
max_slots = MAX_PGSHDEPEND_INSERT_BYTES / sizeof(FormData_pg_shdepend);
816+
max_slots = MAX_CATALOG_MULTI_INSERT_BYTES / sizeof(FormData_pg_shdepend);
824817
slot = palloc(sizeof(TupleTableSlot *) * max_slots);
825818

826819
indstate = CatalogOpenIndexes(sdepRel);
@@ -834,57 +827,62 @@ copyTemplateDependencies(Oid templateDbId, Oid newDbId)
834827
scan = systable_beginscan(sdepRel, SharedDependDependerIndexId, true,
835828
NULL, 1, key);
836829

830+
/* number of slots currently storing tuples */
831+
slot_stored_count = 0;
832+
/* number of slots currently initialized */
833+
slot_init_count = 0;
834+
837835
/*
838836
* Copy the entries of the original database, changing the database Id to
839837
* that of the new database. Note that because we are not copying rows
840838
* with dbId == 0 (ie, rows describing dependent shared objects) we won't
841839
* copy the ownership dependency of the template database itself; this is
842840
* what we want.
843841
*/
844-
slotCount = 0;
845842
while (HeapTupleIsValid(tup = systable_getnext(scan)))
846843
{
847844
Form_pg_shdepend shdep;
848845

849-
if (slot_init)
850-
slot[slotCount] = MakeSingleTupleTableSlot(sdepDesc, &TTSOpsHeapTuple);
846+
if (slot_init_count < max_slots)
847+
{
848+
slot[slot_stored_count] = MakeSingleTupleTableSlot(sdepDesc, &TTSOpsHeapTuple);
849+
slot_init_count++;
850+
}
851851

852-
ExecClearTuple(slot[slotCount]);
852+
ExecClearTuple(slot[slot_stored_count]);
853853

854854
shdep = (Form_pg_shdepend) GETSTRUCT(tup);
855855

856-
slot[slotCount]->tts_values[Anum_pg_shdepend_dbid] = ObjectIdGetDatum(newDbId);
857-
slot[slotCount]->tts_values[Anum_pg_shdepend_classid] = shdep->classid;
858-
slot[slotCount]->tts_values[Anum_pg_shdepend_objid] = shdep->objid;
859-
slot[slotCount]->tts_values[Anum_pg_shdepend_objsubid] = shdep->objsubid;
860-
slot[slotCount]->tts_values[Anum_pg_shdepend_refclassid] = shdep->refclassid;
861-
slot[slotCount]->tts_values[Anum_pg_shdepend_refobjid] = shdep->refobjid;
862-
slot[slotCount]->tts_values[Anum_pg_shdepend_deptype] = shdep->deptype;
856+
slot[slot_stored_count]->tts_values[Anum_pg_shdepend_dbid] = ObjectIdGetDatum(newDbId);
857+
slot[slot_stored_count]->tts_values[Anum_pg_shdepend_classid] = shdep->classid;
858+
slot[slot_stored_count]->tts_values[Anum_pg_shdepend_objid] = shdep->objid;
859+
slot[slot_stored_count]->tts_values[Anum_pg_shdepend_objsubid] = shdep->objsubid;
860+
slot[slot_stored_count]->tts_values[Anum_pg_shdepend_refclassid] = shdep->refclassid;
861+
slot[slot_stored_count]->tts_values[Anum_pg_shdepend_refobjid] = shdep->refobjid;
862+
slot[slot_stored_count]->tts_values[Anum_pg_shdepend_deptype] = shdep->deptype;
863863

864-
ExecStoreVirtualTuple(slot[slotCount]);
865-
slotCount++;
864+
ExecStoreVirtualTuple(slot[slot_stored_count]);
865+
slot_stored_count++;
866866

867867
/* If slots are full, insert a batch of tuples */
868-
if (slotCount == max_slots)
868+
if (slot_stored_count == max_slots)
869869
{
870-
CatalogTuplesMultiInsertWithInfo(sdepRel, slot, slotCount, indstate);
871-
slotCount = 0;
872-
slot_init = false;
870+
CatalogTuplesMultiInsertWithInfo(sdepRel, slot, slot_stored_count, indstate);
871+
slot_stored_count = 0;
873872
}
874873
}
875874

876875
/* Insert any tuples left in the buffer */
877-
if (slotCount > 0)
878-
CatalogTuplesMultiInsertWithInfo(sdepRel, slot, slotCount, indstate);
876+
if (slot_stored_count > 0)
877+
CatalogTuplesMultiInsertWithInfo(sdepRel, slot, slot_stored_count, indstate);
879878

880879
systable_endscan(scan);
881880

882881
CatalogCloseIndexes(indstate);
883882
table_close(sdepRel, RowExclusiveLock);
884883

885884
/* Drop only the number of slots used */
886-
nslots = slot_init ? slotCount : max_slots;
887-
for (int i = 0; i < nslots; i++)
885+
for (int i = 0; i < slot_init_count; i++)
888886
ExecDropSingleTupleTableSlot(slot[i]);
889887
pfree(slot);
890888
}

‎src/include/catalog/indexing.h

+6
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,12 @@
2929
*/
3030
typedef struct ResultRelInfo *CatalogIndexState;
3131

32+
/*
33+
* Cap the maximum amount of bytes allocated for multi-inserts with system
34+
* catalogs, limiting the number of slots used.
35+
*/
36+
#define MAX_CATALOG_MULTI_INSERT_BYTES 65535
37+
3238
/*
3339
* indexing.c prototypes
3440
*/

0 commit comments

Comments
 (0)