Skip to content

Commit db448ce

Browse files
david-rowleydavid-rowley
committed
Optimize alignment calculations in tuple form/deform
Here we convert CompactAttribute.attalign from a char, which is directly derived from pg_attribute.attalign into a uint8, which stores the number of bytes to align the column's value by in the tuple. This allows tuple deformation and tuple size calculations to move away from using the inefficient att_align_nominal() macro, which manually checks each TYPALIGN_* char to translate that into the alignment bytes for the given type. Effectively, this commit changes those to TYPEALIGN calls, which are branchless and only perform some simple arithmetic with some bit-twiddling. The removed branches were often mispredicted by CPUs, especially so in real-world tables which often contain a mishmash of different types with different alignment requirements. Author: David Rowley Reviewed-by: Andres Freund, Victor Yegorov Discussion: https://postgr.es/m/CAApHDvrBztXP3yx=NKNmo3xwFAFhEdyPnvrDg3=M0RhDs+4vYw@mail.gmail.com
1 parent 1f81b48 commit db448ce

File tree

12 files changed

+110
-80
lines changed

12 files changed

+110
-80
lines changed

‎contrib/amcheck/verify_heapam.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1596,7 +1596,7 @@ check_tuple_attribute(HeapCheckContext *ctx)
15961596
/* Skip non-varlena values, but update offset first */
15971597
if (thisatt->attlen != -1)
15981598
{
1599-
ctx->offset = att_align_nominal(ctx->offset, thisatt->attalign);
1599+
ctx->offset = att_nominal_alignby(ctx->offset, thisatt->attalignby);
16001600
ctx->offset = att_addlength_pointer(ctx->offset, thisatt->attlen,
16011601
tp + ctx->offset);
16021602
if (ctx->tuphdr->t_hoff + ctx->offset > ctx->lp_len)
@@ -1612,8 +1612,8 @@ check_tuple_attribute(HeapCheckContext *ctx)
16121612
}
16131613

16141614
/* Ok, we're looking at a varlena attribute. */
1615-
ctx->offset = att_align_pointer(ctx->offset, thisatt->attalign, -1,
1616-
tp + ctx->offset);
1615+
ctx->offset = att_pointer_alignby(ctx->offset, thisatt->attalignby, -1,
1616+
tp + ctx->offset);
16171617

16181618
/* Get the (possibly corrupt) varlena datum */
16191619
attdatum = fetchatt(thisatt, tp + ctx->offset);

‎contrib/pageinspect/heapfuncs.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -357,8 +357,8 @@ tuple_data_split_internal(Oid relid, char *tupdata,
357357

358358
if (attr->attlen == -1)
359359
{
360-
off = att_align_pointer(off, attr->attalign, -1,
361-
tupdata + off);
360+
off = att_pointer_alignby(off, attr->attalignby, -1,
361+
tupdata + off);
362362

363363
/*
364364
* As VARSIZE_ANY throws an exception if it can't properly
@@ -376,7 +376,7 @@ tuple_data_split_internal(Oid relid, char *tupdata,
376376
}
377377
else
378378
{
379-
off = att_align_nominal(off, attr->attalign);
379+
off = att_nominal_alignby(off, attr->attalignby);
380380
len = attr->attlen;
381381
}
382382

‎src/backend/access/brin/brin_tuple.c

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -703,13 +703,15 @@ brin_deconstruct_tuple(BrinDesc *brdesc,
703703

704704
if (thisatt->attlen == -1)
705705
{
706-
off = att_align_pointer(off, thisatt->attalign, -1,
707-
tp + off);
706+
off = att_pointer_alignby(off,
707+
thisatt->attalignby,
708+
-1,
709+
tp + off);
708710
}
709711
else
710712
{
711-
/* not varlena, so safe to use att_align_nominal */
712-
off = att_align_nominal(off, thisatt->attalign);
713+
/* not varlena, so safe to use att_nominal_alignby */
714+
off = att_nominal_alignby(off, thisatt->attalignby);
713715
}
714716

715717
values[stored++] = fetchatt(thisatt, tp + off);

‎src/backend/access/common/attmap.c

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -315,13 +315,13 @@ check_attrmap_match(TupleDesc indesc,
315315

316316
/*
317317
* If it's a dropped column and the corresponding input column is also
318-
* dropped, we don't need a conversion. However, attlen and attalign
319-
* must agree.
318+
* dropped, we don't need a conversion. However, attlen and
319+
* attalignby must agree.
320320
*/
321321
if (attrMap->attnums[i] == 0 &&
322322
inatt->attisdropped &&
323323
inatt->attlen == outatt->attlen &&
324-
inatt->attalign == outatt->attalign)
324+
inatt->attalignby == outatt->attalignby)
325325
continue;
326326

327327
return false;

‎src/backend/access/common/heaptuple.c

Lines changed: 25 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -251,13 +251,13 @@ heap_compute_data_size(TupleDesc tupleDesc,
251251
* we want to flatten the expanded value so that the constructed
252252
* tuple doesn't depend on it
253253
*/
254-
data_length = att_align_nominal(data_length, atti->attalign);
254+
data_length = att_nominal_alignby(data_length, atti->attalignby);
255255
data_length += EOH_get_flat_size(DatumGetEOHP(val));
256256
}
257257
else
258258
{
259-
data_length = att_align_datum(data_length, atti->attalign,
260-
atti->attlen, val);
259+
data_length = att_datum_alignby(data_length, atti->attalignby,
260+
atti->attlen, val);
261261
data_length = att_addlength_datum(data_length, atti->attlen,
262262
val);
263263
}
@@ -308,13 +308,13 @@ fill_val(CompactAttribute *att,
308308
}
309309

310310
/*
311-
* XXX we use the att_align macros on the pointer value itself, not on an
312-
* offset. This is a bit of a hack.
311+
* XXX we use the att_nominal_alignby macro on the pointer value itself,
312+
* not on an offset. This is a bit of a hack.
313313
*/
314314
if (att->attbyval)
315315
{
316316
/* pass-by-value */
317-
data = (char *) att_align_nominal(data, att->attalign);
317+
data = (char *) att_nominal_alignby(data, att->attalignby);
318318
store_att_byval(data, datum, att->attlen);
319319
data_length = att->attlen;
320320
}
@@ -334,8 +334,7 @@ fill_val(CompactAttribute *att,
334334
*/
335335
ExpandedObjectHeader *eoh = DatumGetEOHP(datum);
336336

337-
data = (char *) att_align_nominal(data,
338-
att->attalign);
337+
data = (char *) att_nominal_alignby(data, att->attalignby);
339338
data_length = EOH_get_flat_size(eoh);
340339
EOH_flatten_into(eoh, data, data_length);
341340
}
@@ -363,8 +362,7 @@ fill_val(CompactAttribute *att,
363362
else
364363
{
365364
/* full 4-byte header varlena */
366-
data = (char *) att_align_nominal(data,
367-
att->attalign);
365+
data = (char *) att_nominal_alignby(data, att->attalignby);
368366
data_length = VARSIZE(val);
369367
memcpy(data, val, data_length);
370368
}
@@ -373,14 +371,14 @@ fill_val(CompactAttribute *att,
373371
{
374372
/* cstring ... never needs alignment */
375373
*infomask |= HEAP_HASVARWIDTH;
376-
Assert(att->attalign == TYPALIGN_CHAR);
374+
Assert(att->attalignby == sizeof(char));
377375
data_length = strlen(DatumGetCString(datum)) + 1;
378376
memcpy(data, DatumGetPointer(datum), data_length);
379377
}
380378
else
381379
{
382380
/* fixed-length pass-by-reference */
383-
data = (char *) att_align_nominal(data, att->attalign);
381+
data = (char *) att_nominal_alignby(data, att->attalignby);
384382
Assert(att->attlen > 0);
385383
data_length = att->attlen;
386384
memcpy(data, DatumGetPointer(datum), data_length);
@@ -634,7 +632,7 @@ nocachegetattr(HeapTuple tup,
634632
if (att->attlen <= 0)
635633
break;
636634

637-
off = att_align_nominal(off, att->attalign);
635+
off = att_nominal_alignby(off, att->attalignby);
638636

639637
att->attcacheoff = off;
640638

@@ -683,19 +681,19 @@ nocachegetattr(HeapTuple tup,
683681
* either an aligned or unaligned value.
684682
*/
685683
if (usecache &&
686-
off == att_align_nominal(off, att->attalign))
684+
off == att_nominal_alignby(off, att->attalignby))
687685
att->attcacheoff = off;
688686
else
689687
{
690-
off = att_align_pointer(off, att->attalign, -1,
691-
tp + off);
688+
off = att_pointer_alignby(off, att->attalignby, -1,
689+
tp + off);
692690
usecache = false;
693691
}
694692
}
695693
else
696694
{
697-
/* not varlena, so safe to use att_align_nominal */
698-
off = att_align_nominal(off, att->attalign);
695+
/* not varlena, so safe to use att_nominal_alignby */
696+
off = att_nominal_alignby(off, att->attalignby);
699697

700698
if (usecache)
701699
att->attcacheoff = off;
@@ -898,10 +896,10 @@ expand_tuple(HeapTuple *targetHeapTuple,
898896
{
899897
CompactAttribute *att = TupleDescCompactAttr(tupleDesc, attnum);
900898

901-
targetDataLen = att_align_datum(targetDataLen,
902-
att->attalign,
903-
att->attlen,
904-
attrmiss[attnum].am_value);
899+
targetDataLen = att_datum_alignby(targetDataLen,
900+
att->attalignby,
901+
att->attlen,
902+
attrmiss[attnum].am_value);
905903

906904
targetDataLen = att_addlength_pointer(targetDataLen,
907905
att->attlen,
@@ -1396,19 +1394,19 @@ heap_deform_tuple(HeapTuple tuple, TupleDesc tupleDesc,
13961394
* an aligned or unaligned value.
13971395
*/
13981396
if (!slow &&
1399-
off == att_align_nominal(off, thisatt->attalign))
1397+
off == att_nominal_alignby(off, thisatt->attalignby))
14001398
thisatt->attcacheoff = off;
14011399
else
14021400
{
1403-
off = att_align_pointer(off, thisatt->attalign, -1,
1404-
tp + off);
1401+
off = att_pointer_alignby(off, thisatt->attalignby, -1,
1402+
tp + off);
14051403
slow = true;
14061404
}
14071405
}
14081406
else
14091407
{
1410-
/* not varlena, so safe to use att_align_nominal */
1411-
off = att_align_nominal(off, thisatt->attalign);
1408+
/* not varlena, so safe to use att_nominal_alignby */
1409+
off = att_nominal_alignby(off, thisatt->attalignby);
14121410

14131411
if (!slow)
14141412
thisatt->attcacheoff = off;

‎src/backend/access/common/indextuple.c

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -363,7 +363,7 @@ nocache_index_getattr(IndexTuple tup,
363363
if (att->attlen <= 0)
364364
break;
365365

366-
off = att_align_nominal(off, att->attalign);
366+
off = att_nominal_alignby(off, att->attalignby);
367367

368368
att->attcacheoff = off;
369369

@@ -412,19 +412,19 @@ nocache_index_getattr(IndexTuple tup,
412412
* either an aligned or unaligned value.
413413
*/
414414
if (usecache &&
415-
off == att_align_nominal(off, att->attalign))
415+
off == att_nominal_alignby(off, att->attalignby))
416416
att->attcacheoff = off;
417417
else
418418
{
419-
off = att_align_pointer(off, att->attalign, -1,
420-
tp + off);
419+
off = att_pointer_alignby(off, att->attalignby, -1,
420+
tp + off);
421421
usecache = false;
422422
}
423423
}
424424
else
425425
{
426-
/* not varlena, so safe to use att_align_nominal */
427-
off = att_align_nominal(off, att->attalign);
426+
/* not varlena, so safe to use att_nominal_alignby */
427+
off = att_nominal_alignby(off, att->attalignby);
428428

429429
if (usecache)
430430
att->attcacheoff = off;
@@ -513,19 +513,19 @@ index_deform_tuple_internal(TupleDesc tupleDescriptor,
513513
* an aligned or unaligned value.
514514
*/
515515
if (!slow &&
516-
off == att_align_nominal(off, thisatt->attalign))
516+
off == att_nominal_alignby(off, thisatt->attalignby))
517517
thisatt->attcacheoff = off;
518518
else
519519
{
520-
off = att_align_pointer(off, thisatt->attalign, -1,
521-
tp + off);
520+
off = att_pointer_alignby(off, thisatt->attalignby, -1,
521+
tp + off);
522522
slow = true;
523523
}
524524
}
525525
else
526526
{
527-
/* not varlena, so safe to use att_align_nominal */
528-
off = att_align_nominal(off, thisatt->attalign);
527+
/* not varlena, so safe to use att_nominal_alignby */
528+
off = att_nominal_alignby(off, thisatt->attalignby);
529529

530530
if (!slow)
531531
thisatt->attcacheoff = off;

‎src/backend/access/common/tupdesc.c

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,25 @@ populate_compact_attribute(TupleDesc tupdesc, int attnum)
8080
dst->attgenerated = (src->attgenerated != '\0');
8181
dst->attnotnull = src->attnotnull;
8282

83-
dst->attalign = src->attalign;
83+
switch (src->attalign)
84+
{
85+
case TYPALIGN_INT:
86+
dst->attalignby = ALIGNOF_INT;
87+
break;
88+
case TYPALIGN_CHAR:
89+
dst->attalignby = sizeof(char);
90+
break;
91+
case TYPALIGN_DOUBLE:
92+
dst->attalignby = ALIGNOF_DOUBLE;
93+
break;
94+
case TYPALIGN_SHORT:
95+
dst->attalignby = ALIGNOF_SHORT;
96+
break;
97+
default:
98+
dst->attalignby = 0;
99+
elog(ERROR, "invalid attalign value: %c", src->attalign);
100+
break;
101+
}
84102
}
85103

86104
/*

‎src/backend/executor/execExprInterp.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5304,7 +5304,7 @@ ExecEvalWholeRowVar(ExprState *state, ExprEvalStep *op, ExprContext *econtext)
53045304
if (slot->tts_isnull[i])
53055305
continue; /* null is always okay */
53065306
if (vattr->attlen != sattr->attlen ||
5307-
vattr->attalign != sattr->attalign)
5307+
vattr->attalignby != sattr->attalignby)
53085308
ereport(ERROR,
53095309
(errcode(ERRCODE_DATATYPE_MISMATCH),
53105310
errmsg("table row type and query-specified row type do not match"),

‎src/backend/executor/execTuples.c

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -202,12 +202,12 @@ tts_virtual_materialize(TupleTableSlot *slot)
202202
* We want to flatten the expanded value so that the materialized
203203
* slot doesn't depend on it.
204204
*/
205-
sz = att_align_nominal(sz, att->attalign);
205+
sz = att_nominal_alignby(sz, att->attalignby);
206206
sz += EOH_get_flat_size(DatumGetEOHP(val));
207207
}
208208
else
209209
{
210-
sz = att_align_nominal(sz, att->attalign);
210+
sz = att_nominal_alignby(sz, att->attalignby);
211211
sz = att_addlength_datum(sz, att->attlen, val);
212212
}
213213
}
@@ -242,8 +242,8 @@ tts_virtual_materialize(TupleTableSlot *slot)
242242
*/
243243
ExpandedObjectHeader *eoh = DatumGetEOHP(val);
244244

245-
data = (char *) att_align_nominal(data,
246-
att->attalign);
245+
data = (char *) att_nominal_alignby(data,
246+
att->attalignby);
247247
data_length = EOH_get_flat_size(eoh);
248248
EOH_flatten_into(eoh, data, data_length);
249249

@@ -254,7 +254,7 @@ tts_virtual_materialize(TupleTableSlot *slot)
254254
{
255255
Size data_length = 0;
256256

257-
data = (char *) att_align_nominal(data, att->attalign);
257+
data = (char *) att_nominal_alignby(data, att->attalignby);
258258
data_length = att_addlength_datum(data_length, att->attlen, val);
259259

260260
memcpy(data, DatumGetPointer(val), data_length);
@@ -1067,19 +1067,19 @@ slot_deform_heap_tuple(TupleTableSlot *slot, HeapTuple tuple, uint32 *offp,
10671067
* an aligned or unaligned value.
10681068
*/
10691069
if (!slow &&
1070-
off == att_align_nominal(off, thisatt->attalign))
1070+
off == att_nominal_alignby(off, thisatt->attalignby))
10711071
thisatt->attcacheoff = off;
10721072
else
10731073
{
1074-
off = att_align_pointer(off, thisatt->attalign, -1,
1075-
tp + off);
1074+
off = att_pointer_alignby(off, thisatt->attalignby, -1,
1075+
tp + off);
10761076
slow = true;
10771077
}
10781078
}
10791079
else
10801080
{
1081-
/* not varlena, so safe to use att_align_nominal */
1082-
off = att_align_nominal(off, thisatt->attalign);
1081+
/* not varlena, so safe to use att_nominal_alignby */
1082+
off = att_nominal_alignby(off, thisatt->attalignby);
10831083

10841084
if (!slow)
10851085
thisatt->attcacheoff = off;

0 commit comments

Comments
 (0)