Skip to content

Commit 678a85a

Browse files
vbwagnervbwagner
committed
Merge branch 'pathman_pgpro9_5' into PGPRO9_5
Shared memory handling improvement in the pathman
2 parents a15127b + be5e9d0 commit 678a85a

File tree

9 files changed

+167
-93
lines changed

9 files changed

+167
-93
lines changed

‎contrib/pg_pathman/dsm_array.c

Lines changed: 125 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,46 @@
44
#include "storage/lwlock.h"
55
#include <stdint.h>
66

7-
static Table *table;
7+
88
static dsm_segment *segment = NULL;
99

10+
typedef struct DsmConfig
11+
{
12+
dsm_handle segment_handle;
13+
size_t block_size;
14+
size_t blocks_count;
15+
size_t first_free;
16+
} DsmConfig;
17+
18+
static DsmConfig *dsm_cfg = NULL;
19+
20+
typedef int BlockHeader;
21+
typedef BlockHeader* BlockHeaderPtr;
22+
23+
#define FREE_BIT 0x80000000
24+
#define is_free(header) \
25+
((*header) & FREE_BIT)
26+
#define set_free(header) \
27+
((*header) | FREE_BIT)
28+
#define set_used(header) \
29+
((*header) & ~FREE_BIT)
30+
#define get_length(header) \
31+
((*header) & ~FREE_BIT)
32+
#define set_length(header, length) \
33+
((length) | ((*header) & FREE_BIT))
1034

1135
void
1236
alloc_dsm_table()
1337
{
1438
bool found;
15-
table = (Table *) ShmemInitStruct("dsm table", sizeof(Table), &found);
39+
dsm_cfg = ShmemInitStruct("dsm config", sizeof(DsmConfig), &found);
1640
if (!found)
17-
table->segment_handle = 0;
41+
{
42+
dsm_cfg->segment_handle = 0;
43+
dsm_cfg->block_size = 0;
44+
dsm_cfg->blocks_count = INITIAL_BLOCKS_COUNT;
45+
dsm_cfg->first_free = 0;
46+
}
1847
}
1948

2049

@@ -23,31 +52,33 @@ alloc_dsm_table()
2352
* false if attached to existing segment
2453
*/
2554
bool
26-
init_dsm_segment(size_t block_size)
55+
init_dsm_segment(size_t blocks_count, size_t block_size)
2756
{
2857
bool ret;
29-
dsm_handle handle;
3058

3159
/* lock here */
3260
LWLockAcquire(dsm_init_lock, LW_EXCLUSIVE);
3361

3462
/* if there is already an existing segment then attach to it */
35-
if (table->segment_handle != 0)
63+
if (dsm_cfg->segment_handle != 0)
3664
{
3765
ret = false;
38-
segment = dsm_attach(table->segment_handle);
66+
segment = dsm_attach(dsm_cfg->segment_handle);
3967
}
4068

4169
/*
4270
* If segment hasn't been created yet or has already been destroyed
4371
* (it happens when last session detaches segment) then create new one
4472
*/
45-
if (table->segment_handle == 0 || segment == NULL)
73+
if (dsm_cfg->segment_handle == 0 || segment == NULL)
4674
{
4775
/* create segment */
48-
segment = dsm_create(block_size * BLOCKS_COUNT, 0);
49-
handle = dsm_segment_handle(segment);
50-
init_dsm_table(table, handle, block_size);
76+
segment = dsm_create(block_size * blocks_count, 0);
77+
dsm_cfg->segment_handle = dsm_segment_handle(segment);
78+
dsm_cfg->first_free = 0;
79+
dsm_cfg->block_size = block_size;
80+
dsm_cfg->blocks_count = blocks_count;
81+
init_dsm_table(block_size, 0, dsm_cfg->blocks_count);
5182
ret = true;
5283
}
5384

@@ -63,24 +94,22 @@ init_dsm_segment(size_t block_size)
6394
return ret;
6495
}
6596

97+
/*
98+
* Initialize allocated segment with block structure
99+
*/
66100
void
67-
init_dsm_table(Table *tbl, dsm_handle h, size_t block_size)
101+
init_dsm_table(size_t block_size, size_t start, size_t end)
68102
{
69103
int i;
70-
Block *block;
71-
72-
memset(table, 0, sizeof(Table));
73-
table->segment_handle = h;
74-
table->block_size = block_size;
75-
table->first_free = 0;
104+
BlockHeaderPtr header;
105+
char *ptr = dsm_segment_address(segment);
76106

77107
/* create blocks */
78-
for (i=0; i<BLOCKS_COUNT; i++)
108+
for (i=start; i<end; i++)
79109
{
80-
block = &table->blocks[i];
81-
block->segment = h;
82-
block->offset = i * block_size;
83-
block->is_free = true;
110+
header = (BlockHeaderPtr) &ptr[i * block_size];
111+
*header = set_free(header);
112+
*header = set_length(header, 1);
84113
}
85114

86115
return;
@@ -93,71 +122,115 @@ void
93122
alloc_dsm_array(DsmArray *arr, size_t entry_size, size_t length)
94123
{
95124
int i = 0;
96-
Block *block = NULL;
97-
int free_count = 0;
98125
int size_requested = entry_size * length;
99126
int min_pos = 0;
100127
int max_pos = 0;
101-
102-
for (i = table->first_free; i<BLOCKS_COUNT; i++)
128+
bool found = false;
129+
bool collecting_blocks = false;
130+
size_t offset = -1;
131+
size_t total_length = 0;
132+
BlockHeaderPtr header;
133+
char *ptr = dsm_segment_address(segment);
134+
135+
for (i = dsm_cfg->first_free; i<dsm_cfg->blocks_count; )
103136
{
104-
if (table->blocks[i].is_free)
137+
header = (BlockHeaderPtr) &ptr[i * dsm_cfg->block_size];
138+
if (is_free(header))
105139
{
106-
if (!block)
140+
if (!collecting_blocks)
107141
{
108-
block = &table->blocks[i];
142+
offset = i * dsm_cfg->block_size;
143+
total_length = dsm_cfg->block_size - sizeof(BlockHeader);
109144
min_pos = i;
145+
collecting_blocks = true;
146+
}
147+
else
148+
{
149+
total_length += dsm_cfg->block_size;
110150
}
111-
free_count++;
151+
i++;
112152
}
113153
else
114154
{
115-
free_count = 0;
116-
block = NULL;
155+
collecting_blocks = false;
156+
offset = 0;
157+
total_length = 0;
158+
i += get_length(header);
117159
}
118160

119-
if (free_count * table->block_size >= size_requested)
161+
if (total_length >= size_requested)
120162
{
121-
// return block->offset;
122-
max_pos = i;
163+
max_pos = i-1;
164+
found = true;
123165
break;
124166
}
125167
}
126168

169+
/*
170+
* If dsm segment size is not enough then resize it (or allocate bigger
171+
* for segment SysV and Windows, not implemented yet)
172+
*/
173+
if (!found)
174+
{
175+
size_t new_blocks_count = dsm_cfg->blocks_count * 2;
176+
177+
dsm_resize(segment, new_blocks_count * dsm_cfg->block_size);
178+
init_dsm_table(dsm_cfg->block_size, dsm_cfg->blocks_count, new_blocks_count);
179+
dsm_cfg->blocks_count = new_blocks_count;
180+
181+
/* try again */
182+
return alloc_dsm_array(arr, entry_size, length);
183+
}
184+
127185
/* look up for first free block */
128-
for (i = i+1; i<BLOCKS_COUNT; i++)
129-
if (table->blocks[i].is_free == true)
186+
if (dsm_cfg->first_free == min_pos)
187+
{
188+
for (; i<dsm_cfg->blocks_count; )
130189
{
131-
table->first_free = i;
132-
break;
190+
header = (BlockHeaderPtr) &ptr[i * dsm_cfg->block_size];
191+
if (is_free(header))
192+
{
193+
dsm_cfg->first_free = i;
194+
break;
195+
}
196+
else
197+
{
198+
i += get_length(header);
199+
}
133200
}
201+
}
134202

135203
/* if we found enough of space */
136-
if (free_count * table->block_size >= size_requested)
204+
if (total_length >= size_requested)
137205
{
138-
for(i=min_pos; i<=max_pos; i++)
139-
table->blocks[i].is_free = false;
140-
arr->offset = block->offset;
206+
header = (BlockHeaderPtr) &ptr[min_pos * dsm_cfg->block_size];
207+
*header = set_used(header);
208+
*header = set_length(header, max_pos - min_pos + 1);
209+
210+
arr->offset = offset;
141211
arr->length = length;
142212
}
143213
}
144214

145215
void
146216
free_dsm_array(DsmArray *arr)
147217
{
148-
int start = arr->offset / table->block_size;
218+
int start = arr->offset / dsm_cfg->block_size;
149219
int i = 0;
220+
char *ptr = dsm_segment_address(segment);
221+
BlockHeaderPtr header = (BlockHeaderPtr) &ptr[start * dsm_cfg->block_size];
222+
size_t blocks_count = get_length(header);
150223

151224
/* set blocks free */
152-
for(;; i++)
225+
for(; i < blocks_count; i++)
153226
{
154-
table->blocks[start + i].is_free = true;
155-
if (i * table->block_size >= arr->length)
156-
break;
227+
header = (BlockHeaderPtr) &ptr[(start + i) * dsm_cfg->block_size];
228+
*header = set_free(header);
229+
*header = set_length(header, 1);
157230
}
158231

159-
if (arr->offset < table->first_free)
160-
table->first_free = arr->offset;
232+
if (start < dsm_cfg->first_free)
233+
dsm_cfg->first_free = start;
161234

162235
arr->offset = 0;
163236
arr->length = 0;
@@ -166,5 +239,5 @@ free_dsm_array(DsmArray *arr)
166239
void *
167240
dsm_array_get_pointer(const DsmArray* arr)
168241
{
169-
return (uint8_t *) dsm_segment_address(segment) + arr->offset;
242+
return (char *) dsm_segment_address(segment) + arr->offset + sizeof(BlockHeader);
170243
}

‎contrib/pg_pathman/expected/pg_pathman.out

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,19 +10,31 @@ NOTICE: function test.hash_rel_hash_insert_trigger_func() does not exist, skipp
1010
NOTICE: function test.hash_rel_hash_update_trigger_func() does not exist, skipping
1111
create_hash_partitions
1212
------------------------
13-
13+
3
1414
(1 row)
1515

1616
CREATE TABLE test.range_rel (
1717
id SERIAL PRIMARY KEY,
1818
dt TIMESTAMP,
1919
txt TEXT);
2020
CREATE INDEX ON test.range_rel (dt);
21+
INSERT INTO test.range_rel (dt, txt)
22+
SELECT g, md5(g::TEXT) FROM generate_series('2015-01-01', '2015-04-30', '1 day'::interval) as g;
2123
SELECT pathman.create_range_partitions('test.range_rel', 'dt', '2015-01-01'::DATE, '1 month'::INTERVAL, 3);
2224
NOTICE: sequence "range_rel_seq" does not exist, skipping
2325
create_range_partitions
2426
-------------------------
25-
27+
4
28+
(1 row)
29+
30+
SELECT pathman.partition_data('test.range_rel');
31+
NOTICE: Copying data to test.range_rel_4 (condition: ((dt >= 'Wed Apr 01 00:00:00 2015'::timestamp without time zone) AND (dt < 'Fri May 01 00:00:00 2015'::timestamp without time zone)))
32+
NOTICE: Copying data to test.range_rel_3 (condition: ((dt >= 'Sun Mar 01 00:00:00 2015'::timestamp without time zone) AND (dt < 'Wed Apr 01 00:00:00 2015'::timestamp without time zone)))
33+
NOTICE: Copying data to test.range_rel_2 (condition: ((dt >= 'Sun Feb 01 00:00:00 2015'::timestamp without time zone) AND (dt < 'Sun Mar 01 00:00:00 2015'::timestamp without time zone)))
34+
NOTICE: Copying data to test.range_rel_1 (condition: ((dt >= 'Thu Jan 01 00:00:00 2015'::timestamp without time zone) AND (dt < 'Sun Feb 01 00:00:00 2015'::timestamp without time zone)))
35+
partition_data
36+
----------------
37+
0
2638
(1 row)
2739

2840
CREATE TABLE test.num_range_rel (
@@ -32,13 +44,11 @@ SELECT pathman.create_range_partitions('test.num_range_rel', 'id', 0, 1000, 3);
3244
NOTICE: sequence "num_range_rel_seq" does not exist, skipping
3345
create_range_partitions
3446
-------------------------
35-
47+
4
3648
(1 row)
3749

3850
INSERT INTO test.num_range_rel
3951
SELECT g, md5(g::TEXT) FROM generate_series(1, 3000) as g;
40-
INSERT INTO test.range_rel (dt, txt)
41-
SELECT g, md5(g::TEXT) FROM generate_series('2015-01-01', '2015-04-30', '1 day'::interval) as g;
4252
INSERT INTO test.hash_rel VALUES (1, 1);
4353
INSERT INTO test.hash_rel VALUES (2, 2);
4454
INSERT INTO test.hash_rel VALUES (3, 3);
@@ -449,7 +459,7 @@ NOTICE: function hash_rel_hash_insert_trigger_func() does not exist, skipping
449459
NOTICE: function hash_rel_hash_update_trigger_func() does not exist, skipping
450460
create_hash_partitions
451461
------------------------
452-
462+
3
453463
(1 row)
454464

455465
SELECT partition_data('hash_rel');
@@ -482,7 +492,7 @@ SELECT create_range_partitions('range_rel', 'dt', '2010-01-01'::date, '1 month':
482492
NOTICE: sequence "range_rel_seq" does not exist, skipping
483493
create_range_partitions
484494
-------------------------
485-
495+
13
486496
(1 row)
487497

488498
SELECT partition_data('range_rel');

‎contrib/pg_pathman/init.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ load_config(void)
3131
bool new_segment_created;
3232

3333
initialization_needed = false;
34-
new_segment_created = init_dsm_segment(32);
34+
new_segment_created = init_dsm_segment(INITIAL_BLOCKS_COUNT, 32);
3535

3636
LWLockAcquire(load_config_lock, LW_EXCLUSIVE);
3737
load_relations_hashtable(new_segment_created);
@@ -208,7 +208,7 @@ load_check_constraints(Oid parent_oid)
208208
{
209209
SPITupleTable *tuptable = SPI_tuptable;
210210
Oid *children;
211-
RangeEntry *ranges = NULL;
211+
RangeEntry *ranges;
212212
Datum min;
213213
Datum max;
214214
int hash;

0 commit comments

Comments
 (0)