Skip to content

Commit be5e9d0

Browse files
zilderzilder
committed
pathman: resizable dsm segment
1 parent 0dff116 commit be5e9d0

File tree

6 files changed

+100
-70
lines changed

6 files changed

+100
-70
lines changed

‎contrib/pg_pathman/dsm_array.c

Lines changed: 77 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,18 @@
44
#include "storage/lwlock.h"
55
#include <stdint.h>
66

7-
// static Table *table;
7+
88
static dsm_segment *segment = NULL;
9-
static dsm_handle *segment_handle = 0;
10-
static size_t _first_free = 0;
11-
static size_t _block_size = 0;
9+
10+
typedef struct DsmConfig
11+
{
12+
dsm_handle segment_handle;
13+
size_t block_size;
14+
size_t blocks_count;
15+
size_t first_free;
16+
} DsmConfig;
17+
18+
static DsmConfig *dsm_cfg = NULL;
1219

1320
typedef int BlockHeader;
1421
typedef BlockHeader* BlockHeaderPtr;
@@ -29,9 +36,14 @@ void
2936
alloc_dsm_table()
3037
{
3138
bool found;
32-
segment_handle = ShmemInitStruct("dsm table", sizeof(dsm_handle), &found);
39+
dsm_cfg = ShmemInitStruct("dsm config", sizeof(DsmConfig), &found);
3340
if (!found)
34-
*segment_handle = 0;
41+
{
42+
dsm_cfg->segment_handle = 0;
43+
dsm_cfg->block_size = 0;
44+
dsm_cfg->blocks_count = INITIAL_BLOCKS_COUNT;
45+
dsm_cfg->first_free = 0;
46+
}
3547
}
3648

3749

@@ -40,33 +52,35 @@ alloc_dsm_table()
4052
* false if attached to existing segment
4153
*/
4254
bool
43-
init_dsm_segment(size_t block_size)
55+
init_dsm_segment(size_t blocks_count, size_t block_size)
4456
{
4557
bool ret;
4658

4759
/* lock here */
4860
LWLockAcquire(dsm_init_lock, LW_EXCLUSIVE);
4961

5062
/* if there is already an existing segment then attach to it */
51-
if (*segment_handle != 0)
63+
if (dsm_cfg->segment_handle != 0)
5264
{
5365
ret = false;
54-
segment = dsm_attach(*segment_handle);
66+
segment = dsm_attach(dsm_cfg->segment_handle);
5567
}
5668

5769
/*
5870
* If segment hasn't been created yet or has already been destroyed
5971
* (it happens when last session detaches segment) then create new one
6072
*/
61-
if (*segment_handle == 0 || segment == NULL)
73+
if (dsm_cfg->segment_handle == 0 || segment == NULL)
6274
{
6375
/* create segment */
64-
segment = dsm_create(block_size * BLOCKS_COUNT, 0);
65-
*segment_handle = dsm_segment_handle(segment);
66-
init_dsm_table(block_size);
76+
segment = dsm_create(block_size * blocks_count, 0);
77+
dsm_cfg->segment_handle = dsm_segment_handle(segment);
78+
dsm_cfg->first_free = 0;
79+
dsm_cfg->block_size = block_size;
80+
dsm_cfg->blocks_count = blocks_count;
81+
init_dsm_table(block_size, 0, dsm_cfg->blocks_count);
6782
ret = true;
6883
}
69-
_block_size = block_size;
7084

7185
/*
7286
* Keep mapping till the end of the session. Otherwise it would be
@@ -84,20 +98,19 @@ init_dsm_segment(size_t block_size)
8498
* Initialize allocated segment with block structure
8599
*/
86100
void
87-
init_dsm_table(size_t block_size)
101+
init_dsm_table(size_t block_size, size_t start, size_t end)
88102
{
89103
int i;
90104
BlockHeaderPtr header;
91105
char *ptr = dsm_segment_address(segment);
92106

93107
/* create blocks */
94-
for (i=0; i<BLOCKS_COUNT; i++)
108+
for (i=start; i<end; i++)
95109
{
96110
header = (BlockHeaderPtr) &ptr[i * block_size];
97111
*header = set_free(header);
98112
*header = set_length(header, 1);
99113
}
100-
_first_free = 0;
101114

102115
return;
103116
}
@@ -112,30 +125,34 @@ alloc_dsm_array(DsmArray *arr, size_t entry_size, size_t length)
112125
int size_requested = entry_size * length;
113126
int min_pos = 0;
114127
int max_pos = 0;
115-
size_t offset = 0;
128+
bool found = false;
129+
bool collecting_blocks = false;
130+
size_t offset = -1;
116131
size_t total_length = 0;
117-
char *ptr = dsm_segment_address(segment);
118132
BlockHeaderPtr header;
133+
char *ptr = dsm_segment_address(segment);
119134

120-
for (i = _first_free; i<BLOCKS_COUNT; )
135+
for (i = dsm_cfg->first_free; i<dsm_cfg->blocks_count; )
121136
{
122-
header = (BlockHeaderPtr) &ptr[i * _block_size];
137+
header = (BlockHeaderPtr) &ptr[i * dsm_cfg->block_size];
123138
if (is_free(header))
124139
{
125-
if (!offset)
140+
if (!collecting_blocks)
126141
{
127-
offset = i * _block_size;
128-
total_length = _block_size - sizeof(BlockHeader);
142+
offset = i * dsm_cfg->block_size;
143+
total_length = dsm_cfg->block_size - sizeof(BlockHeader);
129144
min_pos = i;
145+
collecting_blocks = true;
130146
}
131147
else
132148
{
133-
total_length += _block_size;
149+
total_length += dsm_cfg->block_size;
134150
}
135151
i++;
136152
}
137153
else
138154
{
155+
collecting_blocks = false;
139156
offset = 0;
140157
total_length = 0;
141158
i += get_length(header);
@@ -144,29 +161,49 @@ alloc_dsm_array(DsmArray *arr, size_t entry_size, size_t length)
144161
if (total_length >= size_requested)
145162
{
146163
max_pos = i-1;
164+
found = true;
147165
break;
148166
}
149167
}
150168

169+
/*
170+
* If dsm segment size is not enough then resize it (or allocate bigger
171+
* for segment SysV and Windows, not implemented yet)
172+
*/
173+
if (!found)
174+
{
175+
size_t new_blocks_count = dsm_cfg->blocks_count * 2;
176+
177+
dsm_resize(segment, new_blocks_count * dsm_cfg->block_size);
178+
init_dsm_table(dsm_cfg->block_size, dsm_cfg->blocks_count, new_blocks_count);
179+
dsm_cfg->blocks_count = new_blocks_count;
180+
181+
/* try again */
182+
return alloc_dsm_array(arr, entry_size, length);
183+
}
184+
151185
/* look up for first free block */
152-
for (; i<BLOCKS_COUNT; )
186+
if (dsm_cfg->first_free == min_pos)
153187
{
154-
header = (BlockHeaderPtr) &ptr[i * _block_size];
155-
if (is_free(header))
156-
{
157-
_first_free = i;
158-
break;
159-
}
160-
else
188+
for (; i<dsm_cfg->blocks_count; )
161189
{
162-
i += get_length(header);
190+
header = (BlockHeaderPtr) &ptr[i * dsm_cfg->block_size];
191+
if (is_free(header))
192+
{
193+
dsm_cfg->first_free = i;
194+
break;
195+
}
196+
else
197+
{
198+
i += get_length(header);
199+
}
163200
}
164201
}
165202

166203
/* if we found enough of space */
167204
if (total_length >= size_requested)
168205
{
169-
header = (BlockHeaderPtr) &ptr[min_pos * _block_size];
206+
header = (BlockHeaderPtr) &ptr[min_pos * dsm_cfg->block_size];
170207
*header = set_used(header);
171208
*header = set_length(header, max_pos - min_pos + 1);
172209

@@ -178,22 +215,22 @@ alloc_dsm_array(DsmArray *arr, size_t entry_size, size_t length)
178215
void
179216
free_dsm_array(DsmArray *arr)
180217
{
181-
int start = arr->offset / _block_size;
218+
int start = arr->offset / dsm_cfg->block_size;
182219
int i = 0;
183220
char *ptr = dsm_segment_address(segment);
184-
BlockHeaderPtr header = (BlockHeaderPtr) &ptr[start * _block_size];
221+
BlockHeaderPtr header = (BlockHeaderPtr) &ptr[start * dsm_cfg->block_size];
185222
size_t blocks_count = get_length(header);
186223

187224
/* set blocks free */
188225
for(; i < blocks_count; i++)
189226
{
190-
header = (BlockHeaderPtr) &ptr[(start + i) * _block_size];
227+
header = (BlockHeaderPtr) &ptr[(start + i) * dsm_cfg->block_size];
191228
*header = set_free(header);
192229
*header = set_length(header, 1);
193230
}
194231

195-
if (start < _first_free)
196-
_first_free = start;
232+
if (start < dsm_cfg->first_free)
233+
dsm_cfg->first_free = start;
197234

198235
arr->offset = 0;
199236
arr->length = 0;

‎contrib/pg_pathman/expected/pg_pathman.out

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,25 @@ CREATE TABLE test.range_rel (
1818
dt TIMESTAMP,
1919
txt TEXT);
2020
CREATE INDEX ON test.range_rel (dt);
21+
INSERT INTO test.range_rel (dt, txt)
22+
SELECT g, md5(g::TEXT) FROM generate_series('2015-01-01', '2015-04-30', '1 day'::interval) as g;
2123
SELECT pathman.create_range_partitions('test.range_rel', 'dt', '2015-01-01'::DATE, '1 month'::INTERVAL, 3);
2224
NOTICE: sequence "range_rel_seq" does not exist, skipping
2325
create_range_partitions
2426
-------------------------
2527
4
2628
(1 row)
2729

30+
SELECT pathman.partition_data('test.range_rel');
31+
NOTICE: Copying data to test.range_rel_4 (condition: ((dt >= 'Wed Apr 01 00:00:00 2015'::timestamp without time zone) AND (dt < 'Fri May 01 00:00:00 2015'::timestamp without time zone)))
32+
NOTICE: Copying data to test.range_rel_3 (condition: ((dt >= 'Sun Mar 01 00:00:00 2015'::timestamp without time zone) AND (dt < 'Wed Apr 01 00:00:00 2015'::timestamp without time zone)))
33+
NOTICE: Copying data to test.range_rel_2 (condition: ((dt >= 'Sun Feb 01 00:00:00 2015'::timestamp without time zone) AND (dt < 'Sun Mar 01 00:00:00 2015'::timestamp without time zone)))
34+
NOTICE: Copying data to test.range_rel_1 (condition: ((dt >= 'Thu Jan 01 00:00:00 2015'::timestamp without time zone) AND (dt < 'Sun Feb 01 00:00:00 2015'::timestamp without time zone)))
35+
partition_data
36+
----------------
37+
0
38+
(1 row)
39+
2840
CREATE TABLE test.num_range_rel (
2941
id SERIAL PRIMARY KEY,
3042
txt TEXT);
@@ -37,8 +49,6 @@ NOTICE: sequence "num_range_rel_seq" does not exist, skipping
3749

3850
INSERT INTO test.num_range_rel
3951
SELECT g, md5(g::TEXT) FROM generate_series(1, 3000) as g;
40-
INSERT INTO test.range_rel (dt, txt)
41-
SELECT g, md5(g::TEXT) FROM generate_series('2015-01-01', '2015-04-30', '1 day'::interval) as g;
4252
INSERT INTO test.hash_rel VALUES (1, 1);
4353
INSERT INTO test.hash_rel VALUES (2, 2);
4454
INSERT INTO test.hash_rel VALUES (3, 3);

‎contrib/pg_pathman/init.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ load_config(void)
3131
bool new_segment_created;
3232

3333
initialization_needed = false;
34-
new_segment_created = init_dsm_segment(32);
34+
new_segment_created = init_dsm_segment(INITIAL_BLOCKS_COUNT, 32);
3535

3636
LWLockAcquire(load_config_lock, LW_EXCLUSIVE);
3737
load_relations_hashtable(new_segment_created);
@@ -208,7 +208,7 @@ load_check_constraints(Oid parent_oid)
208208
{
209209
SPITupleTable *tuptable = SPI_tuptable;
210210
Oid *children;
211-
RangeEntry *ranges = NULL;
211+
RangeEntry *ranges;
212212
Datum min;
213213
Datum max;
214214
int hash;

‎contrib/pg_pathman/pathman.h

Lines changed: 3 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111

1212
#define ALL NIL
13-
#define BLOCKS_COUNT 10240
13+
#define INITIAL_BLOCKS_COUNT 8192
1414

1515
/*
1616
* Partitioning type
@@ -31,22 +31,6 @@ typedef struct DsmArray
3131
size_t length;
3232
} DsmArray;
3333

34-
typedef struct Block
35-
{
36-
dsm_handle segment;
37-
size_t offset;
38-
bool is_free;
39-
} Block;
40-
41-
typedef struct Table
42-
{
43-
dsm_handle segment_handle;
44-
Block blocks[BLOCKS_COUNT];
45-
size_t block_size;
46-
size_t first_free;
47-
} Table;
48-
49-
5034
/*
5135
* PartRelationInfo
5236
* Per-relation partitioning information
@@ -138,8 +122,8 @@ LWLock *dsm_init_lock;
138122

139123
/* Dynamic shared memory functions */
140124
void alloc_dsm_table(void);
141-
bool init_dsm_segment(size_t block_size);
142-
void init_dsm_table(size_t block_size);
125+
bool init_dsm_segment(size_t blocks_count, size_t block_size);
126+
void init_dsm_table(size_t block_size, size_t start, size_t end);
143127
void alloc_dsm_array(DsmArray *arr, size_t entry_size, size_t length);
144128
void free_dsm_array(DsmArray *arr);
145129
void *dsm_array_get_pointer(const DsmArray* arr);

‎contrib/pg_pathman/sql/init.sql

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,21 +48,20 @@ DECLARE
4848
rec RECORD;
4949
BEGIN
5050
p_parent := @extschema@.validate_relname(p_parent);
51-
FOR rec IN (SELECT child.relname, pg_constraint.consrc
51+
FOR rec IN (SELECT inhrelid as child_id, pg_constraint.consrc
5252
FROM @extschema@.pathman_config as cfg
5353
JOIN pg_class AS parent ON parent.relfilenode = cfg.relname::regclass::oid
5454
JOIN pg_inherits ON inhparent = parent.relfilenode
5555
JOIN pg_constraint ON conrelid = inhrelid AND contype='c'
56-
JOIN pg_class AS child ON child.relfilenode = inhrelid
5756
WHERE cfg.relname = p_parent)
5857
LOOP
59-
RAISE NOTICE 'Copying data to % (condition: %)', rec.relname, rec.consrc;
58+
RAISE NOTICE 'Copying data to % (condition: %)', rec.child_id::regclass::text, rec.consrc;
6059
EXECUTE format('WITH part_data AS (
6160
DELETE FROM ONLY %s WHERE %s RETURNING *)
6261
INSERT INTO %s SELECT * FROM part_data'
6362
, p_parent
6463
, rec.consrc
65-
, rec.relname);
64+
, rec.child_id::regclass::text);
6665
END LOOP;
6766
RETURN 0;
6867
END

‎contrib/pg_pathman/sql/pg_pathman.sql

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,10 @@ CREATE TABLE test.range_rel (
1414
dt TIMESTAMP,
1515
txt TEXT);
1616
CREATE INDEX ON test.range_rel (dt);
17+
INSERT INTO test.range_rel (dt, txt)
18+
SELECT g, md5(g::TEXT) FROM generate_series('2015-01-01', '2015-04-30', '1 day'::interval) as g;
1719
SELECT pathman.create_range_partitions('test.range_rel', 'dt', '2015-01-01'::DATE, '1 month'::INTERVAL, 3);
20+
SELECT pathman.partition_data('test.range_rel');
1821

1922
CREATE TABLE test.num_range_rel (
2023
id SERIAL PRIMARY KEY,
@@ -24,9 +27,6 @@ SELECT pathman.create_range_partitions('test.num_range_rel', 'id', 0, 1000, 3);
2427
INSERT INTO test.num_range_rel
2528
SELECT g, md5(g::TEXT) FROM generate_series(1, 3000) as g;
2629

27-
INSERT INTO test.range_rel (dt, txt)
28-
SELECT g, md5(g::TEXT) FROM generate_series('2015-01-01', '2015-04-30', '1 day'::interval) as g;
29-
3030
INSERT INTO test.hash_rel VALUES (1, 1);
3131
INSERT INTO test.hash_rel VALUES (2, 2);
3232
INSERT INTO test.hash_rel VALUES (3, 3);

0 commit comments

Comments
 (0)