Skip to content

Commit 19f6e9e

Browse files
alubennikovaalubennikova
committed
Fix typos, add comments, rename couple of variables
1 parent 410aa2d commit 19f6e9e

File tree

3 files changed

+134
-57
lines changed

3 files changed

+134
-57
lines changed

‎src/backend/storage/file/cfs.c

Lines changed: 118 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,16 @@
1111
*
1212
* NOTES:
1313
*
14-
* This file implemets compression of file pages.
14+
* This file implements compression of file pages.
1515
* Updated compressed pages are always appended to the end of file segment.
16-
* Garbage collector is used to shrink files when them become tool large.
17-
* GC is spawned as one or more background workers. Them recursively traverse all tablespace directories,
18-
* find out *.cfm files are if logical size of the file is twice larger than physical size of the file
19-
* performs compactification. Locking implemented using atomic operations is used to eliminate race
20-
* conditions.
16+
* Garbage collector is used to reclaim storage occupied by outdated versions of pages.
17+
* GC runs one or more background workers which recursively traverse all tablespace
18+
* directories. If worker finds out that logical size of the file is twice as large as
19+
* physical size of the file, it performs compactification.
20+
* To eliminate race conditions, files are locked during compacification.
21+
* Locks are implemented with atomic operations.
22+
*
23+
* TODO Write a note about *.cfm files.
2124
*/
2225

2326
#include "postgres.h"
@@ -51,13 +54,37 @@
5154
#include "utils/resowner_private.h"
5255
#include "postmaster/bgworker.h"
5356

57+
/*
58+
* GUC variable that defines compression level.
59+
* 0 - no compression, 1 - max speed,
60+
* other possible values depend on the specific algorithm.
61+
* Default value is 1.
62+
*/
63+
int cfs_level;
64+
/*
65+
* GUC variable that defines if encryption of compressed pages is enabled.
66+
* Default value is false.
67+
*/
68+
bool cfs_encryption;
69+
/*
70+
* GUC variable - Verify correctness of data written by GC.
71+
* TODO add description and documentation.
72+
*/
73+
bool cfs_gc_verify_file;
74+
75+
/* GUC variable - Number of garbage collection background workers. Default = 1 */
5476
int cfs_gc_workers;
77+
/*
78+
* GUC variable - Specifies the minimum percent of garbage blocks
79+
* needed to trigger a GC of the file. Default = 50
80+
*/
5581
int cfs_gc_threshold;
82+
/* GUC variable - Time to sleep between GC runs in milliseconds. Default = 5000 */
5683
int cfs_gc_period;
84+
/* GUC variable - Delay in milliseconds between files defragmentation. Default = 0
85+
* TODO What is the purpose of this variable?
86+
*/
5787
int cfs_gc_delay;
58-
int cfs_level;
59-
bool cfs_encryption;
60-
bool cfs_gc_verify_file;
6188

6289
static bool cfs_read_file(int fd, void* data, uint32 size);
6390
static bool cfs_write_file(int fd, void const* data, uint32 size);
@@ -68,90 +95,109 @@ CfsState* cfs_state;
6895
static bool cfs_stop;
6996
static int cfs_processed_segments;
7097

71-
#if CFS_COMPRESSOR == SNAPPY_COMPRESSOR
7298

73-
#include <snappy-c.h>
99+
/* ----------------------------------------------------------------
100+
* Section 1: Various compression algorithms.
101+
* CFS_COMPRESSOR variable can be set at compile time.
102+
* One should define CFS_COMPRESSOR in cfs.h
103+
* Availiable options are:
104+
* - LZ_COMPRESSOR // FIXME. No actual implementation.
105+
* - ZLIB_COMPRESSOR
106+
* - LZ4_COMPRESSOR
107+
* - SNAPPY_COMPRESSOR
108+
* - LCFSE_COMPRESSOR
109+
* - ZSTD_COMPRESSOR
110+
*
111+
* If none of options is chosen, use standard pglz_compress FIXME?, which is
112+
* slow and non-efficient in comparison with others, but doesn't requre
113+
* any extra libraries.
114+
* ----------------------------------------------------------------
115+
*/
116+
117+
#if CFS_COMPRESSOR == ZLIB_COMPRESSOR
118+
119+
#include <zlib.h>
74120

75121
size_t cfs_compress(void* dst, size_t dst_size, void const* src, size_t src_size)
76122
{
77-
return snappy_compress(src, src_size, dst, &dst_size) == SNAPPY_OK ? dst_size : 0;
123+
uLongf compressed_size = dst_size;
124+
int rc = compress2(dst, &compressed_size, src, src_size, cfs_level);
125+
return rc == Z_OK ? compressed_size : rc;
78126
}
79127

80128
size_t cfs_decompress(void* dst, size_t dst_size, void const* src, size_t src_size)
81129
{
82-
return snappy_uncompress(src, src_size, dst, &dst_size) == SNAPPY_OK ? dst_size : 0;
130+
uLongf dest_len = dst_size;
131+
int rc = uncompress(dst, &dest_len, src, src_size);
132+
return rc == Z_OK ? dest_len : rc;
83133
}
84134

85135
char const* cfs_algorithm()
86136
{
87-
return "snappy";
137+
return "zlib";
88138
}
89139

90-
#elif CFS_COMPRESSOR == LCFSE_COMPRESSOR
140+
#elif CFS_COMPRESSOR == LZ4_COMPRESSOR
91141

92-
#include <lcfse.h>
142+
#include <lz4.h>
93143

94144
size_t cfs_compress(void* dst, size_t dst_size, void const* src, size_t src_size)
95145
{
96-
char* scratch_buf = palloc(lcfse_encode_scratch_size());
97-
size_t rc = lcfse_encode_buffer(dst, dst_size, src, src_size, scratch_buf);
98-
pfree(scratch_buf);
99-
return rc;
146+
return LZ4_compress(src, dst, src_size);
100147
}
101148

102149
size_t cfs_decompress(void* dst, size_t dst_size, void const* src, size_t src_size)
103150
{
104-
char* scratch_buf = palloc(lcfse_encode_scratch_size());
105-
size_t rc = lcfse_decode_buffer(dst, dst_size, src, src_size, scratch_buf);
106-
pfree(scratch_buf);
107-
return rc;
151+
return LZ4_decompress_safe(src, dst, src_size, dst_size);
108152
}
109153

110154
char const* cfs_algorithm()
111155
{
112-
return "lcfse";
156+
return "lz4";
113157
}
114158

115-
#elif CFS_COMPRESSOR == LZ4_COMPRESSOR
159+
#elif CFS_COMPRESSOR == SNAPPY_COMPRESSOR
116160

117-
#include <lz4.h>
161+
#include <snappy-c.h>
118162

119163
size_t cfs_compress(void* dst, size_t dst_size, void const* src, size_t src_size)
120164
{
121-
return LZ4_compress(src, dst, src_size);
165+
return snappy_compress(src, src_size, dst, &dst_size) == SNAPPY_OK ? dst_size : 0;
122166
}
123167

124168
size_t cfs_decompress(void* dst, size_t dst_size, void const* src, size_t src_size)
125169
{
126-
return LZ4_decompress_safe(src, dst, src_size, dst_size);
170+
return snappy_uncompress(src, src_size, dst, &dst_size) == SNAPPY_OK ? dst_size : 0;
127171
}
128172

129173
char const* cfs_algorithm()
130174
{
131-
return "lz4";
175+
return "snappy";
132176
}
133177

134-
#elif CFS_COMPRESSOR == ZLIB_COMPRESSOR
178+
#elif CFS_COMPRESSOR == LCFSE_COMPRESSOR
135179

136-
#include <zlib.h>
180+
#include <lcfse.h>
137181

138182
size_t cfs_compress(void* dst, size_t dst_size, void const* src, size_t src_size)
139183
{
140-
uLongf compressed_size = dst_size;
141-
int rc = compress2(dst, &compressed_size, src, src_size, cfs_level);
142-
return rc == Z_OK ? compressed_size : rc;
184+
char* scratch_buf = palloc(lcfse_encode_scratch_size());
185+
size_t rc = lcfse_encode_buffer(dst, dst_size, src, src_size, scratch_buf);
186+
pfree(scratch_buf);
187+
return rc;
143188
}
144189

145190
size_t cfs_decompress(void* dst, size_t dst_size, void const* src, size_t src_size)
146191
{
147-
uLongf dest_len = dst_size;
148-
int rc = uncompress(dst, &dest_len, src, src_size);
149-
return rc == Z_OK ? dest_len : rc;
192+
char* scratch_buf = palloc(lcfse_encode_scratch_size());
193+
size_t rc = lcfse_decode_buffer(dst, dst_size, src, src_size, scratch_buf);
194+
pfree(scratch_buf);
195+
return rc;
150196
}
151197

152198
char const* cfs_algorithm()
153199
{
154-
return "zlib";
200+
return "lcfse";
155201
}
156202

157203
#elif CFS_COMPRESSOR == ZSTD_COMPRESSOR
@@ -195,6 +241,15 @@ char const* cfs_algorithm()
195241
#endif
196242

197243

244+
/* ----------------------------------------------------------------
245+
* Section 2: Encryption related functionality.
246+
*
247+
* TODO
248+
* - replace rc4 algrithm with something more appropriate
249+
* - add more comments
250+
* - what does 'offs' variable for?
251+
* ----------------------------------------------------------------
252+
*/
198253
static void cfs_rc4_encrypt_block(void* block, uint32 offs, uint32 block_size)
199254
{
200255
uint32 i;
@@ -205,7 +260,7 @@ static void cfs_rc4_encrypt_block(void* block, uint32 offs, uint32 block_size)
205260
int x = 0, y = 0;
206261
uint32 skip = (offs / BLCKSZ + block_size) % CFS_CIPHER_KEY_SIZE;
207262

208-
memcpy(state, cfs_state->rc4_init_state, CFS_CIPHER_KEY_SIZE);
263+
memcpy(state, cfs_state->cipher_key, CFS_CIPHER_KEY_SIZE);
209264
for (i = 0; i < skip; i++) {
210265
x = (x + 1) % CFS_CIPHER_KEY_SIZE;
211266
y = (y + state[x]) % CFS_CIPHER_KEY_SIZE;
@@ -224,7 +279,12 @@ static void cfs_rc4_encrypt_block(void* block, uint32 offs, uint32 block_size)
224279
}
225280
}
226281

227-
static void cfs_rc4_init(void)
282+
/*
283+
* Get env variable PG_CIPHER_KEY and initialize encryption state.
284+
* Unset variable afterward.
285+
* Now implements cf4.
286+
*/
287+
static void cfs_encrypt_init(void)
228288
{
229289
int index1 = 0;
230290
int index2 = 0;
@@ -233,13 +293,13 @@ static void cfs_rc4_init(void)
233293
int key_length;
234294
int x = 0, y = 0;
235295
char* cipher_key;
236-
uint8* rc4_init_state = cfs_state->rc4_init_state;
296+
uint8* rc4_init_state = cfs_state->cipher_key;
237297

238298
cipher_key = getenv("PG_CIPHER_KEY");
239299
if (cipher_key == NULL) {
240300
elog(ERROR, "PG_CIPHER_KEY environment variable is not set");
241301
}
242-
unsetenv("PG_CIPHER_KEY"); /* make it not possible to inspect this environment variable through plperl */
302+
unsetenv("PG_CIPHER_KEY"); /* disable inspection of this environment variable */
243303
key_length = strlen(cipher_key);
244304
for (i = 0; i < CFS_CIPHER_KEY_SIZE; ++i) {
245305
rc4_init_state[i] = (uint8)i;
@@ -263,20 +323,21 @@ static void cfs_rc4_init(void)
263323
void cfs_encrypt(void* block, uint32 offs, uint32 size)
264324
{
265325
if (cfs_encryption)
266-
{
267326
cfs_rc4_encrypt_block(block, offs, size);
268-
}
269327
}
270328

271329
void cfs_decrypt(void* block, uint32 offs, uint32 size)
272330
{
273331
if (cfs_encryption)
274-
{
275332
cfs_rc4_encrypt_block(block, offs, size);
276-
}
277333
}
278334

279-
335+
/* ----------------------------------------------------------------
336+
* Section 3: Compression implementation.
337+
*
338+
* TODO add description
339+
* ----------------------------------------------------------------
340+
*/
280341
void cfs_initialize()
281342
{
282343
cfs_state = (CfsState*)ShmemAlloc(sizeof(CfsState));
@@ -287,9 +348,9 @@ void cfs_initialize()
287348
cfs_state->gc_enabled = true;
288349
cfs_state->max_iterations = 0;
289350

290-
if (cfs_encryption) {
291-
cfs_rc4_init();
292-
}
351+
if (cfs_encryption)
352+
cfs_encrypt_init();
353+
293354
elog(LOG, "Start CFS version %s compression algorithm %s encryption %s",
294355
CFS_VERSION, cfs_algorithm(), cfs_encryption ? "enabled" : "disabled");
295356
}
@@ -463,6 +524,12 @@ static int cfs_cmp_page_offs(void const* p1, void const* p2)
463524
return o1 < o2 ? -1 : o1 == o2 ? 0 : 1;
464525
}
465526

527+
/* ----------------------------------------------------------------
528+
* Section 4: Garbage collection functionality.
529+
*
530+
* TODO add description. reorder functions.
531+
* ----------------------------------------------------------------
532+
*/
466533
/*
467534
* Perform garbage collection (if required) of file
468535
* @param map_path path to file map file (*.cfm).

‎src/backend/utils/misc/guc.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2825,7 +2825,7 @@ static struct config_int ConfigureNamesInt[] =
28252825

28262826
{
28272827
{"cfs_gc_threshold", PGC_USERSET, UNGROUPED,
2828-
gettext_noop("Percent of garbage in file after file is comactified"),
2828+
gettext_noop("Minimum percent of garbage blocks in the file prior to garbage collection"),
28292829
NULL,
28302830
0
28312831
},
@@ -2837,7 +2837,7 @@ static struct config_int ConfigureNamesInt[] =
28372837

28382838
{
28392839
{"cfs_gc_period", PGC_USERSET, UNGROUPED,
2840-
gettext_noop("Interval in milliseconds between CFS garbage collection iterations"),
2840+
gettext_noop("Time to sleep between GC runs in milliseconds"),
28412841
NULL,
28422842
GUC_UNIT_MS
28432843
},

‎src/include/storage/cfs.h

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,25 @@
1717
#define CFS_MAX_COMPRESSED_SIZE(size) ((size)*2)
1818
#define CFS_MIN_COMPRESSED_SIZE(size) ((size)*2/3)
1919

20+
/*
21+
* FIXME
22+
*/
2023
#define LZ_COMPRESSOR 1
2124
#define ZLIB_COMPRESSOR 2
2225
#define LZ4_COMPRESSOR 3
2326
#define SNAPPY_COMPRESSOR 4
2427
#define LCFSE_COMPRESSOR 5
2528
#define ZSTD_COMPRESSOR 6
2629

30+
/*
31+
* Set CFS_COMPRESSOR to one of the names above
32+
* to compile postgres with chosen compression algorithm
33+
*/
2734
#ifndef CFS_COMPRESSOR
2835
#define CFS_COMPRESSOR ZLIB_COMPRESSOR
2936
#endif
3037

38+
/* Encryption related variables*/
3139
#define CFS_RC4_DROP_N 3072
3240
#define CFS_CIPHER_KEY_SIZE 256
3341

@@ -54,6 +62,7 @@ typedef struct
5462
uint64 processedBytes;
5563
} CfsStatistic;
5664

65+
/* TODO Add comments */
5766
typedef struct
5867
{
5968
pg_atomic_flag gc_started;
@@ -62,7 +71,7 @@ typedef struct
6271
int max_iterations;
6372
bool gc_enabled;
6473
CfsStatistic gc_stat;
65-
uint8 rc4_init_state[CFS_CIPHER_KEY_SIZE];
74+
uint8 cipher_key[CFS_CIPHER_KEY_SIZE]; /* FIXME Is it the key itself? Or some modification?*/
6675
} CfsState;
6776

6877
typedef struct
@@ -91,13 +100,14 @@ void cfs_decrypt(void* block, uint32 offs, uint32 size);
91100

92101
extern CfsState* cfs_state;
93102

103+
extern int cfs_level;
104+
extern bool cfs_encryption;
105+
extern bool cfs_gc_verify_file;
106+
94107
extern int cfs_gc_delay;
95108
extern int cfs_gc_period;
96109
extern int cfs_gc_workers;
97110
extern int cfs_gc_threshold;
98-
extern int cfs_level;
99-
extern bool cfs_gc_verify_file;
100-
extern bool cfs_encryption;
101111
#endif
102112

103113

0 commit comments

Comments
 (0)