Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit19f6e9e

Browse files
committed
Fix typos, add comments, rename couple of variables
1 parent410aa2d commit19f6e9e

File tree

3 files changed

+134
-57
lines changed

3 files changed

+134
-57
lines changed

‎src/backend/storage/file/cfs.c‎

Lines changed: 118 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,16 @@
1111
*
1212
* NOTES:
1313
*
14-
* This fileimplemets compression of file pages.
14+
* This fileimplements compression of file pages.
1515
* Updated compressed pages are always appended to the end of file segment.
16-
* Garbage collector is used to shrink files when them become tool large.
17-
* GC is spawned as one or more background workers. Them recursively traverse all tablespace directories,
18-
* find out *.cfm files are if logical size of the file is twice larger than physical size of the file
19-
* performs compactification. Locking implemented using atomic operations is used to eliminate race
20-
* conditions.
16+
* Garbage collector is used to reclaim storage occupied by outdated versions of pages.
17+
* GC runs one or more background workers which recursively traverse all tablespace
18+
* directories. If worker finds out that logical size of the file is twice as large as
19+
* physical size of the file, it performs compactification.
20+
* To eliminate race conditions, files are locked during compacification.
21+
* Locks are implemented with atomic operations.
22+
*
23+
* TODO Write a note about *.cfm files.
2124
*/
2225

2326
#include"postgres.h"
@@ -51,13 +54,37 @@
5154
#include"utils/resowner_private.h"
5255
#include"postmaster/bgworker.h"
5356

57+
/*
58+
* GUC variable that defines compression level.
59+
* 0 - no compression, 1 - max speed,
60+
* other possible values depend on the specific algorithm.
61+
* Default value is 1.
62+
*/
63+
intcfs_level;
64+
/*
65+
* GUC variable that defines if encryption of compressed pages is enabled.
66+
* Default value is false.
67+
*/
68+
boolcfs_encryption;
69+
/*
70+
* GUC variable - Verify correctness of data written by GC.
71+
* TODO add description and documentation.
72+
*/
73+
boolcfs_gc_verify_file;
74+
75+
/* GUC variable - Number of garbage collection background workers. Default = 1 */
5476
intcfs_gc_workers;
77+
/*
78+
* GUC variable - Specifies the minimum percent of garbage blocks
79+
* needed to trigger a GC of the file. Default = 50
80+
*/
5581
intcfs_gc_threshold;
82+
/* GUC variable - Time to sleep between GC runs in milliseconds. Default = 5000 */
5683
intcfs_gc_period;
84+
/* GUC variable - Delay in milliseconds between files defragmentation. Default = 0
85+
* TODO What is the purpose of this variable?
86+
*/
5787
intcfs_gc_delay;
58-
intcfs_level;
59-
boolcfs_encryption;
60-
boolcfs_gc_verify_file;
6188

6289
staticboolcfs_read_file(intfd,void*data,uint32size);
6390
staticboolcfs_write_file(intfd,voidconst*data,uint32size);
@@ -68,90 +95,109 @@ CfsState* cfs_state;
6895
staticboolcfs_stop;
6996
staticintcfs_processed_segments;
7097

71-
#ifCFS_COMPRESSOR==SNAPPY_COMPRESSOR
7298

73-
#include<snappy-c.h>
99+
/* ----------------------------------------------------------------
100+
*Section 1: Various compression algorithms.
101+
* CFS_COMPRESSOR variable can be set at compile time.
102+
* One should define CFS_COMPRESSOR in cfs.h
103+
* Availiable options are:
104+
* - LZ_COMPRESSOR // FIXME. No actual implementation.
105+
* - ZLIB_COMPRESSOR
106+
* - LZ4_COMPRESSOR
107+
* - SNAPPY_COMPRESSOR
108+
* - LCFSE_COMPRESSOR
109+
* - ZSTD_COMPRESSOR
110+
*
111+
* If none of options is chosen, use standard pglz_compress FIXME?, which is
112+
* slow and non-efficient in comparison with others, but doesn't requre
113+
* any extra libraries.
114+
* ----------------------------------------------------------------
115+
*/
116+
117+
#ifCFS_COMPRESSOR==ZLIB_COMPRESSOR
118+
119+
#include<zlib.h>
74120

75121
size_tcfs_compress(void*dst,size_tdst_size,voidconst*src,size_tsrc_size)
76122
{
77-
returnsnappy_compress(src,src_size,dst,&dst_size)==SNAPPY_OK ?dst_size :0;
123+
uLongfcompressed_size=dst_size;
124+
intrc=compress2(dst,&compressed_size,src,src_size,cfs_level);
125+
returnrc==Z_OK ?compressed_size :rc;
78126
}
79127

80128
size_tcfs_decompress(void*dst,size_tdst_size,voidconst*src,size_tsrc_size)
81129
{
82-
returnsnappy_uncompress(src,src_size,dst,&dst_size)==SNAPPY_OK ?dst_size :0;
130+
uLongfdest_len=dst_size;
131+
intrc=uncompress(dst,&dest_len,src,src_size);
132+
returnrc==Z_OK ?dest_len :rc;
83133
}
84134

85135
charconst*cfs_algorithm()
86136
{
87-
return"snappy";
137+
return"zlib";
88138
}
89139

90-
#elifCFS_COMPRESSOR==LCFSE_COMPRESSOR
140+
#elifCFS_COMPRESSOR==LZ4_COMPRESSOR
91141

92-
#include<lcfse.h>
142+
#include<lz4.h>
93143

94144
size_tcfs_compress(void*dst,size_tdst_size,voidconst*src,size_tsrc_size)
95145
{
96-
char*scratch_buf=palloc(lcfse_encode_scratch_size());
97-
size_trc=lcfse_encode_buffer(dst,dst_size,src,src_size,scratch_buf);
98-
pfree(scratch_buf);
99-
returnrc;
146+
returnLZ4_compress(src,dst,src_size);
100147
}
101148

102149
size_tcfs_decompress(void*dst,size_tdst_size,voidconst*src,size_tsrc_size)
103150
{
104-
char*scratch_buf=palloc(lcfse_encode_scratch_size());
105-
size_trc=lcfse_decode_buffer(dst,dst_size,src,src_size,scratch_buf);
106-
pfree(scratch_buf);
107-
returnrc;
151+
returnLZ4_decompress_safe(src,dst,src_size,dst_size);
108152
}
109153

110154
charconst*cfs_algorithm()
111155
{
112-
return"lcfse";
156+
return"lz4";
113157
}
114158

115-
#elifCFS_COMPRESSOR==LZ4_COMPRESSOR
159+
#elifCFS_COMPRESSOR==SNAPPY_COMPRESSOR
116160

117-
#include<lz4.h>
161+
#include<snappy-c.h>
118162

119163
size_tcfs_compress(void*dst,size_tdst_size,voidconst*src,size_tsrc_size)
120164
{
121-
returnLZ4_compress(src,dst,src_size);
165+
returnsnappy_compress(src,src_size,dst,&dst_size)==SNAPPY_OK ?dst_size :0;
122166
}
123167

124168
size_tcfs_decompress(void*dst,size_tdst_size,voidconst*src,size_tsrc_size)
125169
{
126-
returnLZ4_decompress_safe(src,dst,src_size,dst_size);
170+
returnsnappy_uncompress(src,src_size,dst,&dst_size)==SNAPPY_OK ?dst_size :0;
127171
}
128172

129173
charconst*cfs_algorithm()
130174
{
131-
return"lz4";
175+
return"snappy";
132176
}
133177

134-
#elifCFS_COMPRESSOR==ZLIB_COMPRESSOR
178+
#elifCFS_COMPRESSOR==LCFSE_COMPRESSOR
135179

136-
#include<zlib.h>
180+
#include<lcfse.h>
137181

138182
size_tcfs_compress(void*dst,size_tdst_size,voidconst*src,size_tsrc_size)
139183
{
140-
uLongfcompressed_size=dst_size;
141-
intrc=compress2(dst,&compressed_size,src,src_size,cfs_level);
142-
returnrc==Z_OK ?compressed_size :rc;
184+
char*scratch_buf=palloc(lcfse_encode_scratch_size());
185+
size_trc=lcfse_encode_buffer(dst,dst_size,src,src_size,scratch_buf);
186+
pfree(scratch_buf);
187+
returnrc;
143188
}
144189

145190
size_tcfs_decompress(void*dst,size_tdst_size,voidconst*src,size_tsrc_size)
146191
{
147-
uLongfdest_len=dst_size;
148-
intrc=uncompress(dst,&dest_len,src,src_size);
149-
returnrc==Z_OK ?dest_len :rc;
192+
char*scratch_buf=palloc(lcfse_encode_scratch_size());
193+
size_trc=lcfse_decode_buffer(dst,dst_size,src,src_size,scratch_buf);
194+
pfree(scratch_buf);
195+
returnrc;
150196
}
151197

152198
charconst*cfs_algorithm()
153199
{
154-
return"zlib";
200+
return"lcfse";
155201
}
156202

157203
#elifCFS_COMPRESSOR==ZSTD_COMPRESSOR
@@ -195,6 +241,15 @@ char const* cfs_algorithm()
195241
#endif
196242

197243

244+
/* ----------------------------------------------------------------
245+
*Section 2: Encryption related functionality.
246+
*
247+
* TODO
248+
* - replace rc4 algrithm with something more appropriate
249+
* - add more comments
250+
* - what does 'offs' variable for?
251+
* ----------------------------------------------------------------
252+
*/
198253
staticvoidcfs_rc4_encrypt_block(void*block,uint32offs,uint32block_size)
199254
{
200255
uint32i;
@@ -205,7 +260,7 @@ static void cfs_rc4_encrypt_block(void* block, uint32 offs, uint32 block_size)
205260
intx=0,y=0;
206261
uint32skip= (offs /BLCKSZ+block_size) %CFS_CIPHER_KEY_SIZE;
207262

208-
memcpy(state,cfs_state->rc4_init_state,CFS_CIPHER_KEY_SIZE);
263+
memcpy(state,cfs_state->cipher_key,CFS_CIPHER_KEY_SIZE);
209264
for (i=0;i<skip;i++) {
210265
x= (x+1) %CFS_CIPHER_KEY_SIZE;
211266
y= (y+state[x]) %CFS_CIPHER_KEY_SIZE;
@@ -224,7 +279,12 @@ static void cfs_rc4_encrypt_block(void* block, uint32 offs, uint32 block_size)
224279
}
225280
}
226281

227-
staticvoidcfs_rc4_init(void)
282+
/*
283+
* Get env variable PG_CIPHER_KEY and initialize encryption state.
284+
* Unset variable afterward.
285+
* Now implements cf4.
286+
*/
287+
staticvoidcfs_encrypt_init(void)
228288
{
229289
intindex1=0;
230290
intindex2=0;
@@ -233,13 +293,13 @@ static void cfs_rc4_init(void)
233293
intkey_length;
234294
intx=0,y=0;
235295
char*cipher_key;
236-
uint8*rc4_init_state=cfs_state->rc4_init_state;
296+
uint8*rc4_init_state=cfs_state->cipher_key;
237297

238298
cipher_key=getenv("PG_CIPHER_KEY");
239299
if (cipher_key==NULL) {
240300
elog(ERROR,"PG_CIPHER_KEY environment variable is not set");
241301
}
242-
unsetenv("PG_CIPHER_KEY");/*make it not possible to inspectthis environment variable through plperl */
302+
unsetenv("PG_CIPHER_KEY");/*disable inspection ofthis environment variable */
243303
key_length=strlen(cipher_key);
244304
for (i=0;i<CFS_CIPHER_KEY_SIZE;++i) {
245305
rc4_init_state[i]= (uint8)i;
@@ -263,20 +323,21 @@ static void cfs_rc4_init(void)
263323
voidcfs_encrypt(void*block,uint32offs,uint32size)
264324
{
265325
if (cfs_encryption)
266-
{
267326
cfs_rc4_encrypt_block(block,offs,size);
268-
}
269327
}
270328

271329
voidcfs_decrypt(void*block,uint32offs,uint32size)
272330
{
273331
if (cfs_encryption)
274-
{
275332
cfs_rc4_encrypt_block(block,offs,size);
276-
}
277333
}
278334

279-
335+
/* ----------------------------------------------------------------
336+
*Section 3: Compression implementation.
337+
*
338+
* TODO add description
339+
* ----------------------------------------------------------------
340+
*/
280341
voidcfs_initialize()
281342
{
282343
cfs_state= (CfsState*)ShmemAlloc(sizeof(CfsState));
@@ -287,9 +348,9 @@ void cfs_initialize()
287348
cfs_state->gc_enabled= true;
288349
cfs_state->max_iterations=0;
289350

290-
if (cfs_encryption) {
291-
cfs_rc4_init();
292-
}
351+
if (cfs_encryption)
352+
cfs_encrypt_init();
353+
293354
elog(LOG,"Start CFS version %s compression algorithm %s encryption %s",
294355
CFS_VERSION,cfs_algorithm(),cfs_encryption ?"enabled" :"disabled");
295356
}
@@ -463,6 +524,12 @@ static int cfs_cmp_page_offs(void const* p1, void const* p2)
463524
returno1<o2 ?-1 :o1==o2 ?0 :1;
464525
}
465526

527+
/* ----------------------------------------------------------------
528+
*Section 4: Garbage collection functionality.
529+
*
530+
* TODO add description. reorder functions.
531+
* ----------------------------------------------------------------
532+
*/
466533
/*
467534
* Perform garbage collection (if required) of file
468535
* @param map_path path to file map file (*.cfm).

‎src/backend/utils/misc/guc.c‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2825,7 +2825,7 @@ static struct config_int ConfigureNamesInt[] =
28252825

28262826
{
28272827
{"cfs_gc_threshold",PGC_USERSET,UNGROUPED,
2828-
gettext_noop("Percentof garbage in fileafter file is comactified"),
2828+
gettext_noop("Minimum percentof garbageblocksinthefileprior to garbage collection"),
28292829
NULL,
28302830
0
28312831
},
@@ -2837,7 +2837,7 @@ static struct config_int ConfigureNamesInt[] =
28372837

28382838
{
28392839
{"cfs_gc_period",PGC_USERSET,UNGROUPED,
2840-
gettext_noop("Interval in milliseconds betweenCFS garbage collection iterations"),
2840+
gettext_noop("Time to sleep betweenGC runs in milliseconds"),
28412841
NULL,
28422842
GUC_UNIT_MS
28432843
},

‎src/include/storage/cfs.h‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,25 @@
1717
#defineCFS_MAX_COMPRESSED_SIZE(size) ((size)*2)
1818
#defineCFS_MIN_COMPRESSED_SIZE(size) ((size)*2/3)
1919

20+
/*
21+
* FIXME
22+
*/
2023
#defineLZ_COMPRESSOR 1
2124
#defineZLIB_COMPRESSOR 2
2225
#defineLZ4_COMPRESSOR 3
2326
#defineSNAPPY_COMPRESSOR 4
2427
#defineLCFSE_COMPRESSOR 5
2528
#defineZSTD_COMPRESSOR 6
2629

30+
/*
31+
* Set CFS_COMPRESSOR to one of the names above
32+
* to compile postgres with chosen compression algorithm
33+
*/
2734
#ifndefCFS_COMPRESSOR
2835
#defineCFS_COMPRESSOR ZLIB_COMPRESSOR
2936
#endif
3037

38+
/* Encryption related variables*/
3139
#defineCFS_RC4_DROP_N 3072
3240
#defineCFS_CIPHER_KEY_SIZE 256
3341

@@ -54,6 +62,7 @@ typedef struct
5462
uint64processedBytes;
5563
}CfsStatistic;
5664

65+
/* TODO Add comments */
5766
typedefstruct
5867
{
5968
pg_atomic_flaggc_started;
@@ -62,7 +71,7 @@ typedef struct
6271
intmax_iterations;
6372
boolgc_enabled;
6473
CfsStatisticgc_stat;
65-
uint8rc4_init_state[CFS_CIPHER_KEY_SIZE];
74+
uint8cipher_key[CFS_CIPHER_KEY_SIZE];/* FIXME Is it the key itself? Or some modification?*/
6675
}CfsState;
6776

6877
typedefstruct
@@ -91,13 +100,14 @@ void cfs_decrypt(void* block, uint32 offs, uint32 size);
91100

92101
externCfsState*cfs_state;
93102

103+
externintcfs_level;
104+
externboolcfs_encryption;
105+
externboolcfs_gc_verify_file;
106+
94107
externintcfs_gc_delay;
95108
externintcfs_gc_period;
96109
externintcfs_gc_workers;
97110
externintcfs_gc_threshold;
98-
externintcfs_level;
99-
externboolcfs_gc_verify_file;
100-
externboolcfs_encryption;
101111
#endif
102112

103113

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp