Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitef5856f

Browse files
committed
Allow BASE_BACKUP to be throttled
A new MAX_RATE option allows imposing a limit to the network transferrate from the server side. This is useful to limit the stress thattaking a base backup has on the server.pg_basebackup is now able to specify a value to the server, too.Author: Antonin HouskaPatch reviewed by Stefan Radomski, Andres Freund, Zoltán Böszörményi,Fujii Masao, and Álvaro Herrera.
1 parent1161d89 commitef5856f

File tree

7 files changed

+306
-20
lines changed

7 files changed

+306
-20
lines changed

‎doc/src/sgml/protocol.sgml

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1772,7 +1772,7 @@ The commands accepted in walsender mode are:
17721772
</varlistentry>
17731773

17741774
<varlistentry>
1775-
<term>BASE_BACKUP [<literal>LABEL</literal> <replaceable>'label'</replaceable>] [<literal>PROGRESS</literal>] [<literal>FAST</literal>] [<literal>WAL</literal>] [<literal>NOWAIT</literal>]</term>
1775+
<term>BASE_BACKUP [<literal>LABEL</literal> <replaceable>'label'</replaceable>] [<literal>PROGRESS</literal>] [<literal>FAST</literal>] [<literal>WAL</literal>] [<literal>NOWAIT</literal>] [<literal>MAX_RATE</literal> <replaceable>rate</replaceable>]</term>
17761776
<listitem>
17771777
<para>
17781778
Instructs the server to start streaming a base backup.
@@ -1840,7 +1840,21 @@ The commands accepted in walsender mode are:
18401840
the waiting and the warning, leaving the client responsible for
18411841
ensuring the required log is available.
18421842
</para>
1843-
</listitem>
1843+
</listitem>
1844+
</varlistentry>
1845+
1846+
<varlistentry>
1847+
<term><literal>MAX_RATE</literal> <replaceable>rate</></term>
1848+
<listitem>
1849+
<para>
1850+
Limit (throttle) the maximum amount of data transferred from server
1851+
to client per unit of time. The expected unit is kilobytes per second.
1852+
If this option is specified, the value must either be equal to zero
1853+
or it must fall within the range from 32 kB through 1 GB (inclusive).
1854+
If zero is passed or the option is not specified, no restriction is
1855+
imposed on the transfer.
1856+
</para>
1857+
</listitem>
18441858
</varlistentry>
18451859
</variablelist>
18461860
</para>

‎doc/src/sgml/ref/pg_basebackup.sgml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,27 @@ PostgreSQL documentation
188188
</listitem>
189189
</varlistentry>
190190

191+
<varlistentry>
192+
<term><option>-r <replaceable class="parameter">rate</replaceable></option></term>
193+
<term><option>--max-rate=<replaceable class="parameter">rate</replaceable></option></term>
194+
<listitem>
195+
<para>
196+
The maximum transfer rate of data transferred from the server. Values are
197+
in kilobytes per second. Use a suffix of <literal>M</> to indicate megabytes
198+
per second. A suffix of <literal>k</> is also accepted, and has no effect.
199+
Valid values are between 32 kilobytes per second and 1024 megabytes per second.
200+
</para>
201+
<para>
202+
The purpose is to limit the impact of <application>pg_basebackup</application>
203+
on the running server.
204+
</para>
205+
<para>
206+
This option always affects transfer of the data directory. Transfer of
207+
WAL files is only affected if the collection method is <literal>fetch</literal>.
208+
</para>
209+
</listitem>
210+
</varlistentry>
211+
191212
<varlistentry>
192213
<term><option>-R</option></term>
193214
<term><option>--write-recovery-conf</option></term>

‎src/backend/replication/basebackup.c

Lines changed: 135 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
#include"libpq/pqformat.h"
2626
#include"miscadmin.h"
2727
#include"nodes/pg_list.h"
28+
#include"pgtar.h"
2829
#include"pgstat.h"
2930
#include"replication/basebackup.h"
3031
#include"replication/walsender.h"
@@ -34,7 +35,8 @@
3435
#include"utils/builtins.h"
3536
#include"utils/elog.h"
3637
#include"utils/ps_status.h"
37-
#include"pgtar.h"
38+
#include"utils/timestamp.h"
39+
3840

3941
typedefstruct
4042
{
@@ -43,6 +45,7 @@ typedef struct
4345
boolfastcheckpoint;
4446
boolnowait;
4547
boolincludewal;
48+
uint32maxrate;
4649
}basebackup_options;
4750

4851

@@ -60,6 +63,7 @@ static void perform_base_backup(basebackup_options *opt, DIR *tblspcdir);
6063
staticvoidparse_basebackup_options(List*options,basebackup_options*opt);
6164
staticvoidSendXlogRecPtrResult(XLogRecPtrptr,TimeLineIDtli);
6265
staticintcompareWalFileNames(constvoid*a,constvoid*b);
66+
staticvoidthrottle(size_tincrement);
6367

6468
/* Was the backup currently in-progress initiated in recovery mode? */
6569
staticboolbackup_started_in_recovery= false;
@@ -72,6 +76,23 @@ static char *statrelpath = NULL;
7276
*/
7377
#defineTAR_SEND_SIZE 32768
7478

79+
/*
80+
* How frequently to throttle, as a fraction of the specified rate-second.
81+
*/
82+
#defineTHROTTLING_FREQUENCY8
83+
84+
/* The actual number of bytes, transfer of which may cause sleep. */
85+
staticuint64throttling_sample;
86+
87+
/* Amount of data already transfered but not yet throttled. */
88+
staticint64throttling_counter;
89+
90+
/* The minimum time required to transfer throttling_sample bytes. */
91+
staticint64elapsed_min_unit;
92+
93+
/* The last check of the transfer rate. */
94+
staticint64throttled_last;
95+
7596
typedefstruct
7697
{
7798
char*oid;
@@ -203,6 +224,29 @@ perform_base_backup(basebackup_options *opt, DIR *tblspcdir)
203224
/* Send tablespace header */
204225
SendBackupHeader(tablespaces);
205226

227+
/* Setup and activate network throttling, if client requested it */
228+
if (opt->maxrate>0)
229+
{
230+
throttling_sample=opt->maxrate*1024 /THROTTLING_FREQUENCY;
231+
232+
/*
233+
* The minimum amount of time for throttling_sample
234+
* bytes to be transfered.
235+
*/
236+
elapsed_min_unit=USECS_PER_SEC /THROTTLING_FREQUENCY;
237+
238+
/* Enable throttling. */
239+
throttling_counter=0;
240+
241+
/* The 'real data' starts now (header was ignored). */
242+
throttled_last=GetCurrentIntegerTimestamp();
243+
}
244+
else
245+
{
246+
/* Disable throttling. */
247+
throttling_counter=-1;
248+
}
249+
206250
/* Send off our tablespaces one by one */
207251
foreach(lc,tablespaces)
208252
{
@@ -430,6 +474,8 @@ perform_base_backup(basebackup_options *opt, DIR *tblspcdir)
430474
(errmsg("base backup could not send data, aborting backup")));
431475

432476
len+=cnt;
477+
throttle(cnt);
478+
433479
if (len==XLogSegSize)
434480
break;
435481
}
@@ -500,6 +546,7 @@ parse_basebackup_options(List *options, basebackup_options *opt)
500546
boolo_fast= false;
501547
boolo_nowait= false;
502548
boolo_wal= false;
549+
boolo_maxrate= false;
503550

504551
MemSet(opt,0,sizeof(*opt));
505552
foreach(lopt,options)
@@ -551,6 +598,25 @@ parse_basebackup_options(List *options, basebackup_options *opt)
551598
opt->includewal= true;
552599
o_wal= true;
553600
}
601+
elseif (strcmp(defel->defname,"max_rate")==0)
602+
{
603+
longmaxrate;
604+
605+
if (o_maxrate)
606+
ereport(ERROR,
607+
(errcode(ERRCODE_SYNTAX_ERROR),
608+
errmsg("duplicate option \"%s\"",defel->defname)));
609+
610+
maxrate=intVal(defel->arg);
611+
if (maxrate<MAX_RATE_LOWER||maxrate>MAX_RATE_UPPER)
612+
ereport(ERROR,
613+
(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
614+
errmsg("%d is outside the valid range for parameter \"%s\" (%d .. %d)",
615+
(int)maxrate,"MAX_RATE",MAX_RATE_LOWER,MAX_RATE_UPPER)));
616+
617+
opt->maxrate= (uint32)maxrate;
618+
o_maxrate= true;
619+
}
554620
else
555621
elog(ERROR,"option \"%s\" not recognized",
556622
defel->defname);
@@ -1112,6 +1178,7 @@ sendFile(char *readfilename, char *tarfilename, struct stat * statbuf,
11121178
(errmsg("base backup could not send data, aborting backup")));
11131179

11141180
len+=cnt;
1181+
throttle(cnt);
11151182

11161183
if (len >=statbuf->st_size)
11171184
{
@@ -1133,10 +1200,14 @@ sendFile(char *readfilename, char *tarfilename, struct stat * statbuf,
11331200
cnt=Min(sizeof(buf),statbuf->st_size-len);
11341201
pq_putmessage('d',buf,cnt);
11351202
len+=cnt;
1203+
throttle(cnt);
11361204
}
11371205
}
11381206

1139-
/* Pad to 512 byte boundary, per tar format requirements */
1207+
/*
1208+
* Pad to 512 byte boundary, per tar format requirements. (This small
1209+
* piece of data is probably not worth throttling.)
1210+
*/
11401211
pad= ((len+511)& ~511)-len;
11411212
if (pad>0)
11421213
{
@@ -1162,3 +1233,65 @@ _tarWriteHeader(const char *filename, const char *linktarget,
11621233

11631234
pq_putmessage('d',h,512);
11641235
}
1236+
1237+
/*
1238+
* Increment the network transfer counter by the given number of bytes,
1239+
* and sleep if necessary to comply with the requested network transfer
1240+
* rate.
1241+
*/
1242+
staticvoid
1243+
throttle(size_tincrement)
1244+
{
1245+
int64elapsed,
1246+
elapsed_min,
1247+
sleep;
1248+
intwait_result;
1249+
1250+
if (throttling_counter<0)
1251+
return;
1252+
1253+
throttling_counter+=increment;
1254+
if (throttling_counter<throttling_sample)
1255+
return;
1256+
1257+
/* Time elapsed since the last measurement (and possible wake up). */
1258+
elapsed=GetCurrentIntegerTimestamp()-throttled_last;
1259+
/* How much should have elapsed at minimum? */
1260+
elapsed_min=elapsed_min_unit* (throttling_counter /throttling_sample);
1261+
sleep=elapsed_min-elapsed;
1262+
/* Only sleep if the transfer is faster than it should be. */
1263+
if (sleep>0)
1264+
{
1265+
ResetLatch(&MyWalSnd->latch);
1266+
1267+
/*
1268+
* (TAR_SEND_SIZE / throttling_sample * elapsed_min_unit) should be
1269+
* the maximum time to sleep. Thus the cast to long is safe.
1270+
*/
1271+
wait_result=WaitLatch(&MyWalSnd->latch,
1272+
WL_LATCH_SET |WL_TIMEOUT |WL_POSTMASTER_DEATH,
1273+
(long) (sleep /1000));
1274+
}
1275+
else
1276+
{
1277+
/*
1278+
* The actual transfer rate is below the limit. A negative value would
1279+
* distort the adjustment of throttled_last.
1280+
*/
1281+
wait_result=0;
1282+
sleep=0;
1283+
}
1284+
1285+
/*
1286+
* Only a whole multiple of throttling_sample was processed. The rest will
1287+
* be done during the next call of this function.
1288+
*/
1289+
throttling_counter %=throttling_sample;
1290+
1291+
/* Once the (possible) sleep has ended, new period starts. */
1292+
if (wait_result&WL_TIMEOUT)
1293+
throttled_last+=elapsed+sleep;
1294+
elseif (sleep>0)
1295+
/* Sleep was necessary but might have been interrupted. */
1296+
throttled_last=GetCurrentIntegerTimestamp();
1297+
}

‎src/backend/replication/repl_gram.y

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ Node *replication_parse_result;
6969
%tokenK_PROGRESS
7070
%tokenK_FAST
7171
%tokenK_NOWAIT
72+
%tokenK_MAX_RATE
7273
%tokenK_WAL
7374
%tokenK_TIMELINE
7475
%tokenK_PHYSICAL
@@ -113,7 +114,7 @@ identify_system:
113114
;
114115

115116
/*
116-
* BASE_BACKUP [LABEL '<label>'] [PROGRESS] [FAST] [WAL] [NOWAIT]
117+
* BASE_BACKUP [LABEL '<label>'] [PROGRESS] [FAST] [WAL] [NOWAIT] [MAX_RATE %d]
117118
*/
118119
base_backup:
119120
K_BASE_BACKUPbase_backup_opt_list
@@ -157,6 +158,11 @@ base_backup_opt:
157158
$$ = makeDefElem("nowait",
158159
(Node *)makeInteger(TRUE));
159160
}
161+
|K_MAX_RATEUCONST
162+
{
163+
$$ = makeDefElem("max_rate",
164+
(Node *)makeInteger($2));
165+
}
160166
;
161167

162168
/* CREATE_REPLICATION_SLOT SLOT slot PHYSICAL*/

‎src/backend/replication/repl_scanner.l

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ IDENTIFY_SYSTEM{ return K_IDENTIFY_SYSTEM; }
8686
LABEL{return K_LABEL; }
8787
NOWAIT{return K_NOWAIT; }
8888
PROGRESS{return K_PROGRESS; }
89+
MAX_RATE{return K_MAX_RATE; }
8990
WAL{return K_WAL; }
9091
TIMELINE{return K_TIMELINE; }
9192
START_REPLICATION{return K_START_REPLICATION; }

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp