Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitad06db2

Browse files
committed
In pg_upgrade, use cached copy of directory listing, rather than calling
scandir() with a pattern for every table.Optimization after report of pg_upgrade slowness with 150k tables.
1 parent07456b4 commitad06db2

File tree

3 files changed

+74
-54
lines changed

3 files changed

+74
-54
lines changed

‎contrib/pg_upgrade/info.c‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ static void map_rel_by_id(Oid oldid, Oid newid,
3030
constchar*old_tablespace,constDbInfo*old_db,
3131
constDbInfo*new_db,constchar*olddata,
3232
constchar*newdata,FileNameMap*map);
33-
staticRelInfo*relarr_lookup_reloid(
34-
RelInfoArr*rel_arr,Oidoid,ClusterwhichCluster);
33+
staticRelInfo*relarr_lookup_reloid(RelInfoArr*rel_arr,
34+
Oidoid,ClusterwhichCluster);
3535
staticRelInfo*relarr_lookup_rel(RelInfoArr*rel_arr,
3636
constchar*nspname,constchar*relname,
3737
ClusterwhichCluster);
@@ -172,18 +172,18 @@ map_rel_by_id(Oid oldid, Oid newid,
172172
* relation belongs to the default tablespace, hence relfiles would
173173
* exist in the data directories.
174174
*/
175-
snprintf(map->old_file,sizeof(map->old_file),"%s/base/%u",olddata,old_db->db_oid);
176-
snprintf(map->new_file,sizeof(map->new_file),"%s/base/%u",newdata,new_db->db_oid);
175+
snprintf(map->old_dir,sizeof(map->old_dir),"%s/base/%u",olddata,old_db->db_oid);
176+
snprintf(map->new_dir,sizeof(map->new_dir),"%s/base/%u",newdata,new_db->db_oid);
177177
}
178178
else
179179
{
180180
/*
181181
* relation belongs to some tablespace, hence copy its physical
182182
* location
183183
*/
184-
snprintf(map->old_file,sizeof(map->old_file),"%s%s/%u",old_tablespace,
184+
snprintf(map->old_dir,sizeof(map->old_dir),"%s%s/%u",old_tablespace,
185185
old_cluster.tablespace_suffix,old_db->db_oid);
186-
snprintf(map->new_file,sizeof(map->new_file),"%s%s/%u",old_tablespace,
186+
snprintf(map->new_dir,sizeof(map->new_dir),"%s%s/%u",old_tablespace,
187187
new_cluster.tablespace_suffix,new_db->db_oid);
188188
}
189189
}
@@ -318,6 +318,7 @@ get_rel_infos(const DbInfo *dbinfo,
318318
* pg_largeobject_loid_pn_index's relfilenode can change if the table was
319319
* reindexed so we get the relfilenode for each database and migrate it as
320320
* a normal user table.
321+
* Order by tablespace so we can cache the directory contents efficiently.
321322
*/
322323

323324
snprintf(query,sizeof(query),
@@ -338,7 +339,7 @@ get_rel_infos(const DbInfo *dbinfo,
338339
"GROUP BY c.oid, n.nspname, c.relname, c.relfilenode,"
339340
"c.reltoastrelid, t.spclocation, "
340341
"n.nspname "
341-
"ORDER BY n.nspname, c.relname;",
342+
"ORDER BYt.spclocation,n.nspname, c.relname;",
342343
FirstNormalObjectId,
343344
/* see the comment at the top of old_8_3_create_sequence_script() */
344345
(GET_MAJOR_VERSION(old_cluster.major_version) <=803) ?

‎contrib/pg_upgrade/pg_upgrade.h‎

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -85,10 +85,10 @@ typedef struct
8585
*/
8686
typedefstruct
8787
{
88+
charold_dir[MAXPGPATH];
89+
charnew_dir[MAXPGPATH];
8890
Oidold_relfilenode;/* Relfilenode of the old relation */
8991
Oidnew_relfilenode;/* Relfilenode of the new relation */
90-
charold_file[MAXPGPATH];
91-
charnew_file[MAXPGPATH];
9292
charold_nspname[NAMEDATALEN];/* old name of the namespace */
9393
charold_relname[NAMEDATALEN];/* old name of the relation */
9494
charnew_nspname[NAMEDATALEN];/* new name of the namespace */
@@ -255,10 +255,8 @@ void check_old_cluster(bool live_check,
255255
char**sequence_script_file_name);
256256
voidcheck_new_cluster(void);
257257
voidreport_clusters_compatible(void);
258-
voidissue_warnings(
259-
char*sequence_script_file_name);
260-
voidoutput_completion_banner(
261-
char*deletion_script_file_name);
258+
voidissue_warnings(char*sequence_script_file_name);
259+
voidoutput_completion_banner(char*deletion_script_file_name);
262260
voidcheck_cluster_versions(void);
263261
voidcheck_cluster_compatibility(boollive_check);
264262
voidcreate_script_for_old_cluster_deletion(char**deletion_script_file_name);
@@ -319,14 +317,12 @@ typedef void *pageCnvCtx;
319317
#endif
320318

321319
intdir_matching_filenames(conststructdirent*scan_ent);
322-
intpg_scandir(constchar*dirname,
323-
structdirent***namelist,
324-
int (*selector) (conststructdirent*));
325-
constchar*copyAndUpdateFile(
326-
pageCnvCtx*pageConverter,constchar*src,
320+
intpg_scandir(constchar*dirname,structdirent***namelist,
321+
int (*selector) (conststructdirent*));
322+
constchar*copyAndUpdateFile(pageCnvCtx*pageConverter,constchar*src,
327323
constchar*dst,boolforce);
328-
constchar*linkAndUpdateFile(
329-
pageCnvCtx*pageConverter,constchar*src,constchar*dst);
324+
constchar*linkAndUpdateFile(pageCnvCtx*pageConverter,constchar*src,
325+
constchar*dst);
330326

331327
voidcheck_hard_link(void);
332328

@@ -374,8 +370,7 @@ PGresult *executeQueryOrDie(PGconn *conn,
374370

375371
voidstart_postmaster(ClusterwhichCluster,boolquiet);
376372
voidstop_postmaster(boolfast,boolquiet);
377-
uint32get_major_server_version(char**verstr,
378-
ClusterwhichCluster);
373+
uint32get_major_server_version(char**verstr,ClusterwhichCluster);
379374
voidcheck_for_libpq_envvars(void);
380375

381376

‎contrib/pg_upgrade/relfilenode.c‎

Lines changed: 56 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -110,20 +110,38 @@ static void
110110
transfer_single_new_db(pageCnvCtx*pageConverter,
111111
FileNameMap*maps,intsize)
112112
{
113+
charold_dir[MAXPGPATH];
114+
structdirent**namelist=NULL;
115+
intnumFiles=0;
113116
intmapnum;
117+
intfileno;
118+
119+
old_dir[0]='\0';
114120

115121
for (mapnum=0;mapnum<size;mapnum++)
116122
{
117123
charold_file[MAXPGPATH];
118124
charnew_file[MAXPGPATH];
119-
structdirent**namelist=NULL;
120-
intnumFiles;
125+
126+
/* Changed tablespaces? Need a new directory scan? */
127+
if (strcmp(maps[mapnum].old_dir,old_dir)!=0)
128+
{
129+
if (numFiles>0)
130+
{
131+
for (fileno=0;fileno<numFiles;fileno++)
132+
pg_free(namelist[fileno]);
133+
pg_free(namelist);
134+
}
135+
136+
snprintf(old_dir,sizeof(old_dir),"%s",maps[mapnum].old_dir);
137+
numFiles=pg_scandir(old_dir,&namelist,NULL);
138+
}
121139

122140
/* Copying files might take some time, so give feedback. */
123141

124-
snprintf(old_file,sizeof(old_file),"%s/%u",maps[mapnum].old_file,
142+
snprintf(old_file,sizeof(old_file),"%s/%u",maps[mapnum].old_dir,
125143
maps[mapnum].old_relfilenode);
126-
snprintf(new_file,sizeof(new_file),"%s/%u",maps[mapnum].new_file,
144+
snprintf(new_file,sizeof(new_file),"%s/%u",maps[mapnum].new_dir,
127145
maps[mapnum].new_relfilenode);
128146
pg_log(PG_REPORT,OVERWRITE_MESSAGE,old_file);
129147

@@ -139,28 +157,27 @@ transfer_single_new_db(pageCnvCtx *pageConverter,
139157
if (GET_MAJOR_VERSION(old_cluster.major_version) >=804)
140158
{
141159
/*
142-
*Now copy/link any fsm and vm files, if they exist
160+
*Copy/link any fsm and vm files, if they exist
143161
*/
144162
snprintf(scandir_file_pattern,sizeof(scandir_file_pattern),"%u_",
145163
maps[mapnum].old_relfilenode);
146-
numFiles=pg_scandir(maps[mapnum].old_file,&namelist,dir_matching_filenames);
147164

148-
while (numFiles--)
165+
for (fileno=0;fileno<numFiles;fileno++)
149166
{
150-
snprintf(old_file,sizeof(old_file),"%s/%s",maps[mapnum].old_file,
151-
namelist[numFiles]->d_name);
152-
snprintf(new_file,sizeof(new_file),"%s/%u%s",maps[mapnum].new_file,
153-
maps[mapnum].new_relfilenode,strchr(namelist[numFiles]->d_name,'_'));
154-
155-
unlink(new_file);
156-
transfer_relfile(pageConverter,old_file,new_file,
157-
maps[mapnum].old_nspname,maps[mapnum].old_relname,
158-
maps[mapnum].new_nspname,maps[mapnum].new_relname);
159-
160-
pg_free(namelist[numFiles]);
167+
if (strncmp(namelist[fileno]->d_name,scandir_file_pattern,
168+
strlen(scandir_file_pattern))==0)
169+
{
170+
snprintf(old_file,sizeof(old_file),"%s/%s",maps[mapnum].old_dir,
171+
namelist[fileno]->d_name);
172+
snprintf(new_file,sizeof(new_file),"%s/%u%s",maps[mapnum].new_dir,
173+
maps[mapnum].new_relfilenode,strchr(namelist[fileno]->d_name,'_'));
174+
175+
unlink(new_file);
176+
transfer_relfile(pageConverter,old_file,new_file,
177+
maps[mapnum].old_nspname,maps[mapnum].old_relname,
178+
maps[mapnum].new_nspname,maps[mapnum].new_relname);
179+
}
161180
}
162-
163-
pg_free(namelist);
164181
}
165182

166183
/*
@@ -172,23 +189,30 @@ transfer_single_new_db(pageCnvCtx *pageConverter,
172189
*/
173190
snprintf(scandir_file_pattern,sizeof(scandir_file_pattern),"%u.",
174191
maps[mapnum].old_relfilenode);
175-
numFiles=pg_scandir(maps[mapnum].old_file,&namelist,dir_matching_filenames);
176192

177-
while (numFiles--)
193+
for (fileno=0;fileno<numFiles;fileno++)
178194
{
179-
snprintf(old_file,sizeof(old_file),"%s/%s",maps[mapnum].old_file,
180-
namelist[numFiles]->d_name);
181-
snprintf(new_file,sizeof(new_file),"%s/%u%s",maps[mapnum].new_file,
182-
maps[mapnum].new_relfilenode,strchr(namelist[numFiles]->d_name,'.'));
183-
184-
unlink(new_file);
185-
transfer_relfile(pageConverter,old_file,new_file,
186-
maps[mapnum].old_nspname,maps[mapnum].old_relname,
187-
maps[mapnum].new_nspname,maps[mapnum].new_relname);
195+
if (strncmp(namelist[fileno]->d_name,scandir_file_pattern,
196+
strlen(scandir_file_pattern))==0)
197+
{
198+
snprintf(old_file,sizeof(old_file),"%s/%s",maps[mapnum].old_dir,
199+
namelist[fileno]->d_name);
200+
snprintf(new_file,sizeof(new_file),"%s/%u%s",maps[mapnum].new_dir,
201+
maps[mapnum].new_relfilenode,strchr(namelist[fileno]->d_name,'.'));
188202

189-
pg_free(namelist[numFiles]);
203+
unlink(new_file);
204+
transfer_relfile(pageConverter,old_file,new_file,
205+
maps[mapnum].old_nspname,maps[mapnum].old_relname,
206+
maps[mapnum].new_nspname,maps[mapnum].new_relname);
207+
}
190208
}
209+
}
191210

211+
212+
if (numFiles>0)
213+
{
214+
for (fileno=0;fileno<numFiles;fileno++)
215+
pg_free(namelist[fileno]);
192216
pg_free(namelist);
193217
}
194218
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp