Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit29add0d

Browse files
committed
In pg_upgrade, copy fsm, vm, and extent files by checking for file
existence via open(), rather than collecting a directory listing andlooking up matching relfilenode files with sequential scans of thearray. This speeds up pg_upgrade by 2x for a large number of tables,e.g. 16k.Per observation by Ants Aasma.
1 parenta235b85 commit29add0d

File tree

3 files changed

+82
-183
lines changed

3 files changed

+82
-183
lines changed

‎contrib/pg_upgrade/file.c

Lines changed: 0 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -221,61 +221,6 @@ copy_file(const char *srcfile, const char *dstfile, bool force)
221221
#endif
222222

223223

224-
/*
225-
* load_directory()
226-
*
227-
* Read all the file names in the specified directory, and return them as
228-
* an array of "char *" pointers. The array address is returned in
229-
* *namelist, and the function result is the count of file names.
230-
*
231-
* To free the result data, free each (char *) array member, then free the
232-
* namelist array itself.
233-
*/
234-
int
235-
load_directory(constchar*dirname,char***namelist)
236-
{
237-
DIR*dirdesc;
238-
structdirent*direntry;
239-
intcount=0;
240-
intallocsize=64;/* initial array size */
241-
242-
*namelist= (char**)pg_malloc(allocsize*sizeof(char*));
243-
244-
if ((dirdesc=opendir(dirname))==NULL)
245-
pg_log(PG_FATAL,"could not open directory \"%s\": %s\n",
246-
dirname,getErrorText(errno));
247-
248-
while (errno=0, (direntry=readdir(dirdesc))!=NULL)
249-
{
250-
if (count >=allocsize)
251-
{
252-
allocsize *=2;
253-
*namelist= (char**)
254-
pg_realloc(*namelist,allocsize*sizeof(char*));
255-
}
256-
257-
(*namelist)[count++]=pg_strdup(direntry->d_name);
258-
}
259-
260-
#ifdefWIN32
261-
/*
262-
* This fix is in mingw cvs (runtime/mingwex/dirent.c rev 1.4), but not in
263-
* released version
264-
*/
265-
if (GetLastError()==ERROR_NO_MORE_FILES)
266-
errno=0;
267-
#endif
268-
269-
if (errno)
270-
pg_log(PG_FATAL,"could not read directory \"%s\": %s\n",
271-
dirname,getErrorText(errno));
272-
273-
closedir(dirdesc);
274-
275-
returncount;
276-
}
277-
278-
279224
void
280225
check_hard_link(void)
281226
{

‎contrib/pg_upgrade/pg_upgrade.h

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@
77

88
#include<unistd.h>
99
#include<assert.h>
10-
#include<dirent.h>
1110
#include<sys/stat.h>
1211
#include<sys/time.h>
1312

@@ -366,7 +365,6 @@ const char *setupPageConverter(pageCnvCtx **result);
366365
typedefvoid*pageCnvCtx;
367366
#endif
368367

369-
intload_directory(constchar*dirname,char***namelist);
370368
constchar*copyAndUpdateFile(pageCnvCtx*pageConverter,constchar*src,
371369
constchar*dst,boolforce);
372370
constchar*linkAndUpdateFile(pageCnvCtx*pageConverter,constchar*src,

‎contrib/pg_upgrade/relfilenode.c

Lines changed: 82 additions & 126 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,8 @@
1717

1818
staticvoidtransfer_single_new_db(pageCnvCtx*pageConverter,
1919
FileNameMap*maps,intsize);
20-
staticvoidtransfer_relfile(pageCnvCtx*pageConverter,
21-
constchar*fromfile,constchar*tofile,
22-
constchar*nspname,constchar*relname);
20+
staticvoidtransfer_relfile(pageCnvCtx*pageConverter,FileNameMap*map,
21+
constchar*suffix);
2322

2423

2524
/*
@@ -131,123 +130,33 @@ static void
131130
transfer_single_new_db(pageCnvCtx*pageConverter,
132131
FileNameMap*maps,intsize)
133132
{
134-
charold_dir[MAXPGPATH];
135-
charfile_pattern[MAXPGPATH];
136-
char**namelist=NULL;
137-
intnumFiles=0;
138133
intmapnum;
139-
intfileno;
140-
boolvm_crashsafe_change= false;
141-
142-
old_dir[0]='\0';
143-
144-
/* Do not copy non-crashsafe vm files for binaries that assume crashsafety */
134+
boolvm_crashsafe_match= true;
135+
136+
/*
137+
* Do the old and new cluster disagree on the crash-safetiness of the vm
138+
* files? If so, do not copy them.
139+
*/
145140
if (old_cluster.controldata.cat_ver<VISIBILITY_MAP_CRASHSAFE_CAT_VER&&
146141
new_cluster.controldata.cat_ver >=VISIBILITY_MAP_CRASHSAFE_CAT_VER)
147-
vm_crashsafe_change=true;
142+
vm_crashsafe_match=false;
148143

149144
for (mapnum=0;mapnum<size;mapnum++)
150145
{
151-
charold_file[MAXPGPATH];
152-
charnew_file[MAXPGPATH];
153-
154-
/* Changed tablespaces? Need a new directory scan? */
155-
if (strcmp(maps[mapnum].old_dir,old_dir)!=0)
156-
{
157-
if (numFiles>0)
158-
{
159-
for (fileno=0;fileno<numFiles;fileno++)
160-
pg_free(namelist[fileno]);
161-
pg_free(namelist);
162-
}
163-
164-
snprintf(old_dir,sizeof(old_dir),"%s",maps[mapnum].old_dir);
165-
numFiles=load_directory(old_dir,&namelist);
166-
}
167-
168-
/* Copying files might take some time, so give feedback. */
169-
170-
snprintf(old_file,sizeof(old_file),"%s/%u",maps[mapnum].old_dir,
171-
maps[mapnum].old_relfilenode);
172-
snprintf(new_file,sizeof(new_file),"%s/%u",maps[mapnum].new_dir,
173-
maps[mapnum].new_relfilenode);
174-
pg_log(PG_REPORT,OVERWRITE_MESSAGE,old_file);
175-
176-
/*
177-
* Copy/link the relation's primary file (segment 0 of main fork)
178-
* to the new cluster
179-
*/
180-
unlink(new_file);
181-
transfer_relfile(pageConverter,old_file,new_file,
182-
maps[mapnum].nspname,maps[mapnum].relname);
146+
/* transfer primary file */
147+
transfer_relfile(pageConverter,&maps[mapnum],"");
183148

184149
/* fsm/vm files added in PG 8.4 */
185150
if (GET_MAJOR_VERSION(old_cluster.major_version) >=804)
186151
{
187152
/*
188153
* Copy/link any fsm and vm files, if they exist
189154
*/
190-
snprintf(file_pattern,sizeof(file_pattern),"%u_",
191-
maps[mapnum].old_relfilenode);
192-
193-
for (fileno=0;fileno<numFiles;fileno++)
194-
{
195-
char*vm_offset=strstr(namelist[fileno],"_vm");
196-
boolis_vm_file= false;
197-
198-
/* Is a visibility map file? (name ends with _vm) */
199-
if (vm_offset&&strlen(vm_offset)==strlen("_vm"))
200-
is_vm_file= true;
201-
202-
if (strncmp(namelist[fileno],file_pattern,
203-
strlen(file_pattern))==0&&
204-
(!is_vm_file|| !vm_crashsafe_change))
205-
{
206-
snprintf(old_file,sizeof(old_file),"%s/%s",maps[mapnum].old_dir,
207-
namelist[fileno]);
208-
snprintf(new_file,sizeof(new_file),"%s/%u%s",maps[mapnum].new_dir,
209-
maps[mapnum].new_relfilenode,strchr(namelist[fileno],'_'));
210-
211-
unlink(new_file);
212-
transfer_relfile(pageConverter,old_file,new_file,
213-
maps[mapnum].nspname,maps[mapnum].relname);
214-
}
215-
}
216-
}
217-
218-
/*
219-
* Now copy/link any related segments as well. Remember, PG breaks
220-
* large files into 1GB segments, the first segment has no extension,
221-
* subsequent segments are named relfilenode.1, relfilenode.2,
222-
* relfilenode.3, ... 'fsm' and 'vm' files use underscores so are not
223-
* copied.
224-
*/
225-
snprintf(file_pattern,sizeof(file_pattern),"%u.",
226-
maps[mapnum].old_relfilenode);
227-
228-
for (fileno=0;fileno<numFiles;fileno++)
229-
{
230-
if (strncmp(namelist[fileno],file_pattern,
231-
strlen(file_pattern))==0)
232-
{
233-
snprintf(old_file,sizeof(old_file),"%s/%s",maps[mapnum].old_dir,
234-
namelist[fileno]);
235-
snprintf(new_file,sizeof(new_file),"%s/%u%s",maps[mapnum].new_dir,
236-
maps[mapnum].new_relfilenode,strchr(namelist[fileno],'.'));
237-
238-
unlink(new_file);
239-
transfer_relfile(pageConverter,old_file,new_file,
240-
maps[mapnum].nspname,maps[mapnum].relname);
241-
}
155+
transfer_relfile(pageConverter,&maps[mapnum],"_fsm");
156+
if (vm_crashsafe_match)
157+
transfer_relfile(pageConverter,&maps[mapnum],"_vm");
242158
}
243159
}
244-
245-
if (numFiles>0)
246-
{
247-
for (fileno=0;fileno<numFiles;fileno++)
248-
pg_free(namelist[fileno]);
249-
pg_free(namelist);
250-
}
251160
}
252161

253162

@@ -257,31 +166,78 @@ transfer_single_new_db(pageCnvCtx *pageConverter,
257166
* Copy or link file from old cluster to new one.
258167
*/
259168
staticvoid
260-
transfer_relfile(pageCnvCtx*pageConverter,constchar*old_file,
261-
constchar*new_file,constchar*nspname,constchar*relname)
169+
transfer_relfile(pageCnvCtx*pageConverter,FileNameMap*map,
170+
constchar*type_suffix)
262171
{
263172
constchar*msg;
264-
265-
if ((user_opts.transfer_mode==TRANSFER_MODE_LINK)&& (pageConverter!=NULL))
266-
pg_log(PG_FATAL,"This upgrade requires page-by-page conversion, "
267-
"you must use copy mode instead of link mode.\n");
268-
269-
if (user_opts.transfer_mode==TRANSFER_MODE_COPY)
173+
charold_file[MAXPGPATH];
174+
charnew_file[MAXPGPATH];
175+
intfd;
176+
intsegno;
177+
charextent_suffix[65];
178+
179+
/*
180+
* Now copy/link any related segments as well. Remember, PG breaks
181+
* large files into 1GB segments, the first segment has no extension,
182+
* subsequent segments are named relfilenode.1, relfilenode.2,
183+
* relfilenode.3.
184+
* copied.
185+
*/
186+
for (segno=0;;segno++)
270187
{
271-
pg_log(PG_VERBOSE,"copying \"%s\" to \"%s\"\n",old_file,new_file);
188+
if (segno==0)
189+
extent_suffix[0]='\0';
190+
else
191+
snprintf(extent_suffix,sizeof(extent_suffix),".%d",segno);
192+
193+
snprintf(old_file,sizeof(old_file),"%s/%u%s%s",map->old_dir,
194+
map->old_relfilenode,type_suffix,extent_suffix);
195+
snprintf(new_file,sizeof(new_file),"%s/%u%s%s",map->new_dir,
196+
map->new_relfilenode,type_suffix,extent_suffix);
197+
198+
/* Is it an extent, fsm, or vm file? */
199+
if (type_suffix[0]!='\0'||segno!=0)
200+
{
201+
/* Did file open fail? */
202+
if ((fd=open(old_file,O_RDONLY))==-1)
203+
{
204+
/* File does not exist? That's OK, just return */
205+
if (errno==ENOENT)
206+
return;
207+
else
208+
pg_log(PG_FATAL,"non-existant file error while copying relation \"%s.%s\" (\"%s\" to \"%s\")\n",
209+
map->nspname,map->relname,old_file,new_file);
210+
}
211+
close(fd);
212+
}
272213

273-
if ((msg=copyAndUpdateFile(pageConverter,old_file,new_file, true))!=NULL)
274-
pg_log(PG_FATAL,"error while copying relation \"%s.%s\" (\"%s\" to \"%s\"): %s\n",
275-
nspname,relname,old_file,new_file,msg);
276-
}
277-
else
278-
{
279-
pg_log(PG_VERBOSE,"linking \"%s\" to \"%s\"\n",old_file,new_file);
214+
unlink(new_file);
215+
216+
/* Copying files might take some time, so give feedback. */
217+
pg_log(PG_REPORT,OVERWRITE_MESSAGE,old_file);
218+
219+
if ((user_opts.transfer_mode==TRANSFER_MODE_LINK)&& (pageConverter!=NULL))
220+
pg_log(PG_FATAL,"This upgrade requires page-by-page conversion, "
221+
"you must use copy mode instead of link mode.\n");
222+
223+
if (user_opts.transfer_mode==TRANSFER_MODE_COPY)
224+
{
225+
pg_log(PG_VERBOSE,"copying \"%s\" to \"%s\"\n",old_file,new_file);
226+
227+
if ((msg=copyAndUpdateFile(pageConverter,old_file,new_file, true))!=NULL)
228+
pg_log(PG_FATAL,"error while copying relation \"%s.%s\" (\"%s\" to \"%s\"): %s\n",
229+
map->nspname,map->relname,old_file,new_file,msg);
230+
}
231+
else
232+
{
233+
pg_log(PG_VERBOSE,"linking \"%s\" to \"%s\"\n",old_file,new_file);
234+
235+
if ((msg=linkAndUpdateFile(pageConverter,old_file,new_file))!=NULL)
236+
pg_log(PG_FATAL,
237+
"error while creating link for relation \"%s.%s\" (\"%s\" to \"%s\"): %s\n",
238+
map->nspname,map->relname,old_file,new_file,msg);
239+
}
240+
}
280241

281-
if ((msg=linkAndUpdateFile(pageConverter,old_file,new_file))!=NULL)
282-
pg_log(PG_FATAL,
283-
"error while creating link for relation \"%s.%s\" (\"%s\" to \"%s\"): %s\n",
284-
nspname,relname,old_file,new_file,msg);
285-
}
286242
return;
287243
}

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp