Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit808e13b

Browse files
author
Amit Kapila
committed
Extend the BufFile interface.
Allow BufFile to support temporary files that can be used by the singlebackend when the corresponding files need to be survived across thetransaction and need to be opened and closed multiple times. Such filesneed to be created as a member of a SharedFileSet.Additionally, this commit implements the interface for BufFileTruncate toallow files to be truncated up to a particular offset and extends theBufFileSeek API to support the SEEK_END case. This also adds an option toprovide a mode while opening the shared BufFiles instead of always openingin read-only mode.These enhancements in BufFile interface are required for the upcomingpatch to allow the replication apply worker, to handle streamedin-progress transactions.Author: Dilip Kumar, Amit KapilaReviewed-by: Amit KapilaTested-by: Neha SharmaDiscussion:https://postgr.es/m/688b0b7f-2f6c-d827-c27b-216a8e3ea700@2ndquadrant.com
1 parentadc8fc6 commit808e13b

File tree

11 files changed

+239
-28
lines changed

11 files changed

+239
-28
lines changed

‎doc/src/sgml/monitoring.sgml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1202,6 +1202,10 @@ postgres 27093 0.0 0.0 30096 2752 ? Ss 11:34 0:00 postgres: ser
12021202
<entry><literal>BufFileWrite</literal></entry>
12031203
<entry>Waiting for a write to a buffered file.</entry>
12041204
</row>
1205+
<row>
1206+
<entry><literal>BufFileTruncate</literal></entry>
1207+
<entry>Waiting for a buffered file to be truncated.</entry>
1208+
</row>
12051209
<row>
12061210
<entry><literal>ControlFileRead</literal></entry>
12071211
<entry>Waiting for a read from the <filename>pg_control</filename>

‎src/backend/postmaster/pgstat.c

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3940,6 +3940,9 @@ pgstat_get_wait_io(WaitEventIO w)
39403940
caseWAIT_EVENT_BUFFILE_WRITE:
39413941
event_name="BufFileWrite";
39423942
break;
3943+
caseWAIT_EVENT_BUFFILE_TRUNCATE:
3944+
event_name="BufFileTruncate";
3945+
break;
39433946
caseWAIT_EVENT_CONTROL_FILE_READ:
39443947
event_name="ControlFileRead";
39453948
break;

‎src/backend/storage/file/buffile.c

Lines changed: 119 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,14 @@
3232
* (by opening multiple fd.c temporary files). This is an essential feature
3333
* for sorts and hashjoins on large amounts of data.
3434
*
35-
* BufFile supports temporary files that can be made read-only and shared with
36-
* other backends, as infrastructure for parallel execution. Such files need
37-
* to be created as a member of a SharedFileSet that all participants are
38-
* attached to.
35+
* BufFile supports temporary files that can be shared with other backends, as
36+
* infrastructure for parallel execution. Such files need to be created as a
37+
* member of a SharedFileSet that all participants are attached to.
38+
*
39+
* BufFile also supports temporary files that can be used by the single backend
40+
* when the corresponding files need to be survived across the transaction and
41+
* need to be opened and closed multiple times. Such files need to be created
42+
* as a member of a SharedFileSet.
3943
*-------------------------------------------------------------------------
4044
*/
4145

@@ -277,7 +281,7 @@ BufFileCreateShared(SharedFileSet *fileset, const char *name)
277281
* backends and render it read-only.
278282
*/
279283
BufFile*
280-
BufFileOpenShared(SharedFileSet*fileset,constchar*name)
284+
BufFileOpenShared(SharedFileSet*fileset,constchar*name,intmode)
281285
{
282286
BufFile*file;
283287
charsegment_name[MAXPGPATH];
@@ -301,7 +305,7 @@ BufFileOpenShared(SharedFileSet *fileset, const char *name)
301305
}
302306
/* Try to load a segment. */
303307
SharedSegmentName(segment_name,name,nfiles);
304-
files[nfiles]=SharedFileSetOpen(fileset,segment_name);
308+
files[nfiles]=SharedFileSetOpen(fileset,segment_name,mode);
305309
if (files[nfiles] <=0)
306310
break;
307311
++nfiles;
@@ -321,7 +325,7 @@ BufFileOpenShared(SharedFileSet *fileset, const char *name)
321325

322326
file=makeBufFileCommon(nfiles);
323327
file->files=files;
324-
file->readOnly=true;/* Can't write to files opened this way */
328+
file->readOnly=(mode==O_RDONLY) ? true : false;
325329
file->fileset=fileset;
326330
file->name=pstrdup(name);
327331

@@ -666,11 +670,21 @@ BufFileSeek(BufFile *file, int fileno, off_t offset, int whence)
666670
newFile=file->curFile;
667671
newOffset= (file->curOffset+file->pos)+offset;
668672
break;
669-
#ifdefNOT_USED
670673
caseSEEK_END:
671-
/* could be implemented, not needed currently */
674+
675+
/*
676+
* The file size of the last file gives us the end offset of that
677+
* file.
678+
*/
679+
newFile=file->numFiles-1;
680+
newOffset=FileSize(file->files[file->numFiles-1]);
681+
if (newOffset<0)
682+
ereport(ERROR,
683+
(errcode_for_file_access(),
684+
errmsg("could not determine size of temporary file \"%s\" from BufFile \"%s\": %m",
685+
FilePathName(file->files[file->numFiles-1]),
686+
file->name)));
672687
break;
673-
#endif
674688
default:
675689
elog(ERROR,"invalid whence: %d",whence);
676690
returnEOF;
@@ -838,3 +852,98 @@ BufFileAppend(BufFile *target, BufFile *source)
838852

839853
returnstartBlock;
840854
}
855+
856+
/*
857+
* Truncate a BufFile created by BufFileCreateShared up to the given fileno and
858+
* the offset.
859+
*/
860+
void
861+
BufFileTruncateShared(BufFile*file,intfileno,off_toffset)
862+
{
863+
intnumFiles=file->numFiles;
864+
intnewFile=fileno;
865+
off_tnewOffset=file->curOffset;
866+
charsegment_name[MAXPGPATH];
867+
inti;
868+
869+
/*
870+
* Loop over all the files up to the given fileno and remove the files
871+
* that are greater than the fileno and truncate the given file up to the
872+
* offset. Note that we also remove the given fileno if the offset is 0
873+
* provided it is not the first file in which we truncate it.
874+
*/
875+
for (i=file->numFiles-1;i >=fileno;i--)
876+
{
877+
if ((i!=fileno||offset==0)&&i!=0)
878+
{
879+
SharedSegmentName(segment_name,file->name,i);
880+
FileClose(file->files[i]);
881+
if (!SharedFileSetDelete(file->fileset,segment_name, true))
882+
ereport(ERROR,
883+
(errcode_for_file_access(),
884+
errmsg("could not delete shared fileset \"%s\": %m",
885+
segment_name)));
886+
numFiles--;
887+
newOffset=MAX_PHYSICAL_FILESIZE;
888+
889+
/*
890+
* This is required to indicate that we have deleted the given
891+
* fileno.
892+
*/
893+
if (i==fileno)
894+
newFile--;
895+
}
896+
else
897+
{
898+
if (FileTruncate(file->files[i],offset,
899+
WAIT_EVENT_BUFFILE_TRUNCATE)<0)
900+
ereport(ERROR,
901+
(errcode_for_file_access(),
902+
errmsg("could not truncate file \"%s\": %m",
903+
FilePathName(file->files[i]))));
904+
newOffset=offset;
905+
}
906+
}
907+
908+
file->numFiles=numFiles;
909+
910+
/*
911+
* If the truncate point is within existing buffer then we can just adjust
912+
* pos within buffer.
913+
*/
914+
if (newFile==file->curFile&&
915+
newOffset >=file->curOffset&&
916+
newOffset <=file->curOffset+file->nbytes)
917+
{
918+
/* No need to reset the current pos if the new pos is greater. */
919+
if (newOffset <=file->curOffset+file->pos)
920+
file->pos= (int) (newOffset-file->curOffset);
921+
922+
/* Adjust the nbytes for the current buffer. */
923+
file->nbytes= (int) (newOffset-file->curOffset);
924+
}
925+
elseif (newFile==file->curFile&&
926+
newOffset<file->curOffset)
927+
{
928+
/*
929+
* The truncate point is within the existing file but prior to the
930+
* current position, so we can forget the current buffer and reset the
931+
* current position.
932+
*/
933+
file->curOffset=newOffset;
934+
file->pos=0;
935+
file->nbytes=0;
936+
}
937+
elseif (newFile<file->curFile)
938+
{
939+
/*
940+
* The truncate point is prior to the current file, so need to reset
941+
* the current position accordingly.
942+
*/
943+
file->curFile=newFile;
944+
file->curOffset=newOffset;
945+
file->pos=0;
946+
file->nbytes=0;
947+
}
948+
/* Nothing to do, if the truncate point is beyond current file. */
949+
}

‎src/backend/storage/file/fd.c

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1743,18 +1743,17 @@ PathNameCreateTemporaryFile(const char *path, bool error_on_failure)
17431743
/*
17441744
* Open a file that was created with PathNameCreateTemporaryFile, possibly in
17451745
* another backend. Files opened this way don't count against the
1746-
* temp_file_limit of the caller, areread-only and are automatically closed
1747-
*at the end of thetransaction but are not deleted on close.
1746+
* temp_file_limit of the caller, areautomatically closed at the end of the
1747+
* transaction but are not deleted on close.
17481748
*/
17491749
File
1750-
PathNameOpenTemporaryFile(constchar*path)
1750+
PathNameOpenTemporaryFile(constchar*path,intmode)
17511751
{
17521752
Filefile;
17531753

17541754
ResourceOwnerEnlargeFiles(CurrentResourceOwner);
17551755

1756-
/* We open the file read-only. */
1757-
file=PathNameOpenFile(path,O_RDONLY |PG_BINARY);
1756+
file=PathNameOpenFile(path,mode |PG_BINARY);
17581757

17591758
/* If no such file, then we don't raise an error. */
17601759
if (file <=0&&errno!=ENOENT)

‎src/backend/storage/file/sharedfileset.c

Lines changed: 97 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@
1313
* files can be discovered by name, and a shared ownership semantics so that
1414
* shared files survive until the last user detaches.
1515
*
16+
* SharedFileSets can be used by backends when the temporary files need to be
17+
* opened/closed multiple times and the underlying files need to survive across
18+
* transactions.
19+
*
1620
*-------------------------------------------------------------------------
1721
*/
1822

@@ -25,25 +29,36 @@
2529
#include"common/hashfn.h"
2630
#include"miscadmin.h"
2731
#include"storage/dsm.h"
32+
#include"storage/ipc.h"
2833
#include"storage/sharedfileset.h"
2934
#include"utils/builtins.h"
3035

36+
staticList*filesetlist=NIL;
37+
3138
staticvoidSharedFileSetOnDetach(dsm_segment*segment,Datumdatum);
39+
staticvoidSharedFileSetDeleteOnProcExit(intstatus,Datumarg);
3240
staticvoidSharedFileSetPath(char*path,SharedFileSet*fileset,Oidtablespace);
3341
staticvoidSharedFilePath(char*path,SharedFileSet*fileset,constchar*name);
3442
staticOidChooseTablespace(constSharedFileSet*fileset,constchar*name);
3543

3644
/*
37-
* Initialize a space for temporary files that can be opened for read-only
38-
* access by other backends. Other backends must attach to it before
39-
* accessing it. Associate this SharedFileSet with 'seg'. Any contained
40-
* files will be deleted when the last backend detaches.
45+
* Initialize a space for temporary files that can be opened by other backends.
46+
* Other backends must attach to it before accessing it. Associate this
47+
* SharedFileSet with 'seg'. Any contained files will be deleted when the
48+
* last backend detaches.
49+
*
50+
* We can also use this interface if the temporary files are used only by
51+
* single backend but the files need to be opened and closed multiple times
52+
* and also the underlying files need to survive across transactions. For
53+
* such cases, dsm segment 'seg' should be passed as NULL. Callers are
54+
* expected to explicitly remove such files by using SharedFileSetDelete/
55+
* SharedFileSetDeleteAll or we remove such files on proc exit.
4156
*
4257
* Files will be distributed over the tablespaces configured in
4358
* temp_tablespaces.
4459
*
4560
* Under the covers the set is one or more directories which will eventually
46-
* be deleted when there are no backends attached.
61+
* be deleted.
4762
*/
4863
void
4964
SharedFileSetInit(SharedFileSet*fileset,dsm_segment*seg)
@@ -84,7 +99,25 @@ SharedFileSetInit(SharedFileSet *fileset, dsm_segment *seg)
8499
}
85100

86101
/* Register our cleanup callback. */
87-
on_dsm_detach(seg,SharedFileSetOnDetach,PointerGetDatum(fileset));
102+
if (seg)
103+
on_dsm_detach(seg,SharedFileSetOnDetach,PointerGetDatum(fileset));
104+
else
105+
{
106+
staticboolregistered_cleanup= false;
107+
108+
if (!registered_cleanup)
109+
{
110+
/*
111+
* We must not have registered any fileset before registering the
112+
* fileset clean up.
113+
*/
114+
Assert(filesetlist==NIL);
115+
on_proc_exit(SharedFileSetDeleteOnProcExit,0);
116+
registered_cleanup= true;
117+
}
118+
119+
filesetlist=lcons((void*)fileset,filesetlist);
120+
}
88121
}
89122

90123
/*
@@ -147,13 +180,13 @@ SharedFileSetCreate(SharedFileSet *fileset, const char *name)
147180
* another backend.
148181
*/
149182
File
150-
SharedFileSetOpen(SharedFileSet*fileset,constchar*name)
183+
SharedFileSetOpen(SharedFileSet*fileset,constchar*name,intmode)
151184
{
152185
charpath[MAXPGPATH];
153186
Filefile;
154187

155188
SharedFilePath(path,fileset,name);
156-
file=PathNameOpenTemporaryFile(path);
189+
file=PathNameOpenTemporaryFile(path,mode);
157190

158191
returnfile;
159192
}
@@ -192,6 +225,9 @@ SharedFileSetDeleteAll(SharedFileSet *fileset)
192225
SharedFileSetPath(dirpath,fileset,fileset->tablespaces[i]);
193226
PathNameDeleteTemporaryDir(dirpath);
194227
}
228+
229+
/* Unregister the shared fileset */
230+
SharedFileSetUnregister(fileset);
195231
}
196232

197233
/*
@@ -222,6 +258,59 @@ SharedFileSetOnDetach(dsm_segment *segment, Datum datum)
222258
SharedFileSetDeleteAll(fileset);
223259
}
224260

261+
/*
262+
* Callback function that will be invoked on the process exit. This will
263+
* process the list of all the registered sharedfilesets and delete the
264+
* underlying files.
265+
*/
266+
staticvoid
267+
SharedFileSetDeleteOnProcExit(intstatus,Datumarg)
268+
{
269+
ListCell*l;
270+
271+
/* Loop over all the pending shared fileset entry */
272+
foreach(l,filesetlist)
273+
{
274+
SharedFileSet*fileset= (SharedFileSet*)lfirst(l);
275+
276+
SharedFileSetDeleteAll(fileset);
277+
}
278+
279+
filesetlist=NIL;
280+
}
281+
282+
/*
283+
* Unregister the shared fileset entry registered for cleanup on proc exit.
284+
*/
285+
void
286+
SharedFileSetUnregister(SharedFileSet*input_fileset)
287+
{
288+
boolfound= false;
289+
ListCell*l;
290+
291+
/*
292+
* If the caller is following the dsm based cleanup then we don't maintain
293+
* the filesetlist so return.
294+
*/
295+
if (filesetlist==NIL)
296+
return;
297+
298+
foreach(l,filesetlist)
299+
{
300+
SharedFileSet*fileset= (SharedFileSet*)lfirst(l);
301+
302+
/* Remove the entry from the list */
303+
if (input_fileset==fileset)
304+
{
305+
filesetlist=list_delete_cell(filesetlist,l);
306+
found= true;
307+
break;
308+
}
309+
}
310+
311+
Assert(found);
312+
}
313+
225314
/*
226315
* Build the path for the directory holding the files backing a SharedFileSet
227316
* in a given tablespace.

‎src/backend/utils/sort/logtape.c

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,8 @@
7878

7979
#include"postgres.h"
8080

81+
#include<fcntl.h>
82+
8183
#include"storage/buffile.h"
8284
#include"utils/builtins.h"
8385
#include"utils/logtape.h"
@@ -551,7 +553,7 @@ ltsConcatWorkerTapes(LogicalTapeSet *lts, TapeShare *shared,
551553
lt=&lts->tapes[i];
552554

553555
pg_itoa(i,filename);
554-
file=BufFileOpenShared(fileset,filename);
556+
file=BufFileOpenShared(fileset,filename,O_RDONLY);
555557
filesize=BufFileSize(file);
556558

557559
/*

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp