Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit54d9e8c

Browse files
committed
Use "transient" files for blind writes
"Blind writes" are a mechanism to push buffers down to disk whenevicting them; since they may belong to different databases than the onea backend is connected to, the backend does not necessarily have arelation to link them to, and thus no way to blow them away. We werekeeping those files open indefinitely, which would cause a problem ifthe underlying table was deleted, because the operating system would notbe able to reclaim the disk space used by those files.To fix, have bufmgr mark such files as transient to smgr; the lowerlayer is allowed to close the file descriptor when the currenttransaction ends. We must be careful to have any other access of thefile to remove the transient markings, to prevent unnecessary expensivesystem calls when evicting buffers belonging to our own database (whichfiles we're likely to require again soon.)
1 parent74b1d29 commit54d9e8c

File tree

6 files changed

+119
-28
lines changed

6 files changed

+119
-28
lines changed

‎src/backend/storage/buffer/bufmgr.c‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1834,7 +1834,10 @@ BufferGetTag(Buffer buffer, RelFileNode *rnode, ForkNumber *forknum,
18341834
* written.)
18351835
*
18361836
* If the caller has an smgr reference for the buffer's relation, pass it
1837-
* as the second parameter. If not, pass NULL.
1837+
* as the second parameter. If not, pass NULL. In the latter case, the
1838+
* relation will be marked as "transient" so that the corresponding
1839+
* kernel-level file descriptors are closed when the current transaction ends,
1840+
* if any.
18381841
*/
18391842
staticvoid
18401843
FlushBuffer(volatileBufferDesc*buf,SMgrRelationreln)
@@ -1856,9 +1859,12 @@ FlushBuffer(volatile BufferDesc *buf, SMgrRelation reln)
18561859
errcontext.previous=error_context_stack;
18571860
error_context_stack=&errcontext;
18581861

1859-
/* Find smgr relation for buffer */
1862+
/* Find smgr relation for buffer, and mark it as transient */
18601863
if (reln==NULL)
1864+
{
18611865
reln=smgropen(buf->tag.rnode,InvalidBackendId);
1866+
smgrsettransient(reln);
1867+
}
18621868

18631869
TRACE_POSTGRESQL_BUFFER_FLUSH_START(buf->tag.forkNum,
18641870
buf->tag.blockNum,

‎src/backend/storage/file/fd.c‎

Lines changed: 82 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -125,12 +125,11 @@ static intmax_safe_fds = 32;/* default if not changed */
125125
/* these are the assigned bits in fdstate below: */
126126
#defineFD_TEMPORARY(1 << 0)/* T = delete when closed */
127127
#defineFD_XACT_TEMPORARY(1 << 1)/* T = delete at eoXact */
128+
#defineFD_XACT_TRANSIENT(1 << 2)/* T = close (not delete) at aoXact,
129+
* but keep VFD */
128130

129-
/*
130-
* Flag to tell whether it's worth scanning VfdCache looking for temp files to
131-
* close
132-
*/
133-
staticboolhave_xact_temporary_files= false;
131+
/* Flag to tell whether there are files to close/delete at end of transaction */
132+
staticboolhave_pending_fd_cleanup= false;
134133

135134
typedefstructvfd
136135
{
@@ -953,7 +952,7 @@ OpenTemporaryFile(bool interXact)
953952
VfdCache[file].resowner=CurrentResourceOwner;
954953

955954
/* ensure cleanup happens at eoxact */
956-
have_xact_temporary_files= true;
955+
have_pending_fd_cleanup= true;
957956
}
958957

959958
returnfile;
@@ -1026,6 +1025,45 @@ OpenTemporaryFileInTablespace(Oid tblspcOid, bool rejectError)
10261025
returnfile;
10271026
}
10281027

1028+
/*
1029+
* Set the transient flag on a file
1030+
*
1031+
* This flag tells CleanupTempFiles to close the kernel-level file descriptor
1032+
* (but not the VFD itself) at end of transaction.
1033+
*/
1034+
void
1035+
FileSetTransient(Filefile)
1036+
{
1037+
Vfd*vfdP;
1038+
1039+
Assert(FileIsValid(file));
1040+
1041+
vfdP=&VfdCache[file];
1042+
vfdP->fdstate |=FD_XACT_TRANSIENT;
1043+
1044+
have_pending_fd_cleanup= true;
1045+
}
1046+
1047+
/*
1048+
* Close a file at the kernel level, but keep the VFD open
1049+
*/
1050+
staticvoid
1051+
FileKernelClose(Filefile)
1052+
{
1053+
Vfd*vfdP;
1054+
1055+
Assert(FileIsValid(file));
1056+
1057+
vfdP=&VfdCache[file];
1058+
1059+
if (!FileIsNotOpen(file))
1060+
{
1061+
if (close(vfdP->fd))
1062+
elog(ERROR,"could not close file \"%s\": %m",vfdP->fileName);
1063+
vfdP->fd=VFD_CLOSED;
1064+
}
1065+
}
1066+
10291067
/*
10301068
* close a file when done with it
10311069
*/
@@ -1778,8 +1816,9 @@ AtEOSubXact_Files(bool isCommit, SubTransactionId mySubid,
17781816
* particularly care which). All still-open per-transaction temporary file
17791817
* VFDs are closed, which also causes the underlying files to be deleted
17801818
* (although they should've been closed already by the ResourceOwner
1781-
* cleanup). Furthermore, all "allocated" stdio files are closed. We also
1782-
* forget any transaction-local temp tablespace list.
1819+
* cleanup). Transient files have their kernel file descriptors closed.
1820+
* Furthermore, all "allocated" stdio files are closed. We also forget any
1821+
* transaction-local temp tablespace list.
17831822
*/
17841823
void
17851824
AtEOXact_Files(void)
@@ -1802,7 +1841,10 @@ AtProcExit_Files(int code, Datum arg)
18021841
}
18031842

18041843
/*
1805-
* Close temporary files and delete their underlying files.
1844+
* General cleanup routine for fd.c.
1845+
*
1846+
* Temporary files are closed, and their underlying files deleted.
1847+
* Transient files are closed.
18061848
*
18071849
* isProcExit: if true, this is being called as the backend process is
18081850
* exiting. If that's the case, we should remove all temporary files; if
@@ -1819,35 +1861,49 @@ CleanupTempFiles(bool isProcExit)
18191861
* Careful here: at proc_exit we need extra cleanup, not just
18201862
* xact_temporary files.
18211863
*/
1822-
if (isProcExit||have_xact_temporary_files)
1864+
if (isProcExit||have_pending_fd_cleanup)
18231865
{
18241866
Assert(FileIsNotOpen(0));/* Make sure ring not corrupted */
18251867
for (i=1;i<SizeVfdCache;i++)
18261868
{
18271869
unsigned shortfdstate=VfdCache[i].fdstate;
18281870

1829-
if ((fdstate&FD_TEMPORARY)&&VfdCache[i].fileName!=NULL)
1871+
if (VfdCache[i].fileName!=NULL)
18301872
{
1831-
/*
1832-
* If we're in the process of exiting a backend process, close
1833-
* all temporary files. Otherwise, only close temporary files
1834-
* local to the current transaction. They should be closed by
1835-
* the ResourceOwner mechanism already, so this is just a
1836-
* debugging cross-check.
1837-
*/
1838-
if (isProcExit)
1839-
FileClose(i);
1840-
elseif (fdstate&FD_XACT_TEMPORARY)
1873+
if (fdstate&FD_TEMPORARY)
1874+
{
1875+
/*
1876+
* If we're in the process of exiting a backend process, close
1877+
* all temporary files. Otherwise, only close temporary files
1878+
* local to the current transaction. They should be closed by
1879+
* the ResourceOwner mechanism already, so this is just a
1880+
* debugging cross-check.
1881+
*/
1882+
if (isProcExit)
1883+
FileClose(i);
1884+
elseif (fdstate&FD_XACT_TEMPORARY)
1885+
{
1886+
elog(WARNING,
1887+
"temporary file %s not closed at end-of-transaction",
1888+
VfdCache[i].fileName);
1889+
FileClose(i);
1890+
}
1891+
}
1892+
elseif (fdstate&FD_XACT_TRANSIENT)
18411893
{
1842-
elog(WARNING,
1843-
"temporary file %s not closed at end-of-transaction",
1844-
VfdCache[i].fileName);
1845-
FileClose(i);
1894+
/*
1895+
* Close the kernel file descriptor, but also remove the
1896+
* flag from the VFD. This is to ensure that if the VFD is
1897+
* reused in the future for non-transient access, we don't
1898+
* close it inappropriately then.
1899+
*/
1900+
FileKernelClose(i);
1901+
VfdCache[i].fdstate &= ~FD_XACT_TRANSIENT;
18461902
}
18471903
}
18481904
}
18491905

1850-
have_xact_temporary_files= false;
1906+
have_pending_fd_cleanup= false;
18511907
}
18521908

18531909
/* Clean up "allocated" stdio files and dirs. */

‎src/backend/storage/smgr/md.c‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,9 @@ mdcreate(SMgrRelation reln, ForkNumber forkNum, bool isRedo)
288288

289289
pfree(path);
290290

291+
if (reln->smgr_transient)
292+
FileSetTransient(fd);
293+
291294
reln->md_fd[forkNum]=_fdvec_alloc();
292295

293296
reln->md_fd[forkNum]->mdfd_vfd=fd;
@@ -542,6 +545,9 @@ mdopen(SMgrRelation reln, ForkNumber forknum, ExtensionBehavior behavior)
542545

543546
pfree(path);
544547

548+
if (reln->smgr_transient)
549+
FileSetTransient(fd);
550+
545551
reln->md_fd[forknum]=mdfd=_fdvec_alloc();
546552

547553
mdfd->mdfd_vfd=fd;
@@ -1556,6 +1562,9 @@ _mdfd_openseg(SMgrRelation reln, ForkNumber forknum, BlockNumber segno,
15561562
if (fd<0)
15571563
returnNULL;
15581564

1565+
if (reln->smgr_transient)
1566+
FileSetTransient(fd);
1567+
15591568
/* allocate an mdfdvec entry for it */
15601569
v=_fdvec_alloc();
15611570

‎src/backend/storage/smgr/smgr.c‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,16 +165,33 @@ smgropen(RelFileNode rnode, BackendId backend)
165165
reln->smgr_targblock=InvalidBlockNumber;
166166
reln->smgr_fsm_nblocks=InvalidBlockNumber;
167167
reln->smgr_vm_nblocks=InvalidBlockNumber;
168+
reln->smgr_transient= false;
168169
reln->smgr_which=0;/* we only have md.c at present */
169170

170171
/* mark it not open */
171172
for (forknum=0;forknum <=MAX_FORKNUM;forknum++)
172173
reln->md_fd[forknum]=NULL;
173174
}
175+
else
176+
/* if it was transient before, it no longer is */
177+
reln->smgr_transient= false;
174178

175179
returnreln;
176180
}
177181

182+
/*
183+
* smgrsettransient() -- mark an SMgrRelation object as transaction-bound
184+
*
185+
* The main effect of this is that all opened files are marked to be
186+
* kernel-level closed (but not necessarily VFD-closed) when the current
187+
* transaction ends.
188+
*/
189+
void
190+
smgrsettransient(SMgrRelationreln)
191+
{
192+
reln->smgr_transient= true;
193+
}
194+
178195
/*
179196
* smgrsetowner() -- Establish a long-lived reference to an SMgrRelation object
180197
*

‎src/include/storage/fd.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ extern intmax_files_per_process;
6161
/* Operations on virtual Files --- equivalent to Unix kernel file ops */
6262
externFilePathNameOpenFile(FileNamefileName,intfileFlags,intfileMode);
6363
externFileOpenTemporaryFile(boolinterXact);
64+
externvoidFileSetTransient(Filefile);
6465
externvoidFileClose(Filefile);
6566
externintFilePrefetch(Filefile,off_toffset,intamount);
6667
externintFileRead(Filefile,char*buffer,intamount);

‎src/include/storage/smgr.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ typedef struct SMgrRelationData
6262
* submodules.Do not touch them from elsewhere.
6363
*/
6464
intsmgr_which;/* storage manager selector */
65+
boolsmgr_transient;/* T if files are to be closed at EOXact */
6566

6667
/* for md.c; NULL for forks that are not open */
6768
struct_MdfdVec*md_fd[MAX_FORKNUM+1];
@@ -74,6 +75,7 @@ typedef SMgrRelationData *SMgrRelation;
7475

7576
externvoidsmgrinit(void);
7677
externSMgrRelationsmgropen(RelFileNodernode,BackendIdbackend);
78+
externvoidsmgrsettransient(SMgrRelationreln);
7779
externboolsmgrexists(SMgrRelationreln,ForkNumberforknum);
7880
externvoidsmgrsetowner(SMgrRelation*owner,SMgrRelationreln);
7981
externvoidsmgrclose(SMgrRelationreln);

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp