Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitfba105b

Browse files
committed
Use "transient" files for blind writes, take 2
"Blind writes" are a mechanism to push buffers down to disk whenevicting them; since they may belong to different databases than the onea backend is connected to, the backend does not necessarily have arelation to link them to, and thus no way to blow them away. We werekeeping those files open indefinitely, which would cause a problem ifthe underlying table was deleted, because the operating system would notbe able to reclaim the disk space used by those files.To fix, have bufmgr mark such files as transient to smgr; the lowerlayer is allowed to close the file descriptor when the currenttransaction ends. We must be careful to have any other access of thefile to remove the transient markings, to prevent unnecessary expensivesystem calls when evicting buffers belonging to our own database (whichfiles we're likely to require again soon.)This commit fixes a bug in the previous one, which neglected to cleanlyhandle the LRU ring that fd.c uses to manage open files, and caused anunacceptable failure just before beta2 and was thus reverted.
1 parent3d114b6 commitfba105b

File tree

6 files changed

+102
-28
lines changed

6 files changed

+102
-28
lines changed

‎src/backend/storage/buffer/bufmgr.c‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1834,7 +1834,10 @@ BufferGetTag(Buffer buffer, RelFileNode *rnode, ForkNumber *forknum,
18341834
* written.)
18351835
*
18361836
* If the caller has an smgr reference for the buffer's relation, pass it
1837-
* as the second parameter. If not, pass NULL.
1837+
* as the second parameter. If not, pass NULL. In the latter case, the
1838+
* relation will be marked as "transient" so that the corresponding
1839+
* kernel-level file descriptors are closed when the current transaction ends,
1840+
* if any.
18381841
*/
18391842
staticvoid
18401843
FlushBuffer(volatileBufferDesc*buf,SMgrRelationreln)
@@ -1856,9 +1859,12 @@ FlushBuffer(volatile BufferDesc *buf, SMgrRelation reln)
18561859
errcontext.previous=error_context_stack;
18571860
error_context_stack=&errcontext;
18581861

1859-
/* Find smgr relation for buffer */
1862+
/* Find smgr relation for buffer, and mark it as transient */
18601863
if (reln==NULL)
1864+
{
18611865
reln=smgropen(buf->tag.rnode,InvalidBackendId);
1866+
smgrsettransient(reln);
1867+
}
18621868

18631869
TRACE_POSTGRESQL_BUFFER_FLUSH_START(buf->tag.forkNum,
18641870
buf->tag.blockNum,

‎src/backend/storage/file/fd.c‎

Lines changed: 65 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -125,12 +125,11 @@ static intmax_safe_fds = 32;/* default if not changed */
125125
/* these are the assigned bits in fdstate below: */
126126
#defineFD_TEMPORARY(1 << 0)/* T = delete when closed */
127127
#defineFD_XACT_TEMPORARY(1 << 1)/* T = delete at eoXact */
128+
#defineFD_XACT_TRANSIENT(1 << 2)/* T = close (not delete) at aoXact,
129+
* but keep VFD */
128130

129-
/*
130-
* Flag to tell whether it's worth scanning VfdCache looking for temp files to
131-
* close
132-
*/
133-
staticboolhave_xact_temporary_files= false;
131+
/* Flag to tell whether there are files to close/delete at end of transaction */
132+
staticboolhave_pending_fd_cleanup= false;
134133

135134
typedefstructvfd
136135
{
@@ -591,6 +590,7 @@ LruDelete(File file)
591590
Vfd*vfdP;
592591

593592
Assert(file!=0);
593+
Assert(!FileIsNotOpen(file));
594594

595595
DO_DB(elog(LOG,"LruDelete %d (%s)",
596596
file,VfdCache[file].fileName));
@@ -953,7 +953,7 @@ OpenTemporaryFile(bool interXact)
953953
VfdCache[file].resowner=CurrentResourceOwner;
954954

955955
/* ensure cleanup happens at eoxact */
956-
have_xact_temporary_files= true;
956+
have_pending_fd_cleanup= true;
957957
}
958958

959959
returnfile;
@@ -1026,6 +1026,25 @@ OpenTemporaryFileInTablespace(Oid tblspcOid, bool rejectError)
10261026
returnfile;
10271027
}
10281028

1029+
/*
1030+
* Set the transient flag on a file
1031+
*
1032+
* This flag tells CleanupTempFiles to close the kernel-level file descriptor
1033+
* (but not the VFD itself) at end of transaction.
1034+
*/
1035+
void
1036+
FileSetTransient(Filefile)
1037+
{
1038+
Vfd*vfdP;
1039+
1040+
Assert(FileIsValid(file));
1041+
1042+
vfdP=&VfdCache[file];
1043+
vfdP->fdstate |=FD_XACT_TRANSIENT;
1044+
1045+
have_pending_fd_cleanup= true;
1046+
}
1047+
10291048
/*
10301049
* close a file when done with it
10311050
*/
@@ -1778,8 +1797,9 @@ AtEOSubXact_Files(bool isCommit, SubTransactionId mySubid,
17781797
* particularly care which). All still-open per-transaction temporary file
17791798
* VFDs are closed, which also causes the underlying files to be deleted
17801799
* (although they should've been closed already by the ResourceOwner
1781-
* cleanup). Furthermore, all "allocated" stdio files are closed. We also
1782-
* forget any transaction-local temp tablespace list.
1800+
* cleanup). Transient files have their kernel file descriptors closed.
1801+
* Furthermore, all "allocated" stdio files are closed. We also forget any
1802+
* transaction-local temp tablespace list.
17831803
*/
17841804
void
17851805
AtEOXact_Files(void)
@@ -1802,7 +1822,10 @@ AtProcExit_Files(int code, Datum arg)
18021822
}
18031823

18041824
/*
1805-
* Close temporary files and delete their underlying files.
1825+
* General cleanup routine for fd.c.
1826+
*
1827+
* Temporary files are closed, and their underlying files deleted.
1828+
* Transient files are closed.
18061829
*
18071830
* isProcExit: if true, this is being called as the backend process is
18081831
* exiting. If that's the case, we should remove all temporary files; if
@@ -1819,35 +1842,51 @@ CleanupTempFiles(bool isProcExit)
18191842
* Careful here: at proc_exit we need extra cleanup, not just
18201843
* xact_temporary files.
18211844
*/
1822-
if (isProcExit||have_xact_temporary_files)
1845+
if (isProcExit||have_pending_fd_cleanup)
18231846
{
18241847
Assert(FileIsNotOpen(0));/* Make sure ring not corrupted */
18251848
for (i=1;i<SizeVfdCache;i++)
18261849
{
18271850
unsigned shortfdstate=VfdCache[i].fdstate;
18281851

1829-
if ((fdstate&FD_TEMPORARY)&&VfdCache[i].fileName!=NULL)
1852+
if (VfdCache[i].fileName!=NULL)
18301853
{
1831-
/*
1832-
* If we're in the process of exiting a backend process, close
1833-
* all temporary files. Otherwise, only close temporary files
1834-
* local to the current transaction. They should be closed by
1835-
* the ResourceOwner mechanism already, so this is just a
1836-
* debugging cross-check.
1837-
*/
1838-
if (isProcExit)
1839-
FileClose(i);
1840-
elseif (fdstate&FD_XACT_TEMPORARY)
1854+
if (fdstate&FD_TEMPORARY)
1855+
{
1856+
/*
1857+
* If we're in the process of exiting a backend process, close
1858+
* all temporary files. Otherwise, only close temporary files
1859+
* local to the current transaction. They should be closed by
1860+
* the ResourceOwner mechanism already, so this is just a
1861+
* debugging cross-check.
1862+
*/
1863+
if (isProcExit)
1864+
FileClose(i);
1865+
elseif (fdstate&FD_XACT_TEMPORARY)
1866+
{
1867+
elog(WARNING,
1868+
"temporary file %s not closed at end-of-transaction",
1869+
VfdCache[i].fileName);
1870+
FileClose(i);
1871+
}
1872+
}
1873+
elseif (fdstate&FD_XACT_TRANSIENT)
18411874
{
1842-
elog(WARNING,
1843-
"temporary file %s not closed at end-of-transaction",
1844-
VfdCache[i].fileName);
1845-
FileClose(i);
1875+
/*
1876+
* Close the FD, and remove the entry from the LRU ring,
1877+
* but also remove the flag from the VFD. This is to
1878+
* ensure that if the VFD is reused in the future for
1879+
* non-transient access, we don't close it inappropriately
1880+
* then.
1881+
*/
1882+
if (!FileIsNotOpen(i))
1883+
LruDelete(i);
1884+
VfdCache[i].fdstate &= ~FD_XACT_TRANSIENT;
18461885
}
18471886
}
18481887
}
18491888

1850-
have_xact_temporary_files= false;
1889+
have_pending_fd_cleanup= false;
18511890
}
18521891

18531892
/* Clean up "allocated" stdio files and dirs. */

‎src/backend/storage/smgr/md.c‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -288,6 +288,9 @@ mdcreate(SMgrRelation reln, ForkNumber forkNum, bool isRedo)
288288

289289
pfree(path);
290290

291+
if (reln->smgr_transient)
292+
FileSetTransient(fd);
293+
291294
reln->md_fd[forkNum]=_fdvec_alloc();
292295

293296
reln->md_fd[forkNum]->mdfd_vfd=fd;
@@ -542,6 +545,9 @@ mdopen(SMgrRelation reln, ForkNumber forknum, ExtensionBehavior behavior)
542545

543546
pfree(path);
544547

548+
if (reln->smgr_transient)
549+
FileSetTransient(fd);
550+
545551
reln->md_fd[forknum]=mdfd=_fdvec_alloc();
546552

547553
mdfd->mdfd_vfd=fd;
@@ -1556,6 +1562,9 @@ _mdfd_openseg(SMgrRelation reln, ForkNumber forknum, BlockNumber segno,
15561562
if (fd<0)
15571563
returnNULL;
15581564

1565+
if (reln->smgr_transient)
1566+
FileSetTransient(fd);
1567+
15591568
/* allocate an mdfdvec entry for it */
15601569
v=_fdvec_alloc();
15611570

‎src/backend/storage/smgr/smgr.c‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,16 +165,33 @@ smgropen(RelFileNode rnode, BackendId backend)
165165
reln->smgr_targblock=InvalidBlockNumber;
166166
reln->smgr_fsm_nblocks=InvalidBlockNumber;
167167
reln->smgr_vm_nblocks=InvalidBlockNumber;
168+
reln->smgr_transient= false;
168169
reln->smgr_which=0;/* we only have md.c at present */
169170

170171
/* mark it not open */
171172
for (forknum=0;forknum <=MAX_FORKNUM;forknum++)
172173
reln->md_fd[forknum]=NULL;
173174
}
175+
else
176+
/* if it was transient before, it no longer is */
177+
reln->smgr_transient= false;
174178

175179
returnreln;
176180
}
177181

182+
/*
183+
* smgrsettransient() -- mark an SMgrRelation object as transaction-bound
184+
*
185+
* The main effect of this is that all opened files are marked to be
186+
* kernel-level closed (but not necessarily VFD-closed) when the current
187+
* transaction ends.
188+
*/
189+
void
190+
smgrsettransient(SMgrRelationreln)
191+
{
192+
reln->smgr_transient= true;
193+
}
194+
178195
/*
179196
* smgrsetowner() -- Establish a long-lived reference to an SMgrRelation object
180197
*

‎src/include/storage/fd.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ extern intmax_files_per_process;
6161
/* Operations on virtual Files --- equivalent to Unix kernel file ops */
6262
externFilePathNameOpenFile(FileNamefileName,intfileFlags,intfileMode);
6363
externFileOpenTemporaryFile(boolinterXact);
64+
externvoidFileSetTransient(Filefile);
6465
externvoidFileClose(Filefile);
6566
externintFilePrefetch(Filefile,off_toffset,intamount);
6667
externintFileRead(Filefile,char*buffer,intamount);

‎src/include/storage/smgr.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ typedef struct SMgrRelationData
6262
* submodules.Do not touch them from elsewhere.
6363
*/
6464
intsmgr_which;/* storage manager selector */
65+
boolsmgr_transient;/* T if files are to be closed at EOXact */
6566

6667
/* for md.c; NULL for forks that are not open */
6768
struct_MdfdVec*md_fd[MAX_FORKNUM+1];
@@ -74,6 +75,7 @@ typedef SMgrRelationData *SMgrRelation;
7475

7576
externvoidsmgrinit(void);
7677
externSMgrRelationsmgropen(RelFileNodernode,BackendIdbackend);
78+
externvoidsmgrsettransient(SMgrRelationreln);
7779
externboolsmgrexists(SMgrRelationreln,ForkNumberforknum);
7880
externvoidsmgrsetowner(SMgrRelation*owner,SMgrRelationreln);
7981
externvoidsmgrclose(SMgrRelationreln);

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp