Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit70fae1f

Browse files
committed
Initial attempt to fix resource usage
Reference counting is now done manually, but it seems that things canstill go wrong at least during testing
1 parente0b0bec commit70fae1f

File tree

5 files changed

+52
-49
lines changed

5 files changed

+52
-49
lines changed

‎doc/source/changes.rst

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@
22
Changelog
33
#########
44

5+
**********
6+
v0.8.6
7+
**********
8+
- Fixed issue with resources never being freed as mmaps were never closed.
9+
- Client counting is now done manually, instead of relying on pyton's reference count
10+
511
**********
612
v0.8.5
713
**********

‎smmap/mman.py

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
buffer,
99
)
1010

11-
fromweakrefimportref
1211
importsys
1312
fromfunctoolsimportreduce
1413

@@ -55,8 +54,7 @@ def _destroy(self):
5554
# Actual client count, which doesn't include the reference kept by the manager, nor ours
5655
# as we are about to be deleted
5756
try:
58-
num_clients=self._rlist.client_count()-2
59-
ifnum_clients==0andlen(self._rlist)==0:
57+
iflen(self._rlist)==0:
6058
# Free all resources associated with the mapped file
6159
self._manager._fdict.pop(self._rlist.path_or_fd())
6260
# END remove regions list from manager
@@ -78,7 +76,7 @@ def _copy_from(self, rhs):
7876
self._size=rhs._size
7977

8078
ifself._regionisnotNone:
81-
self._region.increment_usage_count()
79+
self._region.increment_client_count()
8280
# END handle regions
8381

8482
def__copy__(self):
@@ -126,20 +124,22 @@ def use_region(self, offset=0, size=0, flags=0):
126124

127125
ifneed_region:
128126
self._region=man._obtain_region(self._rlist,offset,size,flags,False)
127+
self._region.increment_client_count()
129128
# END need region handling
130129

131-
self._region.increment_usage_count()
132130
self._ofs=offset-self._region._b
133131
self._size=min(size,self._region.ofs_end()-offset)
134132

135133
returnself
136134

137135
defunuse_region(self):
138-
"""Unuse theucrrent region. Does nothing if we have no current region
136+
"""Unuse thecurrent region. Does nothing if we have no current region
139137
140138
**Note:** the cursor unuses the region automatically upon destruction. It is recommended
141139
to un-use the region once you are done reading from it in persistent cursors as it
142140
helps to free up resource more quickly"""
141+
ifself._regionisnotNone:
142+
self._region.increment_client_count(-1)
143143
self._region=None
144144
# note: should reset ofs and size, but we spare that for performance. Its not
145145
# allowed to query information if we are not valid !
@@ -184,12 +184,10 @@ def size(self):
184184
""":return: amount of bytes we point to"""
185185
returnself._size
186186

187-
defregion_ref(self):
188-
""":return:weak ref to ourmappedregion.
187+
defregion(self):
188+
""":return:our mapped region, or None if nothing ismappedyet
189189
:raise AssertionError: if we have no current region. This is only useful for debugging"""
190-
ifself._regionisNone:
191-
raiseAssertionError("region not set")
192-
returnref(self._region)
190+
returnself._region
193191

194192
defincludes_ofs(self,ofs):
195193
""":return: True if the given absolute offset is contained in the cursors
@@ -311,8 +309,8 @@ def _collect_lru_region(self, size):
311309
lru_list=None
312310
forregionsinself._fdict.values():
313311
forregioninregions:
314-
# check client count -consider that we keep one reference ourselves !
315-
if (region.client_count()-2==0and
312+
# check client count -if it's 1, it's just us
313+
if (region.client_count()==1and
316314
(lru_regionisNoneorregion._uc<lru_region._uc)):
317315
lru_region=region
318316
lru_list=regions
@@ -326,6 +324,7 @@ def _collect_lru_region(self, size):
326324

327325
num_found+=1
328326
del(lru_list[lru_list.index(lru_region)])
327+
lru_region.increment_client_count(-1)
329328
self._memory_size-=lru_region.size()
330329
self._handle_count-=1
331330
# END while there is more memory to free
@@ -449,7 +448,7 @@ def force_map_handle_removal_win(self, base_path):
449448
forpath,rlistinself._fdict.items():
450449
ifpath.startswith(base_path):
451450
forregioninrlist:
452-
region._mf.close()
451+
region.release()
453452
num_closed+=1
454453
# END path matches
455454
# END for each path

‎smmap/test/test_mman.py

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -119,29 +119,29 @@ def test_memman_operation(self):
119119
# window size is 0 for static managers, hence size will be 0. We take that into consideration
120120
size=man.window_size()//2
121121
assertc.use_region(base_offset,size).is_valid()
122-
rr=c.region_ref()
123-
assertrr().client_count()==2# the manager and the cursor and us
122+
rr=c.region()
123+
assertrr.client_count()==2# the manager and the cursor and us
124124

125125
assertman.num_open_files()==1
126126
assertman.num_file_handles()==1
127-
assertman.mapped_memory_size()==rr().size()
127+
assertman.mapped_memory_size()==rr.size()
128128

129129
# assert c.size() == size # the cursor may overallocate in its static version
130130
assertc.ofs_begin()==base_offset
131-
assertrr().ofs_begin()==0# it was aligned and expanded
131+
assertrr.ofs_begin()==0# it was aligned and expanded
132132
ifman.window_size():
133133
# but isn't larger than the max window (aligned)
134-
assertrr().size()==align_to_mmap(man.window_size(),True)
134+
assertrr.size()==align_to_mmap(man.window_size(),True)
135135
else:
136-
assertrr().size()==fc.size
136+
assertrr.size()==fc.size
137137
# END ignore static managers which dont use windows and are aligned to file boundaries
138138

139139
assertc.buffer()[:]==data[base_offset:base_offset+ (sizeorc.size())]
140140

141141
# obtain second window, which spans the first part of the file - it is a still the same window
142142
nsize= (sizeorfc.size)-10
143143
assertc.use_region(0,nsize).is_valid()
144-
assertc.region_ref()()==rr()
144+
assertc.region()==rr
145145
assertman.num_file_handles()==1
146146
assertc.size()==nsize
147147
assertc.ofs_begin()==0
@@ -154,15 +154,15 @@ def test_memman_operation(self):
154154
ifman.window_size():
155155
assertman.num_file_handles()==2
156156
assertc.size()<size
157-
assertc.region_ref()()isnotrr()# old region is still available, but has not curser ref anymore
158-
assertrr().client_count()==1# only held by manager
157+
assertc.region()isnotrr# old region is still available, but has not curser ref anymore
158+
assertrr.client_count()==1# only held by manager
159159
else:
160160
assertc.size()<fc.size
161161
# END ignore static managers which only have one handle per file
162-
rr=c.region_ref()
163-
assertrr().client_count()==2# manager + cursor
164-
assertrr().ofs_begin()<c.ofs_begin()# it should have extended itself to the left
165-
assertrr().ofs_end()<=fc.size# it cannot be larger than the file
162+
rr=c.region()
163+
assertrr.client_count()==2# manager + cursor
164+
assertrr.ofs_begin()<c.ofs_begin()# it should have extended itself to the left
165+
assertrr.ofs_end()<=fc.size# it cannot be larger than the file
166166
assertc.buffer()[:]==data[base_offset:base_offset+ (sizeorc.size())]
167167

168168
# unising a region makes the cursor invalid

‎smmap/test/test_util.py

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -88,12 +88,7 @@ def test_region(self):
8888
# auto-refcount
8989
assertrfull.client_count()==1
9090
rfull2=rfull
91-
assertrfull.client_count()==2
92-
93-
# usage
94-
assertrfull.usage_count()==0
95-
rfull.increment_usage_count()
96-
assertrfull.usage_count()==1
91+
assertrfull.client_count()==1,"no auto-counting"
9792

9893
# window constructor
9994
w=MapWindow.from_region(rfull)
@@ -106,8 +101,6 @@ def test_region_list(self):
106101
foritemin (fc.path,fd):
107102
ml=MapRegionList(item)
108103

109-
assertml.client_count()==1
110-
111104
assertlen(ml)==0
112105
assertml.path_or_fd()==item
113106
assertml.file_size()==fc.size

‎smmap/util.py

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,8 @@ def __init__(self, path_or_fd, ofs, size, flags=0):
177177
os.close(fd)
178178
# END only close it if we opened it
179179
# END close file handle
180+
# We assume the first one to use us keeps us around
181+
self.increment_client_count()
180182

181183
def_read_into_memory(self,fd,offset,size):
182184
""":return: string data as read from the given file descriptor, offset and size """
@@ -222,17 +224,25 @@ def includes_ofs(self, ofs):
222224

223225
defclient_count(self):
224226
""":return: number of clients currently using this region"""
225-
fromsysimportgetrefcount
226-
# -1: self on stack, -1 self in this method, -1 self in getrefcount
227-
returngetrefcount(self)-3
228-
229-
defusage_count(self):
230-
""":return: amount of usages so far"""
231227
returnself._uc
232228

233-
defincrement_usage_count(self):
234-
"""Adjust the usage count by the given positive or negative offset"""
235-
self._uc+=1
229+
defincrement_client_count(self,ofs=1):
230+
"""Adjust the usage count by the given positive or negative offset.
231+
If usage count equals 0, we will auto-release our resources
232+
:return: True if we released resources, False otherwise. In the latter case, we can still be used"""
233+
self._uc+=ofs
234+
assertself._uc>-1,"Increments must match decrements, usage counter negative: %i"%self._uc
235+
236+
ifself.client_count()==0:
237+
self.release()
238+
returnTrue
239+
else:
240+
returnFalse
241+
# end handle release
242+
243+
defrelease(self):
244+
"""Release all resources this instance might hold. Must only be called if there usage_count() is zero"""
245+
self._mf.close()
236246

237247
# re-define all methods which need offset adjustments in compatibility mode
238248
if_need_compat_layer:
@@ -268,11 +278,6 @@ def __init__(self, path_or_fd):
268278
self._path_or_fd=path_or_fd
269279
self._file_size=None
270280

271-
defclient_count(self):
272-
""":return: amount of clients which hold a reference to this instance"""
273-
fromsysimportgetrefcount
274-
returngetrefcount(self)-3
275-
276281
defpath_or_fd(self):
277282
""":return: path or file descriptor we are attached to"""
278283
returnself._path_or_fd

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp