Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commitea68c4e

Browse files
authored
Merge pull request#1024 from effigies/fix/gzip_compression_options
ENH: Create gzip header deterministically by default
2 parents44a1052 +ff5efe4 commitea68c4e

File tree

2 files changed

+147
-7
lines changed

2 files changed

+147
-7
lines changed

‎nibabel/openers.py‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,24 @@
4242
HAVE_INDEXED_GZIP=False
4343

4444

45-
def_gzip_open(filename,mode='rb',compresslevel=9,keep_open=False):
45+
classDeterministicGzipFile(gzip.GzipFile):
46+
""" Deterministic variant of GzipFile
47+
48+
This writer does not add filename information to the header, and defaults
49+
to a modification time (``mtime``) of 0 seconds.
50+
"""
51+
def__init__(self,filename=None,mode=None,compresslevel=9,fileobj=None,mtime=0):
52+
# These two guards are copied from
53+
# https://github.com/python/cpython/blob/6ab65c6/Lib/gzip.py#L171-L174
54+
ifmodeand'b'notinmode:
55+
mode+='b'
56+
iffileobjisNone:
57+
fileobj=self.myfileobj=open(filename,modeor'rb')
58+
returnsuper().__init__(filename="",mode=mode,compresslevel=compresslevel,
59+
fileobj=fileobj,mtime=mtime)
60+
61+
62+
def_gzip_open(filename,mode='rb',compresslevel=9,mtime=0,keep_open=False):
4663

4764
# use indexed_gzip if possible for faster read access. If keep_open ==
4865
# True, we tell IndexedGzipFile to keep the file handle open. Otherwise
@@ -52,7 +69,7 @@ def _gzip_open(filename, mode='rb', compresslevel=9, keep_open=False):
5269

5370
# Fall-back to built-in GzipFile
5471
else:
55-
gzip_file=gzip.GzipFile(filename,mode,compresslevel)
72+
gzip_file=DeterministicGzipFile(filename,mode,compresslevel,mtime=mtime)
5673

5774
returngzip_file
5875

@@ -83,7 +100,7 @@ class Opener(object):
83100
passed to opening method when `fileish` is str. Change of defaults as
84101
for \*args
85102
"""
86-
gz_def= (_gzip_open, ('mode','compresslevel','keep_open'))
103+
gz_def= (_gzip_open, ('mode','compresslevel','mtime','keep_open'))
87104
bz2_def= (BZ2File, ('mode','buffering','compresslevel'))
88105
zstd_def= (_zstd_open, ('mode','level_or_option','zstd_dict'))
89106
compress_ext_map= {
@@ -163,10 +180,7 @@ def name(self):
163180
self._name will be None if object was created with a fileobj, otherwise
164181
it will be the filename.
165182
"""
166-
try:
167-
returnself.fobj.name
168-
exceptAttributeError:
169-
returnself._name
183+
returnself._name
170184

171185
@property
172186
defmode(self):

‎nibabel/tests/test_openers.py‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,12 +12,15 @@
1212
fromgzipimportGzipFile
1313
fromioimportBytesIO,UnsupportedOperation
1414
fromdistutils.versionimportStrictVersion
15+
importhashlib
16+
importtime
1517

1618
fromnumpy.compat.py3kimportasstr,asbytes
1719
from ..openersimport (Opener,
1820
ImageOpener,
1921
HAVE_INDEXED_GZIP,
2022
BZ2File,
23+
DeterministicGzipFile,
2124
)
2225
from ..tmpdirsimportInTemporaryDirectory
2326
from ..volumeutilsimportBinOpener
@@ -367,3 +370,126 @@ def test_iter():
367370
lobj=Opener(Lunk(''))
368371
withpytest.raises(TypeError):
369372
list(lobj)
373+
374+
375+
defmd5sum(fname):
376+
withopen(fname,"rb")asfobj:
377+
returnhashlib.md5(fobj.read()).hexdigest()
378+
379+
380+
deftest_DeterministicGzipFile():
381+
withInTemporaryDirectory():
382+
msg=b"Hello, I'd like to have an argument."
383+
384+
# No filename, no mtime
385+
withopen("ref.gz","wb")asfobj:
386+
withGzipFile(filename="",mode="wb",fileobj=fobj,mtime=0)asgzobj:
387+
gzobj.write(msg)
388+
anon_chksum=md5sum("ref.gz")
389+
390+
withDeterministicGzipFile("default.gz","wb")asfobj:
391+
internal_fobj=fobj.myfileobj
392+
fobj.write(msg)
393+
# Check that myfileobj is being closed by GzipFile.close()
394+
# This is in case GzipFile changes its internal implementation
395+
assertinternal_fobj.closed
396+
397+
assertmd5sum("default.gz")==anon_chksum
398+
399+
# No filename, current mtime
400+
now=time.time()
401+
withopen("ref.gz","wb")asfobj:
402+
withGzipFile(filename="",mode="wb",fileobj=fobj,mtime=now)asgzobj:
403+
gzobj.write(msg)
404+
now_chksum=md5sum("ref.gz")
405+
406+
withDeterministicGzipFile("now.gz","wb",mtime=now)asfobj:
407+
fobj.write(msg)
408+
409+
assertmd5sum("now.gz")==now_chksum
410+
411+
# Change in default behavior
412+
withmock.patch("time.time")ast:
413+
t.return_value=now
414+
415+
# GzipFile will use time.time()
416+
withopen("ref.gz","wb")asfobj:
417+
withGzipFile(filename="",mode="wb",fileobj=fobj)asgzobj:
418+
gzobj.write(msg)
419+
assertmd5sum("ref.gz")==now_chksum
420+
421+
# DeterministicGzipFile will use 0
422+
withDeterministicGzipFile("now.gz","wb")asfobj:
423+
fobj.write(msg)
424+
assertmd5sum("now.gz")==anon_chksum
425+
426+
# GzipFile is filename dependent, DeterministicGzipFile is independent
427+
withGzipFile("filenameA.gz",mode="wb",mtime=0)asgzobj:
428+
gzobj.write(msg)
429+
fnameA_chksum=md5sum("filenameA.gz")
430+
assertfnameA_chksum!=anon_chksum
431+
432+
withDeterministicGzipFile("filenameA.gz","wb")asfobj:
433+
fobj.write(msg)
434+
435+
# But the contents are the same with different filenames
436+
assertmd5sum("filenameA.gz")==anon_chksum
437+
438+
439+
deftest_DeterministicGzipFile_fileobj():
440+
withInTemporaryDirectory():
441+
msg=b"Hello, I'd like to have an argument."
442+
withopen("ref.gz","wb")asfobj:
443+
withGzipFile(filename="",mode="wb",fileobj=fobj,mtime=0)asgzobj:
444+
gzobj.write(msg)
445+
ref_chksum=md5sum("ref.gz")
446+
447+
withopen("test.gz","wb")asfobj:
448+
withDeterministicGzipFile(filename="",mode="wb",fileobj=fobj)asgzobj:
449+
gzobj.write(msg)
450+
md5sum("test.gz")==ref_chksum
451+
452+
withopen("test.gz","wb")asfobj:
453+
withDeterministicGzipFile(fileobj=fobj,mode="wb")asgzobj:
454+
gzobj.write(msg)
455+
md5sum("test.gz")==ref_chksum
456+
457+
withopen("test.gz","wb")asfobj:
458+
withDeterministicGzipFile(filename="test.gz",mode="wb",fileobj=fobj)asgzobj:
459+
gzobj.write(msg)
460+
md5sum("test.gz")==ref_chksum
461+
462+
463+
deftest_bitwise_determinism():
464+
withInTemporaryDirectory():
465+
msg=b"Hello, I'd like to have an argument."
466+
# Canonical reference: No filename, no mtime
467+
# Use default compresslevel
468+
withopen("ref.gz","wb")asfobj:
469+
withGzipFile(filename="",mode="wb",
470+
compresslevel=1,fileobj=fobj,
471+
mtime=0)asgzobj:
472+
gzobj.write(msg)
473+
anon_chksum=md5sum("ref.gz")
474+
475+
# Different times, different filenames
476+
now=time.time()
477+
withmock.patch("time.time")ast:
478+
t.return_value=now
479+
withOpener("a.gz","wb")asfobj:
480+
fobj.write(msg)
481+
t.return_value=now+1
482+
withOpener("b.gz","wb")asfobj:
483+
fobj.write(msg)
484+
485+
assertmd5sum("a.gz")==anon_chksum
486+
assertmd5sum("b.gz")==anon_chksum
487+
488+
# Users can still set mtime, but filenames will not be embedded
489+
withOpener("filenameA.gz","wb",mtime=0xCAFE10C0)asfobj:
490+
fobj.write(msg)
491+
withOpener("filenameB.gz","wb",mtime=0xCAFE10C0)asfobj:
492+
fobj.write(msg)
493+
fnameA_chksum=md5sum("filenameA.gz")
494+
fnameB_chksum=md5sum("filenameB.gz")
495+
assertfnameA_chksum==fnameB_chksum!=anon_chksum

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp