- Notifications
You must be signed in to change notification settings - Fork68
Expand file tree
/
Copy pathtest_stream.py
More file actions
106 lines (85 loc) · 3.55 KB
/
test_stream.py
File metadata and controls
106 lines (85 loc) · 3.55 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
# Copyright (C) 2010, 2011 Sebastian Thiel (byronimo@gmail.com) and contributors
#
# This module is part of GitDB and is released under
# the New BSD License: https://opensource.org/license/bsd-3-clause/
"""Performance data streaming performance"""
fromgitdb.test.performance.libimportTestBigRepoR
fromgitdb.dbimportLooseObjectDB
fromgitdbimportIStream
fromgitdb.utilimportbin_to_hex,remove
fromgitdb.funimportchunk_size
fromtimeimporttime
importos
importsys
fromgitdb.test.libimport (
make_memory_file,
with_rw_directory,
)
#{ Utilities
defread_chunked_stream(stream):
total=0
whileTrue:
chunk=stream.read(chunk_size)
total+=len(chunk)
iflen(chunk)<chunk_size:
break
# END read stream loop
asserttotal==stream.size
returnstream
#} END utilities
classTestObjDBPerformance(TestBigRepoR):
large_data_size_bytes=1000*1000*50# some MiB should do it
moderate_data_size_bytes=1000*1000*1# just 1 MiB
@with_rw_directory
deftest_large_data_streaming(self,path):
ldb=LooseObjectDB(path)
string_ios=list()# list of streams we previously created
# serial mode
forrandomizeinrange(2):
desc= (randomizeand'random ')or''
print("Creating %s data ..."%desc,file=sys.stderr)
st=time()
size,stream=make_memory_file(self.large_data_size_bytes,randomize)
elapsed=time()-st
print("Done (in %f s)"%elapsed,file=sys.stderr)
string_ios.append(stream)
# writing - due to the compression it will seem faster than it is
st=time()
sha=ldb.store(IStream('blob',size,stream)).binsha
elapsed_add=time()-st
assertldb.has_object(sha)
db_file=ldb.readable_db_object_path(bin_to_hex(sha))
fsize_kib=os.path.getsize(db_file)/1000
size_kib=size/1000
print("Added %i KiB (filesize = %i KiB) of %s data to loose odb in %f s ( %f Write KiB / s)"%
(size_kib,fsize_kib,desc,elapsed_add,size_kib/ (elapsed_addor1)),file=sys.stderr)
# reading all at once
st=time()
ostream=ldb.stream(sha)
shadata=ostream.read()
elapsed_readall=time()-st
stream.seek(0)
assertshadata==stream.getvalue()
print("Read %i KiB of %s data at once from loose odb in %f s ( %f Read KiB / s)"%
(size_kib,desc,elapsed_readall,size_kib/ (elapsed_readallor1)),file=sys.stderr)
# reading in chunks of 1 MiB
cs=512*1000
chunks=list()
st=time()
ostream=ldb.stream(sha)
whileTrue:
data=ostream.read(cs)
chunks.append(data)
iflen(data)<cs:
break
# END read in chunks
elapsed_readchunks=time()-st
stream.seek(0)
assertb''.join(chunks)==stream.getvalue()
cs_kib=cs/1000
print("Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)"%
(size_kib,desc,cs_kib,elapsed_readchunks,size_kib/ (elapsed_readchunksor1)),file=sys.stderr)
# del db file so we keep something to do
ostream=None# To release the file handle (win)
remove(db_file)
# END for each randomization factor