Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit4103892

Browse files
committed
Use orjson instead of json, when available
For `mypy -c 'import torch'`, the cache load time goes from 0.44s to0.25s as measured by manager's data_json_load_timeIf I time dump times specifically, I see a saving of 0.65s to 0.07s.Overall, a pretty reasonable perf win -- should we make it a requireddependency?I don't know if the sqlite cache path is used at all, but let me know ifI need a cleverer migration than renaming the table
1 parent1a074b6 commit4103892

File tree

4 files changed

+64
-54
lines changed

4 files changed

+64
-54
lines changed

‎mypy/build.py‎

Lines changed: 18 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@
9595
frommypy.stubinfoimportlegacy_bundled_packages,non_bundled_packages,stub_distribution_name
9696
frommypy.typesimportType
9797
frommypy.typestateimportreset_global_state,type_state
98+
frommypy.utilimportjson_dumps,json_loads
9899
frommypy.versionimport__version__
99100

100101
# Switch to True to produce debug output related to fine-grained incremental
@@ -858,7 +859,7 @@ def load_fine_grained_deps(self, id: str) -> dict[str, set[str]]:
858859
t0=time.time()
859860
ifidinself.fg_deps_meta:
860861
# TODO: Assert deps file wasn't changed.
861-
deps=json.loads(self.metastore.read(self.fg_deps_meta[id]["path"]))
862+
deps=json_loads(self.metastore.read(self.fg_deps_meta[id]["path"]))
862863
else:
863864
deps= {}
864865
val= {k:set(v)fork,vindeps.items()}
@@ -911,8 +912,8 @@ def stats_summary(self) -> Mapping[str, object]:
911912
returnself.stats
912913

913914

914-
defdeps_to_json(x:dict[str,set[str]])->str:
915-
returnjson.dumps({k:list(v)fork,vinx.items()},separators=(",",":"))
915+
defdeps_to_json(x:dict[str,set[str]])->bytes:
916+
returnjson_dumps({k:list(v)fork,vinx.items()})
916917

917918

918919
# File for storing metadata about all the fine-grained dependency caches
@@ -980,7 +981,7 @@ def write_deps_cache(
980981

981982
meta= {"snapshot":meta_snapshot,"deps_meta":fg_deps_meta}
982983

983-
ifnotmetastore.write(DEPS_META_FILE,json.dumps(meta,separators=(",",":"))):
984+
ifnotmetastore.write(DEPS_META_FILE,json_dumps(meta)):
984985
manager.log(f"Error writing fine-grained deps meta JSON file{DEPS_META_FILE}")
985986
error=True
986987

@@ -1048,7 +1049,7 @@ def generate_deps_for_cache(manager: BuildManager, graph: Graph) -> dict[str, di
10481049

10491050
defwrite_plugins_snapshot(manager:BuildManager)->None:
10501051
"""Write snapshot of versions and hashes of currently active plugins."""
1051-
snapshot=json.dumps(manager.plugins_snapshot,separators=(",",":"))
1052+
snapshot=json_dumps(manager.plugins_snapshot)
10521053
ifnotmanager.metastore.write(PLUGIN_SNAPSHOT_FILE,snapshot):
10531054
manager.errors.set_file(_cache_dir_prefix(manager.options),None,manager.options)
10541055
manager.errors.report(0,0,"Error writing plugins snapshot",blocker=True)
@@ -1079,8 +1080,8 @@ def read_quickstart_file(
10791080
# just ignore it.
10801081
raw_quickstart:dict[str,Any]= {}
10811082
try:
1082-
withopen(options.quickstart_file)asf:
1083-
raw_quickstart=json.load(f)
1083+
withopen(options.quickstart_file,"rb")asf:
1084+
raw_quickstart=json_loads(f.read())
10841085

10851086
quickstart= {}
10861087
forfile, (x,y,z)inraw_quickstart.items():
@@ -1148,10 +1149,10 @@ def _load_json_file(
11481149
manager.add_stats(metastore_read_time=time.time()-t0)
11491150
# Only bother to compute the log message if we are logging it, since it could be big
11501151
ifmanager.verbosity()>=2:
1151-
manager.trace(log_success+data.rstrip())
1152+
manager.trace(log_success+data.rstrip().decode())
11521153
try:
11531154
t1=time.time()
1154-
result=json.loads(data)
1155+
result=json_loads(data)
11551156
manager.add_stats(data_json_load_time=time.time()-t1)
11561157
exceptjson.JSONDecodeError:
11571158
manager.errors.set_file(file,None,manager.options)
@@ -1343,8 +1344,8 @@ def find_cache_meta(id: str, path: str, manager: BuildManager) -> CacheMeta | No
13431344
# So that plugins can return data with tuples in it without
13441345
# things silently always invalidating modules, we round-trip
13451346
# the config data. This isn't beautiful.
1346-
plugin_data=json.loads(
1347-
json.dumps(manager.plugin.report_config_data(ReportConfigContext(id,path,is_check=True)))
1347+
plugin_data=json_loads(
1348+
json_dumps(manager.plugin.report_config_data(ReportConfigContext(id,path,is_check=True)))
13481349
)
13491350
ifm.plugin_data!=plugin_data:
13501351
manager.log(f"Metadata abandoned for{id}: plugin configuration differs")
@@ -1478,18 +1479,15 @@ def validate_meta(
14781479
"ignore_all":meta.ignore_all,
14791480
"plugin_data":meta.plugin_data,
14801481
}
1481-
ifmanager.options.debug_cache:
1482-
meta_str=json.dumps(meta_dict,indent=2,sort_keys=True)
1483-
else:
1484-
meta_str=json.dumps(meta_dict,separators=(",",":"))
1482+
meta_bytes=json_dumps(meta_dict,manager.options.debug_cache)
14851483
meta_json,_,_=get_cache_names(id,path,manager.options)
14861484
manager.log(
14871485
"Updating mtime for {}: file {}, meta {}, mtime {}".format(
14881486
id,path,meta_json,meta.mtime
14891487
)
14901488
)
14911489
t1=time.time()
1492-
manager.metastore.write(meta_json,meta_str)# Ignore errors, just an optimization.
1490+
manager.metastore.write(meta_json,meta_bytes)# Ignore errors, just an optimization.
14931491
manager.add_stats(validate_update_time=time.time()-t1,validate_munging_time=t1-t0)
14941492
returnmeta
14951493

@@ -1507,13 +1505,6 @@ def compute_hash(text: str) -> str:
15071505
returnhash_digest(text.encode("utf-8"))
15081506

15091507

1510-
defjson_dumps(obj:Any,debug_cache:bool)->str:
1511-
ifdebug_cache:
1512-
returnjson.dumps(obj,indent=2,sort_keys=True)
1513-
else:
1514-
returnjson.dumps(obj,sort_keys=True,separators=(",",":"))
1515-
1516-
15171508
defwrite_cache(
15181509
id:str,
15191510
path:str,
@@ -1566,8 +1557,8 @@ def write_cache(
15661557

15671558
# Serialize data and analyze interface
15681559
data=tree.serialize()
1569-
data_str=json_dumps(data,manager.options.debug_cache)
1570-
interface_hash=compute_hash(data_str)
1560+
data_bytes=json_dumps(data,manager.options.debug_cache)
1561+
interface_hash=hash_digest(data_bytes)
15711562

15721563
plugin_data=manager.plugin.report_config_data(ReportConfigContext(id,path,is_check=False))
15731564

@@ -1591,7 +1582,7 @@ def write_cache(
15911582
manager.trace(f"Interface for{id} is unchanged")
15921583
else:
15931584
manager.trace(f"Interface for{id} has changed")
1594-
ifnotmetastore.write(data_json,data_str):
1585+
ifnotmetastore.write(data_json,data_bytes):
15951586
# Most likely the error is the replace() call
15961587
# (see https://github.com/python/mypy/issues/3215).
15971588
manager.log(f"Error writing data JSON file{data_json}")
@@ -3566,4 +3557,4 @@ def write_undocumented_ref_info(
35663557
assertnotref_info_file.startswith(".")
35673558

35683559
deps_json=get_undocumented_ref_info_json(state.tree,type_map)
3569-
metastore.write(ref_info_file,json.dumps(deps_json,separators=(",",":")))
3560+
metastore.write(ref_info_file,json_dumps(deps_json))

‎mypy/metastore.py‎

Lines changed: 15 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -33,14 +33,14 @@ def getmtime(self, name: str) -> float:
3333
"""
3434

3535
@abstractmethod
36-
defread(self,name:str)->str:
36+
defread(self,name:str)->bytes:
3737
"""Read the contents of a metadata entry.
3838
3939
Raises FileNotFound if the entry does not exist.
4040
"""
4141

4242
@abstractmethod
43-
defwrite(self,name:str,data:str,mtime:float|None=None)->bool:
43+
defwrite(self,name:str,data:bytes,mtime:float|None=None)->bool:
4444
"""Write a metadata entry.
4545
4646
If mtime is specified, set it as the mtime of the entry. Otherwise,
@@ -86,16 +86,16 @@ def getmtime(self, name: str) -> float:
8686

8787
returnint(os.path.getmtime(os.path.join(self.cache_dir_prefix,name)))
8888

89-
defread(self,name:str)->str:
89+
defread(self,name:str)->bytes:
9090
assertos.path.normpath(name)!=os.path.abspath(name),"Don't use absolute paths!"
9191

9292
ifnotself.cache_dir_prefix:
9393
raiseFileNotFoundError()
9494

95-
withopen(os.path.join(self.cache_dir_prefix,name))asf:
95+
withopen(os.path.join(self.cache_dir_prefix,name),"rb")asf:
9696
returnf.read()
9797

98-
defwrite(self,name:str,data:str,mtime:float|None=None)->bool:
98+
defwrite(self,name:str,data:bytes,mtime:float|None=None)->bool:
9999
assertos.path.normpath(name)!=os.path.abspath(name),"Don't use absolute paths!"
100100

101101
ifnotself.cache_dir_prefix:
@@ -105,7 +105,7 @@ def write(self, name: str, data: str, mtime: float | None = None) -> bool:
105105
tmp_filename=path+"."+random_string()
106106
try:
107107
os.makedirs(os.path.dirname(path),exist_ok=True)
108-
withopen(tmp_filename,"w")asf:
108+
withopen(tmp_filename,"wb")asf:
109109
f.write(data)
110110
os.replace(tmp_filename,path)
111111
ifmtimeisnotNone:
@@ -135,27 +135,20 @@ def list_all(self) -> Iterable[str]:
135135

136136

137137
SCHEMA="""
138-
CREATE TABLE IF NOT EXISTSfiles (
138+
CREATE TABLE IF NOT EXISTSfiles2 (
139139
path TEXT UNIQUE NOT NULL,
140140
mtime REAL,
141-
dataTEXT
141+
dataBLOB
142142
);
143-
CREATE INDEX IF NOT EXISTS path_idx onfiles(path);
143+
CREATE INDEX IF NOT EXISTS path_idx onfiles2(path);
144144
"""
145-
# No migrations yet
146-
MIGRATIONS:list[str]= []
147145

148146

149147
defconnect_db(db_file:str)->sqlite3.Connection:
150148
importsqlite3.dbapi2
151149

152150
db=sqlite3.dbapi2.connect(db_file)
153151
db.executescript(SCHEMA)
154-
formigrinMIGRATIONS:
155-
try:
156-
db.executescript(migr)
157-
exceptsqlite3.OperationalError:
158-
pass
159152
returndb
160153

161154

@@ -188,12 +181,12 @@ def getmtime(self, name: str) -> float:
188181
assertisinstance(mtime,float)
189182
returnmtime
190183

191-
defread(self,name:str)->str:
184+
defread(self,name:str)->bytes:
192185
data=self._query(name,"data")
193-
assertisinstance(data,str)
186+
assertisinstance(data,bytes)
194187
returndata
195188

196-
defwrite(self,name:str,data:str,mtime:float|None=None)->bool:
189+
defwrite(self,name:str,data:bytes,mtime:float|None=None)->bool:
197190
importsqlite3
198191

199192
ifnotself.db:
@@ -202,7 +195,7 @@ def write(self, name: str, data: str, mtime: float | None = None) -> bool:
202195
ifmtimeisNone:
203196
mtime=time.time()
204197
self.db.execute(
205-
"INSERT OR REPLACE INTOfiles(path, mtime, data) VALUES(?, ?, ?)",
198+
"INSERT OR REPLACE INTOfiles2(path, mtime, data) VALUES(?, ?, ?)",
206199
(name,mtime,data),
207200
)
208201
exceptsqlite3.OperationalError:
@@ -213,13 +206,13 @@ def remove(self, name: str) -> None:
213206
ifnotself.db:
214207
raiseFileNotFoundError()
215208

216-
self.db.execute("DELETE FROMfiles WHERE path = ?", (name,))
209+
self.db.execute("DELETE FROMfiles2 WHERE path = ?", (name,))
217210

218211
defcommit(self)->None:
219212
ifself.db:
220213
self.db.commit()
221214

222215
deflist_all(self)->Iterable[str]:
223216
ifself.db:
224-
forrowinself.db.execute("SELECT path FROMfiles"):
217+
forrowinself.db.execute("SELECT path FROMfiles2"):
225218
yieldrow[0]

‎mypy/util.py‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,23 @@
44

55
importhashlib
66
importio
7+
importjson
78
importos
89
importpathlib
910
importre
1011
importshutil
1112
importsys
1213
importtime
1314
fromimportlibimportresourcesasimportlib_resources
14-
fromtypingimportIO,Callable,Container,Final,Iterable,Sequence,Sized,TypeVar
15+
fromtypingimportIO,Any,Callable,Container,Final,Iterable,Sequence,Sized,TypeVar
1516
fromtyping_extensionsimportLiteral
1617

18+
orjson:Any
19+
try:
20+
importorjson# type: ignore[import-not-found, no-redef, unused-ignore]
21+
exceptImportError:
22+
orjson=None
23+
1724
try:
1825
importcurses
1926

@@ -874,3 +881,22 @@ def quote_docstring(docstr: str) -> str:
874881
returnf"''{docstr_repr}''"
875882
else:
876883
returnf'""{docstr_repr}""'
884+
885+
886+
defjson_dumps(obj:object,debug:bool=False)->bytes:
887+
iforjsonisnotNone:
888+
ifdebug:
889+
returnorjson.dumps(obj,option=orjson.OPT_INDENT_2|orjson.OPT_SORT_KEYS)# type: ignore[no-any-return]
890+
else:
891+
returnorjson.dumps(obj)# type: ignore[no-any-return]
892+
893+
ifdebug:
894+
returnjson.dumps(obj,indent=2,sort_keys=True).encode("utf-8")
895+
else:
896+
returnjson.dumps(obj,separators=(",",":")).encode("utf-8")
897+
898+
899+
defjson_loads(data:bytes)->Any:
900+
iforjsonisnotNone:
901+
returnorjson.loads(data)
902+
returnjson.loads(data)

‎mypyc/codegen/emitmodule.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
frommypy.nodesimportMypyFile
2525
frommypy.optionsimportOptions
2626
frommypy.pluginimportPlugin,ReportConfigContext
27-
frommypy.utilimporthash_digest
27+
frommypy.utilimporthash_digest,json_dumps
2828
frommypyc.codegen.cstringimportc_string_initializer
2929
frommypyc.codegen.emitimportEmitter,EmitterContext,HeaderDeclaration,c_array_initializer
3030
frommypyc.codegen.emitclassimportgenerate_class,generate_class_type_decl
@@ -154,7 +154,7 @@ def report_config_data(self, ctx: ReportConfigContext) -> tuple[str | None, list
154154
ir_data=json.loads(ir_json)
155155

156156
# Check that the IR cache matches the metadata cache
157-
ifcompute_hash(meta_json)!=ir_data["meta_hash"]:
157+
ifhash_digest(meta_json)!=ir_data["meta_hash"]:
158158
returnNone
159159

160160
# Check that all of the source files are present and as
@@ -369,11 +369,11 @@ def write_cache(
369369
newpath=get_state_ir_cache_name(st)
370370
ir_data= {
371371
"ir":module.serialize(),
372-
"meta_hash":compute_hash(meta_data),
372+
"meta_hash":hash_digest(meta_data),
373373
"src_hashes":hashes[group_map[id]],
374374
}
375375

376-
result.manager.metastore.write(newpath,json.dumps(ir_data,separators=(",",":")))
376+
result.manager.metastore.write(newpath,json_dumps(ir_data))
377377

378378
result.manager.metastore.commit()
379379

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp