Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit369e564

Browse files
committed
fix(cmd): don't open stdout when fetching
This allows us to use the main thread to parse stderr to get progress,and resolve assertion failures hopefully once and for all.Relates togitpython-developers#301
1 parent36dbe7e commit369e564

File tree

4 files changed

+105
-96
lines changed

4 files changed

+105
-96
lines changed

‎git/cmd.py

Lines changed: 75 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141

4242
execute_kwargs= ('istream','with_keep_cwd','with_extended_output',
4343
'with_exceptions','as_process','stdout_as_string',
44-
'output_stream')
44+
'output_stream','with_stdout')
4545

4646
log=logging.getLogger('git.cmd')
4747
log.addHandler(logging.NullHandler())
@@ -65,84 +65,6 @@ def _bchr(c):
6565
# Documentation
6666
## @{
6767

68-
def_parse_lines_from_buffer(buf):
69-
line=b''
70-
bi=0
71-
lb=len(buf)
72-
whilebi<lb:
73-
char=_bchr(buf[bi])
74-
bi+=1
75-
76-
ifcharin (b'\r',b'\n')andline:
77-
yieldbi,line
78-
line=b''
79-
else:
80-
line+=char
81-
# END process parsed line
82-
# END while file is not done reading
83-
# end
84-
85-
def_read_lines_from_fno(fno,last_buf_list):
86-
buf=os.read(fno,mmap.PAGESIZE)
87-
buf=last_buf_list[0]+buf
88-
89-
bi=0
90-
forbi,linein_parse_lines_from_buffer(buf):
91-
yieldline
92-
# for each line to parse from the buffer
93-
94-
# keep remainder
95-
last_buf_list[0]=buf[bi:]
96-
97-
def_dispatch_single_line(line,handler):
98-
line=line.decode(defenc)
99-
iflineandhandler:
100-
try:
101-
handler(line)
102-
exceptException:
103-
# Keep reading, have to pump the lines empty nontheless
104-
log.error("Line handler exception on line: %s",line,exc_info=True)
105-
# end
106-
# end dispatch helper
107-
# end single line helper
108-
109-
def_dispatch_lines(fno,handler,buf_list):
110-
lc=0
111-
last_buf=buf_list[0]
112-
whileTrue:
113-
forlinein_read_lines_from_fno(fno,buf_list):
114-
_dispatch_single_line(line,handler)
115-
lc+=1
116-
# for each line
117-
118-
iflast_buf==buf_list[0]:
119-
break
120-
121-
last_buf=buf_list[0]
122-
# end endless loop
123-
returnlc
124-
# end
125-
126-
def_deplete_buffer(fno,handler,buf_list,wg=None):
127-
lc=0
128-
whileTrue:
129-
line_count=_dispatch_lines(fno,handler,buf_list)
130-
lc+=line_count
131-
ifline_count==0:
132-
break
133-
# end deplete buffer
134-
135-
ifbuf_list[0]:
136-
_dispatch_single_line(buf_list[0],handler)
137-
lc+=1
138-
# end
139-
140-
ifwg:
141-
wg.done()
142-
143-
returnlc
144-
# end
145-
14668
defhandle_process_output(process,stdout_handler,stderr_handler,finalizer):
14769
"""Registers for notifications to lean that process output is ready to read, and dispatches lines to
14870
the respective line handlers. We are able to handle carriage returns in case progress is sent by that
@@ -156,6 +78,76 @@ def handle_process_output(process, stdout_handler, stderr_handler, finalizer):
15678
fdmap= {process.stdout.fileno(): (stdout_handler, [b'']),
15779
process.stderr.fileno(): (stderr_handler, [b''])}
15880

81+
def_parse_lines_from_buffer(buf):
82+
line=b''
83+
bi=0
84+
lb=len(buf)
85+
whilebi<lb:
86+
char=_bchr(buf[bi])
87+
bi+=1
88+
89+
ifcharin (b'\r',b'\n')andline:
90+
yieldbi,line
91+
line=b''
92+
else:
93+
line+=char
94+
# END process parsed line
95+
# END while file is not done reading
96+
# end
97+
98+
def_read_lines_from_fno(fno,last_buf_list):
99+
buf=os.read(fno,mmap.PAGESIZE)
100+
buf=last_buf_list[0]+buf
101+
102+
bi=0
103+
forbi,linein_parse_lines_from_buffer(buf):
104+
yieldline
105+
# for each line to parse from the buffer
106+
107+
# keep remainder
108+
last_buf_list[0]=buf[bi:]
109+
110+
def_dispatch_single_line(line,handler):
111+
line=line.decode(defenc)
112+
iflineandhandler:
113+
try:
114+
handler(line)
115+
exceptException:
116+
# Keep reading, have to pump the lines empty nontheless
117+
log.error("Line handler exception on line: %s",line,exc_info=True)
118+
# end
119+
# end dispatch helper
120+
# end single line helper
121+
122+
def_dispatch_lines(fno,handler,buf_list):
123+
lc=0
124+
forlinein_read_lines_from_fno(fno,buf_list):
125+
_dispatch_single_line(line,handler)
126+
lc+=1
127+
# for each line
128+
returnlc
129+
# end
130+
131+
def_deplete_buffer(fno,handler,buf_list,wg=None):
132+
lc=0
133+
whileTrue:
134+
line_count=_dispatch_lines(fno,handler,buf_list)
135+
lc+=line_count
136+
ifline_count==0:
137+
break
138+
# end deplete buffer
139+
140+
ifbuf_list[0]:
141+
_dispatch_single_line(buf_list[0],handler)
142+
lc+=1
143+
# end
144+
145+
ifwg:
146+
wg.done()
147+
148+
returnlc
149+
# end
150+
159151
ifhasattr(select,'poll'):
160152
# poll is preferred, as select is limited to file handles up to 1024 ... . This could otherwise be
161153
# an issue for us, as it matters how many handles our own process has
@@ -483,6 +475,7 @@ def execute(self, command,
483475
as_process=False,
484476
output_stream=None,
485477
stdout_as_string=True,
478+
with_stdout=True,
486479
**subprocess_kwargs
487480
):
488481
"""Handles executing the command on the shell and consumes and returns
@@ -536,6 +529,8 @@ def execute(self, command,
536529
some of the valid kwargs are already set by this method, the ones you
537530
specify may not be the same ones.
538531
532+
:param with_stdout: If True, default True, we open stdout on the created process
533+
539534
:return:
540535
* str(output) if extended_output = False (Default)
541536
* tuple(int(status), str(stdout), str(stderr)) if extended_output = True
@@ -586,7 +581,7 @@ def execute(self, command,
586581
cwd=cwd,
587582
stdin=istream,
588583
stderr=PIPE,
589-
stdout=PIPE,
584+
stdout=with_stdoutandPIPEorNone,
590585
shell=self.USE_SHELL,
591586
close_fds=(os.name=='posix'),# unsupported on windows
592587
**subprocess_kwargs

‎git/remote.py

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -550,7 +550,7 @@ def _get_fetch_info_from_stderr(self, proc, progress):
550550

551551
progress_handler=progress.new_message_handler()
552552

553-
defmy_progress_handler(line):
553+
forlineinproc.stderr.readlines():
554554
forplineinprogress_handler(line):
555555
ifline.startswith('fatal:')orline.startswith('error:'):
556556
raiseGitCommandError(("Error when fetching: %s"%line,),2)
@@ -563,9 +563,7 @@ def my_progress_handler(line):
563563
# end for each comand code we know
564564
# end for each line progress didn't handle
565565
# end
566-
567-
# We are only interested in stderr here ...
568-
handle_process_output(proc,None,my_progress_handler,finalize_process)
566+
finalize_process(proc)
569567

570568
# read head information
571569
fp=open(join(self.repo.git_dir,'FETCH_HEAD'),'rb')
@@ -651,7 +649,8 @@ def fetch(self, refspec=None, progress=None, **kwargs):
651649
else:
652650
args= [refspec]
653651

654-
proc=self.repo.git.fetch(self,*args,with_extended_output=True,as_process=True,v=True,**kwargs)
652+
proc=self.repo.git.fetch(self,*args,with_extended_output=True,as_process=True,with_stdout=True,v=True,
653+
**kwargs)
655654
res=self._get_fetch_info_from_stderr(proc,progressorRemoteProgress())
656655
ifhasattr(self.repo.odb,'update_cache'):
657656
self.repo.odb.update_cache()

‎git/test/fixtures/cat_file.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
importsys
2+
3+
forlineinopen(sys.argv[1]).readlines():
4+
sys.stdout.write(line)
5+
sys.stderr.write(line)

‎git/test/test_git.py

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@
55
# This module is part of GitPython and is released under
66
# the BSD License: http://www.opensource.org/licenses/bsd-license.php
77
importos
8+
importsys
89
importmock
10+
importsubprocess
911

1012
fromgit.test.libimport (
1113
TestBase,
@@ -22,7 +24,6 @@
2224
GitCommandNotFound,
2325
Repo
2426
)
25-
fromgit.cmdimport_deplete_buffer
2627
fromgitdb.test.libimportwith_rw_directory
2728

2829
fromgit.compatimportPY3
@@ -206,16 +207,25 @@ def test_environment(self, rw_dir):
206207
# end
207208
# end if select.poll exists
208209

209-
deftest_dispatch_lines(self):
210+
deftest_handle_process_output(self):
211+
fromgit.cmdimporthandle_process_output
212+
210213
line_count=5002
211-
count= [0]
212-
defcounter(line):
213-
count[0]+=1
214+
count= [None,0,0]
215+
216+
defcounter_stdout(line):
217+
count[1]+=1
218+
219+
defcounter_stderr(line):
220+
count[2]+=1
221+
222+
proc=subprocess.Popen([sys.executable,fixture_path('cat_file.py'),str(fixture_path('issue-301_stderr'))],
223+
stdin=None,
224+
stdout=subprocess.PIPE,
225+
stderr=subprocess.PIPE,
226+
shell=False)
214227

215-
fd=os.open(fixture_path('issue-301_stderr'),os.O_RDONLY)
216-
buf_list= [b'']
217-
lines_parsed=_deplete_buffer(fd,counter,buf_list)
218-
os.close(fd)
228+
handle_process_output(proc,counter_stdout,counter_stderr,lambdaproc:proc.wait())
219229

220-
assertlines_parsed==line_count
221-
assertcount[0]==line_count
230+
assertcount[1]==line_count
231+
assertcount[2]==line_count

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp