Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit0401087

Browse files
committed
fix(cmd): line parsing
* Previously we could fail to parse the last line within a read buffer, which is now fixed.* Added a test to verify our *slow* line parsing works as expected.
1 parent3008310 commit0401087

File tree

4 files changed

+10101
-68
lines changed

4 files changed

+10101
-68
lines changed

‎git/cmd.py

Lines changed: 81 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,84 @@ def _bchr(c):
6565
# Documentation
6666
## @{
6767

68+
def_parse_lines_from_buffer(buf):
69+
line=b''
70+
bi=0
71+
lb=len(buf)
72+
whilebi<lb:
73+
char=_bchr(buf[bi])
74+
bi+=1
75+
76+
ifcharin (b'\r',b'\n')andline:
77+
yieldbi,line
78+
line=b''
79+
else:
80+
line+=char
81+
# END process parsed line
82+
# END while file is not done reading
83+
# end
84+
85+
def_read_lines_from_fno(fno,last_buf_list):
86+
buf=os.read(fno,mmap.PAGESIZE)
87+
buf=last_buf_list[0]+buf
88+
89+
bi=0
90+
forbi,linein_parse_lines_from_buffer(buf):
91+
yieldline
92+
# for each line to parse from the buffer
93+
94+
# keep remainder
95+
last_buf_list[0]=buf[bi:]
96+
97+
def_dispatch_single_line(line,handler):
98+
line=line.decode(defenc)
99+
iflineandhandler:
100+
try:
101+
handler(line)
102+
exceptException:
103+
# Keep reading, have to pump the lines empty nontheless
104+
log.error("Line handler exception on line: %s",line,exc_info=True)
105+
# end
106+
# end dispatch helper
107+
# end single line helper
108+
109+
def_dispatch_lines(fno,handler,buf_list):
110+
lc=0
111+
last_buf=buf_list[0]
112+
whileTrue:
113+
forlinein_read_lines_from_fno(fno,buf_list):
114+
_dispatch_single_line(line,handler)
115+
lc+=1
116+
# for each line
117+
118+
iflast_buf==buf_list[0]:
119+
break
120+
121+
last_buf=buf_list[0]
122+
# end endless loop
123+
returnlc
124+
# end
125+
126+
def_deplete_buffer(fno,handler,buf_list,wg=None):
127+
lc=0
128+
whileTrue:
129+
line_count=_dispatch_lines(fno,handler,buf_list)
130+
lc+=line_count
131+
ifline_count==0:
132+
break
133+
# end deplete buffer
134+
135+
ifbuf_list[0]:
136+
_dispatch_single_line(buf_list[0],handler)
137+
lc+=1
138+
# end
139+
140+
ifwg:
141+
wg.done()
142+
143+
returnlc
144+
# end
145+
68146
defhandle_process_output(process,stdout_handler,stderr_handler,finalizer):
69147
"""Registers for notifications to lean that process output is ready to read, and dispatches lines to
70148
the respective line handlers. We are able to handle carriage returns in case progress is sent by that
@@ -75,71 +153,6 @@ def handle_process_output(process, stdout_handler, stderr_handler, finalizer):
75153
:param stdout_handler: f(stdout_line_string), or None
76154
:param stderr_hanlder: f(stderr_line_string), or None
77155
:param finalizer: f(proc) - wait for proc to finish"""
78-
defparse_lines_from_buffer(fno,buf):
79-
line=b''
80-
bi=0
81-
lb=len(buf)
82-
whilebi<lb:
83-
char=_bchr(buf[bi])
84-
bi+=1
85-
86-
ifcharin (b'\r',b'\n')andline:
87-
yieldbi,line
88-
line=b''
89-
else:
90-
line+=char
91-
# END process parsed line
92-
# END while file is not done reading
93-
# end
94-
95-
defread_lines_from_fno(fno,last_buf_list):
96-
buf=os.read(fno,mmap.PAGESIZE)
97-
buf=last_buf_list[0]+buf
98-
99-
bi=0
100-
forbi,lineinparse_lines_from_buffer(fno,buf):
101-
yieldline
102-
# for each line to parse from the buffer
103-
104-
# keep remainder
105-
last_buf_list[0]=buf[bi:]
106-
107-
defdispatch_single_line(line,handler):
108-
line=line.decode(defenc)
109-
iflineandhandler:
110-
try:
111-
handler(line)
112-
exceptException:
113-
# Keep reading, have to pump the lines empty nontheless
114-
log.error("Line handler exception on line: %s",line,exc_info=True)
115-
# end
116-
# end dispatch helper
117-
# end single line helper
118-
119-
defdispatch_lines(fno,handler,buf_list):
120-
lc=0
121-
forlineinread_lines_from_fno(fno,buf_list):
122-
dispatch_single_line(line,handler)
123-
lc+=1
124-
# for each line
125-
returnlc
126-
# end
127-
128-
defdeplete_buffer(fno,handler,buf_list,wg=None):
129-
whileTrue:
130-
line_count=dispatch_lines(fno,handler,buf_list)
131-
ifline_count==0:
132-
break
133-
# end deplete buffer
134-
135-
ifbuf_list[0]:
136-
dispatch_single_line(buf_list[0],handler)
137-
# end
138-
139-
ifwg:
140-
wg.done()
141-
# end
142-
143156
fdmap= {process.stdout.fileno(): (stdout_handler, [b'']),
144157
process.stderr.fileno(): (stderr_handler, [b''])}
145158

@@ -169,7 +182,7 @@ def deplete_buffer(fno, handler, buf_list, wg=None):
169182
ifresult&CLOSED:
170183
closed_streams.add(fd)
171184
else:
172-
dispatch_lines(fd,*fdmap[fd])
185+
_dispatch_lines(fd,*fdmap[fd])
173186
# end handle closed stream
174187
# end for each poll-result tuple
175188

@@ -180,7 +193,7 @@ def deplete_buffer(fno, handler, buf_list, wg=None):
180193

181194
# Depelete all remaining buffers
182195
forfno, (handler,buf_list)infdmap.items():
183-
deplete_buffer(fno,handler,buf_list)
196+
_deplete_buffer(fno,handler,buf_list)
184197
# end for each file handle
185198

186199
forfnoinfdmap.keys():
@@ -194,7 +207,7 @@ def deplete_buffer(fno, handler, buf_list, wg=None):
194207
wg=WaitGroup()
195208
forfno, (handler,buf_list)infdmap.items():
196209
wg.add(1)
197-
t=threading.Thread(target=lambda:deplete_buffer(fno,handler,buf_list,wg))
210+
t=threading.Thread(target=lambda:_deplete_buffer(fno,handler,buf_list,wg))
198211
t.start()
199212
# end
200213
# NOTE: Just joining threads can possibly fail as there is a gap between .start() and when it's

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp