Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit400d728

Browse files
committed
Implemented RemoteProgress parsing for git-fetch, which might become available at some point natively, within the git suite
Progress parsing now deals properly with Ascii_Escape characters that are meant for the tty - git might stop sending this at some point, but we can deal with it no matter what
1 parent77cde00 commit400d728

File tree

2 files changed

+115
-50
lines changed

2 files changed

+115
-50
lines changed

‎lib/git/remote.py‎

Lines changed: 92 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -38,32 +38,46 @@ def _call_config(self, method, *args, **kwargs):
3838
returngetattr(self._config,method)(self._section_name,*args,**kwargs)
3939

4040

41-
classPushProgress(object):
41+
classRemoteProgress(object):
4242
"""
4343
Handler providing an interface to parse progress information emitted by git-push
44-
and to dispatch callbacks allowing subclasses to react to the progress.
44+
andgit-fetch andto dispatch callbacks allowing subclasses to react to the progress.
4545
"""
4646
BEGIN,END,COUNTING,COMPRESSING,WRITING= [1<<xforxinrange(5) ]
4747
STAGE_MASK=BEGIN|END
4848
OP_MASK=COUNTING|COMPRESSING|WRITING
4949

5050
__slots__= ("_cur_line","_seen_ops")
51-
re_op_absolute=re.compile("([\w\s]+):\s+()(\d+)()(.*)")
52-
re_op_relative=re.compile("([\w\s]+):\s+(\d+)% \((\d+)/(\d+)\)(.*)")
51+
re_op_absolute=re.compile("(remote: )?([\w\s]+):\s+()(\d+)()(.*)")
52+
re_op_relative=re.compile("(remote: )?([\w\s]+):\s+(\d+)% \((\d+)/(\d+)\)(.*)")
5353

5454
def__init__(self):
5555
self._seen_ops=list()
5656

5757
def_parse_progress_line(self,line):
5858
"""
5959
Parse progress information from the given line as retrieved by git-push
60-
"""
60+
or git-fetch
61+
@return: list(line, ...) list of lines that could not be processed"""
6162
# handle
6263
# Counting objects: 4, done.
6364
# Compressing objects: 50% (1/2) \rCompressing objects: 100% (2/2) \rCompressing objects: 100% (2/2), done.
6465
self._cur_line=line
6566
sub_lines=line.split('\r')
67+
failed_lines=list()
6668
forslineinsub_lines:
69+
# find esacpe characters and cut them away - regex will not work with
70+
# them as they are non-ascii. As git might expect a tty, it will send them
71+
last_valid_index=None
72+
fori,cinenumerate(reversed(sline)):
73+
iford(c)<32:
74+
# its a slice index
75+
last_valid_index=-i-1
76+
# END character was non-ascii
77+
# END for each character in sline
78+
iflast_valid_indexisnotNone:
79+
sline=sline[:last_valid_index]
80+
# END cut away invalid part
6781
sline=sline.rstrip()
6882

6983
cur_count,max_count=None,None
@@ -73,11 +87,13 @@ def _parse_progress_line(self, line):
7387

7488
ifnotmatch:
7589
self.line_dropped(sline)
90+
failed_lines.append(sline)
7691
continue
7792
# END could not get match
7893

7994
op_code=0
80-
op_name,percent,cur_count,max_count,message=match.groups()
95+
remote,op_name,percent,cur_count,max_count,message=match.groups()
96+
8197
# get operation id
8298
ifop_name=="Counting objects":
8399
op_code|=self.COUNTING
@@ -106,8 +122,8 @@ def _parse_progress_line(self, line):
106122
# END end message handling
107123

108124
self.update(op_code,cur_count,max_count,message)
109-
110125
# END for each sub line
126+
returnfailed_lines
111127

112128
defline_dropped(self,line):
113129
"""
@@ -574,38 +590,75 @@ def update(self, **kwargs):
574590
self.repo.git.remote("update",self.name)
575591
returnself
576592

577-
def_get_fetch_info_from_stderr(self,stderr):
593+
def_digest_process_messages(self,fh,progress):
594+
"""Read progress messages from file-like object fh, supplying the respective
595+
progress messages to the progress instance.
596+
@return: list(line, ...) list of lines without linebreaks that did
597+
not contain progress information"""
598+
line_so_far=''
599+
dropped_lines=list()
600+
whileTrue:
601+
char=fh.read(1)
602+
ifnotchar:
603+
break
604+
605+
ifcharin ('\r','\n'):
606+
dropped_lines.extend(progress._parse_progress_line(line_so_far))
607+
line_so_far=''
608+
else:
609+
line_so_far+=char
610+
# END process parsed line
611+
# END while file is not done reading
612+
returndropped_lines
613+
614+
615+
def_finalize_proc(self,proc):
616+
"""Wait for the process (fetch, pull or push) and handle its errors accordingly"""
617+
try:
618+
proc.wait()
619+
exceptGitCommandError,e:
620+
# if a push has rejected items, the command has non-zero return status
621+
# a return status of 128 indicates a connection error - reraise the previous one
622+
ifproc.poll()==128:
623+
raise
624+
pass
625+
# END exception handling
626+
627+
628+
def_get_fetch_info_from_stderr(self,proc,progress):
578629
# skip first line as it is some remote info we are not interested in
579630
output=IterableList('name')
580-
err_info=stderr.splitlines()[1:]
631+
632+
633+
# lines which are no progress are fetch info lines
634+
# this also waits for the command to finish
635+
# Skip some progress lines that don't provide relevant information
636+
fetch_info_lines=list()
637+
forlineinself._digest_process_messages(proc.stderr,progress):
638+
ifline.startswith('From')orline.startswith('remote: Total'):
639+
continue
640+
fetch_info_lines.append(line)
641+
# END for each line
581642

582643
# read head information
583644
fp=open(os.path.join(self.repo.git_dir,'FETCH_HEAD'),'r')
584645
fetch_head_info=fp.readlines()
585646
fp.close()
586647

648+
assertlen(fetch_info_lines)==len(fetch_head_info)
649+
587650
output.extend(FetchInfo._from_line(self.repo,err_line,fetch_line)
588-
forerr_line,fetch_lineinzip(err_info,fetch_head_info))
651+
forerr_line,fetch_lineinzip(fetch_info_lines,fetch_head_info))
652+
653+
self._finalize_proc(proc)
589654
returnoutput
590655

591656
def_get_push_info(self,proc,progress):
592657
# read progress information from stderr
593658
# we hope stdout can hold all the data, it should ...
594659
# read the lines manually as it will use carriage returns between the messages
595660
# to override the previous one. This is why we read the bytes manually
596-
line_so_far=''
597-
whileTrue:
598-
char=proc.stderr.read(1)
599-
ifnotchar:
600-
break
601-
602-
ifcharin ('\r','\n'):
603-
progress._parse_progress_line(line_so_far)
604-
line_so_far=''
605-
else:
606-
line_so_far+=char
607-
# END process parsed line
608-
# END for each progress line
661+
self._digest_process_messages(proc.stderr,progress)
609662

610663
output=IterableList('name')
611664
forlineinproc.stdout.readlines():
@@ -616,19 +669,12 @@ def _get_push_info(self, proc, progress):
616669
pass
617670
# END exception handling
618671
# END for each line
619-
try:
620-
proc.wait()
621-
exceptGitCommandError,e:
622-
# if a push has rejected items, the command has non-zero return status
623-
# a return status of 128 indicates a connection error - reraise the previous one
624-
ifproc.poll()==128:
625-
raise
626-
pass
627-
# END exception handling
672+
673+
self._finalize_proc(proc)
628674
returnoutput
629675

630676

631-
deffetch(self,refspec=None,**kwargs):
677+
deffetch(self,refspec=None,progress=None,**kwargs):
632678
"""
633679
Fetch the latest changes for this remote
634680
@@ -643,7 +689,9 @@ def fetch(self, refspec=None, **kwargs):
643689
See also git-push(1).
644690
645691
Taken from the git manual
646-
692+
``progress``
693+
See 'push' method
694+
647695
``**kwargs``
648696
Additional arguments to be passed to git-fetch
649697
@@ -655,25 +703,28 @@ def fetch(self, refspec=None, **kwargs):
655703
As fetch does not provide progress information to non-ttys, we cannot make
656704
it available here unfortunately as in the 'push' method.
657705
"""
658-
status,stdout,stderr=self.repo.git.fetch(self,refspec,with_extended_output=True,v=True,**kwargs)
659-
returnself._get_fetch_info_from_stderr(stderr)
706+
proc=self.repo.git.fetch(self,refspec,with_extended_output=True,as_process=True,v=True,**kwargs)
707+
returnself._get_fetch_info_from_stderr(proc,progressorRemoteProgress())
660708

661-
defpull(self,refspec=None,**kwargs):
709+
defpull(self,refspec=None,progress=None,**kwargs):
662710
"""
663711
Pull changes from the given branch, being the same as a fetch followed
664712
by a merge of branch with your local branch.
665713
666714
``refspec``
667715
see 'fetch' method
716+
717+
``progress``
718+
see 'push' method
668719
669720
``**kwargs``
670721
Additional arguments to be passed to git-pull
671722
672723
Returns
673724
Please see 'fetch' method
674725
"""
675-
status,stdout,stderr=self.repo.git.pull(self,refspec,with_extended_output=True,v=True,**kwargs)
676-
returnself._get_fetch_info_from_stderr(stderr)
726+
proc=self.repo.git.pull(self,refspec,with_extended_output=True,as_process=True,v=True,**kwargs)
727+
returnself._get_fetch_info_from_stderr(proc,progressorRemoteProgress())
677728

678729
defpush(self,refspec=None,progress=None,**kwargs):
679730
"""
@@ -683,7 +734,7 @@ def push(self, refspec=None, progress=None, **kwargs):
683734
see 'fetch' method
684735
685736
``progress``
686-
Instance of typePushProgress allowing the caller to receive
737+
Instance of typeRemoteProgress allowing the caller to receive
687738
progress information until the method returns.
688739
If None, progress information will be discarded
689740
@@ -700,7 +751,7 @@ def push(self, refspec=None, progress=None, **kwargs):
700751
be null.
701752
"""
702753
proc=self.repo.git.push(self,refspec,porcelain=True,as_process=True,**kwargs)
703-
returnself._get_push_info(proc,progressorPushProgress())
754+
returnself._get_push_info(proc,progressorRemoteProgress())
704755

705756
@property
706757
defconfig_reader(self):

‎test/git/test_remote.py‎

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,19 +14,21 @@
1414
# assure we have repeatable results
1515
random.seed(0)
1616

17-
classTestPushProgress(PushProgress):
18-
__slots__= ("_seen_lines","_stages_per_op" )
17+
classTestRemoteProgress(RemoteProgress):
18+
__slots__= ("_seen_lines","_stages_per_op",'_num_progress_messages' )
1919
def__init__(self):
20-
super(TestPushProgress,self).__init__()
20+
super(TestRemoteProgress,self).__init__()
2121
self._seen_lines=list()
2222
self._stages_per_op=dict()
23+
self._num_progress_messages=0
2324

2425
def_parse_progress_line(self,line):
2526
# we may remove the line later if it is dropped
2627
# Keep it for debugging
2728
self._seen_lines.append(line)
28-
super(TestPushProgress,self)._parse_progress_line(line)
29+
rval=super(TestRemoteProgress,self)._parse_progress_line(line)
2930
assertlen(line)>1,"line %r too short"%line
31+
returnrval
3032

3133
defline_dropped(self,line):
3234
try:
@@ -44,11 +46,15 @@ def update(self, op_code, cur_count, max_count=None, message=''):
4446

4547
ifop_code& (self.WRITING|self.END)== (self.WRITING|self.END):
4648
assertmessage
47-
# END check we get message
49+
# END check we get message
50+
51+
self._num_progress_messages+=1
52+
4853

4954
defmake_assertion(self):
55+
# we don't always receive messages
5056
ifnotself._seen_lines:
51-
return
57+
return
5258

5359
# sometimes objects are not compressed which is okay
5460
assertlen(self._seen_ops)in (2,3)
@@ -59,6 +65,10 @@ def make_assertion(self):
5965
assertstages&self.STAGE_MASK==self.STAGE_MASK
6066
# END for each op/stage
6167

68+
defassert_received_message(self):
69+
assertself._num_progress_messages
70+
71+
6272
classTestRemote(TestBase):
6373

6474
def_print_fetchhead(self,repo):
@@ -124,7 +134,10 @@ def _test_fetch(self,remote, rw_repo, remote_repo):
124134
self._test_fetch_info(rw_repo)
125135

126136
deffetch_and_test(remote,**kwargs):
137+
progress=TestRemoteProgress()
138+
kwargs['progress']=progress
127139
res=remote.fetch(**kwargs)
140+
progress.make_assertion()
128141
self._test_fetch_result(res,remote)
129142
returnres
130143
# END fetch and check
@@ -257,7 +270,7 @@ def _test_push_and_pull(self,remote, rw_repo, remote_repo):
257270

258271
# simple file push
259272
self._commit_random_file(rw_repo)
260-
progress=TestPushProgress()
273+
progress=TestRemoteProgress()
261274
res=remote.push(lhead.reference,progress)
262275
assertisinstance(res,IterableList)
263276
self._test_push_result(res,remote)
@@ -281,7 +294,7 @@ def _test_push_and_pull(self,remote, rw_repo, remote_repo):
281294
assertlen(res)==0
282295

283296
# push new tags
284-
progress=TestPushProgress()
297+
progress=TestRemoteProgress()
285298
to_be_updated="my_tag.1.0RV"
286299
new_tag=TagReference.create(rw_repo,to_be_updated)
287300
other_tag=TagReference.create(rw_repo,"my_obj_tag.2.1aRV",message="my message")
@@ -305,10 +318,11 @@ def _test_push_and_pull(self,remote, rw_repo, remote_repo):
305318
res=remote.push(":%s"%new_tag.path)
306319
self._test_push_result(res,remote)
307320
assertres[0].flags&PushInfo.DELETED
321+
progress.assert_received_message()
308322

309323
# push new branch
310324
new_head=Head.create(rw_repo,"my_new_branch")
311-
progress=TestPushProgress()
325+
progress=TestRemoteProgress()
312326
res=remote.push(new_head,progress)
313327
assertres[0].flags&PushInfo.NEW_HEAD
314328
progress.make_assertion()

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp