Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

Commit45eb728

Browse files
committed
Merge branch 'nonblocking-ops'
Fixesgitpython-developers#145
2 parentsd83f6e8 +87a6ffa commit45eb728

File tree

6 files changed

+201
-59
lines changed

6 files changed

+201
-59
lines changed

‎doc/source/changes.rst

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@
22
Changelog
33
=========
44

5+
0.3.5 - Bugfixes
6+
================
7+
* push/pull/fetch operations will not block anymore
8+
* A list of all fixed issues can be found here: https://github.com/gitpython-developers/GitPython/issues?q=milestone%3A%22v0.3.5+-+bugfixes%22+
9+
510
0.3.4 - Python 3 Support
611
========================
712
* Internally, hexadecimal SHA1 are treated as ascii encoded strings. Binary SHA1 are treated as bytes.

‎git/cmd.py

Lines changed: 116 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@
66

77
importos
88
importsys
9+
importselect
910
importlogging
11+
importthreading
1012
fromsubprocessimport (
1113
call,
1214
Popen,
@@ -16,7 +18,8 @@
1618

1719
from .utilimport (
1820
LazyMixin,
19-
stream_copy
21+
stream_copy,
22+
WaitGroup
2023
)
2124
from .excimportGitCommandError
2225
fromgit.compatimport (
@@ -36,9 +39,121 @@
3639
__all__= ('Git', )
3740

3841

42+
# ==============================================================================
43+
## @name Utilities
44+
# ------------------------------------------------------------------------------
45+
# Documentation
46+
## @{
47+
48+
defhandle_process_output(process,stdout_handler,stderr_handler,finalizer):
49+
"""Registers for notifications to lean that process output is ready to read, and dispatches lines to
50+
the respective line handlers. We are able to handle carriage returns in case progress is sent by that
51+
mean. For performance reasons, we only apply this to stderr.
52+
This function returns once the finalizer returns
53+
:return: result of finalizer
54+
:param process: subprocess.Popen instance
55+
:param stdout_handler: f(stdout_line_string), or None
56+
:param stderr_hanlder: f(stderr_line_string), or None
57+
:param finalizer: f(proc) - wait for proc to finish"""
58+
defread_line_fast(stream):
59+
returnstream.readline()
60+
61+
defread_line_slow(stream):
62+
line=b''
63+
whileTrue:
64+
char=stream.read(1)# reads individual single byte strings
65+
ifnotchar:
66+
break
67+
68+
ifcharin (b'\r',b'\n')andline:
69+
break
70+
else:
71+
line+=char
72+
# END process parsed line
73+
# END while file is not done reading
74+
returnline
75+
# end
76+
77+
defdispatch_line(fno):
78+
stream,handler,readline=fdmap[fno]
79+
# this can possibly block for a while, but since we wake-up with at least one or more lines to handle,
80+
# we are good ...
81+
line=readline(stream).decode(defenc)
82+
iflineandhandler:
83+
handler(line)
84+
returnline
85+
# end dispatch helper
86+
# end
87+
88+
defdeplete_buffer(fno,wg=None):
89+
whileTrue:
90+
line=dispatch_line(fno)
91+
ifnotline:
92+
break
93+
# end deplete buffer
94+
ifwg:
95+
wg.done()
96+
# end
97+
98+
fdmap= {process.stdout.fileno(): (process.stdout,stdout_handler,read_line_fast),
99+
process.stderr.fileno(): (process.stderr,stderr_handler,read_line_slow)}
100+
101+
ifhasattr(select,'poll'):
102+
# poll is preferred, as select is limited to file handles up to 1024 ... . This could otherwise be
103+
# an issue for us, as it matters how many handles or own process has
104+
poll=select.poll()
105+
READ_ONLY=select.POLLIN|select.POLLPRI|select.POLLHUP|select.POLLERR
106+
CLOSED=select.POLLHUP|select.POLLERR
107+
108+
poll.register(process.stdout,READ_ONLY)
109+
poll.register(process.stderr,READ_ONLY)
110+
111+
closed_streams=set()
112+
whileTrue:
113+
# no timeout
114+
poll_result=poll.poll()
115+
forfd,resultinpoll_result:
116+
ifresult&CLOSED:
117+
closed_streams.add(fd)
118+
else:
119+
dispatch_line(fd)
120+
# end handle closed stream
121+
# end for each poll-result tuple
122+
123+
iflen(closed_streams)==len(fdmap):
124+
break
125+
# end its all done
126+
# end endless loop
127+
128+
# Depelete all remaining buffers
129+
forfnoinfdmap.keys():
130+
deplete_buffer(fno)
131+
# end for each file handle
132+
else:
133+
# Oh ... probably we are on windows. select.select() can only handle sockets, we have files
134+
# The only reliable way to do this now is to use threads and wait for both to finish
135+
# Since the finalizer is expected to wait, we don't have to introduce our own wait primitive
136+
# NO: It's not enough unfortunately, and we will have to sync the threads
137+
wg=WaitGroup()
138+
forfnoinfdmap.keys():
139+
wg.add(1)
140+
t=threading.Thread(target=lambda:deplete_buffer(fno,wg))
141+
t.start()
142+
# end
143+
# NOTE: Just joining threads can possibly fail as there is a gap between .start() and when it's
144+
# actually started, which could make the wait() call to just return because the thread is not yet
145+
# active
146+
wg.wait()
147+
# end
148+
149+
returnfinalizer(process)
150+
151+
39152
defdashify(string):
40153
returnstring.replace('_','-')
41154

155+
## -- End Utilities -- @}
156+
42157

43158
classGit(LazyMixin):
44159

‎git/index/base.py

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -287,11 +287,11 @@ def from_tree(cls, repo, *treeish, **kwargs):
287287
changes according to the amount of trees.
288288
If 1 Tree is given, it will just be read into a new index
289289
If 2 Trees are given, they will be merged into a new index using a
290-
two way merge algorithm. Tree 1 is the 'current' tree, tree 2 is the 'other'
291-
one. It behaves like a fast-forward.
292-
If 3 Trees are given, a 3-way merge will be performed with the first tree
293-
being the common ancestor of tree 2 and tree 3. Tree 2 is the 'current' tree,
294-
tree 3 is the 'other' one
290+
two way merge algorithm. Tree 1 is the 'current' tree, tree 2 is the 'other'
291+
one. It behaves like a fast-forward.
292+
If 3 Trees are given, a 3-way merge will be performed with the first tree
293+
being the common ancestor of tree 2 and tree 3. Tree 2 is the 'current' tree,
294+
tree 3 is the 'other' one
295295
296296
:param kwargs:
297297
Additional arguments passed to git-read-tree
@@ -882,14 +882,11 @@ def move(self, items, skip_errors=False, **kwargs):
882882

883883
defcommit(self,message,parent_commits=None,head=True,author=None,committer=None):
884884
"""Commit the current default index file, creating a commit object.
885-
886885
For more information on the arguments, see tree.commit.
887-
:note:
888-
If you have manually altered the .entries member of this instance,
889-
don't forget to write() your changes to disk beforehand.
890886
891-
:return:
892-
Commit object representing the new commit"""
887+
:note: If you have manually altered the .entries member of this instance,
888+
don't forget to write() your changes to disk beforehand.
889+
:return: Commit object representing the new commit"""
893890
tree=self.write_tree()
894891
returnCommit.create_from_tree(self.repo,tree,message,parent_commits,
895892
head,author=author,committer=committer)

‎git/remote.py

Lines changed: 25 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
join_path,
3232
finalize_process
3333
)
34+
fromgit.cmdimporthandle_process_output
3435
fromgitdb.utilimportjoin
3536
fromgit.compatimportdefenc
3637

@@ -40,30 +41,6 @@
4041
#{ Utilities
4142

4243

43-
defdigest_process_messages(fh,progress):
44-
"""Read progress messages from file-like object fh, supplying the respective
45-
progress messages to the progress instance.
46-
47-
:param fh: File handle to read from
48-
:return: list(line, ...) list of lines without linebreaks that did
49-
not contain progress information"""
50-
line_so_far=b''
51-
dropped_lines=list()
52-
whileTrue:
53-
char=fh.read(1)# reads individual single byte strings
54-
ifnotchar:
55-
break
56-
57-
ifcharin (b'\r',b'\n')andline_so_far:
58-
dropped_lines.extend(progress._parse_progress_line(line_so_far.decode(defenc)))
59-
line_so_far=b''
60-
else:
61-
line_so_far+=char
62-
# END process parsed line
63-
# END while file is not done reading
64-
returndropped_lines
65-
66-
6744
defadd_progress(kwargs,git,progress):
6845
"""Add the --progress flag to the given kwargs dict if supported by the
6946
git command. If the actual progress in the given progress instance is not
@@ -532,17 +509,25 @@ def _get_fetch_info_from_stderr(self, proc, progress):
532509
# Basically we want all fetch info lines which appear to be in regular form, and thus have a
533510
# command character. Everything else we ignore,
534511
cmds=set(PushInfo._flag_map.keys())&set(FetchInfo._flag_map.keys())
535-
forlineindigest_process_messages(proc.stderr,progress):
536-
ifline.startswith('fatal:'):
537-
raiseGitCommandError(("Error when fetching: %s"%line,),2)
538-
# END handle special messages
539-
forcmdincmds:
540-
ifline[1]==cmd:
541-
fetch_info_lines.append(line)
542-
continue
543-
# end find command code
544-
# end for each comand code we know
545-
# END for each line
512+
513+
progress_handler=progress.new_message_handler()
514+
515+
defmy_progress_handler(line):
516+
forplineinprogress_handler(line):
517+
ifline.startswith('fatal:'):
518+
raiseGitCommandError(("Error when fetching: %s"%line,),2)
519+
# END handle special messages
520+
forcmdincmds:
521+
ifline[1]==cmd:
522+
fetch_info_lines.append(line)
523+
continue
524+
# end find command code
525+
# end for each comand code we know
526+
# end for each line progress didn't handle
527+
# end
528+
529+
# We are only interested in stderr here ...
530+
handle_process_output(proc,None,my_progress_handler,finalize_process)
546531

547532
# read head information
548533
fp=open(join(self.repo.git_dir,'FETCH_HEAD'),'rb')
@@ -555,7 +540,6 @@ def _get_fetch_info_from_stderr(self, proc, progress):
555540

556541
output.extend(FetchInfo._from_line(self.repo,err_line,fetch_line)
557542
forerr_line,fetch_lineinzip(fetch_info_lines,fetch_head_info))
558-
finalize_process(proc)
559543
returnoutput
560544

561545
def_get_push_info(self,proc,progress):
@@ -564,19 +548,19 @@ def _get_push_info(self, proc, progress):
564548
# read the lines manually as it will use carriage returns between the messages
565549
# to override the previous one. This is why we read the bytes manually
566550
# TODO: poll() on file descriptors to know what to read next, process streams concurrently
567-
digest_process_messages(proc.stderr,progress)
568-
551+
progress_handler=progress.new_message_handler()
569552
output=IterableList('name')
570-
forlineinproc.stdout.readlines():
571-
line=line.decode(defenc)
553+
554+
defstdout_handler(line):
572555
try:
573556
output.append(PushInfo._from_line(self,line))
574557
exceptValueError:
575558
# if an error happens, additional info is given which we cannot parse
576559
pass
577560
# END exception handling
578561
# END for each line
579-
finalize_process(proc)
562+
563+
handle_process_output(proc,stdout_handler,progress_handler,finalize_process)
580564
returnoutput
581565

582566
deffetch(self,refspec=None,progress=None,**kwargs):

‎git/repo/base.py

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,10 @@
55
# the BSD License: http://www.opensource.org/licenses/bsd-license.php
66

77
fromgit.excimportInvalidGitRepositoryError,NoSuchPathError
8-
fromgit.cmdimportGit
8+
fromgit.cmdimport (
9+
Git,
10+
handle_process_output
11+
)
912
fromgit.refsimport (
1013
HEAD,
1114
Head,
@@ -25,7 +28,6 @@
2528
fromgit.configimportGitConfigParser
2629
fromgit.remoteimport (
2730
Remote,
28-
digest_process_messages,
2931
add_progress
3032
)
3133

@@ -711,9 +713,10 @@ def _clone(cls, git, url, path, odb_default_type, progress, **kwargs):
711713
proc=git.clone(url,path,with_extended_output=True,as_process=True,
712714
v=True,**add_progress(kwargs,git,progress))
713715
ifprogress:
714-
digest_process_messages(proc.stderr,progress)
715-
# END handle progress
716-
finalize_process(proc)
716+
handle_process_output(proc,None,progress.new_message_handler(),finalize_process)
717+
else:
718+
finalize_process(proc)
719+
# end handle progress
717720
finally:
718721
ifprev_cwdisnotNone:
719722
os.chdir(prev_cwd)

‎git/util.py

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
importshutil
1313
importplatform
1414
importgetpass
15+
importthreading
1516

1617
# NOTE: Some of the unused imports might be used/imported by others.
1718
# Handle once test-cases are back up and running.
@@ -32,7 +33,7 @@
3233
__all__= ("stream_copy","join_path","to_native_path_windows","to_native_path_linux",
3334
"join_path_native","Stats","IndexFileSHA1Writer","Iterable","IterableList",
3435
"BlockingLockFile","LockFile",'Actor','get_user_id','assure_directory_exists',
35-
'RemoteProgress','rmtree')
36+
'RemoteProgress','rmtree','WaitGroup')
3637

3738
#{ Utility Methods
3839

@@ -249,6 +250,14 @@ def _parse_progress_line(self, line):
249250
# END for each sub line
250251
returnfailed_lines
251252

253+
defnew_message_handler(self):
254+
""":return: a progress handler suitable for handle_process_output(), passing lines on to this Progress
255+
handler in a suitable format"""
256+
defhandler(line):
257+
returnself._parse_progress_line(line.rstrip())
258+
# end
259+
returnhandler
260+
252261
defline_dropped(self,line):
253262
"""Called whenever a line could not be understood and was therefore dropped."""
254263
pass
@@ -691,3 +700,32 @@ def iter_items(cls, repo, *args, **kwargs):
691700
raiseNotImplementedError("To be implemented by Subclass")
692701

693702
#} END classes
703+
704+
705+
classWaitGroup(object):
706+
"""WaitGroup is like Go sync.WaitGroup.
707+
708+
Without all the useful corner cases.
709+
By Peter Teichman, taken from https://gist.github.com/pteichman/84b92ae7cef0ab98f5a8
710+
"""
711+
def__init__(self):
712+
self.count=0
713+
self.cv=threading.Condition()
714+
715+
defadd(self,n):
716+
self.cv.acquire()
717+
self.count+=n
718+
self.cv.release()
719+
720+
defdone(self):
721+
self.cv.acquire()
722+
self.count-=1
723+
ifself.count==0:
724+
self.cv.notify_all()
725+
self.cv.release()
726+
727+
defwait(self):
728+
self.cv.acquire()
729+
whileself.count>0:
730+
self.cv.wait()
731+
self.cv.release()

0 commit comments

Comments
 (0)
[8]ページ先頭
©2009-2025 Movatter.jp