Fully removed all async dependencies

Byron · Byron · commit bf942a913d69 · 2014-11-13T13:31:32.000+01:00
diff --git a/.gitmodules b/.gitmodules
@@ -1,6 +1,3 @@
-[submodule "async"]
-	path = gitdb/ext/async
-	url = https://github.com/gitpython-developers/async.git
 [submodule "smmap"]
 	path = gitdb/ext/smmap
 	url = https://github.com/Byron/smmap.git
diff --git a/doc/source/changes.rst b/doc/source/changes.rst
@@ -2,6 +2,13 @@
 Changelog
 #########
 
+*****
+0.6.0
+*****
+
+* Added support got python 3.X
+* Removed all `async` dependencies and all `*_async` versions of methods with it.
+
 *****
 0.5.4
 *****
diff --git a/gitdb/__init__.py b/gitdb/__init__.py
@@ -10,7 +10,7 @@
 #{ Initialization
 def _init_externals():
 """Initialize external projects by putting them into the path"""
- for module in ('async', 'smmap'):
+ for module in ('smmap',):
 sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module))
 
 try:
@@ -27,7 +27,7 @@ def _init_externals():
 __author__ = "Sebastian Thiel"
 __contact__ = "byronimo@gmail.com"
 __homepage__ = "https://github.com/gitpython-developers/gitdb"
-version_info = (0, 5, 5)
+version_info = (0, 6, 0)
 __version__ = '.'.join(str(i) for i in version_info)
 
 
diff --git a/gitdb/db/base.py b/gitdb/db/base.py
@@ -4,7 +4,6 @@
 # the New BSD License: http://www.opensource.org/licenses/bsd-license.php
 """Contains implementations of database retrieveing objects"""
 from gitdb.util import (
- pool,
 join,
 LazyMixin,
 hex_to_bin
@@ -15,10 +14,6 @@
 AmbiguousObjectName
 )
 
-from async import (
- ChannelThreadTask
-)
-
 from itertools import chain
 from functools import reduce
 
@@ -41,47 +36,18 @@ def has_object(self, sha):
 binary sha is contained in the database"""
 raise NotImplementedError("To be implemented in subclass")
 
- def has_object_async(self, reader):
- """Return a reader yielding information about the membership of objects
- as identified by shas
- :param reader: Reader yielding 20 byte shas.
- :return: async.Reader yielding tuples of (sha, bool) pairs which indicate
- whether the given sha exists in the database or not"""
- task = ChannelThreadTask(reader, str(self.has_object_async), lambda sha: (sha, self.has_object(sha)))
- return pool.add_task(task)
-
 def info(self, sha):
 """ :return: OInfo instance
 :param sha: bytes binary sha
 :raise BadObject:"""
 raise NotImplementedError("To be implemented in subclass")
 
- def info_async(self, reader):
- """Retrieve information of a multitude of objects asynchronously
- :param reader: Channel yielding the sha's of the objects of interest
- :return: async.Reader yielding OInfo|InvalidOInfo, in any order"""
- task = ChannelThreadTask(reader, str(self.info_async), self.info)
- return pool.add_task(task)
-
 def stream(self, sha):
 """:return: OStream instance
 :param sha: 20 bytes binary sha
 :raise BadObject:"""
 raise NotImplementedError("To be implemented in subclass")
 
- def stream_async(self, reader):
- """Retrieve the OStream of multiple objects
- :param reader: see ``info``
- :param max_threads: see ``ObjectDBW.store``
- :return: async.Reader yielding OStream|InvalidOStream instances in any order
-
- **Note:** depending on the system configuration, it might not be possible to
- read all OStreams at once. Instead, read them individually using reader.read(x)
- where x is small enough."""
- # base implementation just uses the stream method repeatedly
- task = ChannelThreadTask(reader, str(self.stream_async), self.stream)
- return pool.add_task(task)
-
 def size(self):
 """:return: amount of objects in this database"""
 raise NotImplementedError()
@@ -129,27 +95,6 @@ def store(self, istream):
 :raise IOError: if data could not be written"""
 raise NotImplementedError("To be implemented in subclass")
 
- def store_async(self, reader):
- """
- Create multiple new objects in the database asynchronously. The method will
- return right away, returning an output channel which receives the results as
- they are computed.
-
- :return: Channel yielding your IStream which served as input, in any order.
- The IStreams sha will be set to the sha it received during the process,
- or its error attribute will be set to the exception informing about the error.
-
- :param reader: async.Reader yielding IStream instances.
- The same instances will be used in the output channel as were received
- in by the Reader.
-
- **Note:** As some ODB implementations implement this operation atomic, they might
- abort the whole operation if one item could not be processed. Hence check how
- many items have actually been produced."""
- # base implementation uses store to perform the work
- task = ChannelThreadTask(reader, str(self.store_async), self.store)
- return pool.add_task(task)
-
 #} END edit interface
 
 
diff --git a/gitdb/db/mem.py b/gitdb/db/mem.py
@@ -32,10 +32,7 @@ class MemoryDB(ObjectDBR, ObjectDBW):
 """A memory database stores everything to memory, providing fast IO and object
 retrieval. It should be used to buffer results and obtain SHAs before writing
 it to the actual physical storage, as it allows to query whether object already
- exists in the target storage before introducing actual IO
-
- **Note:** memory is currently not threadsafe, hence the async methods cannot be used
- for storing"""
+ exists in the target storage before introducing actual IO"""
 
 def __init__(self):
 super(MemoryDB, self).__init__()
@@ -62,9 +59,6 @@ def store(self, istream):
 
 return istream
 
- def store_async(self, reader):
- raise UnsupportedOperation("MemoryDBs cannot currently be used for async write access")
-
 def has_object(self, sha):
 return sha in self._cache
 
diff --git a/gitdb/db/pack.py b/gitdb/db/pack.py
@@ -125,10 +125,6 @@ def store(self, istream):
 inefficient"""
 raise UnsupportedOperation()
 
- def store_async(self, reader):
- # TODO: add ObjectDBRW before implementing this
- raise NotImplementedError()
-
 #} END object db write
 
 
diff --git a/gitdb/ext/async b/gitdb/ext/async
diff --git a/gitdb/test/__init__.py b/gitdb/test/__init__.py
@@ -2,15 +2,3 @@
 #
 # This module is part of GitDB and is released under
 # the New BSD License: http://www.opensource.org/licenses/bsd-license.php
-
-import gitdb.util
-
-#{ Initialization
-def _init_pool():
- """Assure the pool is actually threaded"""
- size = 2
- print("Setting ThreadPool to %i" % size)
- gitdb.util.pool.set_size(size)
-
-
-#} END initialization
diff --git a/gitdb/test/db/lib.py b/gitdb/test/db/lib.py
@@ -10,8 +10,6 @@
 TestBase
 )
 
-
-
 from gitdb.stream import (
 Sha1Writer,
 ZippedStoreShaWriter
@@ -28,8 +26,6 @@
 from gitdb.utils.encoding import force_bytes
 from gitdb.utils.compat import xrange
 
-from async import IteratorReader
-
 from io import BytesIO
 
 from struct import pack
@@ -132,82 +128,3 @@ def _assert_object_writing(self, db):
 # END for each data set
 # END for each dry_run mode
 
- def _assert_object_writing_async(self, db):
- """Test generic object writing using asynchronous access"""
- ni = 5000
- def istream_generator(offset=0, ni=ni):
- for data_src in xrange(ni):
- data = bytes(data_src + offset)
- yield IStream(str_blob_type, len(data), BytesIO(data))
- # END for each item
- # END generator utility
-
- # for now, we are very trusty here as we expect it to work if it worked
- # in the single-stream case
-
- # write objects
- reader = IteratorReader(istream_generator())
- istream_reader = db.store_async(reader)
- istreams = istream_reader.read() # read all
- assert istream_reader.task().error() is None
- assert len(istreams) == ni
-
- for stream in istreams:
- assert stream.error is None
- assert len(stream.binsha) == 20
- assert isinstance(stream, IStream)
- # END assert each stream
-
- # test has-object-async - we must have all previously added ones
- reader = IteratorReader( istream.binsha for istream in istreams )
- hasobject_reader = db.has_object_async(reader)
- count = 0
- for sha, has_object in hasobject_reader:
- assert has_object
- count += 1
- # END for each sha
- assert count == ni
-
- # read the objects we have just written
- reader = IteratorReader( istream.binsha for istream in istreams )
- ostream_reader = db.stream_async(reader)
-
- # read items individually to prevent hitting possible sys-limits
- count = 0
- for ostream in ostream_reader:
- assert isinstance(ostream, OStream)
- count += 1
- # END for each ostream
- assert ostream_reader.task().error() is None
- assert count == ni
-
- # get info about our items
- reader = IteratorReader( istream.binsha for istream in istreams )
- info_reader = db.info_async(reader)
-
- count = 0
- for oinfo in info_reader:
- assert isinstance(oinfo, OInfo)
- count += 1
- # END for each oinfo instance
- assert count == ni
-
-
- # combined read-write using a converter
- # add 2500 items, and obtain their output streams
- nni = 2500
- reader = IteratorReader(istream_generator(offset=ni, ni=nni))
- istream_to_sha = lambda istreams: [ istream.binsha for istream in istreams ]
-
- istream_reader = db.store_async(reader)
- istream_reader.set_post_cb(istream_to_sha)
-
- ostream_reader = db.stream_async(istream_reader)
-
- count = 0
- # read it individually, otherwise we might run into the ulimit
- for ostream in ostream_reader:
- assert isinstance(ostream, OStream)
- count += 1
- # END for each ostream
- assert count == nni
diff --git a/gitdb/test/db/test_git.py b/gitdb/test/db/test_git.py
@@ -48,4 +48,3 @@ def test_writing(self, path):
 
 # its possible to write objects
 self._assert_object_writing(gdb)
- self._assert_object_writing_async(gdb)
diff --git a/gitdb/test/db/test_loose.py b/gitdb/test/db/test_loose.py
@@ -18,7 +18,6 @@ def test_basics(self, path):
 
 # write data
 self._assert_object_writing(ldb)
- self._assert_object_writing_async(ldb)
 
 # verify sha iteration and size
 shas = list(ldb.sha_iter())
diff --git a/gitdb/test/performance/test_stream.py b/gitdb/test/performance/test_stream.py
diff --git a/gitdb/test/test_example.py b/gitdb/test/test_example.py
diff --git a/gitdb/util.py b/gitdb/util.py
diff --git a/setup.py b/setup.py

-Original file line number
+Diff line change
@@ @@ -1,6 +1,3 @@ @@
 -[submodule "async"]
 -path=gitdb/ext/async
 -url=https://github.com/gitpython-developers/async.git
 [submodule "smmap"]
 path=gitdb/ext/smmap
 url=https://github.com/Byron/smmap.git
-Original file line number
+Diff line change
 #{ Initialization
 def_init_externals():
 """Initialize external projects by putting them into the path"""
 -formodulein ('async', 'smmap'):
 +formodulein ('smmap',):
 sys.path.append(os.path.join(os.path.dirname(__file__), 'ext', module))
 try:
 __author__="Sebastian Thiel"
 __contact__="byronimo@gmail.com"
 __homepage__="https://github.com/gitpython-developers/gitdb"
 -version_info= (0, 5, 5)
 +version_info= (0, 6, 0)
 __version__='.'.join(str(i) foriinversion_info)
-Original file line number
+Diff line change
 TestBase
+)
+-
+-
 fromgitdb.streamimport (
 Sha1Writer,
 ZippedStoreShaWriter
 fromgitdb.utils.encodingimportforce_bytes
 fromgitdb.utils.compatimportxrange
 -fromasyncimportIteratorReader
+-
 fromioimportBytesIO
 fromstructimportpack
 # END for each data set
 # END for each dry_run mode
 -def_assert_object_writing_async(self, db):
 -"""Test generic object writing using asynchronous access"""
 -ni=5000
 -defistream_generator(offset=0, ni=ni):
 -fordata_srcinxrange(ni):
 -data=bytes(data_src+offset)
 -yieldIStream(str_blob_type, len(data), BytesIO(data))
 -# END for each item
 -# END generator utility
+-
 -# for now, we are very trusty here as we expect it to work if it worked
 -# in the single-stream case
+-
 -# write objects
 -reader=IteratorReader(istream_generator())
 -istream_reader=db.store_async(reader)
 -istreams=istream_reader.read() # read all
 -assertistream_reader.task().error() isNone
 -assertlen(istreams) ==ni
+-
 -forstreaministreams:
 -assertstream.errorisNone
 -assertlen(stream.binsha) ==20
 -assertisinstance(stream, IStream)
 -# END assert each stream
+-
 -# test has-object-async - we must have all previously added ones
 -reader=IteratorReader( istream.binshaforistreaministreams )
 -hasobject_reader=db.has_object_async(reader)
 -count=0
 -forsha, has_objectinhasobject_reader:
 -asserthas_object
 -count+=1
 -# END for each sha
 -assertcount==ni
+-
 -# read the objects we have just written
 -reader=IteratorReader( istream.binshaforistreaministreams )
 -ostream_reader=db.stream_async(reader)
+-
 -# read items individually to prevent hitting possible sys-limits
 -count=0
 -forostreaminostream_reader:
 -assertisinstance(ostream, OStream)
 -count+=1
 -# END for each ostream
 -assertostream_reader.task().error() isNone
 -assertcount==ni
+-
 -# get info about our items
 -reader=IteratorReader( istream.binshaforistreaministreams )
 -info_reader=db.info_async(reader)
+-
 -count=0
 -foroinfoininfo_reader:
 -assertisinstance(oinfo, OInfo)
 -count+=1
 -# END for each oinfo instance
 -assertcount==ni
+-
+-
 -# combined read-write using a converter
 -# add 2500 items, and obtain their output streams
 -nni=2500
 -reader=IteratorReader(istream_generator(offset=ni, ni=nni))
 -istream_to_sha=lambdaistreams: [ istream.binshaforistreaministreams ]
+-
 -istream_reader=db.store_async(reader)
 -istream_reader.set_post_cb(istream_to_sha)
+-
 -ostream_reader=db.stream_async(istream_reader)
+-
 -count=0
 -# read it individually, otherwise we might run into the ulimit
 -forostreaminostream_reader:
 -assertisinstance(ostream, OStream)
 -count+=1
 -# END for each ostream
 -assertcount==nni
Original file line number	Diff line number	Diff line change
`@@ -48,4 +48,3 @@ def test_writing(self, path):`
`48`	`48`
`49`	`49`	`# its possible to write objects`
`50`	`50`	`self._assert_object_writing(gdb)`
`51`		`-self._assert_object_writing_async(gdb)`