Visitar URL original
Merge pull request #358 from tseaver/refactor-move_bucket_iterator_to… · googleapis/google-cloud-python@e3f6ae4 · GitHub
Skip to content

Commit e3f6ae4

Browse files
tseavertseaver
committed
Merge pull request #358 from tseaver/refactor-move_bucket_iterator_to_connection
Move 'bucket.BucketIterator' -> 'connection._BucketIterator'.
2 parents 95e80e1 + d0cff2c commit e3f6ae4

6 files changed

Lines changed: 122 additions & 116 deletions

File tree

‎gcloud/storage/bucket.py‎

Lines changed: 29 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,31 @@
99
from gcloud.storage.acl import DefaultObjectACL
1010
from gcloud.storage.iterator import Iterator
1111
from gcloud.storage.key import Key
12-
from gcloud.storage.key import _KeyIterator
12+
13+
14+
class _KeyIterator(Iterator):
15+
"""An iterator listing keys in a bucket
16+
17+
You shouldn't have to use this directly, but instead should use the
18+
helper methods on :class:`gcloud.storage.key.Bucket` objects.
19+
20+
:type bucket: :class:`gcloud.storage.bucket.Bucket`
21+
:param bucket: The bucket from which to list keys.
22+
"""
23+
def __init__(self, bucket, connection=None, extra_params=None):
24+
self.bucket = bucket
25+
super(_KeyIterator, self).__init__(
26+
connection=bucket.connection, path=bucket.path + '/o',
27+
extra_params=extra_params)
28+
29+
def get_items_from_response(self, response):
30+
"""Yield :class:`.storage.key.Key` items from response.
31+
32+
:type response: dict
33+
:param response: The JSON API response for a page of keys.
34+
"""
35+
for item in response.get('items', []):
36+
yield Key.from_dict(item, bucket=self.bucket)
1337

1438

1539
class Bucket(_PropertyMixin):
@@ -21,6 +45,7 @@ class Bucket(_PropertyMixin):
2145
:type name: string
2246
:param name: The name of the bucket.
2347
"""
48+
_iterator_class = _KeyIterator
2449

2550
CUSTOM_PROPERTY_ACCESSORS = {
2651
'acl': 'get_acl()',
@@ -66,7 +91,7 @@ def __repr__(self):
6691
return '<Bucket: %s>' % self.name
6792

6893
def __iter__(self):
69-
return iter(_KeyIterator(bucket=self))
94+
return iter(self._iterator_class(bucket=self))
7095

7196
def __contains__(self, key):
7297
return self.get_key(key) is not None
@@ -640,29 +665,7 @@ def make_public(self, recursive=False, future=False):
640665
doa.save()
641666

642667
if recursive:
643-
for key in self:
668+
iterator = self._iterator_class(self)
669+
for key in iterator:
644670
key.get_acl().all().grant_read()
645671
key.save_acl()
646-
647-
648-
class BucketIterator(Iterator):
649-
"""An iterator listing all buckets.
650-
651-
You shouldn't have to use this directly, but instead should use the helper
652-
methods on :class:`gcloud.storage.connection.Connection` objects.
653-
654-
:type connection: :class:`gcloud.storage.connection.Connection`
655-
:param connection: The connection to use for querying the list of buckets.
656-
"""
657-
658-
def __init__(self, connection):
659-
super(BucketIterator, self).__init__(connection=connection, path='/b')
660-
661-
def get_items_from_response(self, response):
662-
"""Factory method which yields :class:`.Bucket` items from a response.
663-
664-
:type response: dict
665-
:param response: The JSON API response for a page of buckets.
666-
"""
667-
for item in response.get('items', []):
668-
yield Bucket.from_dict(item, connection=self.connection)

‎gcloud/storage/connection.py‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,10 @@
1212
from OpenSSL import crypto
1313
import pytz
1414

15-
from gcloud import connection
15+
from gcloud.connection import Connection as _Base
1616
from gcloud.storage import exceptions
1717
from gcloud.storage.bucket import Bucket
18-
from gcloud.storage.bucket import BucketIterator
18+
from gcloud.storage.iterator import Iterator
1919

2020

2121
def _utcnow(): # pragma: NO COVER testing replaces
@@ -26,7 +26,7 @@ def _utcnow(): # pragma: NO COVER testing replaces
2626
return datetime.datetime.utcnow()
2727

2828

29-
class Connection(connection.Connection):
29+
class Connection(_Base):
3030
"""A connection to Google Cloud Storage via the JSON REST API.
3131
3232
This class should understand only the basic types (and protobufs)
@@ -84,7 +84,7 @@ def __init__(self, project, *args, **kwargs):
8484
self.project = project
8585

8686
def __iter__(self):
87-
return iter(BucketIterator(connection=self))
87+
return iter(_BucketIterator(connection=self))
8888

8989
def __contains__(self, bucket_name):
9090
return self.lookup(bucket_name) is not None
@@ -488,6 +488,29 @@ def generate_signed_url(self, resource, expiration,
488488
querystring=urllib.urlencode(query_params))
489489

490490

491+
class _BucketIterator(Iterator):
492+
"""An iterator listing all buckets.
493+
494+
You shouldn't have to use this directly, but instead should use the helper
495+
methods on :class:`gcloud.storage.connection.Connection` objects.
496+
497+
:type connection: :class:`gcloud.storage.connection.Connection`
498+
:param connection: The connection to use for querying the list of buckets.
499+
"""
500+
501+
def __init__(self, connection):
502+
super(_BucketIterator, self).__init__(connection=connection, path='/b')
503+
504+
def get_items_from_response(self, response):
505+
"""Factory method which yields :class:`.Bucket` items from a response.
506+
507+
:type response: dict
508+
:param response: The JSON API response for a page of buckets.
509+
"""
510+
for item in response.get('items', []):
511+
yield Bucket.from_dict(item, connection=self.connection)
512+
513+
491514
def _get_expiration_seconds(expiration):
492515
"""Convert 'expiration' to a number of seconds in the future.
493516

‎gcloud/storage/key.py‎

Lines changed: 0 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
from gcloud.storage._helpers import _scalar_property
1010
from gcloud.storage.acl import ObjectACL
1111
from gcloud.storage.exceptions import StorageError
12-
from gcloud.storage.iterator import Iterator
1312

1413

1514
class Key(_PropertyMixin):
@@ -595,31 +594,6 @@ def updated(self):
595594
return self.properties['updated']
596595

597596

598-
class _KeyIterator(Iterator):
599-
"""An iterator listing keys.
600-
601-
You shouldn't have to use this directly, but instead should use the
602-
helper methods on :class:`gcloud.storage.key.Key` objects.
603-
604-
:type bucket: :class:`gcloud.storage.bucket.Bucket`
605-
:param bucket: The bucket from which to list keys.
606-
"""
607-
def __init__(self, bucket, extra_params=None):
608-
self.bucket = bucket
609-
super(_KeyIterator, self).__init__(
610-
connection=bucket.connection, path=bucket.path + '/o',
611-
extra_params=extra_params)
612-
613-
def get_items_from_response(self, response):
614-
"""Yield :class:`.storage.key.Key` items from response.
615-
616-
:type response: dict
617-
:param response: The JSON API response for a page of keys.
618-
"""
619-
for item in response.get('items', []):
620-
yield Key.from_dict(item, bucket=self.bucket)
621-
622-
623597
class _KeyDataIterator(object):
624598
"""An iterator listing data stored in a key.
625599

‎gcloud/storage/test_bucket.py‎

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -832,9 +832,7 @@ def test_make_public_w_future(self):
832832

833833
def test_make_public_recursive(self):
834834
from gcloud.storage.acl import _ACLEntity
835-
from gcloud._testing import _Monkey
836-
from gcloud.storage import key
837-
from gcloud.storage import bucket as MUT
835+
from gcloud.storage.bucket import _KeyIterator
838836
_saved = []
839837

840838
class _Key(object):
@@ -856,7 +854,7 @@ def grant_read(self):
856854
def save_acl(self):
857855
_saved.append((self._bucket, self._name, self._granted))
858856

859-
class _KeyIterator(key._KeyIterator):
857+
class _Iterator(_KeyIterator):
860858
def get_items_from_response(self, response):
861859
for item in response.get('items', []):
862860
yield _Key(self.bucket, item['name'])
@@ -868,8 +866,8 @@ def get_items_from_response(self, response):
868866
connection = _Connection(after, {'items': [{'name': KEY}]})
869867
bucket = self._makeOne(connection, NAME)
870868
bucket.acl.loaded = True
871-
with _Monkey(MUT, _KeyIterator=_KeyIterator):
872-
bucket.make_public(recursive=True)
869+
bucket._iterator_class = _Iterator
870+
bucket.make_public(recursive=True)
873871
self.assertEqual(list(bucket.acl), permissive)
874872
self.assertEqual(list(bucket.default_object_acl), [])
875873
self.assertEqual(_saved, [(bucket, KEY, True)])
@@ -884,40 +882,44 @@ def get_items_from_response(self, response):
884882
self.assertEqual(kw[1]['query_params'], {})
885883

886884

887-
class TestBucketIterator(unittest2.TestCase):
885+
class Test__KeyIterator(unittest2.TestCase):
888886

889887
def _getTargetClass(self):
890-
from gcloud.storage.bucket import BucketIterator
891-
return BucketIterator
888+
from gcloud.storage.bucket import _KeyIterator
889+
return _KeyIterator
892890

893891
def _makeOne(self, *args, **kw):
894892
return self._getTargetClass()(*args, **kw)
895893

896894
def test_ctor(self):
897895
connection = _Connection()
898-
iterator = self._makeOne(connection)
896+
bucket = _Bucket(connection)
897+
iterator = self._makeOne(bucket)
898+
self.assertTrue(iterator.bucket is bucket)
899899
self.assertTrue(iterator.connection is connection)
900-
self.assertEqual(iterator.path, '/b')
900+
self.assertEqual(iterator.path, '%s/o' % bucket.path)
901901
self.assertEqual(iterator.page_number, 0)
902902
self.assertEqual(iterator.next_page_token, None)
903903

904904
def test_get_items_from_response_empty(self):
905905
connection = _Connection()
906-
iterator = self._makeOne(connection)
906+
bucket = _Bucket(connection)
907+
iterator = self._makeOne(bucket)
907908
self.assertEqual(list(iterator.get_items_from_response({})), [])
908909

909910
def test_get_items_from_response_non_empty(self):
910-
from gcloud.storage.bucket import Bucket
911+
from gcloud.storage.key import Key
911912
KEY = 'key'
912913
response = {'items': [{'name': KEY}]}
913914
connection = _Connection()
914-
iterator = self._makeOne(connection)
915-
buckets = list(iterator.get_items_from_response(response))
916-
self.assertEqual(len(buckets), 1)
917-
bucket = buckets[0]
918-
self.assertTrue(isinstance(bucket, Bucket))
919-
self.assertTrue(bucket.connection is connection)
920-
self.assertEqual(bucket.name, KEY)
915+
bucket = _Bucket(connection)
916+
iterator = self._makeOne(bucket)
917+
keys = list(iterator.get_items_from_response(response))
918+
self.assertEqual(len(keys), 1)
919+
key = keys[0]
920+
self.assertTrue(isinstance(key, Key))
921+
self.assertTrue(key.connection is connection)
922+
self.assertEqual(key.name, KEY)
921923

922924

923925
class _Connection(object):
@@ -947,6 +949,14 @@ def delete_bucket(self, bucket, force=False):
947949
return True
948950

949951

952+
class _Bucket(object):
953+
path = '/b/name'
954+
name = 'name'
955+
956+
def __init__(self, connection):
957+
self.connection = connection
958+
959+
950960
class MockFile(io.StringIO):
951961
name = None
952962

‎gcloud/storage/test_connection.py‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -610,6 +610,42 @@ def test_generate_signed_url_w_expiration_int(self):
610610
self.assertEqual(frag, '')
611611

612612

613+
class Test__BucketIterator(unittest2.TestCase):
614+
615+
def _getTargetClass(self):
616+
from gcloud.storage.connection import _BucketIterator
617+
return _BucketIterator
618+
619+
def _makeOne(self, *args, **kw):
620+
return self._getTargetClass()(*args, **kw)
621+
622+
def test_ctor(self):
623+
connection = object()
624+
iterator = self._makeOne(connection)
625+
self.assertTrue(iterator.connection is connection)
626+
self.assertEqual(iterator.path, '/b')
627+
self.assertEqual(iterator.page_number, 0)
628+
self.assertEqual(iterator.next_page_token, None)
629+
630+
def test_get_items_from_response_empty(self):
631+
connection = object()
632+
iterator = self._makeOne(connection)
633+
self.assertEqual(list(iterator.get_items_from_response({})), [])
634+
635+
def test_get_items_from_response_non_empty(self):
636+
from gcloud.storage.bucket import Bucket
637+
KEY = 'key'
638+
response = {'items': [{'name': KEY}]}
639+
connection = object()
640+
iterator = self._makeOne(connection)
641+
buckets = list(iterator.get_items_from_response(response))
642+
self.assertEqual(len(buckets), 1)
643+
bucket = buckets[0]
644+
self.assertTrue(isinstance(bucket, Bucket))
645+
self.assertTrue(bucket.connection is connection)
646+
self.assertEqual(bucket.name, KEY)
647+
648+
613649
class Test__get_expiration_seconds(unittest2.TestCase):
614650

615651
def _callFUT(self, expiration):

‎gcloud/storage/test_key.py‎

Lines changed: 0 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -662,46 +662,6 @@ def test_updated(self):
662662
self.assertEqual(key.updated, UPDATED)
663663

664664

665-
class Test__KeyIterator(unittest2.TestCase):
666-
667-
def _getTargetClass(self):
668-
from gcloud.storage.key import _KeyIterator
669-
return _KeyIterator
670-
671-
def _makeOne(self, *args, **kw):
672-
return self._getTargetClass()(*args, **kw)
673-
674-
def test_ctor(self):
675-
connection = _Connection()
676-
bucket = _Bucket(connection)
677-
iterator = self._makeOne(bucket)
678-
self.assertTrue(iterator.bucket is bucket)
679-
self.assertTrue(iterator.connection is connection)
680-
self.assertEqual(iterator.path, '%s/o' % bucket.path)
681-
self.assertEqual(iterator.page_number, 0)
682-
self.assertEqual(iterator.next_page_token, None)
683-
684-
def test_get_items_from_response_empty(self):
685-
connection = _Connection()
686-
bucket = _Bucket(connection)
687-
iterator = self._makeOne(bucket)
688-
self.assertEqual(list(iterator.get_items_from_response({})), [])
689-
690-
def test_get_items_from_response_non_empty(self):
691-
from gcloud.storage.key import Key
692-
KEY = 'key'
693-
response = {'items': [{'name': KEY}]}
694-
connection = _Connection()
695-
bucket = _Bucket(connection)
696-
iterator = self._makeOne(bucket)
697-
keys = list(iterator.get_items_from_response(response))
698-
self.assertEqual(len(keys), 1)
699-
key = keys[0]
700-
self.assertTrue(isinstance(key, Key))
701-
self.assertTrue(key.connection is connection)
702-
self.assertEqual(key.name, KEY)
703-
704-
705665
class Test__KeyDataIterator(unittest2.TestCase):
706666

707667
def _getTargetClass(self):

0 commit comments

Comments
 (0)