Visitar URL original
Merge pull request #1699 from tseaver/1554-add_method_based_paging_it… · googleapis/google-cloud-python@71d2845 · GitHub
Skip to content

Commit 71d2845

Browse files
committed
Merge pull request #1699 from tseaver/1554-add_method_based_paging_iterator
Create a generic iterator wrapping 'list_*' methods of client-side objects
2 parents fde68df + 88e3869 commit 71d2845

2 files changed

Lines changed: 132 additions & 0 deletions

File tree

‎gcloud/iterator.py‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,3 +141,47 @@ def get_items_from_response(self, response):
141141
:returns: Items that the iterator should yield.
142142
"""
143143
raise NotImplementedError
144+
145+
146+
class MethodIterator(object):
147+
"""Method-based iterator iterating through Cloud JSON APIs list responses.
148+
149+
:type method: instance method
150+
:param method: ``list_foo`` method of a domain object, taking as arguments
151+
``page_token``, ``page_size``, and optional additional
152+
keyword arguments.
153+
154+
:type page_token: string or ``NoneType``
155+
:param page_token: Initial page token to pass. if ``None``, fetch the
156+
first page from the ``method`` API call.
157+
158+
:type page_size: integer or ``NoneType``
159+
:param page_size: Maximum number of items to return from the ``method``
160+
API call; if ``None``, uses the default for the API.
161+
162+
:type max_calls: integer or ``NoneType``
163+
:param max_calls: Maximum number of times to make the ``method``
164+
API call; if ``None``, applies no limit.
165+
166+
:type kw: dict
167+
:param kw: optional keyword argments to be passed to ``method``.
168+
"""
169+
def __init__(self, method, page_token=None, page_size=None,
170+
max_calls=None, **kw):
171+
self._method = method
172+
self._token = page_token
173+
self._page_size = page_size
174+
self._kw = kw
175+
self._max_calls = max_calls
176+
self._page_num = 0
177+
178+
def __iter__(self):
179+
while self._max_calls is None or self._page_num < self._max_calls:
180+
items, new_token = self._method(
181+
page_token=self._token, page_size=self._page_size, **self._kw)
182+
for item in items:
183+
yield item
184+
if new_token is None:
185+
return
186+
self._page_num += 1
187+
self._token = new_token

‎gcloud/test_iterator.py‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,94 @@ def test_get_items_from_response_raises_NotImplementedError(self):
172172
iterator.get_items_from_response, object())
173173

174174

175+
class TestMethodIterator(unittest2.TestCase):
176+
177+
def _getTargetClass(self):
178+
from gcloud.iterator import MethodIterator
179+
return MethodIterator
180+
181+
def _makeOne(self, *args, **kw):
182+
return self._getTargetClass()(*args, **kw)
183+
184+
def test_ctor_defaults(self):
185+
wlm = _WithListMethod()
186+
iterator = self._makeOne(wlm.list_foo)
187+
self.assertEqual(iterator._method, wlm.list_foo)
188+
self.assertEqual(iterator._token, None)
189+
self.assertEqual(iterator._page_size, None)
190+
self.assertEqual(iterator._kw, {})
191+
self.assertEqual(iterator._max_calls, None)
192+
self.assertEqual(iterator._page_num, 0)
193+
194+
def test_ctor_explicit(self):
195+
wlm = _WithListMethod()
196+
TOKEN = wlm._letters
197+
SIZE = 4
198+
CALLS = 2
199+
iterator = self._makeOne(wlm.list_foo, TOKEN, SIZE, CALLS,
200+
foo_type='Bar')
201+
self.assertEqual(iterator._method, wlm.list_foo)
202+
self.assertEqual(iterator._token, TOKEN)
203+
self.assertEqual(iterator._page_size, SIZE)
204+
self.assertEqual(iterator._kw, {'foo_type': 'Bar'})
205+
self.assertEqual(iterator._max_calls, CALLS)
206+
self.assertEqual(iterator._page_num, 0)
207+
208+
def test___iter___defaults(self):
209+
import string
210+
wlm = _WithListMethod()
211+
iterator = self._makeOne(wlm.list_foo)
212+
found = []
213+
for char in iterator:
214+
found.append(char)
215+
self.assertEqual(found, list(string.printable))
216+
self.assertEqual(len(wlm._called_with), len(found) // 10)
217+
for i, (token, size, kw) in enumerate(wlm._called_with):
218+
if i == 0:
219+
self.assertEqual(token, None)
220+
else:
221+
self.assertEqual(token, string.printable[i * 10:])
222+
self.assertEqual(size, None)
223+
self.assertEqual(kw, {})
224+
225+
def test___iter___explicit_size_and_maxcalls_and_kw(self):
226+
import string
227+
wlm = _WithListMethod()
228+
iterator = self._makeOne(wlm.list_foo, page_size=2, max_calls=3,
229+
foo_type='Bar')
230+
found = []
231+
for char in iterator:
232+
found.append(char)
233+
self.assertEqual(found, list(string.printable[:2 * 3]))
234+
self.assertEqual(len(wlm._called_with), len(found) // 2)
235+
for i, (token, size, kw) in enumerate(wlm._called_with):
236+
if i == 0:
237+
self.assertEqual(token, None)
238+
else:
239+
self.assertEqual(token, string.printable[i * 2:])
240+
self.assertEqual(size, 2)
241+
self.assertEqual(kw, {'foo_type': 'Bar'})
242+
243+
244+
class _WithListMethod(object):
245+
246+
def __init__(self):
247+
import string
248+
self._called_with = []
249+
self._letters = string.printable
250+
251+
def list_foo(self, page_token, page_size, **kw):
252+
if page_token is not None:
253+
assert page_token == self._letters
254+
self._called_with.append((page_token, page_size, kw))
255+
if page_size is None:
256+
page_size = 10
257+
page, self._letters = (
258+
self._letters[:page_size], self._letters[page_size:])
259+
token = self._letters or None
260+
return page, token
261+
262+
175263
class _Connection(object):
176264

177265
def __init__(self, *responses):

0 commit comments

Comments
 (0)