Visitar URL original
feat: Mutations Batcher shim (#1309) · googleapis/google-cloud-python@54a9ba3 · GitHub
Skip to content

Commit 54a9ba3

Browse files
authored
feat: Mutations Batcher shim (#1309)
**Changes Made:** - Replaced mutations batcher implementation with one based off of the data client. - Reworked unit tests. - Added additional system tests.
1 parent b7ec06b commit 54a9ba3

13 files changed

Lines changed: 557 additions & 287 deletions

File tree

‎packages/google-cloud-bigtable/google/cloud/bigtable/batcher.py‎

Lines changed: 53 additions & 167 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,21 @@
1313
# limitations under the License.
1414

1515
"""User friendly container for Google Cloud Bigtable MutationBatcher."""
16+
<<<<<<< ours
1617

1718
import atexit
1819
import concurrent.futures
1920
import queue
2021
import threading
2122
from dataclasses import dataclass
23+
=======
24+
import queue
25+
import atexit
26+
27+
28+
from google.cloud.bigtable.data.exceptions import MutationsExceptionGroup
29+
from google.cloud.bigtable.data.mutations import RowMutationEntry
30+
>>>>>>> theirs
2231

2332
from google.api_core.exceptions import from_grpc_status
2433

@@ -40,131 +49,6 @@ def __init__(self, message, exc):
4049
super().__init__(self.message)
4150

4251

43-
class _MutationsBatchQueue(object):
44-
"""Private Threadsafe Queue to hold rows for batching."""
45-
46-
def __init__(self, max_mutation_bytes=MAX_MUTATION_SIZE, flush_count=FLUSH_COUNT):
47-
"""Specify the queue constraints"""
48-
self._queue = queue.Queue()
49-
self.total_mutation_count = 0
50-
self.total_size = 0
51-
self.max_mutation_bytes = max_mutation_bytes
52-
self.flush_count = flush_count
53-
54-
def get(self):
55-
"""
56-
Retrieve an item from the queue. Recalculate queue size.
57-
58-
If the queue is empty, return None.
59-
"""
60-
try:
61-
row = self._queue.get_nowait()
62-
mutation_size = row.get_mutations_size()
63-
self.total_mutation_count -= len(row._get_mutations())
64-
self.total_size -= mutation_size
65-
return row
66-
except queue.Empty:
67-
return None
68-
69-
def put(self, item):
70-
"""Insert an item to the queue. Recalculate queue size."""
71-
72-
mutation_count = len(item._get_mutations())
73-
74-
self._queue.put(item)
75-
76-
self.total_size += item.get_mutations_size()
77-
self.total_mutation_count += mutation_count
78-
79-
def full(self):
80-
"""Check if the queue is full."""
81-
if (
82-
self.total_mutation_count >= self.flush_count
83-
or self.total_size >= self.max_mutation_bytes
84-
):
85-
return True
86-
return False
87-
88-
89-
@dataclass
90-
class _BatchInfo:
91-
"""Keeping track of size of a batch"""
92-
93-
mutations_count: int = 0
94-
rows_count: int = 0
95-
mutations_size: int = 0
96-
97-
98-
class _FlowControl(object):
99-
def __init__(
100-
self,
101-
max_mutations=MAX_OUTSTANDING_ELEMENTS,
102-
max_mutation_bytes=MAX_OUTSTANDING_BYTES,
103-
):
104-
"""Control the inflight requests. Keep track of the mutations, row bytes and row counts.
105-
As requests to backend are being made, adjust the number of mutations being processed.
106-
107-
If threshold is reached, block the flow.
108-
Reopen the flow as requests are finished.
109-
"""
110-
self.max_mutations = max_mutations
111-
self.max_mutation_bytes = max_mutation_bytes
112-
self.inflight_mutations = 0
113-
self.inflight_size = 0
114-
self.event = threading.Event()
115-
self.event.set()
116-
self._lock = threading.Lock()
117-
118-
def is_blocked(self):
119-
"""Returns True if:
120-
121-
- inflight mutations >= max_mutations, or
122-
- inflight bytes size >= max_mutation_bytes, or
123-
"""
124-
125-
return (
126-
self.inflight_mutations >= self.max_mutations
127-
or self.inflight_size >= self.max_mutation_bytes
128-
)
129-
130-
def control_flow(self, batch_info):
131-
"""
132-
Calculate the resources used by this batch
133-
"""
134-
135-
with self._lock:
136-
self.inflight_mutations += batch_info.mutations_count
137-
self.inflight_size += batch_info.mutations_size
138-
self.set_flow_control_status()
139-
140-
def wait(self):
141-
"""
142-
Wait until flow control pushback has been released.
143-
It awakens as soon as `event` is set.
144-
"""
145-
self.event.wait()
146-
147-
def set_flow_control_status(self):
148-
"""Check the inflight mutations and size.
149-
150-
If values exceed the allowed threshold, block the event.
151-
"""
152-
if self.is_blocked():
153-
self.event.clear() # sleep
154-
else:
155-
self.event.set() # awaken the threads
156-
157-
def release(self, batch_info):
158-
"""
159-
Release the resources.
160-
Decrement the row size to allow enqueued mutations to be run.
161-
"""
162-
with self._lock:
163-
self.inflight_mutations -= batch_info.mutations_count
164-
self.inflight_size -= batch_info.mutations_size
165-
self.set_flow_control_status()
166-
167-
16852
class MutationsBatcher(object):
16953
"""A MutationsBatcher is used in batch cases where the number of mutations
17054
is large or unknown. It will store :class:`DirectRow` in memory until one of the
@@ -226,10 +110,8 @@ def __init__(
226110
flush_interval=1,
227111
batch_completed_callback=None,
228112
):
229-
self._rows = _MutationsBatchQueue(
230-
max_mutation_bytes=max_row_bytes, flush_count=flush_count
231-
)
232113
self.table = table
114+
<<<<<<< ours
233115
self._executor = concurrent.futures.ThreadPoolExecutor()
234116
atexit.register(self.close)
235117
# ``flush_interval`` is retained for backwards compatibility but is no
@@ -244,15 +126,42 @@ def __init__(
244126
)
245127
self.futures_mapping = {}
246128
self.exceptions = queue.Queue()
129+
=======
130+
self._batcher_kwargs = {
131+
"flush_interval": flush_interval,
132+
"flush_limit_mutation_count": flush_count,
133+
"flush_limit_bytes": max_row_bytes,
134+
"flow_control_max_mutation_count": MAX_OUTSTANDING_ELEMENTS,
135+
"flow_control_max_bytes": MAX_OUTSTANDING_BYTES,
136+
}
137+
>>>>>>> theirs
247138
self._user_batch_completed_callback = batch_completed_callback
139+
self._init_batcher()
140+
atexit.register(self.close)
141+
self._exceptions = queue.Queue()
248142

249143
@property
250144
def flush_count(self):
251-
return self._rows.flush_count
145+
return self._flush_count
252146

253147
@property
254148
def max_row_bytes(self):
255-
return self._rows.max_mutation_bytes
149+
return self._max_row_bytes
150+
151+
def _init_batcher(self):
152+
self._batcher = self.table._table_impl.mutations_batcher(**self._batcher_kwargs)
153+
self._batcher._user_batch_completed_callback = (
154+
self._user_batch_completed_callback
155+
)
156+
157+
def _close_batcher(self):
158+
try:
159+
self._batcher.close()
160+
except MutationsExceptionGroup as exc_group:
161+
for error in exc_group.exceptions:
162+
# Unpack the root cause of the FailedMutationEntryError
163+
# and return that error to the user.
164+
self._exceptions.put(error.__cause__)
256165

257166
def __enter__(self):
258167
"""Starting the MutationsBatcher as a context manager"""
@@ -276,10 +185,7 @@ def mutate(self, row):
276185
* :exc:`~.table._BigtableRetryableError` if any row returned a transient error.
277186
* :exc:`RuntimeError` if the number of responses doesn't match the number of rows that were retried
278187
"""
279-
self._rows.put(row)
280-
281-
if self._rows.full():
282-
self._flush_async()
188+
self._batcher.append(RowMutationEntry(row.row_key, row._get_mutations()))
283189

284190
def mutate_rows(self, rows):
285191
"""Add multiple rows to the batch. If the current batch meets one of the size
@@ -312,6 +218,7 @@ def flush(self):
312218
:dedent: 4
313219
314220
:raises:
221+
<<<<<<< ours
315222
* :exc:`.batcherMutationsBatchError` if there's any error in the mutations.
316223
"""
317224
rows_to_flush = []
@@ -399,37 +306,12 @@ def _row_fits_in_batch(self, row, batch_info):
399306
400307
:rtype: bool
401308
:returns: True if the row can fit in the current batch.
309+
=======
310+
* :exc:`~batcher.MutationsBatchError` if there's any error in the mutations.
311+
>>>>>>> theirs
402312
"""
403-
new_rows_count = batch_info.rows_count + 1
404-
new_mutations_count = batch_info.mutations_count + len(row._get_mutations())
405-
new_mutations_size = batch_info.mutations_size + row.get_mutations_size()
406-
return (
407-
new_rows_count <= self.flush_count
408-
and new_mutations_size <= self.max_row_bytes
409-
and new_mutations_count <= self.flow_control.max_mutations
410-
and new_mutations_size <= self.flow_control.max_mutation_bytes
411-
)
412-
413-
def _flush_rows(self, rows_to_flush):
414-
"""Mutate the specified rows.
415-
416-
:raises:
417-
* :exc:`.batcherMutationsBatchError` if there's any error in the mutations.
418-
"""
419-
responses = []
420-
if len(rows_to_flush) > 0:
421-
response = self.table.mutate_rows(rows_to_flush)
422-
423-
if self._user_batch_completed_callback:
424-
self._user_batch_completed_callback(response)
425-
426-
for result in response:
427-
if result.code != 0:
428-
exc = from_grpc_status(result.code, result.message)
429-
self.exceptions.put(exc)
430-
responses.append(result)
431-
432-
return responses
313+
self._close_batcher()
314+
self._init_batcher()
433315

434316
def __exit__(self, exc_type, exc_value, exc_traceback):
435317
"""Clean up resources. Flush and shutdown the ThreadPoolExecutor."""
@@ -440,8 +322,9 @@ def close(self):
440322
Any errors will be raised.
441323
442324
:raises:
443-
* :exc:`.batcherMutationsBatchError` if there's any error in the mutations.
325+
* :exc:`~batcher.MutationsBatchError` if there's any error in the mutations.
444326
"""
327+
<<<<<<< ours
445328
try:
446329
self.flush()
447330
except MutationsBatchError as exc:
@@ -456,7 +339,10 @@ def close(self):
456339
# Record it like any other batch failure and continue.
457340
self.exceptions.put(exc)
458341
self._executor.shutdown(wait=True)
342+
=======
343+
self._close_batcher()
344+
>>>>>>> theirs
459345
atexit.unregister(self.close)
460-
if self.exceptions.qsize() > 0:
461-
exc = list(self.exceptions.queue)
346+
if self._exceptions.qsize() > 0:
347+
exc = list(self._exceptions.queue)
462348
raise MutationsBatchError("Errors in batch mutations.", exc=exc)

‎packages/google-cloud-bigtable/google/cloud/bigtable/data/_async/_mutate_rows.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ async def start(self):
128128
Raises:
129129
MutationsExceptionGroup: if any mutations failed
130130
"""
131+
<<<<<<< ours
131132
with self._operation_metric:
132133
try:
133134
# trigger mutate_rows
@@ -156,6 +157,23 @@ async def start(self):
156157
if all_errors:
157158
raise bt_exceptions.MutationsExceptionGroup(
158159
all_errors, len(self.mutations)
160+
=======
161+
try:
162+
# trigger mutate_rows
163+
await self._operation()
164+
except Exception as exc:
165+
# exceptions raised by retryable are added to the list of exceptions for all unfinalized mutations
166+
incomplete_indices = self.remaining_indices.copy()
167+
for idx in incomplete_indices:
168+
self._handle_entry_error(idx, exc)
169+
finally:
170+
# raise exception detailing incomplete mutations
171+
all_errors: list[bt_exceptions.FailedMutationEntryError] = []
172+
for idx, exc_list in self.errors.items():
173+
if len(exc_list) == 0:
174+
raise core_exceptions.ClientError(
175+
f"Mutation {idx} failed with no associated errors"
176+
>>>>>>> theirs
159177
)
160178

161179
@CrossSync.convert

‎packages/google-cloud-bigtable/google/cloud/bigtable/data/_async/client.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,11 @@
2424
import warnings
2525
from functools import partial
2626
from typing import (
27+
<<<<<<< ours
2728
TYPE_CHECKING,
29+
=======
30+
cast,
31+
>>>>>>> theirs
2832
Any,
2933
AsyncIterable,
3034
Callable,
@@ -146,7 +150,12 @@
146150
)
147151

148152
if TYPE_CHECKING:
153+
<<<<<<< ours
149154
from google.cloud.bigtable.data._helpers import RowKeySamples, ShardedQuery
155+
=======
156+
from google.cloud.bigtable.data._helpers import RowKeySamples
157+
from google.cloud.bigtable.data._helpers import ShardedQuery
158+
>>>>>>> theirs
150159

151160
if CrossSync.is_async:
152161
from google.cloud.bigtable.data._async.mutations_batcher import (

0 commit comments

Comments
 (0)