1313# limitations under the License.
1414
1515"""User friendly container for Google Cloud Bigtable MutationBatcher."""
16+ < << << << ours
1617
1718import atexit
1819import concurrent .futures
1920import queue
2021import threading
2122from dataclasses import dataclass
23+ == == == =
24+ import queue
25+ import atexit
26+
27+
28+ from google .cloud .bigtable .data .exceptions import MutationsExceptionGroup
29+ from google .cloud .bigtable .data .mutations import RowMutationEntry
30+ > >> >> >> theirs
2231
2332from google .api_core .exceptions import from_grpc_status
2433
@@ -40,131 +49,6 @@ def __init__(self, message, exc):
4049 super ().__init__ (self .message )
4150
4251
43- class _MutationsBatchQueue (object ):
44- """Private Threadsafe Queue to hold rows for batching."""
45-
46- def __init__ (self , max_mutation_bytes = MAX_MUTATION_SIZE , flush_count = FLUSH_COUNT ):
47- """Specify the queue constraints"""
48- self ._queue = queue .Queue ()
49- self .total_mutation_count = 0
50- self .total_size = 0
51- self .max_mutation_bytes = max_mutation_bytes
52- self .flush_count = flush_count
53-
54- def get (self ):
55- """
56- Retrieve an item from the queue. Recalculate queue size.
57-
58- If the queue is empty, return None.
59- """
60- try :
61- row = self ._queue .get_nowait ()
62- mutation_size = row .get_mutations_size ()
63- self .total_mutation_count -= len (row ._get_mutations ())
64- self .total_size -= mutation_size
65- return row
66- except queue .Empty :
67- return None
68-
69- def put (self , item ):
70- """Insert an item to the queue. Recalculate queue size."""
71-
72- mutation_count = len (item ._get_mutations ())
73-
74- self ._queue .put (item )
75-
76- self .total_size += item .get_mutations_size ()
77- self .total_mutation_count += mutation_count
78-
79- def full (self ):
80- """Check if the queue is full."""
81- if (
82- self .total_mutation_count >= self .flush_count
83- or self .total_size >= self .max_mutation_bytes
84- ):
85- return True
86- return False
87-
88-
89- @dataclass
90- class _BatchInfo :
91- """Keeping track of size of a batch"""
92-
93- mutations_count : int = 0
94- rows_count : int = 0
95- mutations_size : int = 0
96-
97-
98- class _FlowControl (object ):
99- def __init__ (
100- self ,
101- max_mutations = MAX_OUTSTANDING_ELEMENTS ,
102- max_mutation_bytes = MAX_OUTSTANDING_BYTES ,
103- ):
104- """Control the inflight requests. Keep track of the mutations, row bytes and row counts.
105- As requests to backend are being made, adjust the number of mutations being processed.
106-
107- If threshold is reached, block the flow.
108- Reopen the flow as requests are finished.
109- """
110- self .max_mutations = max_mutations
111- self .max_mutation_bytes = max_mutation_bytes
112- self .inflight_mutations = 0
113- self .inflight_size = 0
114- self .event = threading .Event ()
115- self .event .set ()
116- self ._lock = threading .Lock ()
117-
118- def is_blocked (self ):
119- """Returns True if:
120-
121- - inflight mutations >= max_mutations, or
122- - inflight bytes size >= max_mutation_bytes, or
123- """
124-
125- return (
126- self .inflight_mutations >= self .max_mutations
127- or self .inflight_size >= self .max_mutation_bytes
128- )
129-
130- def control_flow (self , batch_info ):
131- """
132- Calculate the resources used by this batch
133- """
134-
135- with self ._lock :
136- self .inflight_mutations += batch_info .mutations_count
137- self .inflight_size += batch_info .mutations_size
138- self .set_flow_control_status ()
139-
140- def wait (self ):
141- """
142- Wait until flow control pushback has been released.
143- It awakens as soon as `event` is set.
144- """
145- self .event .wait ()
146-
147- def set_flow_control_status (self ):
148- """Check the inflight mutations and size.
149-
150- If values exceed the allowed threshold, block the event.
151- """
152- if self .is_blocked ():
153- self .event .clear () # sleep
154- else :
155- self .event .set () # awaken the threads
156-
157- def release (self , batch_info ):
158- """
159- Release the resources.
160- Decrement the row size to allow enqueued mutations to be run.
161- """
162- with self ._lock :
163- self .inflight_mutations -= batch_info .mutations_count
164- self .inflight_size -= batch_info .mutations_size
165- self .set_flow_control_status ()
166-
167-
16852class MutationsBatcher (object ):
16953 """A MutationsBatcher is used in batch cases where the number of mutations
17054 is large or unknown. It will store :class:`DirectRow` in memory until one of the
@@ -226,10 +110,8 @@ def __init__(
226110 flush_interval = 1 ,
227111 batch_completed_callback = None ,
228112 ):
229- self ._rows = _MutationsBatchQueue (
230- max_mutation_bytes = max_row_bytes , flush_count = flush_count
231- )
232113 self .table = table
114+ < << << << ours
233115 self ._executor = concurrent .futures .ThreadPoolExecutor ()
234116 atexit .register (self .close )
235117 # ``flush_interval`` is retained for backwards compatibility but is no
@@ -244,15 +126,42 @@ def __init__(
244126 )
245127 self .futures_mapping = {}
246128 self .exceptions = queue .Queue ()
129+ == == == =
130+ self ._batcher_kwargs = {
131+ "flush_interval" : flush_interval ,
132+ "flush_limit_mutation_count" : flush_count ,
133+ "flush_limit_bytes" : max_row_bytes ,
134+ "flow_control_max_mutation_count" : MAX_OUTSTANDING_ELEMENTS ,
135+ "flow_control_max_bytes" : MAX_OUTSTANDING_BYTES ,
136+ }
137+ >> >> >> > theirs
247138 self ._user_batch_completed_callback = batch_completed_callback
139+ self ._init_batcher ()
140+ atexit .register (self .close )
141+ self ._exceptions = queue .Queue ()
248142
249143 @property
250144 def flush_count (self ):
251- return self ._rows . flush_count
145+ return self ._flush_count
252146
253147 @property
254148 def max_row_bytes (self ):
255- return self ._rows .max_mutation_bytes
149+ return self ._max_row_bytes
150+
151+ def _init_batcher (self ):
152+ self ._batcher = self .table ._table_impl .mutations_batcher (** self ._batcher_kwargs )
153+ self ._batcher ._user_batch_completed_callback = (
154+ self ._user_batch_completed_callback
155+ )
156+
157+ def _close_batcher (self ):
158+ try :
159+ self ._batcher .close ()
160+ except MutationsExceptionGroup as exc_group :
161+ for error in exc_group .exceptions :
162+ # Unpack the root cause of the FailedMutationEntryError
163+ # and return that error to the user.
164+ self ._exceptions .put (error .__cause__ )
256165
257166 def __enter__ (self ):
258167 """Starting the MutationsBatcher as a context manager"""
@@ -276,10 +185,7 @@ def mutate(self, row):
276185 * :exc:`~.table._BigtableRetryableError` if any row returned a transient error.
277186 * :exc:`RuntimeError` if the number of responses doesn't match the number of rows that were retried
278187 """
279- self ._rows .put (row )
280-
281- if self ._rows .full ():
282- self ._flush_async ()
188+ self ._batcher .append (RowMutationEntry (row .row_key , row ._get_mutations ()))
283189
284190 def mutate_rows (self , rows ):
285191 """Add multiple rows to the batch. If the current batch meets one of the size
@@ -312,6 +218,7 @@ def flush(self):
312218 :dedent: 4
313219
314220 :raises:
221+ <<<<<<< ours
315222 * :exc:`.batcherMutationsBatchError` if there's any error in the mutations.
316223 """
317224 rows_to_flush = []
@@ -399,37 +306,12 @@ def _row_fits_in_batch(self, row, batch_info):
399306
400307 :rtype: bool
401308 :returns: True if the row can fit in the current batch.
309+ =======
310+ * :exc:`~batcher.MutationsBatchError` if there's any error in the mutations.
311+ >>>>>>> theirs
402312 """
403- new_rows_count = batch_info .rows_count + 1
404- new_mutations_count = batch_info .mutations_count + len (row ._get_mutations ())
405- new_mutations_size = batch_info .mutations_size + row .get_mutations_size ()
406- return (
407- new_rows_count <= self .flush_count
408- and new_mutations_size <= self .max_row_bytes
409- and new_mutations_count <= self .flow_control .max_mutations
410- and new_mutations_size <= self .flow_control .max_mutation_bytes
411- )
412-
413- def _flush_rows (self , rows_to_flush ):
414- """Mutate the specified rows.
415-
416- :raises:
417- * :exc:`.batcherMutationsBatchError` if there's any error in the mutations.
418- """
419- responses = []
420- if len (rows_to_flush ) > 0 :
421- response = self .table .mutate_rows (rows_to_flush )
422-
423- if self ._user_batch_completed_callback :
424- self ._user_batch_completed_callback (response )
425-
426- for result in response :
427- if result .code != 0 :
428- exc = from_grpc_status (result .code , result .message )
429- self .exceptions .put (exc )
430- responses .append (result )
431-
432- return responses
313+ self ._close_batcher ()
314+ self ._init_batcher ()
433315
434316 def __exit__ (self , exc_type , exc_value , exc_traceback ):
435317 """Clean up resources. Flush and shutdown the ThreadPoolExecutor."""
@@ -440,8 +322,9 @@ def close(self):
440322 Any errors will be raised.
441323
442324 :raises:
443- * :exc:`.batcherMutationsBatchError ` if there's any error in the mutations.
325+ * :exc:`~batcher.MutationsBatchError ` if there's any error in the mutations.
444326 """
327+ < << << << ours
445328 try :
446329 self .flush ()
447330 except MutationsBatchError as exc :
@@ -456,7 +339,10 @@ def close(self):
456339 # Record it like any other batch failure and continue.
457340 self .exceptions .put (exc )
458341 self ._executor .shutdown (wait = True )
342+ == == == =
343+ self ._close_batcher ()
344+ >> >> >> > theirs
459345 atexit .unregister (self .close )
460- if self .exceptions .qsize () > 0 :
461- exc = list (self .exceptions .queue )
346+ if self ._exceptions .qsize () > 0 :
347+ exc = list (self ._exceptions .queue )
462348 raise MutationsBatchError ("Errors in batch mutations." , exc = exc )
0 commit comments