Visitar URL original
deps: update hbase-client version (#4526) · googleapis/java-bigtable-hbase@ed3a802 · GitHub
Skip to content

Commit ed3a802

Browse files
mutianfmutianf
authored
deps: update hbase-client version (#4526)
* deps: update hbase-client version * ignore deps * update * add ctor * add ctor * add ctor * remove unused ctor
1 parent 886567f commit ed3a802

14 files changed

Lines changed: 130 additions & 6 deletions

File tree

‎bigtable-client-core-parent/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/BigtableBufferedMutatorHelper.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import com.google.cloud.bigtable.hbase.wrappers.BulkMutationWrapper;
2727
import com.google.cloud.bigtable.hbase.wrappers.DataClientWrapper;
2828
import java.io.IOException;
29+
import java.time.Duration;
2930
import java.util.ArrayList;
3031
import java.util.List;
3132
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -127,6 +128,14 @@ public long getWriteBufferSize() {
127128
return this.settings.getBatchingMaxRequestSize();
128129
}
129130

131+
public int getMaxRowKeyCount() {
132+
return this.settings.getBulkMaxRowCount();
133+
}
134+
135+
public Duration getAutoFlushInterval() {
136+
return this.settings.getAutoFlushInterval();
137+
}
138+
130139
public List<ApiFuture<?>> mutate(List<? extends Mutation> mutations) {
131140
closedReadLock.lock();
132141
try {

‎bigtable-client-core-parent/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/wrappers/BigtableHBaseSettings.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import com.google.common.annotations.VisibleForTesting;
2828
import com.google.common.base.Preconditions;
2929
import java.io.IOException;
30+
import java.time.Duration;
3031
import java.util.Map;
3132
import org.apache.hadoop.conf.Configuration;
3233

@@ -101,6 +102,8 @@ public int getTtlSecondsForBackup() {
101102

102103
public abstract long getBatchingMaxRequestSize();
103104

105+
public abstract Duration getAutoFlushInterval();
106+
104107
// This is equivalent to allow server-side timestamp.
105108
public abstract boolean isRetriesWithoutTimestampAllowed();
106109

‎bigtable-client-core-parent/bigtable-hbase/src/main/java/com/google/cloud/bigtable/hbase/wrappers/veneer/BigtableHBaseVeneerSettings.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,15 @@ public long getBatchingMaxRequestSize() {
241241
return batchingMaxMemory;
242242
}
243243

244+
@Override
245+
public java.time.Duration getAutoFlushInterval() {
246+
return dataSettings
247+
.getStubSettings()
248+
.bulkMutateRowsSettings()
249+
.getBatchingSettings()
250+
.getDelayThresholdDuration();
251+
}
252+
244253
@Override
245254
public boolean isRetriesWithoutTimestampAllowed() {
246255
return allowRetriesWithoutTimestamp;

‎bigtable-dataflow-parent/bigtable-beam-import/src/test/java/com/google/cloud/bigtable/beam/hbasesnapshots/EndToEndIT.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import com.google.cloud.bigtable.beam.validation.SyncTableJob;
2727
import com.google.cloud.bigtable.beam.validation.SyncTableJob.SyncTableOptions;
2828
import com.google.cloud.bigtable.hbase.BigtableConfiguration;
29+
import com.google.cloud.bigtable.hbase.BigtableOptionsFactory;
2930
import com.google.common.base.Preconditions;
3031
import com.google.common.io.ByteStreams;
3132
import java.io.BufferedReader;
@@ -54,6 +55,7 @@
5455
import org.apache.beam.sdk.metrics.MetricQueryResults;
5556
import org.apache.beam.sdk.options.PipelineOptionsFactory;
5657
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
58+
import org.apache.hadoop.conf.Configuration;
5759
import org.apache.hadoop.hbase.Cell;
5860
import org.apache.hadoop.hbase.HColumnDescriptor;
5961
import org.apache.hadoop.hbase.HConstants;
@@ -122,9 +124,13 @@ public void setup() throws Exception {
122124

123125
uploadFixture(gcsUtil, SNAPSHOT_FIXTURE_NAME, fixtureDir);
124126

127+
// Disable CSM to reduce noise in the test output
128+
Configuration config =
129+
BigtableConfiguration.configure(properties.getProjectId(), properties.getInstanceId());
130+
config.set(BigtableOptionsFactory.BIGTABLE_ENABLE_CLIENT_SIDE_METRICS, "false");
131+
125132
// Bigtable config
126-
connection =
127-
BigtableConfiguration.connect(properties.getProjectId(), properties.getInstanceId());
133+
connection = BigtableConfiguration.connect(config);
128134
// TODO: use timebased names to allow for gc
129135
tableId = "test_" + UUID.randomUUID();
130136

‎bigtable-dataflow-parent/bigtable-beam-import/src/test/java/com/google/cloud/bigtable/beam/it/CloudBigtableBeamIT.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import com.google.cloud.bigtable.beam.test_env.EnvSetup;
2727
import com.google.cloud.bigtable.beam.test_env.TestProperties;
2828
import com.google.cloud.bigtable.hbase.BigtableConfiguration;
29+
import com.google.cloud.bigtable.hbase.BigtableOptionsFactory;
2930
import java.io.IOException;
3031
import java.io.Serializable;
3132
import java.util.ArrayList;
@@ -112,6 +113,8 @@ public void setUp() throws IOException {
112113
properties
113114
.getAdminEndpoint()
114115
.ifPresent(endpoint -> config.set(BIGTABLE_ADMIN_HOST_KEY, endpoint));
116+
// Disable CSM to reduce noise in the test output
117+
config.set(BigtableOptionsFactory.BIGTABLE_ENABLE_CLIENT_SIDE_METRICS, "false");
115118

116119
connection = BigtableConfiguration.connect(config);
117120

‎bigtable-dataflow-parent/bigtable-beam-import/src/test/java/com/google/cloud/bigtable/beam/sequencefiles/EndToEndIT.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import com.google.cloud.bigtable.beam.test_env.EnvSetup;
2323
import com.google.cloud.bigtable.beam.test_env.TestProperties;
2424
import com.google.cloud.bigtable.hbase.BigtableConfiguration;
25+
import com.google.cloud.bigtable.hbase.BigtableOptionsFactory;
2526
import com.google.common.collect.ImmutableList;
2627
import com.google.common.collect.Sets;
2728
import com.google.common.truth.Correspondence;
@@ -38,6 +39,7 @@
3839
import org.apache.beam.sdk.extensions.gcp.util.gcsfs.GcsPath;
3940
import org.apache.beam.sdk.options.PipelineOptionsFactory;
4041
import org.apache.beam.sdk.options.ValueProvider.StaticValueProvider;
42+
import org.apache.hadoop.conf.Configuration;
4143
import org.apache.hadoop.hbase.Cell;
4244
import org.apache.hadoop.hbase.CellUtil;
4345
import org.apache.hadoop.hbase.TableName;
@@ -89,9 +91,13 @@ public void setup() throws Exception {
8991
properties.applyTo(gcpOptions);
9092
gcsUtil = new GcsUtil.GcsUtilFactory().create(gcpOptions);
9193

94+
// Disable CSM to reduce noise in the test output
95+
Configuration config =
96+
BigtableConfiguration.configure(properties.getProjectId(), properties.getInstanceId());
97+
config.set(BigtableOptionsFactory.BIGTABLE_ENABLE_CLIENT_SIDE_METRICS, "false");
98+
9299
// Bigtable config
93-
connection =
94-
BigtableConfiguration.connect(properties.getProjectId(), properties.getInstanceId());
100+
connection = BigtableConfiguration.connect(config);
95101
// TODO: support endpoints
96102
}
97103

‎bigtable-hbase-2.x-parent/bigtable-hbase-2.x-mapreduce/pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,12 @@ limitations under the License.
188188
<targetDependencies>
189189
<targetDependency>org.apache.hbase:hbase-mapreduce</targetDependency>
190190
</targetDependencies>
191+
<ignoredDependencies>
192+
<!-- TODO: version mismatches after upgrading hbase client to 2.6.4 -->
193+
<ignoredDependency>javax.activation:activation</ignoredDependency>
194+
<ignoredDependency>javax.xml.bind:jaxb-api</ignoredDependency>
195+
<ignoredDependency>javax.xml.stream:stax-api</ignoredDependency>
196+
</ignoredDependencies>
191197
</configuration>
192198
</execution>
193199
</executions>

‎bigtable-hbase-2.x-parent/bigtable-hbase-2.x-shaded/pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,7 @@ limitations under the License.
247247
<exclude>org.apache.htrace:htrace-core4</exclude>
248248
<exclude>commons-logging:commons-logging</exclude>
249249
<exclude>org.apache.yetus:audience-annotations</exclude>
250+
<exclude>jakarta.activation:jakarta.activation-api</exclude>
250251
</excludes>
251252
</artifactSet>
252253
<relocations>

‎bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableAsyncAdmin.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -695,7 +695,13 @@ public int getStoreFileCount() {
695695
public Size getStoreFileSize() {
696696
return new Size(size, Unit.BYTE);
697697
}
698+
699+
@Override
700+
public Size getMemStoreSize() {
701+
return new Size(size, Unit.BYTE);
702+
}
698703
}
704+
699705
/** Handler for unsupported operations for generating Admin class at runtime. */
700706
public static class UnsupportedOperationsHandler implements InvocationHandler {
701707
@Override

‎bigtable-hbase-2.x-parent/bigtable-hbase-2.x/src/main/java/com/google/cloud/bigtable/hbase2_x/BigtableAsyncBufferedMutator.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@
2424
import com.google.cloud.bigtable.hbase.wrappers.BigtableHBaseSettings;
2525
import java.io.IOException;
2626
import java.util.List;
27+
import java.util.Map;
2728
import java.util.concurrent.CompletableFuture;
29+
import java.util.concurrent.TimeUnit;
2830
import java.util.stream.Collectors;
2931
import org.apache.hadoop.conf.Configuration;
3032
import org.apache.hadoop.hbase.TableName;
@@ -88,6 +90,24 @@ public long getWriteBufferSize() {
8890
return helper.getWriteBufferSize();
8991
}
9092

93+
/** {@inheritDoc} */
94+
@Override
95+
public int getMaxMutations() {
96+
return helper.getMaxRowKeyCount();
97+
}
98+
99+
/** {@inheritDoc} */
100+
@Override
101+
public long getPeriodicalFlushTimeout(TimeUnit unit) {
102+
return unit.convert(helper.getAutoFlushInterval().toNanos(), TimeUnit.NANOSECONDS);
103+
}
104+
105+
/** {@inheritDoc} */
106+
@Override
107+
public Map<String, byte[]> getRequestAttributes() {
108+
throw new UnsupportedOperationException("not implemented");
109+
}
110+
91111
/** {@inheritDoc} */
92112
@Override
93113
public List<CompletableFuture<Void>> mutate(List<? extends Mutation> mutations) {

0 commit comments

Comments
 (0)