Visitar URL original
feat: add ability to write Range values with JSONStreamWriter by PhongChuong · Pull Request #2498 · googleapis/java-bigquerystorage · GitHub
Skip to content
This repository was archived by the owner on Feb 24, 2026. It is now read-only.

feat: add ability to write Range values with JSONStreamWriter #2498

Merged
PhongChuong merged 20 commits into from
May 31, 2024
Merged

feat: add ability to write Range values with JSONStreamWriter #2498

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
7326d2c
feat: add ability to write Range values with JSONStreamWriter
PhongChuong May 10, 2024
5492c55
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] May 10, 2024
63dda7c
Update tests to include mixed case and string values
PhongChuong May 30, 2024
cf020e4
Merge remote-tracking branch 'origin/rangeJSONStreamWriter' into rang…
PhongChuong May 30, 2024
f427f12
🦉 Updates from OwlBot post-processor
gcf-owl-bot[bot] May 30, 2024
4ae730e
Add Arrow processor to validate reading range values
PhongChuong May 30, 2024
deefb3d
Merge remote-tracking branch 'origin/rangeJSONStreamWriter' into rang…
PhongChuong May 30, 2024
8c9ec04
Add test scope to Arrow dependencies
PhongChuong May 30, 2024
6aaa6a9
Fix unit test
PhongChuong May 30, 2024
534f7b0
Fix maven dependencies check failure for arrow
PhongChuong May 30, 2024
54df209
Add arrow-memory-netty to fix sample exception issue with no DefaultA…
PhongChuong May 30, 2024
4e16603
Temp remove arrow test dependencies to test sample failure
PhongChuong May 30, 2024
266f7b9
Further testing
PhongChuong May 30, 2024
44e95b3
Test by hard coding test arrow version
PhongChuong May 31, 2024
31032c8
Revert changes for testing
PhongChuong May 31, 2024
5fcef45
Pass dep. check
PhongChuong May 31, 2024
9c04ef8
Merge branch 'main' into rangeJSONStreamWriter
PhongChuong May 31, 2024
aa0ca7a
Updated test for case sensitivity
PhongChuong May 31, 2024
c9d3be2
Update integration test for case sensitivity for all cases
PhongChuong May 31, 2024
60f5b07
Update JsonToProtoMessageTest to include mixed case fields
PhongChuong May 31, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions google-cloud-bigquerystorage/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
</parent>
<properties>
<site.installationModule>google-cloud-bigquerystorage</site.installationModule>
<arrow.version>15.0.2</arrow.version>
</properties>
<build>
<extensions>
Expand Down Expand Up @@ -197,6 +198,18 @@
<version>1.11.3</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
<version>${arrow.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-memory-core</artifactId>
<version>${arrow.version}</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>io.grpc</groupId>
Expand Down
105 changes: 81 additions & 24 deletions ...ge/src/main/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*/
package com.google.cloud.bigquery.storage.v1;

import com.google.cloud.bigquery.storage.v1.TableFieldSchema.Mode;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
Expand Down Expand Up @@ -61,6 +62,7 @@ public class BQTableSchemaToProtoDescriptor {
.put(TableFieldSchema.Type.TIMESTAMP, FieldDescriptorProto.Type.TYPE_INT64)
.put(TableFieldSchema.Type.JSON, FieldDescriptorProto.Type.TYPE_STRING)
.put(TableFieldSchema.Type.INTERVAL, FieldDescriptorProto.Type.TYPE_STRING)
.put(TableFieldSchema.Type.RANGE, FieldDescriptorProto.Type.TYPE_MESSAGE)
.build();

/**
Expand Down Expand Up @@ -89,7 +91,7 @@ private static Descriptor convertBQTableSchemaToProtoDescriptorImpl(
TableSchema BQTableSchema,
String scope,
HashMap<ImmutableList<TableFieldSchema>, Descriptor> dependencyMap)
throws Descriptors.DescriptorValidationException {
throws Descriptors.DescriptorValidationException, IllegalArgumentException {
List<FileDescriptor> dependenciesList = new ArrayList<FileDescriptor>();
List<FieldDescriptorProto> fields = new ArrayList<FieldDescriptorProto>();
int index = 1;
Expand All @@ -99,25 +101,72 @@ private static Descriptor convertBQTableSchemaToProtoDescriptorImpl(
? BQTableField.getName()
: BigQuerySchemaUtil.generatePlaceholderFieldName(BQTableField.getName());
String currentScope = scope + "__" + scopeName;
if (BQTableField.getType() == TableFieldSchema.Type.STRUCT) {
ImmutableList<TableFieldSchema> fieldList =
ImmutableList.copyOf(BQTableField.getFieldsList());
if (dependencyMap.containsKey(fieldList)) {
Descriptor descriptor = dependencyMap.get(fieldList);
dependenciesList.add(descriptor.getFile());
fields.add(convertBQTableFieldToProtoField(BQTableField, index++, descriptor.getName()));
} else {
Descriptor descriptor =
convertBQTableSchemaToProtoDescriptorImpl(
TableSchema.newBuilder().addAllFields(fieldList).build(),
currentScope,
dependencyMap);
dependenciesList.add(descriptor.getFile());
dependencyMap.put(fieldList, descriptor);
switch (BQTableField.getType()) {
case STRUCT:
ImmutableList<TableFieldSchema> fieldList =
ImmutableList.copyOf(BQTableField.getFieldsList());
if (dependencyMap.containsKey(fieldList)) {
Descriptor descriptor = dependencyMap.get(fieldList);
dependenciesList.add(descriptor.getFile());
fields.add(
convertBQTableFieldToProtoField(BQTableField, index++, descriptor.getName()));
} else {
Descriptor descriptor =
convertBQTableSchemaToProtoDescriptorImpl(
TableSchema.newBuilder().addAllFields(fieldList).build(),
currentScope,
dependencyMap);
dependenciesList.add(descriptor.getFile());
dependencyMap.put(fieldList, descriptor);
fields.add(convertBQTableFieldToProtoField(BQTableField, index++, currentScope));
}
break;
case RANGE:
switch (BQTableField.getRangeElementType().getType()) {
case DATE:
case DATETIME:
case TIMESTAMP:
break;
default:
throw new IllegalArgumentException(
String.format(
"Error: %s of type RANGE requires range element type (DATE, DATETIME, TIMESTAMP)",
currentScope));
}
// For RANGE type, expliclitly add the fields start and end of the same FieldElementType
// as it is not expliclity defined in the TableSchema.
ImmutableList<TableFieldSchema> rangeFields =
ImmutableList.of(
TableFieldSchema.newBuilder()
.setType(BQTableField.getRangeElementType().getType())
.setName("start")
.setMode(Mode.NULLABLE)
.build(),
TableFieldSchema.newBuilder()
.setType(BQTableField.getRangeElementType().getType())
.setName("end")
.setMode(Mode.NULLABLE)
.build());

if (dependencyMap.containsKey(rangeFields)) {
Descriptor descriptor = dependencyMap.get(rangeFields);
dependenciesList.add(descriptor.getFile());
fields.add(
convertBQTableFieldToProtoField(BQTableField, index++, descriptor.getName()));
} else {
Descriptor descriptor =
convertBQTableSchemaToProtoDescriptorImpl(
TableSchema.newBuilder().addAllFields(rangeFields).build(),
currentScope,
dependencyMap);
dependenciesList.add(descriptor.getFile());
dependencyMap.put(rangeFields, descriptor);
fields.add(convertBQTableFieldToProtoField(BQTableField, index++, currentScope));
}
break;
default:
fields.add(convertBQTableFieldToProtoField(BQTableField, index++, currentScope));
}
} else {
fields.add(convertBQTableFieldToProtoField(BQTableField, index++, currentScope));
break;
}
}
FileDescriptor[] dependenciesArray = new FileDescriptor[dependenciesList.size()];
Expand Down Expand Up @@ -150,11 +199,19 @@ private static FieldDescriptorProto convertBQTableFieldToProtoField(
.setNumber(index)
.setLabel((FieldDescriptorProto.Label) BQTableSchemaModeMap.get(mode));

if (BQTableField.getType() == TableFieldSchema.Type.STRUCT) {
fieldDescriptor.setTypeName(scope);
} else {
fieldDescriptor.setType(
(FieldDescriptorProto.Type) BQTableSchemaTypeMap.get(BQTableField.getType()));
switch (BQTableField.getType()) {
case STRUCT:
fieldDescriptor.setTypeName(scope);
break;
case RANGE:
fieldDescriptor.setType(
(FieldDescriptorProto.Type) BQTableSchemaTypeMap.get(BQTableField.getType()));
fieldDescriptor.setTypeName(scope);
break;
default:
fieldDescriptor.setType(
(FieldDescriptorProto.Type) BQTableSchemaTypeMap.get(BQTableField.getType()));
break;
}

// Sets columnName annotation when field name is not proto comptaible.
Expand Down
34 changes: 34 additions & 0 deletions ...igquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,40 @@ private FieldDescriptorAndFieldTableSchema computeDescriptorAndSchema(
if (tableFieldSchemaList != null) {
// protoSchema is generated from tableSchema so their field ordering should match.
fieldSchema = tableFieldSchemaList.get(field.getIndex());
// For RANGE type, expliclitly add the fields start and end of the same FieldElementType as it
// is not expliclity defined in the TableFieldSchema.
if (fieldSchema.getType() == TableFieldSchema.Type.RANGE) {
switch (fieldSchema.getRangeElementType().getType()) {
case DATE:
case DATETIME:
case TIMESTAMP:
fieldSchema =
fieldSchema
.toBuilder()
.addFields(
TableFieldSchema.newBuilder()
.setName("start")
.setType(fieldSchema.getRangeElementType().getType())
.build())
.addFields(
TableFieldSchema.newBuilder()
.setName("end")
.setType(fieldSchema.getRangeElementType().getType())
.build())
.build();
break;
default:
throw new ValidationException(
"Field at index "
+ field.getIndex()
+ " with name ("
+ fieldSchema.getName()
+ ") with type (RANGE) has an unsupported range element type ("
+ fieldSchema.getRangeElementType()
+ ")");
}
}

if (!fieldSchema.getName().toLowerCase().equals(BigQuerySchemaUtil.getFieldName(field))) {
throw new ValidationException(
"Field at index "
Expand Down
70 changes: 70 additions & 0 deletions ...rc/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,76 @@ public void testSimpleTypes() throws Exception {
}
}

@Test
public void testRange() throws Exception {
final TableSchema tableSchema =
TableSchema.newBuilder()
.addFields(
TableFieldSchema.newBuilder()
.setName("range_date")
.setType(TableFieldSchema.Type.RANGE)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setRangeElementType(
TableFieldSchema.FieldElementType.newBuilder()
.setType(TableFieldSchema.Type.DATE)
.build())
.build())
.addFields(
TableFieldSchema.newBuilder()
.setName("range_datetime")
.setType(TableFieldSchema.Type.RANGE)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setRangeElementType(
TableFieldSchema.FieldElementType.newBuilder()
.setType(TableFieldSchema.Type.DATETIME)
.build())
.build())
.addFields(
TableFieldSchema.newBuilder()
.setName("range_timestamp")
.setType(TableFieldSchema.Type.RANGE)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setRangeElementType(
TableFieldSchema.FieldElementType.newBuilder()
.setType(TableFieldSchema.Type.TIMESTAMP)
.build())
.build())
.addFields(
TableFieldSchema.newBuilder()
.setName("range_date_miXEd_caSE")
.setType(TableFieldSchema.Type.RANGE)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setRangeElementType(
TableFieldSchema.FieldElementType.newBuilder()
.setType(TableFieldSchema.Type.DATE)
.build())
.build())
.addFields(
TableFieldSchema.newBuilder()
.setName("range_datetime_miXEd_caSE")
.setType(TableFieldSchema.Type.RANGE)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setRangeElementType(
TableFieldSchema.FieldElementType.newBuilder()
.setType(TableFieldSchema.Type.DATETIME)
.build())
.build())
.addFields(
TableFieldSchema.newBuilder()
.setName("range_timestamp_miXEd_caSE")
.setType(TableFieldSchema.Type.RANGE)
.setMode(TableFieldSchema.Mode.NULLABLE)
.setRangeElementType(
TableFieldSchema.FieldElementType.newBuilder()
.setType(TableFieldSchema.Type.TIMESTAMP)
.build())
.build())
.build();
final Descriptor descriptor =
BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema);
isDescriptorEqual(descriptor, TestRange.getDescriptor());
}

@Test
public void testStructSimple() throws Exception {
final TableFieldSchema stringType =
Expand Down
Loading