Visitar URL original
feat(storage): implement response-based checksum validation in JSON r… · googleapis/google-cloud-java@8604291 · GitHub
Skip to content

Commit 8604291

Browse files
nidhiii-27nidhiii-27
committed
feat(storage): implement response-based checksum validation in JSON read channel
- Refactor ApiaryUnbufferedReadableByteChannel to use response-based shouldValidate helper to decide whether to wrap and validate CRC32C. - Extract expected CRC32C checksum from headers on every request to ensure it's available after retries or seeks. - Add unit tests verifying validation on full range downloads (including suffix range downloads) and verify that partial range downloads are not validated. [Generated-by: AI]
1 parent a599c73 commit 8604291

2 files changed

Lines changed: 92 additions & 5 deletions

File tree

‎java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannel.java‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,10 @@ private ScatteringByteChannel open() {
199199

200200
HttpResponse media = get.executeMedia();
201201
InputStream content = media.getContent();
202+
203+
Map<String, String> hashes = ChecksumResponseParser.extractHashesFromHeader(media);
204+
this.expectedCrc32cBase64 = hashes.get("crc32c");
205+
202206
if (xGoogGeneration == null) {
203207
HttpHeaders responseHeaders = media.getHeaders();
204208

@@ -230,15 +234,13 @@ private ScatteringByteChannel open() {
230234
if (!result.isDone()) {
231235
result.set(clone);
232236
}
233-
234-
Map<String, String> hashes = ChecksumResponseParser.extractHashesFromHeader(media);
235-
this.expectedCrc32cBase64 = hashes.get("crc32c");
236237
}
237238
}
238239

239240
boolean isHasherEnabled = !(hasher instanceof Hasher.NoOpHasher);
240-
boolean isFullObjectDownload = (request.getByteRangeSpec().getHttpRangeHeader() == null);
241-
if (isHasherEnabled && isFullObjectDownload && expectedCrc32cBase64 != null) {
241+
boolean shouldValidate =
242+
isHasherEnabled && HttpStorageRpcHasherHelper.INSTANCE.shouldValidate(media);
243+
if (shouldValidate && expectedCrc32cBase64 != null) {
242244
this.hashingInputStream = new HashingInputStream(Hashing.crc32c(), content);
243245
content = this.hashingInputStream;
244246
}

‎java-storage/google-cloud-storage/src/test/java/com/google/cloud/storage/ApiaryUnbufferedReadableByteChannelTest.java‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import java.nio.ByteBuffer;
3535
import java.nio.channels.Channels;
3636
import java.nio.channels.WritableByteChannel;
37+
import java.util.Map;
3738
import org.junit.Test;
3839

3940
public class ApiaryUnbufferedReadableByteChannelTest {
@@ -43,6 +44,11 @@ public class ApiaryUnbufferedReadableByteChannelTest {
4344
private static final String WRONG_CRC32C_BASE64 = "AAAAAA==";
4445

4546
private Storage createMockStorageClient(String googHashHeader) {
47+
return createMockStorageClient(200, googHashHeader, null);
48+
}
49+
50+
private Storage createMockStorageClient(
51+
int statusCode, String googHashHeader, Map<String, String> extraHeaders) {
4652
HttpTransport transport =
4753
new HttpTransport() {
4854
@Override
@@ -53,13 +59,19 @@ protected com.google.api.client.http.LowLevelHttpRequest buildRequest(
5359
public com.google.api.client.http.LowLevelHttpResponse execute() throws IOException {
5460
MockLowLevelHttpResponse lowLevelResponse =
5561
new MockLowLevelHttpResponse()
62+
.setStatusCode(statusCode)
5663
.setContent(CONTENT_BYTES)
5764
.setContentLength(CONTENT_BYTES.length)
5865
.addHeader("Content-Length", String.valueOf(CONTENT_BYTES.length))
5966
.addHeader("x-goog-generation", "12345");
6067
if (googHashHeader != null) {
6168
lowLevelResponse.addHeader("x-goog-hash", googHashHeader);
6269
}
70+
if (extraHeaders != null) {
71+
for (Map.Entry<String, String> entry : extraHeaders.entrySet()) {
72+
lowLevelResponse.addHeader(entry.getKey(), entry.getValue());
73+
}
74+
}
6375
return lowLevelResponse;
6476
}
6577
};
@@ -133,4 +145,77 @@ public void testRead_mismatchedCrc32cValidation_throwsChecksumMismatch() throws
133145
assertTrue(expected.getMessage().contains("Mismatch checksum value"));
134146
}
135147
}
148+
149+
@Test
150+
public void testRead_suffixRangeFullObjectCrc32cValidation() throws IOException {
151+
// Suffix range request resulting in content-range: bytes 0-12/13
152+
Map<String, String> extraHeaders = ImmutableMap.of("Content-Range", "bytes 0-12/13");
153+
Storage storageClient =
154+
createMockStorageClient(206, "crc32c=" + CORRECT_CRC32C_BASE64, extraHeaders);
155+
156+
StorageObject from = new StorageObject().setBucket("bucket").setName("blob");
157+
ApiaryReadRequest apiaryReadRequest =
158+
new ApiaryReadRequest(from, ImmutableMap.of(), ByteRangeSpec.nullRange());
159+
160+
SettableApiFuture<StorageObject> resultFuture = SettableApiFuture.create();
161+
try (ApiaryUnbufferedReadableByteChannel channel =
162+
new ApiaryUnbufferedReadableByteChannel(
163+
apiaryReadRequest,
164+
storageClient,
165+
resultFuture,
166+
RetrierWithAlg.attemptOnce(),
167+
Retrying.neverRetry(),
168+
Hasher.defaultHasher()); ) {
169+
170+
ByteArrayOutputStream out = new ByteArrayOutputStream();
171+
try (WritableByteChannel w = Channels.newChannel(out)) {
172+
ByteBuffer buf = ByteBuffer.allocate(4096);
173+
while (channel.read(new ByteBuffer[] {buf}, 0, 1) != -1) {
174+
buf.flip();
175+
w.write(buf);
176+
buf.clear();
177+
}
178+
}
179+
180+
assertArrayEquals(CONTENT_BYTES, out.toByteArray());
181+
}
182+
}
183+
184+
@Test
185+
public void testRead_partialRangeNoCrc32cValidation() throws IOException {
186+
// Partial range request resulting in content-range: bytes 1-12/13
187+
Map<String, String> extraHeaders = ImmutableMap.of("Content-Range", "bytes 1-12/13");
188+
// Even if checksum is wrong, it shouldn't throw ChecksumMismatchException because validation is
189+
// skipped for partial downloads.
190+
Storage storageClient =
191+
createMockStorageClient(206, "crc32c=" + WRONG_CRC32C_BASE64, extraHeaders);
192+
193+
StorageObject from = new StorageObject().setBucket("bucket").setName("blob");
194+
ApiaryReadRequest apiaryReadRequest =
195+
new ApiaryReadRequest(from, ImmutableMap.of(), ByteRangeSpec.nullRange());
196+
197+
SettableApiFuture<StorageObject> resultFuture = SettableApiFuture.create();
198+
try (ApiaryUnbufferedReadableByteChannel channel =
199+
new ApiaryUnbufferedReadableByteChannel(
200+
apiaryReadRequest,
201+
storageClient,
202+
resultFuture,
203+
RetrierWithAlg.attemptOnce(),
204+
Retrying.neverRetry(),
205+
Hasher.defaultHasher()); ) {
206+
207+
ByteArrayOutputStream out = new ByteArrayOutputStream();
208+
try (WritableByteChannel w = Channels.newChannel(out)) {
209+
ByteBuffer buf = ByteBuffer.allocate(4096);
210+
while (channel.read(new ByteBuffer[] {buf}, 0, 1) != -1) {
211+
buf.flip();
212+
w.write(buf);
213+
buf.clear();
214+
}
215+
}
216+
217+
// We read the entire response content successfully (no exception thrown)
218+
assertArrayEquals(CONTENT_BYTES, out.toByteArray());
219+
}
220+
}
136221
}

0 commit comments

Comments
 (0)