feat: add Storage#downloadTo (#1354)

sydney-munro · web-flow · commit 5a565a74cd6a · 2022-04-19T12:40:12.000-04:00
diff --git a/google-cloud-storage/clirr-ignored-differences.xml b/google-cloud-storage/clirr-ignored-differences.xml
@@ -1,4 +1,14 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!-- see https://www.mojohaus.org/clirr-maven-plugin/examples/ignored-differences.html -->
 <differences>
+  <difference>
+    <className>com/google/cloud/storage/Storage*</className>
+    <differenceType>7012</differenceType>
+    <method>* downloadTo(com.google.cloud.storage.BlobId, java.io.OutputStream, com.google.cloud.storage.Storage$BlobSourceOption[])</method>
+  </difference>
+  <difference>
+    <className>com/google/cloud/storage/Storage*</className>
+    <differenceType>7012</differenceType>
+    <method>* downloadTo(com.google.cloud.storage.BlobId, java.nio.file.Path, com.google.cloud.storage.Storage$BlobSourceOption[])</method>
+  </difference>
 </differences>
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Blob.java
@@ -19,9 +19,7 @@
 import static com.google.cloud.storage.Blob.BlobSourceOption.toGetOptions;
 import static com.google.cloud.storage.Blob.BlobSourceOption.toSourceOptions;
 import static com.google.common.base.Preconditions.checkNotNull;
-import static java.util.concurrent.Executors.callable;
 
-import com.google.api.gax.retrying.ResultRetryAlgorithm;
 import com.google.api.services.storage.model.StorageObject;
 import com.google.auth.ServiceAccountSigner;
 import com.google.auth.ServiceAccountSigner.SigningException;
@@ -34,20 +32,17 @@
 import com.google.cloud.storage.Storage.SignUrlOption;
 import com.google.cloud.storage.spi.v1.StorageRpc;
 import com.google.common.io.BaseEncoding;
-import com.google.common.io.CountingOutputStream;
 import java.io.IOException;
 import java.io.ObjectInputStream;
 import java.io.OutputStream;
 import java.net.URL;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.security.Key;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
 
 /**
  * An object in Google Cloud Storage. A {@code Blob} object includes the {@code BlobId} instance,
@@ -231,11 +226,7 @@ static Storage.BlobGetOption[] toGetOptions(BlobInfo blobInfo, BlobSourceOption.
    * @throws StorageException upon failure
    */
   public void downloadTo(Path path, BlobSourceOption... options) {
-    try (OutputStream outputStream = Files.newOutputStream(path)) {
-      downloadTo(outputStream, options);
-    } catch (IOException e) {
-      throw new StorageException(e);
-    }
+    storage.downloadTo(getBlobId(), path, BlobSourceOption.toSourceOptions(this, options));
   }
 
   /**
@@ -245,20 +236,7 @@ public void downloadTo(Path path, BlobSourceOption... options) {
    * @param options
    */
   public void downloadTo(OutputStream outputStream, BlobSourceOption... options) {
-    final CountingOutputStream countingOutputStream = new CountingOutputStream(outputStream);
-    final StorageRpc storageRpc = this.options.getStorageRpcV1();
-    StorageObject pb = getBlobId().toPb();
-    final Map<StorageRpc.Option, ?> requestOptions = StorageImpl.optionMap(getBlobId(), options);
-    ResultRetryAlgorithm<?> algorithm = retryAlgorithmManager.getForObjectsGet(pb, requestOptions);
-    Retrying.run(
-        this.options,
-        algorithm,
-        callable(
-            () -> {
-              storageRpc.read(
-                  pb, requestOptions, countingOutputStream.getCount(), countingOutputStream);
-            }),
-        Function.identity());
+    storage.downloadTo(getBlobId(), outputStream, BlobSourceOption.toSourceOptions(this, options));
   }
 
   /**
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java
@@ -31,6 +31,7 @@
 import com.google.cloud.Tuple;
 import com.google.cloud.WriteChannel;
 import com.google.cloud.storage.Acl.Entity;
+import com.google.cloud.storage.Blob.BlobSourceOption;
 import com.google.cloud.storage.HmacKey.HmacKeyMetadata;
 import com.google.cloud.storage.PostPolicyV4.PostConditionsV4;
 import com.google.cloud.storage.PostPolicyV4.PostFieldsV4;
@@ -41,6 +42,7 @@
 import com.google.common.io.BaseEncoding;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.io.Serializable;
 import java.net.URL;
 import java.nio.file.Path;
@@ -2672,6 +2674,45 @@ Blob createFrom(
    */
   ReadChannel reader(BlobId blob, BlobSourceOption... options);
 
+  /**
+   * Downloads the given blob to the given path using specified blob read options.
+   *
+   * <pre>{@code
+   * String bucketName = "my-unique-bucket";
+   * String blobName = "my-blob-name";
+   * BlobId blobId = BlobId.of(bucketName, blobName);
+   * Path destination = Paths.get("my-blob-destination.txt");
+   * downloadTo(blobId, destination);
+   * // do stuff with destination
+   * }</pre>
+   *
+   * @param blob
+   * @param path
+   * @param options
+   * @throws StorageException upon failure
+   */
+  void downloadTo(BlobId blob, Path path, BlobSourceOption... options);
+
+  /**
+   * Downloads the given blob to the given output stream using specified blob read options.
+   *
+   * <pre>{@code
+   * String bucketName = "my-unique-bucket";
+   * String blobName = "my-blob-name";
+   * BlobId blobId = BlobId.of(bucketName, blobName);
+   * Path destination = Paths.get("my-blob-destination.txt");
+   * try (OutputStream outputStream = Files.newOutputStream(path)) {
+   *  downloadTo(blob, outputStream);
+   *  // do stuff with destination
+   * }
+   * }</pre>
+   *
+   * @param blob
+   * @param outputStream
+   * @param options
+   */
+  void downloadTo(BlobId blob, OutputStream outputStream, BlobSourceOption... options);
+
   /**
    * Creates a blob and returns a channel for writing its content. By default any MD5 and CRC32C
    * values in the given {@code blobInfo} are ignored unless requested via the {@code
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageImpl.java
@@ -32,6 +32,7 @@
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
 import static java.nio.charset.StandardCharsets.UTF_8;
+import static java.util.concurrent.Executors.callable;
 
 import com.google.api.gax.paging.Page;
 import com.google.api.gax.retrying.ResultRetryAlgorithm;
@@ -49,6 +50,7 @@
 import com.google.cloud.Tuple;
 import com.google.cloud.WriteChannel;
 import com.google.cloud.storage.Acl.Entity;
+import com.google.cloud.storage.Blob.BlobSourceOption;
 import com.google.cloud.storage.HmacKey.HmacKeyMetadata;
 import com.google.cloud.storage.PostPolicyV4.ConditionV4Type;
 import com.google.cloud.storage.PostPolicyV4.PostConditionsV4;
@@ -67,10 +69,12 @@
 import com.google.common.collect.Maps;
 import com.google.common.hash.Hashing;
 import com.google.common.io.BaseEncoding;
+import com.google.common.io.CountingOutputStream;
 import com.google.common.primitives.Ints;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.io.UnsupportedEncodingException;
 import java.net.MalformedURLException;
 import java.net.URI;
@@ -543,6 +547,32 @@ public ReadChannel reader(BlobId blob, BlobSourceOption... options) {
     return new BlobReadChannel(getOptions(), blob, optionsMap);
   }
 
+  @Override
+  public void downloadTo(BlobId blob, Path path, BlobSourceOption... options) {
+    try (OutputStream outputStream = Files.newOutputStream(path)) {
+      downloadTo(blob, outputStream, options);
+    } catch (IOException e) {
+      throw new StorageException(e);
+    }
+  }
+
+  @Override
+  public void downloadTo(BlobId blob, OutputStream outputStream, BlobSourceOption... options) {
+    final CountingOutputStream countingOutputStream = new CountingOutputStream(outputStream);
+    final StorageObject pb = blob.toPb();
+    final Map<StorageRpc.Option, ?> requestOptions = optionMap(blob, options);
+    ResultRetryAlgorithm<?> algorithm = retryAlgorithmManager.getForObjectsGet(pb, requestOptions);
+    Retrying.run(
+        getOptions(),
+        algorithm,
+        callable(
+            () -> {
+              storageRpc.read(
+                  pb, requestOptions, countingOutputStream.getCount(), countingOutputStream);
+            }),
+        Function.identity());
+  }
+
   @Override
   public BlobWriteChannel writer(BlobInfo blobInfo, BlobWriteOption... options) {
     Tuple<BlobInfo, BlobTargetOption[]> targetOptions = BlobTargetOption.convert(blobInfo, options);
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/BlobTest.java
@@ -16,14 +16,11 @@
 
 package com.google.cloud.storage;
 
-import static org.easymock.EasyMock.anyObject;
 import static org.easymock.EasyMock.capture;
 import static org.easymock.EasyMock.createMock;
-import static org.easymock.EasyMock.createNiceMock;
 import static org.easymock.EasyMock.createStrictMock;
 import static org.easymock.EasyMock.eq;
 import static org.easymock.EasyMock.expect;
-import static org.easymock.EasyMock.getCurrentArguments;
 import static org.easymock.EasyMock.replay;
 import static org.easymock.EasyMock.verify;
 import static org.junit.Assert.assertArrayEquals;
@@ -32,11 +29,9 @@
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 import com.google.api.core.ApiClock;
 import com.google.api.gax.retrying.RetrySettings;
-import com.google.api.services.storage.model.StorageObject;
 import com.google.cloud.ReadChannel;
 import com.google.cloud.storage.Acl.Project;
 import com.google.cloud.storage.Acl.Project.ProjectRole;
@@ -45,21 +40,16 @@
 import com.google.cloud.storage.Blob.BlobSourceOption;
 import com.google.cloud.storage.Storage.BlobWriteOption;
 import com.google.cloud.storage.Storage.CopyRequest;
-import com.google.cloud.storage.spi.v1.StorageRpc;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.io.BaseEncoding;
-import java.io.File;
-import java.io.OutputStream;
 import java.net.URL;
-import java.nio.file.Files;
 import java.security.Key;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import javax.crypto.spec.SecretKeySpec;
 import org.easymock.Capture;
-import org.easymock.IAnswer;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
@@ -612,102 +602,4 @@ public void testBuilder() {
     assertNull(blob.getCustomTime());
     assertTrue(blob.isDirectory());
   }
-
-  private StorageRpc prepareForDownload() {
-    StorageRpc mockStorageRpc = createNiceMock(StorageRpc.class);
-    expect(storage.getOptions()).andReturn(mockOptions).anyTimes();
-    replay(storage);
-    expect(mockOptions.getStorageRpcV1()).andReturn(mockStorageRpc);
-    expect(mockOptions.getRetrySettings()).andReturn(RETRY_SETTINGS);
-    expect(mockOptions.getClock()).andReturn(API_CLOCK);
-    expect(mockOptions.getRetryAlgorithmManager()).andReturn(retryAlgorithmManager).anyTimes();
-    replay(mockOptions);
-    blob = new Blob(storage, new BlobInfo.BuilderImpl(BLOB_INFO));
-    return mockStorageRpc;
-  }
-
-  @Test
-  public void testDownloadTo() throws Exception {
-    final byte[] expected = {1, 2};
-    StorageRpc mockStorageRpc = prepareForDownload();
-    expect(
-            mockStorageRpc.read(
-                anyObject(StorageObject.class),
-                anyObject(Map.class),
-                eq(0l),
-                anyObject(OutputStream.class)))
-        .andAnswer(
-            new IAnswer<Long>() {
-              @Override
-              public Long answer() throws Throwable {
-                ((OutputStream) getCurrentArguments()[3]).write(expected);
-                return 2l;
-              }
-            });
-    replay(mockStorageRpc);
-    File file = File.createTempFile("blob", ".tmp");
-    blob.downloadTo(file.toPath());
-    byte actual[] = Files.readAllBytes(file.toPath());
-    assertArrayEquals(expected, actual);
-  }
-
-  @Test
-  public void testDownloadToWithRetries() throws Exception {
-    final byte[] expected = {1, 2};
-    StorageRpc mockStorageRpc = prepareForDownload();
-    expect(
-            mockStorageRpc.read(
-                anyObject(StorageObject.class),
-                anyObject(Map.class),
-                eq(0l),
-                anyObject(OutputStream.class)))
-        .andAnswer(
-            new IAnswer<Long>() {
-              @Override
-              public Long answer() throws Throwable {
-                ((OutputStream) getCurrentArguments()[3]).write(expected[0]);
-                throw new StorageException(504, "error");
-              }
-            });
-    expect(
-            mockStorageRpc.read(
-                anyObject(StorageObject.class),
-                anyObject(Map.class),
-                eq(1l),
-                anyObject(OutputStream.class)))
-        .andAnswer(
-            new IAnswer<Long>() {
-              @Override
-              public Long answer() throws Throwable {
-                ((OutputStream) getCurrentArguments()[3]).write(expected[1]);
-                return 1l;
-              }
-            });
-    replay(mockStorageRpc);
-    File file = File.createTempFile("blob", ".tmp");
-    blob.downloadTo(file.toPath());
-    byte actual[] = Files.readAllBytes(file.toPath());
-    assertArrayEquals(expected, actual);
-  }
-
-  @Test
-  public void testDownloadToWithException() throws Exception {
-    StorageRpc mockStorageRpc = prepareForDownload();
-    Exception exception = new IllegalStateException("test");
-    expect(
-            mockStorageRpc.read(
-                anyObject(StorageObject.class),
-                anyObject(Map.class),
-                eq(0l),
-                anyObject(OutputStream.class)))
-        .andThrow(exception);
-    replay(mockStorageRpc);
-    File file = File.createTempFile("blob", ".tmp");
-    try {
-      blob.downloadTo(file.toPath());
-      fail();
-    } catch (StorageException e) {
-      assertSame(exception, e.getCause());
-    }
-  }
 }
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplTest.java
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITStorageTest.java
