feat: Add matchGlob parameter to BlobListOption (#1965)

briantruong777 · BenWhitehead · web-flow · commit 93be97a9ae06 · 2023-04-05T12:48:36.000-04:00
This is a new feature (b/236167515) where List Objects allows filtering
results based on a provided glob pattern.

Currently only the JSON API supports the matchGlob parameter, so using
the option with gRPC will throw an exception. gRPC support is planned
for later.

---------

Co-authored-by: BenWhitehead &lt;BenWhitehead@users.noreply.github.com&gt;
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java
@@ -20,6 +20,7 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 import static java.util.Objects.requireNonNull;
 
+import com.google.api.core.BetaApi;
 import com.google.api.core.InternalExtensionOnly;
 import com.google.api.gax.paging.Page;
 import com.google.auth.ServiceAccountSigner;
@@ -1300,6 +1301,18 @@ public static BlobListOption endOffset(@NonNull String endOffset) {
       return new BlobListOption(UnifiedOpts.endOffset(endOffset));
     }
 
+    /**
+     * Returns an option to set a glob pattern to filter results to blobs that match the pattern.
+     *
+     * @see <a href="https://cloud.google.com/storage/docs/json_api/v1/objects/list">List
+     *     Objects</a>
+     */
+    @BetaApi
+    @TransportCompatibility({Transport.HTTP})
+    public static BlobListOption matchGlob(@NonNull String glob) {
+      return new BlobListOption(UnifiedOpts.matchGlob(glob));
+    }
+
     /**
      * Returns an option to define the billing user project. This option is required by buckets with
      * `requester_pays` flag enabled to assign operation costs.
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java
@@ -413,6 +413,11 @@ static KmsKeyName kmsKeyName(@NonNull String kmsKeyName) {
     return new KmsKeyName(kmsKeyName);
   }
 
+  static MatchGlob matchGlob(@NonNull String glob) {
+    requireNonNull(glob, "glob must be non null");
+    return new MatchGlob(glob);
+  }
+
   static Md5Match md5Match(@NonNull String md5) {
     requireNonNull(md5, "md5 must be non null");
     return new Md5Match(md5);
@@ -1070,6 +1075,20 @@ public Mapper<RewriteObjectRequest.Builder> rewriteObject() {
     }
   }
 
+  static final class MatchGlob extends RpcOptVal<String> implements ObjectListOpt {
+    private static final long serialVersionUID = 8819855597395473178L;
+
+    private MatchGlob(String val) {
+      super(StorageRpc.Option.MATCH_GLOB, val);
+    }
+
+    @Override
+    public Mapper<ListObjectsRequest.Builder> listObjects() {
+      return GrpcStorageImpl.throwHttpJsonOnly(
+          com.google.cloud.storage.Storage.BlobListOption.class, "matchGlob(String)");
+    }
+  }
+
   @Deprecated
   static final class Md5Match implements ObjectTargetOpt {
     private static final long serialVersionUID = 5237207911268363887L;
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java
@@ -423,6 +423,7 @@ public Tuple<String, Iterable<StorageObject>> list(final String bucket, Map<Opti
               .setDelimiter(Option.DELIMITER.getString(options))
               .setStartOffset(Option.START_OFF_SET.getString(options))
               .setEndOffset(Option.END_OFF_SET.getString(options))
+              .setMatchGlob(Option.MATCH_GLOB.getString(options))
               .setPrefix(Option.PREFIX.getString(options))
               .setMaxResults(Option.MAX_RESULTS.getLong(options))
               .setPageToken(Option.PAGE_TOKEN.getString(options))
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java
@@ -61,6 +61,7 @@ enum Option {
     DELIMITER("delimiter"),
     START_OFF_SET("startOffset"),
     END_OFF_SET("endOffset"),
+    MATCH_GLOB("matchGlob"),
     VERSIONS("versions"),
     FIELDS("fields"),
     CUSTOMER_SUPPLIED_KEY("customerSuppliedKey"),
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/StorageImplMockitoTest.java
@@ -1308,6 +1308,27 @@ public void testListBlobsWithOffset() {
     assertArrayEquals(blobList.toArray(), Iterables.toArray(page.getValues(), Blob.class));
   }
 
+  @Test
+  public void testListBlobsMatchGlob() {
+    String cursor = "cursor";
+    String matchGlob = "foo*bar";
+    Map<StorageRpc.Option, ?> options = ImmutableMap.of(StorageRpc.Option.MATCH_GLOB, matchGlob);
+    ImmutableList<BlobInfo> blobInfoList = ImmutableList.of(BLOB_INFO1, BLOB_INFO2);
+    Tuple<String, Iterable<com.google.api.services.storage.model.StorageObject>> result =
+        Tuple.of(
+            cursor, Iterables.transform(blobInfoList, Conversions.apiary().blobInfo()::encode));
+    doReturn(result)
+        .doThrow(UNEXPECTED_CALL_EXCEPTION)
+        .when(storageRpcMock)
+        .list(BUCKET_NAME1, options);
+
+    initializeService();
+    ImmutableList<Blob> blobList = ImmutableList.of(expectedBlob1, expectedBlob2);
+    Page<Blob> page = storage.list(BUCKET_NAME1, Storage.BlobListOption.matchGlob(matchGlob));
+    assertEquals(cursor, page.getNextPageToken());
+    assertArrayEquals(blobList.toArray(), Iterables.toArray(page.getValues(), Blob.class));
+  }
+
   @Test
   public void testListBlobsWithException() {
     doThrow(STORAGE_FAILURE).when(storageRpcMock).list(BUCKET_NAME1, EMPTY_RPC_OPTIONS);
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITObjectTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITObjectTest.java
@@ -16,6 +16,7 @@
 
 package com.google.cloud.storage.it;
 
+import static com.google.cloud.storage.TestUtils.assertAll;
 import static com.google.common.truth.Truth.assertThat;
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.junit.Assert.assertArrayEquals;
@@ -65,6 +66,7 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Iterators;
 import com.google.common.hash.Hashing;
 import com.google.common.io.BaseEncoding;
@@ -542,6 +544,60 @@ public void testListBlobsCurrentDirectoryIncludesBothObjectsAndSyntheticDirector
     assertThat(actual).contains(PackagePrivateMethodWorkarounds.noAcl(obj2Gen1));
   }
 
+  @Test
+  // When gRPC support is added for matchGlob, enable this test for gRPC.
+  @Exclude(transports = Transport.GRPC)
+  public void testListBlobsWithMatchGlob() throws Exception {
+    BucketInfo bucketInfo = BucketInfo.newBuilder(generator.randomBucketName()).build();
+    try (TemporaryBucket tempBucket =
+        TemporaryBucket.newBuilder().setBucketInfo(bucketInfo).setStorage(storage).build()) {
+      BucketInfo bucket = tempBucket.getBucket();
+      assertNotNull(storage.create(BlobInfo.newBuilder(bucket, "foo/bar").build()));
+      assertNotNull(storage.create(BlobInfo.newBuilder(bucket, "foo/baz").build()));
+      assertNotNull(storage.create(BlobInfo.newBuilder(bucket, "foo/foobar").build()));
+      assertNotNull(storage.create(BlobInfo.newBuilder(bucket, "foobar").build()));
+
+      Page<Blob> page1 = storage.list(bucket.getName(), BlobListOption.matchGlob("foo*bar"));
+      Page<Blob> page2 = storage.list(bucket.getName(), BlobListOption.matchGlob("foo**bar"));
+      Page<Blob> page3 = storage.list(bucket.getName(), BlobListOption.matchGlob("**/foobar"));
+      Page<Blob> page4 = storage.list(bucket.getName(), BlobListOption.matchGlob("*/ba[rz]"));
+      Page<Blob> page5 = storage.list(bucket.getName(), BlobListOption.matchGlob("*/ba[!a-y]"));
+      Page<Blob> page6 =
+          storage.list(bucket.getName(), BlobListOption.matchGlob("**/{foobar,baz}"));
+      Page<Blob> page7 =
+          storage.list(bucket.getName(), BlobListOption.matchGlob("foo/{foo*,*baz}"));
+      assertAll(
+          () ->
+              assertThat(Iterables.transform(page1.iterateAll(), blob -> blob.getName()))
+                  .containsExactly("foobar")
+                  .inOrder(),
+          () ->
+              assertThat(Iterables.transform(page2.iterateAll(), blob -> blob.getName()))
+                  .containsExactly("foo/bar", "foo/foobar", "foobar")
+                  .inOrder(),
+          () ->
+              assertThat(Iterables.transform(page3.iterateAll(), blob -> blob.getName()))
+                  .containsExactly("foo/foobar", "foobar")
+                  .inOrder(),
+          () ->
+              assertThat(Iterables.transform(page4.iterateAll(), blob -> blob.getName()))
+                  .containsExactly("foo/bar", "foo/baz")
+                  .inOrder(),
+          () ->
+              assertThat(Iterables.transform(page5.iterateAll(), blob -> blob.getName()))
+                  .containsExactly("foo/baz")
+                  .inOrder(),
+          () ->
+              assertThat(Iterables.transform(page6.iterateAll(), blob -> blob.getName()))
+                  .containsExactly("foo/baz", "foo/foobar", "foobar")
+                  .inOrder(),
+          () ->
+              assertThat(Iterables.transform(page7.iterateAll(), blob -> blob.getName()))
+                  .containsExactly("foo/baz", "foo/foobar")
+                  .inOrder());
+    }
+  }
+
   @Test
   public void testListBlobsMultiplePages() {
     String basePath = generator.randomObjectName();
