feat: add hierarchical namespace and folders features (#2445)

JesseLovelace · gcf-owl-bot[bot] · web-flow · commit 8074fffed520 · 2024-03-14T10:37:47.000-07:00
* feat: support includeFolders list option * Add grpc and HNS * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix tests * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fix tests again * fix clirr --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
diff --git a/google-cloud-storage/clirr-ignored-differences.xml b/google-cloud-storage/clirr-ignored-differences.xml
@@ -15,6 +15,12 @@
     <method>* writeAndClose(*)</method>
   </difference>
 
+  <difference>
+    <differenceType>7013</differenceType>
+    <className>com/google/cloud/storage/BucketInfo$Builder</className>
+    <method>com.google.cloud.storage.BucketInfo$Builder setHierarchicalNamespace(com.google.cloud.storage.BucketInfo$HierarchicalNamespace)</method>
+  </difference>
+
   <difference>
     <differenceType>7013</differenceType>
     <className>com/google/cloud/storage/BlobInfo$Builder</className>
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Bucket.java
@@ -748,6 +748,12 @@ Builder setObjectRetention(ObjectRetention objectRetention) {
       return this;
     }
 
+    @Override
+    public Builder setHierarchicalNamespace(HierarchicalNamespace hierarchicalNamespace) {
+      infoBuilder.setHierarchicalNamespace(hierarchicalNamespace);
+      return this;
+    }
+
     @Override
     public Bucket build() {
       return new Bucket(storage, infoBuilder);
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/BucketInfo.java
@@ -118,6 +118,7 @@ public class BucketInfo implements Serializable {
   private final Logging logging;
   private final CustomPlacementConfig customPlacementConfig;
   private final ObjectRetention objectRetention;
+  private final HierarchicalNamespace hierarchicalNamespace;
 
   private final transient ImmutableSet<NamedField> modifiedFields;
 
@@ -713,6 +714,71 @@ public Logging build() {
     }
   }
 
+  /** The bucket's hierarchical namespace (Folders) configuration. Enable this to use HNS. */
+  public static final class HierarchicalNamespace implements Serializable {
+
+    private static final long serialVersionUID = 5932926691444613101L;
+    private Boolean enabled;
+
+    public Boolean getEnabled() {
+      return enabled;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+      if (this == o) {
+        return true;
+      }
+      if (!(o instanceof HierarchicalNamespace)) {
+        return false;
+      }
+      HierarchicalNamespace that = (HierarchicalNamespace) o;
+      return Objects.equals(enabled, that.enabled);
+    }
+
+    @Override
+    public int hashCode() {
+      return Objects.hash(enabled);
+    }
+
+    @Override
+    public String toString() {
+      return MoreObjects.toStringHelper(this).add("enabled", enabled).toString();
+    }
+
+    private HierarchicalNamespace() {}
+
+    private HierarchicalNamespace(Builder builder) {
+      this.enabled = builder.enabled;
+    }
+
+    public static Builder newBuilder() {
+      return new Builder();
+    }
+
+    public Builder toBuilder() {
+      return newBuilder().setEnabled(enabled);
+    }
+
+    public static final class Builder {
+      private Boolean enabled;
+
+      /**
+       * Sets whether Hierarchical Namespace (Folders) is enabled for this bucket. This can only be
+       * enabled at bucket create time. If this is enabled, Uniform Bucket-Level Access must also be
+       * enabled.
+       */
+      public Builder setEnabled(Boolean enabled) {
+        this.enabled = enabled;
+        return this;
+      }
+
+      public HierarchicalNamespace build() {
+        return new HierarchicalNamespace(this);
+      }
+    }
+  }
+
   /**
    * Lifecycle rule for a bucket. Allows supported Actions, such as deleting and changing storage
    * class, to be executed when certain Conditions are met.
@@ -1683,6 +1749,8 @@ public Builder setRetentionPeriodDuration(Duration retentionPeriod) {
 
     public abstract Builder setCustomPlacementConfig(CustomPlacementConfig customPlacementConfig);
 
+    public abstract Builder setHierarchicalNamespace(HierarchicalNamespace hierarchicalNamespace);
+
     abstract Builder setObjectRetention(ObjectRetention objectRetention);
 
     /** Creates a {@code BucketInfo} object. */
@@ -1783,6 +1851,7 @@ static final class BuilderImpl extends Builder {
     private Logging logging;
     private CustomPlacementConfig customPlacementConfig;
     private ObjectRetention objectRetention;
+    private HierarchicalNamespace hierarchicalNamespace;
     private final ImmutableSet.Builder<NamedField> modifiedFields = ImmutableSet.builder();
 
     BuilderImpl(String name) {
@@ -1822,6 +1891,7 @@ static final class BuilderImpl extends Builder {
       logging = bucketInfo.logging;
       customPlacementConfig = bucketInfo.customPlacementConfig;
       objectRetention = bucketInfo.objectRetention;
+      hierarchicalNamespace = bucketInfo.hierarchicalNamespace;
     }
 
     @Override
@@ -2187,6 +2257,15 @@ Builder setObjectRetention(ObjectRetention objectRetention) {
       return this;
     }
 
+    @Override
+    public Builder setHierarchicalNamespace(HierarchicalNamespace hierarchicalNamespace) {
+      if (!Objects.equals(this.hierarchicalNamespace, hierarchicalNamespace)) {
+        modifiedFields.add(BucketField.HIERARCHICAL_NAMESPACE);
+      }
+      this.hierarchicalNamespace = hierarchicalNamespace;
+      return this;
+    }
+
     @Override
     Builder setLocationType(String locationType) {
       if (!Objects.equals(this.locationType, locationType)) {
@@ -2428,6 +2507,7 @@ private Builder clearDeleteLifecycleRules() {
     logging = builder.logging;
     customPlacementConfig = builder.customPlacementConfig;
     objectRetention = builder.objectRetention;
+    hierarchicalNamespace = builder.hierarchicalNamespace;
     modifiedFields = builder.modifiedFields.build();
   }
 
@@ -2768,6 +2848,11 @@ public ObjectRetention getObjectRetention() {
     return objectRetention;
   }
 
+  /** Returns the Hierarchical Namespace (Folders) Configuration */
+  public HierarchicalNamespace getHierarchicalNamespace() {
+    return hierarchicalNamespace;
+  }
+
   /** Returns a builder for the current bucket. */
   public Builder toBuilder() {
     return new BuilderImpl(this);
@@ -2805,6 +2890,7 @@ public int hashCode() {
         autoclass,
         locationType,
         objectRetention,
+        hierarchicalNamespace,
         logging);
   }
 
@@ -2846,6 +2932,7 @@ public boolean equals(Object o) {
         && Objects.equals(autoclass, that.autoclass)
         && Objects.equals(locationType, that.locationType)
         && Objects.equals(objectRetention, that.objectRetention)
+        && Objects.equals(hierarchicalNamespace, that.hierarchicalNamespace)
         && Objects.equals(logging, that.logging);
   }
 
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/GrpcConversions.java
@@ -110,6 +110,10 @@ final class GrpcConversions {
   private final Codec<Condition, Expr> iamConditionCodec =
       Codec.of(this::conditionEncode, this::conditionDecode);
 
+  private final Codec<BucketInfo.HierarchicalNamespace, Bucket.HierarchicalNamespace>
+      hierarchicalNamespaceCodec =
+          Codec.of(this::hierarchicalNamespaceEncode, this::hierarchicalNamespaceDecode);
+
   @VisibleForTesting
   final Codec<OffsetDateTime, Timestamp> timestampCodec =
       Codec.of(
@@ -297,6 +301,10 @@ private BucketInfo bucketInfoDecode(Bucket from) {
               .setDataLocations(customPlacementConfig.getDataLocationsList())
               .build());
     }
+    if (from.hasHierarchicalNamespace()) {
+      to.setHierarchicalNamespace(
+          hierarchicalNamespaceCodec.decode(from.getHierarchicalNamespace()));
+    }
     // TODO(frankyn): Add SelfLink when the field is available
     if (!from.getEtag().isEmpty()) {
       to.setEtag(from.getEtag());
@@ -382,6 +390,10 @@ private Bucket bucketInfoEncode(BucketInfo from) {
               .addAllDataLocations(customPlacementConfig.getDataLocations())
               .build());
     }
+    ifNonNull(
+        from.getHierarchicalNamespace(),
+        hierarchicalNamespaceCodec::encode,
+        to::setHierarchicalNamespace);
     // TODO(frankyn): Add SelfLink when the field is available
     ifNonNull(from.getEtag(), to::setEtag);
     return to.build();
@@ -589,6 +601,20 @@ private Bucket.Autoclass autoclassEncode(BucketInfo.Autoclass from) {
     return to.build();
   }
 
+  private Bucket.HierarchicalNamespace hierarchicalNamespaceEncode(
+      BucketInfo.HierarchicalNamespace from) {
+    Bucket.HierarchicalNamespace.Builder to = Bucket.HierarchicalNamespace.newBuilder();
+    ifNonNull(from.getEnabled(), to::setEnabled);
+    return to.build();
+  }
+
+  private BucketInfo.HierarchicalNamespace hierarchicalNamespaceDecode(
+      Bucket.HierarchicalNamespace from) {
+    BucketInfo.HierarchicalNamespace.Builder to = BucketInfo.HierarchicalNamespace.newBuilder();
+    to.setEnabled(from.getEnabled());
+    return to.build();
+  }
+
   private Bucket.IamConfig iamConfigEncode(BucketInfo.IamConfiguration from) {
     Bucket.IamConfig.Builder to = Bucket.IamConfig.newBuilder();
     to.setUniformBucketLevelAccess(ublaEncode(from));
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/JsonConversions.java
@@ -137,6 +137,10 @@ final class JsonConversions {
   private final Codec<BlobInfo, StorageObject> blobInfoCodec =
       Codec.of(this::blobInfoEncode, this::blobInfoDecode);
 
+  private final Codec<BucketInfo.HierarchicalNamespace, Bucket.HierarchicalNamespace>
+      hierarchicalNamespaceCodec =
+          Codec.of(this::hierarchicalNamespaceEncode, this::hierarchicalNamespaceDecode);
+
   private final Codec<NotificationInfo, com.google.api.services.storage.model.Notification>
       notificationInfoCodec = Codec.of(this::notificationEncode, this::notificationDecode);
   private final Codec<CustomPlacementConfig, Bucket.CustomPlacementConfig>
@@ -437,6 +441,10 @@ private Bucket bucketInfoEncode(BucketInfo from) {
         this::customPlacementConfigEncode,
         to::setCustomPlacementConfig);
     ifNonNull(from.getObjectRetention(), this::objectRetentionEncode, to::setObjectRetention);
+    ifNonNull(
+        from.getHierarchicalNamespace(),
+        this::hierarchicalNamespaceEncode,
+        to::setHierarchicalNamespace);
     return to;
   }
 
@@ -487,6 +495,10 @@ private BucketInfo bucketInfoDecode(com.google.api.services.storage.model.Bucket
         from.getCustomPlacementConfig(),
         this::customPlacementConfigDecode,
         to::setCustomPlacementConfig);
+    ifNonNull(
+        from.getHierarchicalNamespace(),
+        this::hierarchicalNamespaceDecode,
+        to::setHierarchicalNamespace);
     ifNonNull(from.getObjectRetention(), this::objectRetentionDecode, to::setObjectRetention);
     return to.build();
   }
@@ -861,6 +873,20 @@ private com.google.api.services.storage.model.Notification notificationEncode(
     return to;
   }
 
+  private Bucket.HierarchicalNamespace hierarchicalNamespaceEncode(
+      BucketInfo.HierarchicalNamespace from) {
+    Bucket.HierarchicalNamespace to = new Bucket.HierarchicalNamespace();
+    ifNonNull(from.getEnabled(), to::setEnabled);
+    return to;
+  }
+
+  private BucketInfo.HierarchicalNamespace hierarchicalNamespaceDecode(
+      Bucket.HierarchicalNamespace from) {
+    BucketInfo.HierarchicalNamespace.Builder to = BucketInfo.HierarchicalNamespace.newBuilder();
+    to.setEnabled(from.getEnabled());
+    return to.build();
+  }
+
   private NotificationInfo notificationDecode(
       com.google.api.services.storage.model.Notification from) {
     NotificationInfo.Builder builder = new NotificationInfo.BuilderImpl(from.getTopic());
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/Storage.java
@@ -159,6 +159,9 @@ enum BucketField implements FieldSelector, NamedField {
     CUSTOM_PLACEMENT_CONFIG("customPlacementConfig", "custom_placement_config"),
     @TransportCompatibility({Transport.HTTP, Transport.GRPC})
     AUTOCLASS("autoclass"),
+
+    @TransportCompatibility({Transport.HTTP, Transport.GRPC})
+    HIERARCHICAL_NAMESPACE("hierarchicalNamespace", "hierarchical_namespace"),
     @TransportCompatibility({Transport.HTTP})
     OBJECT_RETENTION("objectRetention");
 
@@ -1788,6 +1791,14 @@ public static BlobListOption matchGlob(@NonNull String glob) {
       return new BlobListOption(UnifiedOpts.matchGlob(glob));
     }
 
+    /**
+     * Returns an option for whether to include all Folders (including empty Folders) in response.
+     */
+    @TransportCompatibility({Transport.HTTP, Transport.GRPC})
+    public static BlobListOption includeFolders(boolean includeFolders) {
+      return new BlobListOption(UnifiedOpts.includeFoldersAsPrefixes(includeFolders));
+    }
+
     /**
      * Returns an option to define the billing user project. This option is required by buckets with
      * `requester_pays` flag enabled to assign operation costs.
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/UnifiedOpts.java
@@ -370,6 +370,10 @@ static Delimiter delimiter(@NonNull String delimiter) {
     return new Delimiter(delimiter);
   }
 
+  static IncludeFoldersAsPrefixes includeFoldersAsPrefixes(boolean includeFoldersAsPrefixes) {
+    return new IncludeFoldersAsPrefixes(includeFoldersAsPrefixes);
+  }
+
   @Deprecated
   static DetectContentType detectContentType() {
     return DetectContentType.INSTANCE;
@@ -636,6 +640,20 @@ public Mapper<RewriteObjectRequest.Builder> rewriteObject() {
     }
   }
 
+  static final class IncludeFoldersAsPrefixes extends RpcOptVal<Boolean> implements ObjectListOpt {
+
+    private static final long serialVersionUID = 321916692864878282L;
+
+    private IncludeFoldersAsPrefixes(boolean val) {
+      super(StorageRpc.Option.INCLUDE_FOLDERS_AS_PREFIXES, val);
+    }
+
+    @Override
+    public Mapper<ListObjectsRequest.Builder> listObjects() {
+      return b -> b.setIncludeFoldersAsPrefixes(val);
+    }
+  }
+
   static final class Delimiter extends RpcOptVal<String> implements ObjectListOpt {
     private static final long serialVersionUID = -3789556789947615714L;
 
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/HttpStorageRpc.java
@@ -459,6 +459,7 @@ public Tuple<String, Iterable<StorageObject>> list(final String bucket, Map<Opti
               .setPageToken(Option.PAGE_TOKEN.getString(options))
               .setFields(Option.FIELDS.getString(options))
               .setUserProject(Option.USER_PROJECT.getString(options))
+              .setIncludeFoldersAsPrefixes(Option.INCLUDE_FOLDERS_AS_PREFIXES.getBoolean(options))
               .execute();
       Iterable<StorageObject> storageObjects =
           Iterables.concat(
diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/spi/v1/StorageRpc.java
@@ -73,7 +73,9 @@ enum Option {
     DETECT_CONTENT_TYPE("detectContentType"),
     ENABLE_OBJECT_RETENTION("enableObjectRetention"),
     RETURN_RAW_INPUT_STREAM("returnRawInputStream"),
-    OVERRIDE_UNLOCKED_RETENTION("overrideUnlockedRetention");
+    OVERRIDE_UNLOCKED_RETENTION("overrideUnlockedRetention"),
+    INCLUDE_FOLDERS_AS_PREFIXES("includeFoldersAsPrefixes");
+    ;
 
     private final String value;
 
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketReadMaskTest.java
@@ -133,6 +133,7 @@ public ImmutableList<?> parameters() {
               new Args<>(BucketField.TIME_CREATED, LazyAssertion.equal()),
               new Args<>(BucketField.UPDATED, LazyAssertion.equal()),
               new Args<>(BucketField.VERSIONING, LazyAssertion.equal()),
+              new Args<>(BucketField.HIERARCHICAL_NAMESPACE, LazyAssertion.equal()),
               new Args<>(BucketField.WEBSITE, LazyAssertion.equal()));
 
       List<String> argsDefined =
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITBucketTest.java
diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITOptionRegressionTest.java
