Add object-storage-mock test utility

LICENSE

@@ -329,6 +329,75 @@ This product includes code from Project Nessie.
 * tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioAccess.java
 * tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioContainer.java
 * tools/minio-testcontainer/src/main/java/org/apache/polaris/test/minio/MinioExtension.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/AccessCheckHandler.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/adlsgen2/DataLakeStorageError.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/adlsgen2/DataLakeStorageErrorObj.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/adlsgen2/FilesystemResourceType.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/adlsgen2/GetPropertiesAction.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/adlsgen2/package-info.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/adlsgen2/Path.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/adlsgen2/PathList.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/AdlsGen2Resource.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/adlsgen2/UpdateAction.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/AwsChunkedInputStream.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/Bucket.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/gcs/ErrorResponse.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/gcs/ListResponse.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/gcs/ObjectAlt.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/gcs/package-info.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/gcs/Projection.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/GcsResource.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/gcs/StorageObject.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/gcs/UploadType.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/HeapStorageBucket.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/InterceptingBucket.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/MockObject.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/ObjectRetriever.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/ObjectStorageMock.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/Range.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/BatchDeleteRequest.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/BatchDeleteResponse.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/Bucket.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/Buckets.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/CompletedPartBase.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/CompletedPart.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/DeletedS3Object.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/ErrorObj.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/ErrorResponse.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/EtagDeserializer.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/EtagSerializer.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/ListAllMyBucketsResult.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/ListBucketResultBase.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/ListBucketResult.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/ListBucketResultV2.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/ListPartsResult.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/ObjectIdentifier.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/Owner.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/Part.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/Prefix.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/S3Resource.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/S3Constants.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/S3ObjectIdentifier.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/S3Object.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/StorageClass.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/Tagging.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/s3/Tag.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/sts/AssumeRoleHandler.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/sts/AssumeRoleResponse.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/sts/AssumeRoleResult.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/sts/Credentials.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/StsResource.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/sts/ResponseMetadata.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/sts/RoleUser.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/util/Holder.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/util/PrefixSpliterator.java
+* tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/util/StartAfterSpliterator.java
+* tools/object-storage-mock/src/test/java/org/apache/polaris/objectstoragemock/AbstractObjectStorageMockServer.java
+* tools/object-storage-mock/src/test/java/org/apache/polaris/objectstoragemock/TestAwsChunkedInputStream.java
+* tools/object-storage-mock/src/test/java/org/apache/polaris/objectstoragemock/TestMockServer.java
+* tools/object-storage-mock/src/test/java/org/apache/polaris/objectstoragemock/TestWithAdlsGen2.java
+* tools/object-storage-mock/src/test/java/org/apache/polaris/objectstoragemock/TestWithGcs.java
+* tools/object-storage-mock/src/test/java/org/apache/polaris/objectstoragemock/TestWithS3Client.java
 * tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/Rustfs.java
 * tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsAccess.java
 * tools/rustfs-testcontainer/src/main/java/org/apache/polaris/test/rustfs/RustfsContainer.java

bom/build.gradle.kts

@@ -85,6 +85,8 @@ dependencies {
     api(project(":polaris-config-docs-annotations"))
     api(project(":polaris-config-docs-generator"))
 
+    api(project(":polaris-object-storage-mock"))
+
     api(project(":polaris-core"))
 
     api(project(":polaris-relational-jdbc"))

gradle/libs.versions.toml

@@ -79,6 +79,7 @@ jakarta-servlet-api = { module = "jakarta.servlet:jakarta.servlet-api", version
 jakarta-validation-api = { module = "jakarta.validation:jakarta.validation-api", version = "3.1.1" }
 jakarta-ws-rs-api = { module = "jakarta.ws.rs:jakarta.ws.rs-api", version = "4.0.0" }
 javax-servlet-api = { module = "javax.servlet:javax.servlet-api", version = "4.0.1" }
+jersey-bom = { module = "org.glassfish.jersey:jersey-bom", version = "4.0.2" }
 jspecify = { module = "org.jspecify:jspecify", version = "1.0.0" }
 jcstress-core = { module = "org.openjdk.jcstress:jcstress-core", version = "0.16" }
 jmh-core = { module = "org.openjdk.jmh:jmh-core", version.ref = "jmh" }
@@ -89,6 +90,7 @@ keycloak-admin-client = { module = "org.keycloak:keycloak-admin-client", version
 logback-classic = { module = "ch.qos.logback:logback-classic", version = "1.5.33" }
 micrometer-bom = { module = "io.micrometer:micrometer-bom", version = "1.16.5" }
 microprofile-fault-tolerance-api = { module = "org.eclipse.microprofile.fault-tolerance:microprofile-fault-tolerance-api", version = "4.1.2" }
+microprofile-openapi = { module = "org.eclipse.microprofile.openapi:microprofile-openapi-api", version = "4.1.1" }
 mockito-core = { module = "org.mockito:mockito-core", version = "5.23.0" }
 mockito-junit-jupiter = { module = "org.mockito:mockito-junit-jupiter", version = "5.23.0" }
 mongodb-driver-sync = { module = "org.mongodb:mongodb-driver-sync", version = "5.7.1" }

gradle/projects.main.properties

@@ -54,6 +54,8 @@ polaris-config-docs-annotations=tools/config-docs/annotations
 polaris-config-docs-generator=tools/config-docs/generator
 polaris-config-docs-site=tools/config-docs/site
 
+polaris-object-storage-mock=tools/object-storage-mock
+
 # executor abstraction
 polaris-async-api=persistence/nosql/async/api
 polaris-async-java=persistence/nosql/async/java

tools/object-storage-mock/README.md

@@ -0,0 +1,152 @@
+<!--
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Object Storage Mock
+
+`polaris-object-storage-mock` provides an embeddable HTTP server that looks like
+cloud object storage to client code. It implements the subset of S3, Google Cloud
+Storage, Azure Data Lake Storage Gen2, and AWS STS behavior that Polaris tests
+need, while keeping the actual object backing store under test control.
+
+The mock is intended for tests that should exercise real storage client
+libraries, request paths, response metadata, conditional requests, listing
+pagination, and error handling without depending on external cloud services or
+containerized storage services.
+
+## Why Not Just Use a File System or a Map?
+
+A file-system directory or an in-memory `Map<String, byte[]>` can store bytes,
+but it does not behave like an object storage service. Object storage clients
+interact with an HTTP API, not directly with files or maps, and much of the
+behavior Polaris cares about lives in that protocol layer:
+
+- Provider-specific URLs, query parameters, headers, response bodies, and status
+  codes.
+- Object metadata such as ETags, content types, content lengths, last-modified
+  timestamps, and storage classes.
+- Conditional reads using headers such as `If-Match` and `If-None-Match`.
+- Prefix listing, continuation tokens, page sizes, and provider-specific list
+  response shapes.
+- Writes, appends, deletes, and batch deletes as real SDKs issue them.
+- STS `AssumeRole` responses for code paths that obtain temporary S3
+  credentials.
+
+This module separates those concerns. The HTTP resources translate provider
+protocols into a small bucket abstraction, and tests decide how each bucket
+behaves. A bucket can be heap-backed, generated on demand, wrapped by
+interceptors, or connected to a custom fixture.
+
+## What This Enables
+
+The mock lets tests use the same SDKs and configuration paths that production
+code uses. For example, tests can point an AWS S3 client, Google Cloud Storage
+client, Azure Data Lake Storage Gen2 client, or Iceberg file IO configuration at
+the mock server and verify behavior through that client rather than bypassing it
+with direct fixture access.
+
+This is useful for:
+
+- Fast unit and integration tests that do not require cloud credentials.
+- Cross-provider tests that reuse the same logical objects through S3, GCS, and
+  ADLS Gen2 APIs.
+- Tests that need large synthetic listings without materializing all objects in
+  memory.
+- Tests that need precise failures, metadata, access checks, or write
+  interception.
+- Tests for Polaris storage configuration that need real endpoint URLs.
+
+## Basic Usage
+
+Create one or more buckets, start the server, and configure a client with the
+provider-specific endpoint returned by the server:
+
+```java
+import static org.apache.polaris.objectstoragemock.HeapStorageBucket.newHeapStorageBucket;
+
+import java.util.Map;
+import org.apache.polaris.objectstoragemock.Bucket;
+import org.apache.polaris.objectstoragemock.ObjectStorageMock;
+import org.apache.polaris.objectstoragemock.ObjectStorageMock.MockServer;
+
+Bucket bucket = newHeapStorageBucket().bucket();
+
+try (MockServer server =
+    ObjectStorageMock.builder().putBuckets("warehouse", bucket).build().start()) {
+  // Configure S3 clients with server.getS3BaseUri().
+  // Configure GCS clients with server.getGcsBaseUri().
+  // Configure ADLS Gen2 clients with server.getAdlsGen2BaseUri().
+  // Configure STS clients with server.getStsEndpointURI().
+}
+```
+
+For S3 and Iceberg, `MockServer.icebergProperties()` returns a minimal property
+set that points S3 file IO at the mock endpoint and enables path-style access:
+
+```java
+try (MockServer server =
+    ObjectStorageMock.builder()
+        .putBuckets("warehouse", newHeapStorageBucket().bucket())
+        .build()
+        .start()) {
+  Map<String, String> properties = server.icebergProperties();
+}
+```
+
+When configuring S3 clients directly, use path-style access. The mock exposes
+buckets as path segments and does not implement virtual-host-style bucket
+routing.
+
+## Bucket Backends
+
+The core test-facing type is `Bucket`. It contains handlers for retrieving,
+listing, updating, and deleting objects:
+
+- `object()` returns a `MockObject` for a key.
+- `lister()` streams keys and metadata for list operations.
+- `updater()` receives writes and commits resulting objects.
+- `deleter()` removes keys.
+
+`HeapStorageBucket` is the simplest backend. It stores objects in memory and is
+useful when a test needs ordinary read-after-write behavior.
+
+For more controlled tests, build a `Bucket` directly. This allows synthetic
+listings, custom metadata, write capture, and targeted failures without
+preloading a backing store.
+
+`InterceptingBucket` wraps another bucket and lets a test override selected
+operations while falling back to the wrapped bucket for everything else. This is
+useful when most behavior should be normal but one key, prefix, or operation
+needs special handling.
+
+## Supported Protocol Surfaces
+
+The mock implements only the storage API surface needed by Polaris tests. It is
+not a complete replacement for S3, GCS, ADLS Gen2, or STS.
+
+Supported behavior includes:
+
+- S3 bucket listing, object listing, head, get, put, delete, batch delete, and
+  STS assume-role responses.
+- GCS object listing, metadata reads, media reads, writes, and deletes.
+- ADLS Gen2 path listing, properties, reads, creates, appends, flushes, and
+  deletes.
+- Range and conditional request handling where implemented by the resources.
+
+Unsupported operations generally return a not-implemented response or are absent
+from the mock API. Add only the protocol behavior a test actually needs.

tools/object-storage-mock/build.gradle.kts

@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+  id("org.kordamp.gradle.jandex")
+  id("polaris-server")
+}
+
+dependencies {
+  compileOnly(libs.jakarta.ws.rs.api)
+  compileOnly(libs.jakarta.validation.api)
+  compileOnly(libs.jakarta.annotation.api)
+  compileOnly(libs.jakarta.inject.api)
+
+  compileOnly(libs.microprofile.openapi)
+
+  compileOnly("io.quarkus:quarkus-arc:${libs.versions.quarkus.get()}")
+
+  implementation(platform(libs.jersey.bom))
+  implementation("org.glassfish.jersey.core:jersey-server")
+  implementation("org.glassfish.jersey.containers:jersey-container-servlet")
+  implementation("org.glassfish.jersey.containers:jersey-container-jetty-http")
+  implementation("org.glassfish.jersey.inject:jersey-hk2")
+  implementation("org.glassfish.jersey.media:jersey-media-json-jackson")
+
+  compileOnly(project(":polaris-immutables"))
+  annotationProcessor(project(":polaris-immutables", configuration = "processor"))
+
+  implementation(libs.guava)
+
+  implementation(platform(libs.jackson.bom))
+  implementation("com.fasterxml.jackson.core:jackson-annotations")
+  implementation("com.fasterxml.jackson.core:jackson-databind")
+  implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-xml")
+  implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310")
+  implementation("com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-json-provider")
+  implementation("com.fasterxml.jackson.jakarta.rs:jackson-jakarta-rs-xml-provider")
+
+  implementation(libs.slf4j.api)
+
+  testRuntimeOnly(libs.logback.classic)
+
+  testImplementation(platform(libs.awssdk.bom))
+  testImplementation("software.amazon.awssdk:s3")
+  testImplementation("software.amazon.awssdk:url-connection-client")
+  testImplementation("software.amazon.awssdk:sts")
+
+  testImplementation(platform(libs.azuresdk.bom))
+  testImplementation("com.azure:azure-storage-file-datalake")
+
+  testImplementation(platform(libs.google.cloud.storage.bom))
+  testImplementation("com.google.cloud:google-cloud-storage")
+
+  testCompileOnly(libs.microprofile.openapi)
+
+  testCompileOnly(project(":polaris-immutables"))
+  testAnnotationProcessor(project(":polaris-immutables", configuration = "processor"))
+}
+
+tasks.withType(Test::class.java).configureEach { systemProperty("aws.region", "us-east-1") }

tools/object-storage-mock/src/main/java/org/apache/polaris/objectstoragemock/AccessCheckHandler.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright (C) 2024 Dremio
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.polaris.objectstoragemock;
+
+public interface AccessCheckHandler {
+  boolean accessAllowed(String objectKey);
+}