Skip to content

Commit f4d6178

Browse files
roryqiroryqi
authored
[#10851] improvement(iceberg): Change default Iceberg JDBC schema version to v1 (#10852)
### What changes were proposed in this pull request? Change default Iceberg JDBC schema version to v1 ### Why are the changes needed? Fix: #10851 ### Does this PR introduce _any_ user-facing change? Modified the document ### How was this patch tested? Added the UT.
1 parent 2646f8d commit f4d6178

5 files changed

Lines changed: 70 additions & 21 deletions

File tree

‎clients/client-java/src/test/java/org/apache/gravitino/client/integration/test/authorization/TopicAuthorizationIT.java‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import java.util.HashMap;
2929
import java.util.List;
3030
import java.util.Map;
31+
import java.util.concurrent.TimeUnit;
3132
import org.apache.gravitino.Catalog;
3233
import org.apache.gravitino.MetadataObject;
3334
import org.apache.gravitino.MetadataObjects;
@@ -44,6 +45,7 @@
4445
import org.apache.gravitino.messaging.Topic;
4546
import org.apache.gravitino.messaging.TopicCatalog;
4647
import org.apache.gravitino.messaging.TopicChange;
48+
import org.awaitility.Awaitility;
4749
import org.junit.jupiter.api.BeforeAll;
4850
import org.junit.jupiter.api.MethodOrderer;
4951
import org.junit.jupiter.api.Order;
@@ -245,9 +247,15 @@ public void testDropTopic() {
245247
TopicCatalog topicCatalog = client.loadMetalake(METALAKE).loadCatalog(CATALOG).asTopicCatalog();
246248
topicCatalog.dropTopic(NameIdentifier.of(SCHEMA, "topic1"));
247249
// check topics are dropped
248-
NameIdentifier[] topicsList = topicCatalog.listTopics(Namespace.of(SCHEMA));
249-
assertArrayEquals(new NameIdentifier[] {}, topicsList);
250-
NameIdentifier[] topicsListNormalUser = topicCatalogNormalUser.listTopics(Namespace.of(SCHEMA));
251-
assertArrayEquals(new NameIdentifier[] {}, topicsListNormalUser);
250+
Awaitility.await()
251+
.atMost(30, TimeUnit.SECONDS)
252+
.untilAsserted(
253+
() -> {
254+
NameIdentifier[] topicsList = topicCatalog.listTopics(Namespace.of(SCHEMA));
255+
assertArrayEquals(new NameIdentifier[] {}, topicsList);
256+
NameIdentifier[] topicsListNormalUser =
257+
topicCatalogNormalUser.listTopics(Namespace.of(SCHEMA));
258+
assertArrayEquals(new NameIdentifier[] {}, topicsListNormalUser);
259+
});
252260
}
253261
}

‎docs/iceberg-rest-service.md‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -107,17 +107,17 @@ The Gravitino Iceberg REST catalog service uses the memory catalog backend by de
107107

108108
#### JDBC backend configuration
109109

110-
| Configuration item | Description | Default value | Required | Since Version |
111-
|-----------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------|----------|---------------|
112-
| `gravitino.iceberg-rest.catalog-backend` | The Catalog backend of the Gravitino Iceberg REST catalog service. Use the value **`jdbc`** for the JDBC catalog backend. | `memory` | Yes | 0.2.0 |
113-
| `gravitino.iceberg-rest.uri` | The JDBC connection address, such as `jdbc:postgresql://127.0.0.1:5432` for Postgres, or `jdbc:mysql://127.0.0.1:3306/` for mysql. | (none) | Yes | 0.2.0 |
114-
| `gravitino.iceberg-rest.warehouse` | The warehouse directory of JDBC catalog. Set the HDFS prefix if using HDFS, such as `hdfs://127.0.0.1:9000/user/hive/warehouse-jdbc` | (none) | Yes | 0.2.0 |
115-
| `gravitino.iceberg-rest.catalog-backend-name` | The catalog name passed to underlying Iceberg catalog backend. Catalog name in JDBC backend is used to isolate namespace and tables. | `jdbc` for JDBC backend | No | 0.5.2 |
116-
| `gravitino.iceberg-rest.jdbc-user` | The username of the JDBC connection. | (none) | No | 0.2.0 |
117-
| `gravitino.iceberg-rest.jdbc-password` | The password of the JDBC connection. | (none) | No | 0.2.0 |
118-
| `gravitino.iceberg-rest.jdbc-initialize` | Whether to initialize the meta tables when creating the JDBC catalog. | `true` | No | 0.2.0 |
119-
| `gravitino.iceberg-rest.jdbc-driver` | `com.mysql.jdbc.Driver` or `com.mysql.cj.jdbc.Driver` for MySQL, `org.postgresql.Driver` for PostgreSQL. | (none) | Yes | 0.3.0 |
120-
| `gravitino.iceberg-rest.jdbc-schema-version` | The schema version of the JDBC catalog. Set to `V1` to enable view support. Once the underlying database is migrated to V1, this property is no longer required on subsequent restarts. | `V0` | No | 1.2.0 |
110+
| Configuration item | Description | Default value | Required | Since Version |
111+
|-----------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------|----------|---------------|
112+
| `gravitino.iceberg-rest.catalog-backend` | The Catalog backend of the Gravitino Iceberg REST catalog service. Use the value **`jdbc`** for the JDBC catalog backend. | `memory` | Yes | 0.2.0 |
113+
| `gravitino.iceberg-rest.uri` | The JDBC connection address, such as `jdbc:postgresql://127.0.0.1:5432` for Postgres, or `jdbc:mysql://127.0.0.1:3306/` for mysql. | (none) | Yes | 0.2.0 |
114+
| `gravitino.iceberg-rest.warehouse` | The warehouse directory of JDBC catalog. Set the HDFS prefix if using HDFS, such as `hdfs://127.0.0.1:9000/user/hive/warehouse-jdbc` | (none) | Yes | 0.2.0 |
115+
| `gravitino.iceberg-rest.catalog-backend-name` | The catalog name passed to underlying Iceberg catalog backend. Catalog name in JDBC backend is used to isolate namespace and tables. | `jdbc` for JDBC backend | No | 0.5.2 |
116+
| `gravitino.iceberg-rest.jdbc-user` | The username of the JDBC connection. | (none) | No | 0.2.0 |
117+
| `gravitino.iceberg-rest.jdbc-password` | The password of the JDBC connection. | (none) | No | 0.2.0 |
118+
| `gravitino.iceberg-rest.jdbc-initialize` | Whether to initialize the meta tables when creating the JDBC catalog. | `true` | No | 0.2.0 |
119+
| `gravitino.iceberg-rest.jdbc-driver` | `com.mysql.jdbc.Driver` or `com.mysql.cj.jdbc.Driver` for MySQL, `org.postgresql.Driver` for PostgreSQL. | (none) | Yes | 0.3.0 |
120+
| `gravitino.iceberg-rest.jdbc-schema-version` | The schema version of the JDBC catalog. Defaults to `V1` to enable view support. Set to `V0` only if you need to opt out of view support. Once the underlying database is migrated to V1, this property is no longer required on subsequent restarts. | `V1` | No | 1.2.0 |
121121

122122
If you have a JDBC Iceberg catalog prior, you must set `catalog-backend-name` to keep consistent with your Jdbc Iceberg catalog name to operate the prior namespace and tables.
123123

@@ -471,11 +471,11 @@ Please set the `gravitino.iceberg-rest.warehouse` parameter to `{storage_prefix}
471471

472472
### View support
473473

474-
View operations are supported when using the JDBC catalog backend with schema version `V1`. Iceberg will automatically migrate the database schema on the first restart and detect the migration on all subsequent restarts.
474+
View operations are supported when using the JDBC catalog backend with schema version `V1`. The default schema version is now `V1`, so view support is enabled out of the box. Iceberg will automatically migrate the database schema on the first restart and detect the migration on all subsequent restarts.
475475

476-
| Configuration item | Description | Default value | Required | Since Version |
477-
|----------------------------------------------|--------------------------------------------------------------------------------------------|---------------|----------|---------------|
478-
| `gravitino.iceberg-rest.jdbc-schema-version` | The schema version of the JDBC catalog backend. Set to `V1` to enable view operations. | `V0` | No | 1.2.0 |
476+
| Configuration item | Description | Default value | Required | Since Version |
477+
|----------------------------------------------|----------------------------------------------------------------------------------------------------------------------|---------------|----------|---------------|
478+
| `gravitino.iceberg-rest.jdbc-schema-version` | The schema version of the JDBC catalog backend. Defaults to `V1` to enable view operations. Set to `V0` to opt out. | `V1` | No | 1.2.0 |
479479

480480
### Other Apache Iceberg catalog properties
481481

‎iceberg/iceberg-common/src/main/java/org/apache/gravitino/iceberg/common/utils/IcebergCatalogUtil.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,9 @@ private static JdbcCatalog loadJdbcCatalog(IcebergConfig icebergConfig) {
108108
new JdbcCatalogWithMetadataLocationSupport(
109109
icebergConfig.get(IcebergConfig.JDBC_INIT_TABLES));
110110

111+
// Default to V1 schema to support view operations; can be overridden by explicit config.
112+
properties.putIfAbsent(IcebergConstants.ICEBERG_JDBC_SCHEMA_VERSION, "V1");
113+
111114
HdfsConfiguration hdfsConfiguration = new HdfsConfiguration();
112115
properties.forEach(hdfsConfiguration::set);
113116
jdbcCatalog.setConf(hdfsConfiguration);

‎iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/ops/TestIcebergCatalogWrapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ private static Map<String, String> metadataConfig(Path warehouseDir) {
100100
config.put(IcebergConstants.GRAVITINO_JDBC_DRIVER, "org.sqlite.JDBC");
101101
config.put(IcebergConstants.ICEBERG_JDBC_USER, "test");
102102
config.put(IcebergConstants.ICEBERG_JDBC_PASSWORD, "test");
103-
config.put(IcebergConstants.ICEBERG_JDBC_INITIALIZE, "false");
103+
config.put(IcebergConstants.ICEBERG_JDBC_INITIALIZE, "true");
104104
config.put(
105105
IcebergConstants.TABLE_METADATA_CACHE_IMPL, TrackingTableMetadataCache.class.getName());
106106
return config;

‎iceberg/iceberg-common/src/test/java/org/apache/gravitino/iceberg/common/utils/TestIcebergCatalogUtil.java‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.apache.iceberg.hive.HiveCatalog;
3030
import org.apache.iceberg.inmemory.InMemoryCatalog;
3131
import org.apache.iceberg.jdbc.JdbcCatalog;
32+
import org.apache.iceberg.jdbc.JdbcCatalogWithMetadataLocationSupport;
3233
import org.junit.jupiter.api.Assertions;
3334
import org.junit.jupiter.api.Test;
3435

@@ -66,7 +67,7 @@ void testLoadCatalog() {
6667
properties.put(IcebergConstants.GRAVITINO_JDBC_DRIVER, "org.sqlite.JDBC");
6768
properties.put(IcebergConstants.ICEBERG_JDBC_USER, "test");
6869
properties.put(IcebergConstants.ICEBERG_JDBC_PASSWORD, "test");
69-
properties.put(IcebergConstants.ICEBERG_JDBC_INITIALIZE, "false");
70+
properties.put(IcebergConstants.ICEBERG_JDBC_INITIALIZE, "true");
7071
catalog =
7172
IcebergCatalogUtil.loadCatalogBackend(
7273
IcebergCatalogBackend.JDBC, new IcebergConfig(properties));
@@ -79,6 +80,43 @@ void testLoadCatalog() {
7980
});
8081
}
8182

83+
@Test
84+
void testJdbcCatalogDefaultSchemaVersionIsV1() {
85+
Map<String, String> properties = new HashMap<>();
86+
properties.put(CatalogProperties.URI, "jdbc:sqlite::memory:");
87+
properties.put(CatalogProperties.WAREHOUSE_LOCATION, "test");
88+
properties.put(IcebergConstants.GRAVITINO_JDBC_DRIVER, "org.sqlite.JDBC");
89+
properties.put(IcebergConstants.ICEBERG_JDBC_USER, "test");
90+
properties.put(IcebergConstants.ICEBERG_JDBC_PASSWORD, "test");
91+
// jdbc.schema-version is intentionally not set; default should be V1
92+
Catalog catalog =
93+
IcebergCatalogUtil.loadCatalogBackend(
94+
IcebergCatalogBackend.JDBC, new IcebergConfig(properties));
95+
Assertions.assertInstanceOf(JdbcCatalogWithMetadataLocationSupport.class, catalog);
96+
Assertions.assertTrue(
97+
((JdbcCatalogWithMetadataLocationSupport) catalog).supportsViewsWithSchemaVersion(),
98+
"JDBC catalog should default to V1 schema and support view operations");
99+
}
100+
101+
@Test
102+
void testJdbcCatalogExplicitSchemaVersionNotOverridden() {
103+
Map<String, String> properties = new HashMap<>();
104+
properties.put(CatalogProperties.URI, "jdbc:sqlite::memory:");
105+
properties.put(CatalogProperties.WAREHOUSE_LOCATION, "test");
106+
properties.put(IcebergConstants.GRAVITINO_JDBC_DRIVER, "org.sqlite.JDBC");
107+
properties.put(IcebergConstants.ICEBERG_JDBC_USER, "test");
108+
properties.put(IcebergConstants.ICEBERG_JDBC_PASSWORD, "test");
109+
// Explicitly set V0; loadJdbcCatalog must not override it with V1
110+
properties.put(IcebergConstants.ICEBERG_JDBC_SCHEMA_VERSION, "V0");
111+
Catalog catalog =
112+
IcebergCatalogUtil.loadCatalogBackend(
113+
IcebergCatalogBackend.JDBC, new IcebergConfig(properties));
114+
Assertions.assertInstanceOf(JdbcCatalogWithMetadataLocationSupport.class, catalog);
115+
Assertions.assertFalse(
116+
((JdbcCatalogWithMetadataLocationSupport) catalog).supportsViewsWithSchemaVersion(),
117+
"Explicitly configured V0 schema should not be overridden to V1");
118+
}
119+
82120
@Test
83121
void testValidLoadCustomCatalog() {
84122
Catalog catalog;

0 commit comments

Comments
 (0)