diff --git a/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 57f7cadac181..156d7c7ebe76 100644 --- a/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -445,6 +445,7 @@ public enum ErrorMsg { @Deprecated // kept for backwards reference REPLACE_VIEW_WITH_MATERIALIZED(10400, "Attempt to replace view {0} with materialized view", true), REPLACE_MATERIALIZED_WITH_VIEW(10401, "Attempt to replace materialized view {0} with view", true), + VIEW_STORAGE_HANDLER_UNSUPPORTED(10448, "Storage handler {0} doesn't support external logical views", true), UPDATE_DELETE_VIEW(10402, "You cannot update or delete records in a view"), MATERIALIZED_VIEW_DEF_EMPTY(10403, "Query for the materialized view rebuild could not be retrieved"), MERGE_PREDIACTE_REQUIRED(10404, "MERGE statement with both UPDATE and DELETE clauses " + diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HMSTablePropertyHelper.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HMSTablePropertyHelper.java index f7de4dd07196..43098c9913a5 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HMSTablePropertyHelper.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HMSTablePropertyHelper.java @@ -161,6 +161,7 @@ public static void updateHmsTableForIcebergView( HiveOperationsBase.ICEBERG_VIEW_TYPE_VALUE.toUpperCase(Locale.ENGLISH), metadata.schema(), maxHiveTablePropertySize); + parameters.put(hive_metastoreConstants.META_TABLE_STORAGE, HIVE_ICEBERG_STORAGE_HANDLER); tbl.setParameters(parameters); } diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveOperationsBase.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveOperationsBase.java index 937939b48160..a5a0657cc12e 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveOperationsBase.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveOperationsBase.java @@ -44,7 +44,7 @@ import org.slf4j.LoggerFactory; /** All the HMS operations like table,view,materialized_view should implement this. */ -interface HiveOperationsBase { +public interface HiveOperationsBase { Logger LOG = LoggerFactory.getLogger(HiveOperationsBase.class); // The max size is based on HMS backend database. For Hive versions below 2.3, the max table diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveViewOperations.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveViewOperations.java index c0c959974a61..10f6736c3538 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveViewOperations.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveViewOperations.java @@ -302,7 +302,7 @@ private Table newHMSView(ViewMetadata metadata) { tableType().name()); } - private String sqlFor(ViewMetadata metadata) { + public static String sqlFor(ViewMetadata metadata) { SQLViewRepresentation closest = null; for (ViewRepresentation representation : metadata.currentVersion().representations()) { if (representation instanceof SQLViewRepresentation) { diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergLogicalViewSupport.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergLogicalViewSupport.java new file mode 100644 index 000000000000..9f11c308ddc4 --- /dev/null +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergLogicalViewSupport.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.hive; + +import java.io.Closeable; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.catalog.ViewCatalog; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.view.ViewBuilder; + +/** + * Commits a native Iceberg view through the configured default Iceberg catalog (HiveCatalog or REST + * catalog, etc.) when {@code Catalog} also implements {@link ViewCatalog}. + */ +public final class IcebergLogicalViewSupport { + + private IcebergLogicalViewSupport() { + } + + /** + * Loads the native Iceberg logical view definition and applies SQL, schema, and Iceberg params to {@code hmsTable} + */ + public static void enrichHmsTableFromIcebergView( + org.apache.hadoop.hive.metastore.api.Table hmsTable, Configuration conf) { + TableIdentifier identifier = TableIdentifier.of(hmsTable.getDbName(), hmsTable.getTableName()); + String catalogName = IcebergCatalogProperties.getCatalogName(conf); + Map catalogProps = IcebergCatalogProperties.getCatalogProperties(conf, catalogName); + Catalog catalog = CatalogUtil.buildIcebergCatalog(catalogName, catalogProps, conf); + + try { + if (catalog instanceof Closeable closeable) { + try (Closeable ignored = closeable) { + loadAndApplyView(hmsTable, conf, catalog, catalogName, identifier); + } + } else { + loadAndApplyView(hmsTable, conf, catalog, catalogName, identifier); + } + } catch (IOException e) { + throw new UncheckedIOException("Failed to close Iceberg catalog", e); + } + } + + private static void loadAndApplyView( + org.apache.hadoop.hive.metastore.api.Table hmsTable, + Configuration conf, + Catalog catalog, + String catalogName, + TableIdentifier identifier) { + ViewCatalog viewCatalog = asViewCatalog(catalog, catalogName); + MetastoreUtil.applyIcebergViewToHmsTable(hmsTable, viewCatalog.loadView(identifier), conf); + } + + /** Creates or replaces a view in the Iceberg catalog. */ + public static void createOrReplaceView( + Configuration conf, + String databaseName, + String viewName, + List fieldSchemas, + String viewSql, + Map tblProperties, + String comment) { + + TableIdentifier identifier = TableIdentifier.of(databaseName, viewName); + String catalogName = IcebergCatalogProperties.getCatalogName(conf); + Map catalogProps = IcebergCatalogProperties.getCatalogProperties(conf, catalogName); + Catalog catalog = CatalogUtil.buildIcebergCatalog(catalogName, catalogProps, conf); + + if (catalog instanceof Closeable closeable) { + try (Closeable ignored = closeable) { + commitView(catalog, catalogName, identifier, fieldSchemas, viewSql, tblProperties, comment); + } catch (IOException e) { + throw new UncheckedIOException("Failed to close Iceberg catalog", e); + } + } else { + commitView(catalog, catalogName, identifier, fieldSchemas, viewSql, tblProperties, comment); + } + } + + private static void commitView( + Catalog catalog, + String catalogName, + TableIdentifier identifier, + List fieldSchemas, + String viewSql, + Map tblProperties, + String comment) { + ViewCatalog viewCatalog = asViewCatalog(catalog, catalogName); + + ViewBuilder builder = + viewCatalog + .buildView(identifier) + .withSchema(HiveSchemaUtil.convert(fieldSchemas, Collections.emptyMap(), true)) + .withDefaultNamespace(Namespace.of(identifier.namespace().level(0))) + .withQuery("hive", viewSql); + + if (StringUtils.isNotBlank(comment)) { + builder = builder.withProperty("comment", comment); + } + + Map tblProps = + tblProperties == null ? Maps.newHashMap() : Maps.newHashMap(tblProperties); + + builder.withProperties(tblProps); + + builder.createOrReplace(); + } + + private static ViewCatalog asViewCatalog(Catalog catalog, String catalogName) { + if (catalog instanceof ViewCatalog viewCatalog) { + return viewCatalog; + } + throw new UnsupportedOperationException( + String.format( + "Iceberg catalog '%s' does not implement ViewCatalog.", + catalogName) + + " Iceberg views require a catalog that implements ViewCatalog (e.g. HiveCatalog or REST)."); + } +} diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java index 95e1e5b36623..cf57c00d5b4e 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java @@ -36,8 +36,10 @@ import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.BaseTable; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.Schema; @@ -46,6 +48,11 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.util.PropertyUtil; +import org.apache.iceberg.view.BaseView; +import org.apache.iceberg.view.SQLViewRepresentation; +import org.apache.iceberg.view.View; +import org.apache.iceberg.view.ViewMetadata; import org.apache.thrift.TException; public class MetastoreUtil { @@ -134,7 +141,11 @@ public static Table toHiveTable(org.apache.iceberg.Table table, Configuration co result.setDbName(tableName.getDb()); result.setTableName(tableName.getTable()); result.setTableType(TableType.EXTERNAL_TABLE.toString()); - result.setPartitionKeys(getPartitionKeys(table, table.spec().specId())); + + // TODO: Revert after HIVE-29633 is fixed + // result.setPartitionKeys(getPartitionKeys(table, table.spec().specId())); + result.setPartitionKeys(Lists.newArrayList()); + TableMetadata metadata = ((BaseTable) table).operations().current(); long maxHiveTablePropertySize = conf.getLong(HiveOperationsBase.HIVE_TABLE_PROPERTY_MAX_SIZE, HiveOperationsBase.HIVE_TABLE_PROPERTY_MAX_SIZE_DEFAULT); @@ -142,12 +153,89 @@ public static Table toHiveTable(org.apache.iceberg.Table table, Configuration co null, true, maxHiveTablePropertySize, null); String catalogType = IcebergCatalogProperties.getCatalogType(conf); if (!StringUtils.isEmpty(catalogType) && !IcebergCatalogProperties.NO_CATALOG_TYPE.equals(catalogType)) { - result.getParameters().put(CatalogUtil.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.getCatalogType(conf)); + result.getParameters().put(CatalogUtil.ICEBERG_CATALOG_TYPE, catalogType); } result.setSd(getHiveStorageDescriptor(table)); return result; } + /** + * Builds a minimal HMS {@link Table} shell for a native Iceberg logical view (identity, view type, + * and Iceberg storage-handler markers only). The storage handler {@code postGetTable} hook enriches + * this object via {@link IcebergLogicalViewSupport#enrichHmsTableFromIcebergView} (view SQL, + * schema, and Iceberg parameters). + */ + public static Table buildMinimalHMSView(String catName, String dbName, String tableName) { + Table result = new Table(); + result.setCatName(catName); + result.setDbName(dbName); + result.setTableName(tableName); + result.setTableType(TableType.VIRTUAL_VIEW.toString()); + + Map parameters = Maps.newHashMap(); + parameters.put( + BaseMetastoreTableOperations.TABLE_TYPE_PROP, HiveOperationsBase.ICEBERG_VIEW_TYPE_VALUE); + parameters.put( + hive_metastoreConstants.META_TABLE_STORAGE, HMSTablePropertyHelper.HIVE_ICEBERG_STORAGE_HANDLER); + result.setParameters(parameters); + return result; + } + + /** + * Applies Iceberg view metadata (SQL, schema, params) onto an existing HMS {@link Table}. + */ + public static void applyIcebergViewToHmsTable(Table hmsTable, View view, Configuration conf) { + ViewMetadata metadata = ((BaseView) view).operations().current(); + String sqlText = viewSqlText(view, metadata); + + boolean hiveEngineEnabled = false; + hmsTable.setSd(HiveOperationsBase.storageDescriptor(metadata.schema(), metadata.location(), hiveEngineEnabled)); + StorageDescriptor sd = hmsTable.getSd(); + + if (sd.getBucketCols() == null) { + sd.setBucketCols(Lists.newArrayList()); + } + + if (sd.getSortCols() == null) { + sd.setSortCols(Lists.newArrayList()); + } + + long maxHiveTablePropertySize = + conf.getLong( + HiveOperationsBase.HIVE_TABLE_PROPERTY_MAX_SIZE, + HiveOperationsBase.HIVE_TABLE_PROPERTY_MAX_SIZE_DEFAULT); + HMSTablePropertyHelper.updateHmsTableForIcebergView( + metadata.metadataFileLocation(), + hmsTable, + metadata, + Collections.emptySet(), + maxHiveTablePropertySize, + null); + + hmsTable.setCreateTime((int) (metadata.version(1).timestampMillis() / 1000)); + hmsTable.setLastAccessTime((int) (metadata.currentVersion().timestampMillis() / 1000)); + hmsTable.setOwner( + PropertyUtil.propertyAsString( + metadata.properties(), HiveCatalog.HMS_TABLE_OWNER, HiveHadoopUtil.currentUser())); + + // In-memory overlay for compile/describe: authoritative SQL comes from Iceberg metadata. + hmsTable.setViewOriginalText(sqlText); + hmsTable.setViewExpandedText(sqlText); + + String catalogType = IcebergCatalogProperties.getCatalogType(conf); + if (!StringUtils.isEmpty(catalogType) && !IcebergCatalogProperties.NO_CATALOG_TYPE.equals(catalogType)) { + hmsTable.getParameters().put(CatalogUtil.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.getCatalogType(conf)); + } + } + + private static String viewSqlText(View view, ViewMetadata metadata) { + SQLViewRepresentation hiveRepr = view.sqlFor("hive"); + if (hiveRepr != null) { + return hiveRepr.sql(); + } + return HiveViewOperations.sqlFor(metadata); + } + private static StorageDescriptor getHiveStorageDescriptor(org.apache.iceberg.Table table) { var result = new StorageDescriptor(); result.setCols(HiveSchemaUtil.convert(table.schema())); diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java index 4390d5a0bca1..d8684b768e11 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java @@ -21,12 +21,15 @@ import java.io.IOException; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; +import java.util.Set; import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.CreateTableRequest; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.DropDatabaseRequest; @@ -38,15 +41,19 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.client.BaseMetaStoreClient; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.Schema; import org.apache.iceberg.SortOrder; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.catalog.ViewCatalog; import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.hive.HMSTablePropertyHelper; +import org.apache.iceberg.hive.HiveOperationsBase; import org.apache.iceberg.hive.HiveSchemaUtil; import org.apache.iceberg.hive.IcebergCatalogProperties; +import org.apache.iceberg.hive.IcebergLogicalViewSupport; import org.apache.iceberg.hive.IcebergTableProperties; import org.apache.iceberg.hive.MetastoreUtil; import org.apache.iceberg.hive.RuntimeMetaException; @@ -123,10 +130,20 @@ public List getTables(String catName, String dbName, String tablePattern Pattern pattern = Pattern.compile(regex); // List tables from the specific database (namespace) and filter them. - return restCatalog.listTables(Namespace.of(dbName)).stream() + Set names = new LinkedHashSet<>(); + restCatalog.listTables(Namespace.of(dbName)).stream() .map(TableIdentifier::name) .filter(pattern.asPredicate()) - .toList(); + .forEach(names::add); + + if (restCatalog instanceof ViewCatalog viewCatalog) { + viewCatalog + .listViews(Namespace.of(dbName)).stream() + .map(TableIdentifier::name) + .filter(pattern.asPredicate()) + .forEach(names::add); + } + return Lists.newArrayList(names); } @Override @@ -136,7 +153,12 @@ public List getAllTables(String catName, String dbName) { @Override public void dropTable(Table table, boolean deleteData, boolean ignoreUnknownTab, boolean ifPurge) throws TException { - restCatalog.dropTable(TableIdentifier.of(table.getDbName(), table.getTableName())); + TableIdentifier id = TableIdentifier.of(table.getDbName(), table.getTableName()); + if (restCatalog instanceof ViewCatalog viewCatalog && viewCatalog.viewExists(id)) { + viewCatalog.dropView(id); + } else { + restCatalog.dropTable(id); + } } private void validateCurrentCatalog(String catName) { @@ -149,7 +171,11 @@ private void validateCurrentCatalog(String catName) { @Override public boolean tableExists(String catName, String dbName, String tableName) { validateCurrentCatalog(catName); - return restCatalog.tableExists(TableIdentifier.of(dbName, tableName)); + TableIdentifier id = TableIdentifier.of(dbName, tableName); + if (restCatalog.tableExists(id)) { + return true; + } + return restCatalog instanceof ViewCatalog viewCatalog && viewCatalog.viewExists(id); } @Override @@ -178,25 +204,58 @@ public Database getDatabase(String catName, String dbName) throws NoSuchObjectEx @Override public Table getTable(GetTableRequest tableRequest) throws TException { validateCurrentCatalog(tableRequest.getCatName()); - org.apache.iceberg.Table icebergTable; + TableIdentifier id = + TableIdentifier.of(tableRequest.getDbName(), tableRequest.getTblName()); try { - icebergTable = restCatalog.loadTable(TableIdentifier.of(tableRequest.getDbName(), - tableRequest.getTblName())); - } catch (NoSuchTableException exception) { + org.apache.iceberg.Table icebergTable = restCatalog.loadTable(id); + return MetastoreUtil.toHiveTable(icebergTable, conf); + } catch (NoSuchTableException tableMissing) { + if (restCatalog instanceof ViewCatalog viewCatalog) { + if (!viewCatalog.viewExists(id)) { + throw new NoSuchObjectException(); + } + return MetastoreUtil.buildMinimalHMSView( + tableRequest.getCatName(), tableRequest.getDbName(), tableRequest.getTblName()); + } throw new NoSuchObjectException(); } - return MetastoreUtil.toHiveTable(icebergTable, conf); + } + + private static boolean hasIcebergViewTableType(Table table) { + if (!TableType.VIRTUAL_VIEW.toString().equals(table.getTableType())) { + return false; + } + Map params = table.getParameters(); + if (params == null) { + return false; + } + return HiveOperationsBase.ICEBERG_VIEW_TYPE_VALUE.equalsIgnoreCase( + params.get(BaseMetastoreTableOperations.TABLE_TYPE_PROP)); + } + + @Override + public void alter_table(String catName, String dbName, String tblName, Table newTable, + EnvironmentContext envContext, String validWriteIdList) { + validateCurrentCatalog(catName); + if (hasIcebergViewTableType(newTable) && restCatalog instanceof ViewCatalog) { + createOrReplaceIcebergView(newTable, dbName, tblName); + } } @Override public void createTable(CreateTableRequest request) throws TException { Table table = request.getTable(); - List cols = Lists.newArrayList(table.getSd().getCols()); - if (table.isSetPartitionKeys() && !table.getPartitionKeys().isEmpty()) { - cols.addAll(table.getPartitionKeys()); + if (hasIcebergViewTableType(table) && restCatalog instanceof ViewCatalog) { + createOrReplaceIcebergView(table, table.getDbName(), table.getTableName()); + } else { + createIcebergTable(request); } + } + + private void createIcebergTable(CreateTableRequest request) { + Table table = request.getTable(); Properties tableProperties = IcebergTableProperties.getTableProperties(table, conf); - Schema schema = HiveSchemaUtil.convert(cols, Collections.emptyMap(), true); + Schema schema = HiveSchemaUtil.convert(hmsTableColumns(table), Collections.emptyMap(), true); Map envCtxProps = Optional.ofNullable(request.getEnvContext()) .map(EnvironmentContext::getProperties) .orElse(Collections.emptyMap()); @@ -204,7 +263,8 @@ public void createTable(CreateTableRequest request) throws TException { HMSTablePropertyHelper.getPartitionSpec(envCtxProps, schema); SortOrder sortOrder = HMSTablePropertyHelper.getSortOrder(tableProperties, schema); - restCatalog.buildTable(TableIdentifier.of(table.getDbName(), table.getTableName()), schema) + restCatalog + .buildTable(TableIdentifier.of(table.getDbName(), table.getTableName()), schema) .withPartitionSpec(partitionSpec) .withLocation(tableProperties.getProperty(IcebergTableProperties.LOCATION)) .withSortOrder(sortOrder) @@ -212,6 +272,22 @@ public void createTable(CreateTableRequest request) throws TException { .create(); } + private void createOrReplaceIcebergView(Table table, String dbName, String tableName) { + Map tblProps = + table.getParameters() == null ? Maps.newHashMap() : Maps.newHashMap(table.getParameters()); + String comment = tblProps.get("comment"); + IcebergLogicalViewSupport.createOrReplaceView( + conf, dbName, tableName, hmsTableColumns(table), table.getViewExpandedText(), tblProps, comment); + } + + private static List hmsTableColumns(Table table) { + List cols = Lists.newArrayList(table.getSd().getCols()); + if (table.isSetPartitionKeys() && !table.getPartitionKeys().isEmpty()) { + cols.addAll(table.getPartitionKeys()); + } + return cols; + } + @Override public void createDatabase(Database db) { validateCurrentCatalog(db.getCatalogName()); diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergLogicalViewSupport.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergLogicalViewSupport.java new file mode 100644 index 000000000000..e48a2613f74c --- /dev/null +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergLogicalViewSupport.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.hive; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.view.BaseView; +import org.apache.iceberg.view.View; +import org.apache.iceberg.view.ViewMetadata; +import org.apache.thrift.TException; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE; +import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE; +import static org.assertj.core.api.Assertions.assertThat; + +class TestIcebergLogicalViewSupport { + + private static final String DB = "native_vw_db"; + private static final String VIEW = "native_vw"; + + @RegisterExtension + private static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION = + HiveMetastoreExtension.builder().withDatabase(DB).build(); + + @AfterEach + void dropView() { + HiveCatalog cat = loadCatalog(); + TableIdentifier id = TableIdentifier.of(DB, VIEW); + cat.dropView(id); + } + + private HiveConf nativeViewConf() { + HiveConf conf = new HiveConf(HIVE_METASTORE_EXTENSION.hiveConf()); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CATALOG_DEFAULT, "hive"); + conf.set( + IcebergCatalogProperties.catalogPropertyConfigKey("hive", ICEBERG_CATALOG_TYPE), + ICEBERG_CATALOG_TYPE_HIVE); + return conf; + } + + private HiveCatalog loadCatalog() { + return (HiveCatalog) + CatalogUtil.loadCatalog( + HiveCatalog.class.getName(), + ICEBERG_CATALOG_TYPE_HIVE, + ImmutableMap.of( + CatalogProperties.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS, + String.valueOf(TimeUnit.SECONDS.toMillis(10))), + HIVE_METASTORE_EXTENSION.hiveConf()); + } + + @Test + void testCreateCommitsNativeViewWithUserProperties() { + HiveConf conf = nativeViewConf(); + List cols = + Arrays.asList(new FieldSchema("id", "int", null), new FieldSchema("name", "string", null)); + String sql = String.format("select id, name from %s.src_tbl", DB); + Map props = Collections.singletonMap("k1", "v1"); + + IcebergLogicalViewSupport.createOrReplaceView( + conf, DB, VIEW, cols, sql, props, "hello-view"); + + HiveCatalog cat = loadCatalog(); + TableIdentifier id = TableIdentifier.of(DB, VIEW); + assertThat(cat.viewExists(id)).isTrue(); + View view = cat.loadView(id); + assertThat(view.properties()) + .containsEntry("comment", "hello-view") + .containsEntry("k1", "v1") + .doesNotContainKey("hive.storage.external.logical.view.handler"); + HiveViewOperations ops = (HiveViewOperations) ((BaseView) view).operations(); + assertThat(ops.current().currentVersion().representations()).isNotEmpty(); + } + + @Test + void testCreateOrReplaceViewReplacesExisting() { + HiveConf conf = nativeViewConf(); + List cols = Collections.singletonList(new FieldSchema("id", "int", null)); + TableIdentifier id = TableIdentifier.of(DB, VIEW); + + IcebergLogicalViewSupport.createOrReplaceView( + conf, DB, VIEW, cols, "select 1 as id", null, null); + View afterCreate = loadCatalog().loadView(id); + assertThat(afterCreate.sqlFor("hive").sql().trim()).isEqualTo("select 1 as id"); + + IcebergLogicalViewSupport.createOrReplaceView( + conf, DB, VIEW, cols, "select 2 as id", null, null); + + assertThat(loadCatalog().viewExists(id)).isTrue(); + View afterReplace = loadCatalog().loadView(id); + assertThat(afterReplace.sqlFor("hive").sql().trim()).isEqualTo("select 2 as id"); + } + + @Test + void testEnrichHmsTableFromIcebergViewOverridesStaleHmsSql() throws TException { + HiveConf conf = nativeViewConf(); + List cols = Collections.singletonList(new FieldSchema("id", "int", null)); + String sql = "select 42 as id"; + + IcebergLogicalViewSupport.createOrReplaceView( + conf, DB, VIEW, cols, sql, null, null); + + org.apache.hadoop.hive.metastore.api.Table hmsTable = + HIVE_METASTORE_EXTENSION.metastoreClient().getTable(DB, VIEW); + hmsTable.setViewOriginalText("select 0"); + hmsTable.setViewExpandedText("select 0"); + + IcebergLogicalViewSupport.enrichHmsTableFromIcebergView(hmsTable, conf); + assertThat(hmsTable.getViewExpandedText()).isEqualTo(sql); + assertThat(hmsTable.getViewOriginalText()).isEqualTo(sql); + + ViewMetadata metadata = ((BaseView) loadCatalog().loadView(TableIdentifier.of(DB, VIEW))).operations().current(); + assertThat(hmsTable.getCreateTime()).isEqualTo((int) (metadata.version(1).timestampMillis() / 1000)); + } +} diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/client/TestHiveRESTCatalogClient.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/client/TestHiveRESTCatalogClient.java index 1ae7e742774c..fd10b13df02b 100644 --- a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/client/TestHiveRESTCatalogClient.java +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/client/TestHiveRESTCatalogClient.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.CreateTableRequest; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; @@ -32,6 +33,7 @@ import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.BaseTable; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.PartitionSpec; @@ -43,7 +45,10 @@ import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.catalog.ViewCatalog; +import org.apache.iceberg.hive.HiveOperationsBase; import org.apache.iceberg.hive.HiveSchemaUtil; +import org.apache.iceberg.hive.IcebergLogicalViewSupport; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.LocationProvider; import org.apache.iceberg.relocated.com.google.common.collect.Maps; @@ -59,8 +64,9 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; -public class TestHiveRESTCatalogClient { +class TestHiveRESTCatalogClient { private static HiveRESTCatalogClient spyHiveRESTCatalogClient; private static RESTCatalog mockRestCatalog; @@ -137,13 +143,13 @@ public void after() { } @Test - public void testGetTable() throws TException { + void testGetTable() throws TException { spyHiveRESTCatalogClient.getTable("default", "tableName"); Mockito.verify(mockRestCatalog).loadTable(TableIdentifier.of("default", "tableName")); } @Test - public void testCreateTable() throws TException { + void testCreateTable() throws TException { Table table = new Table(); table.setTableName("tableName"); table.setDbName("default"); @@ -155,7 +161,7 @@ public void testCreateTable() throws TException { } @Test - public void testCreatePartitionedTable() throws TException { + void testCreatePartitionedTable() throws TException { Table table = new Table(); table.setTableName("tableName"); table.setDbName("default"); @@ -192,9 +198,54 @@ public void testCreatePartitionedTable() throws TException { } @Test - public void testGetDatabase() throws TException { + void testGetDatabase() throws TException { Database aDefault = spyHiveRESTCatalogClient.getDatabase("default"); assertThat(aDefault.getName()).isEqualTo("default"); Mockito.verify(mockRestCatalog).listNamespaces(Namespace.empty()); } + + @Test + void testAlterIcebergLogicalView() { + RESTCatalog viewCapableCatalog = + Mockito.mock(RESTCatalog.class, Mockito.withSettings().extraInterfaces(ViewCatalog.class)); + Mockito.doReturn("hive").when(viewCapableCatalog).name(); + mockCatalogUtil.when(() -> CatalogUtil.buildIcebergCatalog(any(), any(), any())) + .thenReturn(viewCapableCatalog); + + Configuration configuration = new Configuration(); + configuration.set("iceberg.catalog", "ice01"); + configuration.set("iceberg.catalog.ice01.uri", "http://localhost"); + HiveRESTCatalogClient client = new HiveRESTCatalogClient(configuration); + + try (MockedStatic viewSupport = + Mockito.mockStatic(IcebergLogicalViewSupport.class)) { + client.alter_table("hive", "ice_db", "ice_v1", createLogicalView(), null, null); + viewSupport.verify( + () -> + IcebergLogicalViewSupport.createOrReplaceView( + any(), + eq("ice_db"), + eq("ice_v1"), + any(), + eq("select 1"), + any(), + eq(null))); + } + } + + private static Table createLogicalView() { + Table view = new Table(); + view.setTableName("ice_v1"); + view.setDbName("ice_db"); + view.setTableType(TableType.VIRTUAL_VIEW.toString()); + view.setViewExpandedText("select 1"); + view.setSd(new StorageDescriptor()); + view.getSd().setCols(Collections.singletonList(new FieldSchema("x", "int", ""))); + view.setParameters( + Maps.newHashMap( + Map.of( + BaseMetastoreTableOperations.TABLE_TYPE_PROP, + HiveOperationsBase.ICEBERG_VIEW_TYPE_VALUE))); + return view; + } } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java index 69fbe5bf99c2..be02c1ececc2 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java @@ -34,6 +34,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.metastore.HiveMetaHook; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.CreateTableRequest; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -60,8 +61,10 @@ import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.exceptions.NotFoundException; import org.apache.iceberg.hive.HMSTablePropertyHelper; +import org.apache.iceberg.hive.HiveOperationsBase; import org.apache.iceberg.hive.HiveSchemaUtil; import org.apache.iceberg.hive.IcebergCatalogProperties; +import org.apache.iceberg.hive.IcebergLogicalViewSupport; import org.apache.iceberg.hive.IcebergTableProperties; import org.apache.iceberg.mr.Catalogs; import org.apache.iceberg.mr.InputFormatConfig; @@ -115,6 +118,16 @@ public BaseHiveIcebergMetaHook(Configuration conf) { this.conf = conf; } + public static boolean isIcebergLogicalView(org.apache.hadoop.hive.metastore.api.Table hmsTable) { + if (hmsTable == null || + hmsTable.getParameters() == null || + !TableType.VIRTUAL_VIEW.toString().equals(hmsTable.getTableType())) { + return false; + } + String storageHandler = hmsTable.getParameters().get(hive_metastoreConstants.META_TABLE_STORAGE); + return HiveMetaHook.HIVE_ICEBERG_STORAGE_HANDLER.equals(storageHandler); + } + @Override public void preCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) { CreateTableRequest request = new CreateTableRequest(hmsTable); @@ -127,6 +140,10 @@ public void preCreateTable(CreateTableRequest request) { if (hmsTable.isTemporary()) { throw new UnsupportedOperationException("Creation of temporary iceberg tables is not supported."); } + if (isIcebergLogicalView(hmsTable)) { + preCreateIcebergLogicalView(request); + return; + } this.tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); // Set the table type even for non HiveCatalog based tables @@ -204,6 +221,18 @@ public void preCreateTable(CreateTableRequest request) { setSortOrder(hmsTable, schema, tableProperties); } + private void preCreateIcebergLogicalView(CreateTableRequest request) { + + org.apache.hadoop.hive.metastore.api.Table hmsTable = request.getTable(); + tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); + + hmsTable + .getParameters() + .put( + BaseMetastoreTableOperations.TABLE_TYPE_PROP, + HiveOperationsBase.ICEBERG_VIEW_TYPE_VALUE); + } + /** * Method for verification that necessary catalog configs are defined in Session Conf. * @@ -504,6 +533,10 @@ protected void setWriteModeDefaults(Table icebergTbl, Map newPro public void postGetTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) { if (hmsTable != null) { try { + if (isIcebergLogicalView(hmsTable)) { + IcebergLogicalViewSupport.enrichHmsTableFromIcebergView(hmsTable, conf); + return; + } Table tbl = IcebergTableUtil.getTable(conf, hmsTable); String formatVersion = String.valueOf(TableUtil.formatVersion(tbl)); hmsTable.getParameters().put(TableProperties.FORMAT_VERSION, formatVersion); @@ -531,4 +564,5 @@ private static boolean isHiveIcebergStorageHandler(String storageHandler) { throw new RuntimeException("Error checking storage handler class", e); } } + } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index 58c2d19373dd..5e280a388a6f 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -117,6 +117,7 @@ import org.apache.iceberg.hive.HiveLock; import org.apache.iceberg.hive.HiveSchemaUtil; import org.apache.iceberg.hive.HiveTableOperations; +import org.apache.iceberg.hive.IcebergLogicalViewSupport; import org.apache.iceberg.hive.IcebergTableProperties; import org.apache.iceberg.hive.MetastoreLock; import org.apache.iceberg.hive.NoLock; @@ -176,13 +177,23 @@ public HiveIcebergMetaHook(Configuration conf) { super(conf); } - @Override - public void rollbackCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) { - // do nothing - } - @Override public void commitCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) { + if (isIcebergLogicalView(hmsTable)) { + tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); + Map tblProps = + hmsTable.getParameters() == null ? Maps.newHashMap() : Maps.newHashMap(hmsTable.getParameters()); + String comment = tblProps.get("comment"); + IcebergLogicalViewSupport.createOrReplaceView( + conf, + hmsTable.getDbName(), + hmsTable.getTableName(), + hmsTable.getSd().getCols(), + hmsTable.getViewExpandedText(), + tblProps, + comment); + return; + } if (icebergTable == null) { setFileFormat(tableProperties.getProperty(TableProperties.DEFAULT_FILE_FORMAT)); @@ -208,11 +219,6 @@ public void commitCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTabl } } - @Override - public void preDropTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) { - // do nothing - } - @Override public void preDropTable(org.apache.hadoop.hive.metastore.api.Table hmsTable, boolean deleteData) { this.tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); @@ -235,11 +241,6 @@ public void preDropTable(org.apache.hadoop.hive.metastore.api.Table hmsTable, bo } } - @Override - public void rollbackDropTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) { - // do nothing - } - @Override public void commitDropTable(org.apache.hadoop.hive.metastore.api.Table hmsTable, boolean deleteData) { if (deleteData && deleteIcebergTable) { @@ -265,6 +266,15 @@ public void commitDropTable(org.apache.hadoop.hive.metastore.api.Table hmsTable, @Override public void preAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable, EnvironmentContext context) throws MetaException { + if (BaseHiveIcebergMetaHook.isIcebergLogicalView(hmsTable)) { + currentAlterTableOp = null; + if (commitLock == null) { + commitLock = new NoLock(); + } + commitLock.lock(); + tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); + return; + } tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); setupAlterOperationType(hmsTable, context); if (AlterTableType.RENAME.equals(currentAlterTableOp)) { @@ -493,6 +503,21 @@ public void commitAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable if (commitLock == null) { throw new IllegalStateException("Hive commit lock should already be set"); } + if (BaseHiveIcebergMetaHook.isIcebergLogicalView(hmsTable)) { + tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); + Map tblProps = + hmsTable.getParameters() == null ? Maps.newHashMap() : Maps.newHashMap(hmsTable.getParameters()); + String comment = tblProps.get("comment"); + IcebergLogicalViewSupport.createOrReplaceView( + conf, + hmsTable.getDbName(), + hmsTable.getTableName(), + hmsTable.getSd().getCols(), + hmsTable.getViewExpandedText(), + tblProps, + comment); + return; + } commitLock.unlock(); if (isTableMigration) { tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 65852f1a8553..166e5e548281 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -383,6 +383,11 @@ public boolean supportsPartitioning() { return true; } + @Override + public boolean supportsExternalLogicalViewCatalog() { + return true; + } + /** * @param jobConf Job configuration for InputFormat to access * @param deserializer Deserializer @@ -428,6 +433,9 @@ public boolean canProvidePartitionStatistics(org.apache.hadoop.hive.ql.metadata. if (!getStatsSource().equals(HiveMetaHook.ICEBERG)) { return false; } + if (BaseHiveIcebergMetaHook.isIcebergLogicalView(hmsTable.getTTable())) { + return false; + } Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); Snapshot snapshot = IcebergTableUtil.getTableSnapshot(table, hmsTable); if (snapshot != null) { @@ -891,6 +899,9 @@ public boolean supportsPartitionTransform() { @Override public List getPartitionTransformSpec(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { + if (BaseHiveIcebergMetaHook.isIcebergLogicalView(hmsTable.getTTable())) { + return Collections.emptyList(); + } Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); return table.spec().fields().stream() .filter(f -> !f.transform().isVoid()) @@ -905,6 +916,9 @@ public List getPartitionTransformSpec(org.apache.hadoop.hive.ql.m @Override public Map> getPartitionTransformSpecs( org.apache.hadoop.hive.ql.metadata.Table hmsTable) { + if (BaseHiveIcebergMetaHook.isIcebergLogicalView(hmsTable.getTTable())) { + return Collections.emptyMap(); + } Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); return table.specs().entrySet().stream().flatMap(e -> e.getValue().fields().stream() @@ -1570,6 +1584,9 @@ public boolean supportsSortColumns() { @Override public List sortColumns(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { + if (BaseHiveIcebergMetaHook.isIcebergLogicalView(hmsTable.getTTable())) { + return Collections.emptyList(); + } TableDesc tableDesc = Utilities.getTableDesc(hmsTable); Table table = IcebergTableUtil.getTable(conf, tableDesc.getProperties()); if (table.sortOrder().isUnsorted()) { @@ -2133,6 +2150,10 @@ public List getPartitions(org.apache.hadoop.hive.ql.metadata.Table hm } public boolean isPartitioned(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { + if (BaseHiveIcebergMetaHook.isIcebergLogicalView(hmsTable.getTTable())) { + List partCols = hmsTable.getPartCols(); + return partCols != null && !partCols.isEmpty(); + } if (!hmsTable.getTTable().isSetId()) { return false; } @@ -2278,6 +2299,10 @@ public boolean canPerformMetadataDelete(org.apache.hadoop.hive.ql.metadata.Table @Override public List getPartitionKeys(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { + if (BaseHiveIcebergMetaHook.isIcebergLogicalView(hmsTable.getTTable())) { + List partCols = hmsTable.getPartCols(); + return partCols != null ? partCols : Collections.emptyList(); + } if (!hmsTable.getTTable().isSetId()) { return Collections.emptyList(); } diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_logical_view.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_logical_view.q new file mode 100644 index 000000000000..037b67371b13 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_logical_view.q @@ -0,0 +1,87 @@ +-- SORT_QUERY_RESULTS +-- Mask random uuid +--! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/ + +create database ice_native_view_db; +use ice_native_view_db; + +create table src_ice ( + first_name string, + last_name string + ) +partitioned by (dept_id bigint) +stored by iceberg stored as orc; + +INSERT INTO src_ice VALUES + ('fn1','ln1', 1), + ('fn2','ln2', 1), + ('fn3','ln3', 1), + ('fn4','ln4', 1), + ('fn5','ln5', 2), + ('fn6','ln6', 2), + ('fn7','ln7', 2); + +------------------------------------------------------------------------------- +-- Native Iceberg view via TBLPROPERTIES +------------------------------------------------------------------------------- + +-- TEST VIEW CREATION -- + +create view v_ice tblproperties ('view-format'='iceberg') +as select * from src_ice; + +select * from v_ice; + +-- TEST VIEW REPLACEMENT -- + +create or replace view v_ice tblproperties ('view-format'='iceberg') +as select first_name || '-' || dept_id from src_ice where dept_id = 1; + +select * from v_ice; +desc formatted v_ice; + +------------------------------------------------------------------------------- +-- Native Iceberg view when default storage handler is Iceberg +-- and no 'view-format' property in TBLPROPERTIES +------------------------------------------------------------------------------- + +set hive.default.storage.handler.class=org.apache.iceberg.mr.hive.HiveIcebergStorageHandler; + +-- TEST VIEW CREATION WITH IF EXISTS -- + +create view if not exists v_def +as select first_name, last_name, dept_id from src_ice where dept_id = 2; + +select * from v_def; + +-- TEST VIEW IS NOT CREATED BECAUSE IT ALREADY EXISTS -- + +create view if not exists v_def +as select first_name, last_name, dept_id from src_ice; + +select * from v_def; + +desc formatted v_def; +drop view v_def; + +----------------------------------------------------------------------------------------- +-- Classic Hive view when the base table is Iceberg and default storage handler is unset +----------------------------------------------------------------------------------------- + +set hive.default.storage.handler.class=; + +create view v_hive as select * from src_ice; +select * from v_hive; +desc formatted v_hive; +drop view v_hive; + +----------------------------------------------------------------------------------------- +-- Replace Iceberg logical view with a Hive-native logical view +----------------------------------------------------------------------------------------- + +create or replace view v_ice +as select first_name from src_ice where dept_id = 2; + +select * from v_ice; +desc formatted v_ice; +drop view v_ice; \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q index 81982ca44d98..5cdab1267f20 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q @@ -15,8 +15,6 @@ --! qt:replace:/(\s+current-snapshot-timestamp-ms\s+)\S+(\s*)/$1#Masked#$2/ --! qt:replace:/(MAJOR\s+succeeded\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/ --! qt:replace:/(MAJOR\s+refused\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/ --- Mask compaction id as they will be allocated in parallel threads ---! qt:replace:/^[0-9]/#Masked#/ -- Mask removed file size --! qt:replace:/(\S\"removed-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/ -- Mask iceberg version @@ -47,7 +45,7 @@ partitioned by (company_id bigint) stored by iceberg stored as orc; ----------------------------------------------------------------------------- ---! Creating table with a valid catalog name in table properties +--! Creating a table with a valid catalog name in table properties ----------------------------------------------------------------------------- create table ice_orc2 ( @@ -74,6 +72,51 @@ VALUES ('fn1','ln1', 1, 10), ('fn2','ln2', 2, 20), ('fn3','ln3', 3, 30); describe formatted ice_orc2; select * from ice_orc2; +--------------------------------------------------------------------------------------------------------------------- +--! Iceberg Native View tests +--------------------------------------------------------------------------------------------------------------------- + +----------------------------------------------------------------------------------------------------- +--! Native Iceberg logical view with TBLPROPERTIES ('view-format'='iceberg') on a REST catalog table +----------------------------------------------------------------------------------------------------- + +create view ice_v1 tblproperties ('view-format'='iceberg') +as select first_name, last_name from ice_orc2 where dept_id in (1, 3); + +select * from ice_v1; +desc formatted ice_v1; + +------- if-not-exists view test - view should not change ------------------------- + +create view if not exists ice_v1 tblproperties ('view-format'='iceberg') +as select * from ice_orc2 where dept_id = 10000; + +select * from ice_v1; +desc formatted ice_v1; + +------- replace view test - view should be replaced ------------------------------ + +create or replace view ice_v1 tblproperties ('view-format'='iceberg') +as select first_name || '-' || dept_id from ice_orc2 where dept_id = 2; + +select * from ice_v1; +desc formatted ice_v1; + +drop view ice_v1; + +----------------------------------------------------------------------------------------------- +--! Native Iceberg logical view with default Iceberg storage handler and REST catalog table +----------------------------------------------------------------------------------------------- + +set hive.default.storage.handler.class=org.apache.iceberg.mr.hive.HiveIcebergStorageHandler; + +create view ice_v2 +as select first_name, last_name || '-' || dept_id from ice_orc2 where team_id in (20, 30); + +select * from ice_v2; +desc formatted ice_v2; +drop view ice_v2; + ----------------------------------------------------------------------------- show tables; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q index 27f23122240b..47e080ec8881 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q @@ -15,8 +15,6 @@ --! qt:replace:/(\s+current-snapshot-timestamp-ms\s+)\S+(\s*)/$1#Masked#$2/ --! qt:replace:/(MAJOR\s+succeeded\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/ --! qt:replace:/(MAJOR\s+refused\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/ --- Mask compaction id as they will be allocated in parallel threads ---! qt:replace:/^[0-9]/#Masked#/ -- Mask removed file size --! qt:replace:/(\S\"removed-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/ -- Mask iceberg version @@ -47,7 +45,7 @@ partitioned by (company_id bigint) stored by iceberg stored as orc; ----------------------------------------------------------------------------- ---! Creating table with a valid catalog name in table properties +--! Creating a table with a valid catalog name in table properties ----------------------------------------------------------------------------- create table ice_orc2 ( @@ -69,6 +67,31 @@ VALUES ('fn1','ln1', 1, 10), ('fn2','ln2', 2, 20), ('fn3','ln3', 3, 30); describe formatted ice_orc2; select * from ice_orc2; +----------------------------------------------------------------------------------------------- +--! Native Iceberg logical view with TBLPROPERTIES ('view-format'='iceberg') on a REST catalog table +----------------------------------------------------------------------------------------------- + +create view ice_v1 tblproperties ('view-format'='iceberg') +as select first_name, last_name from ice_orc2 where dept_id in (1,2); + +select * from ice_v1; +desc formatted ice_v1; +drop view ice_v1; + +----------------------------------------------------------------------------------------------- +--! Native Iceberg logical view: 'view-format' table properly omitted, Hive config 'hive.default.storage.handler.class' +--! set to 'HiveIcebergStorageHandler' +----------------------------------------------------------------------------------------------- + +set hive.default.storage.handler.class=org.apache.iceberg.mr.hive.HiveIcebergStorageHandler; + +create view ice_v2 +as select dept_id, team_id from ice_orc2 where company_id = 100; + +select * from ice_v2; +desc formatted ice_v2; +drop view ice_v2; + ----------------------------------------------------------------------------- show tables; diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_native_logical_view.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_native_logical_view.q.out new file mode 100644 index 000000000000..ad072ca1c23b --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_native_logical_view.q.out @@ -0,0 +1,380 @@ +PREHOOK: query: create database ice_native_view_db +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:ice_native_view_db +POSTHOOK: query: create database ice_native_view_db +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:ice_native_view_db +PREHOOK: query: use ice_native_view_db +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:ice_native_view_db +POSTHOOK: query: use ice_native_view_db +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:ice_native_view_db +PREHOOK: query: create table src_ice ( + first_name string, + last_name string + ) +partitioned by (dept_id bigint) +stored by iceberg stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: create table src_ice ( + first_name string, + last_name string + ) +partitioned by (dept_id bigint) +stored by iceberg stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: INSERT INTO src_ice VALUES + ('fn1','ln1', 1), + ('fn2','ln2', 1), + ('fn3','ln3', 1), + ('fn4','ln4', 1), + ('fn5','ln5', 2), + ('fn6','ln6', 2), + ('fn7','ln7', 2) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: INSERT INTO src_ice VALUES + ('fn1','ln1', 1), + ('fn2','ln2', 1), + ('fn3','ln3', 1), + ('fn4','ln4', 1), + ('fn5','ln5', 2), + ('fn6','ln6', 2), + ('fn7','ln7', 2) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: create view v_ice tblproperties ('view-format'='iceberg') +as select * from src_ice +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@v_ice +POSTHOOK: query: create view v_ice tblproperties ('view-format'='iceberg') +as select * from src_ice +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_ice +POSTHOOK: Lineage: v_ice.dept_id SIMPLE [(src_ice)src_ice.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: v_ice.first_name SIMPLE [(src_ice)src_ice.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: v_ice.last_name SIMPLE [(src_ice)src_ice.FieldSchema(name:last_name, type:string, comment:null), ] +PREHOOK: query: select * from v_ice +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_ice +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from v_ice +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_ice +POSTHOOK: Output: hdfs://### HDFS PATH ### +fn1 ln1 1 +fn2 ln2 1 +fn3 ln3 1 +fn4 ln4 1 +fn5 ln5 2 +fn6 ln6 2 +fn7 ln7 2 +PREHOOK: query: create or replace view v_ice tblproperties ('view-format'='iceberg') +as select first_name || '-' || dept_id from src_ice where dept_id = 1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@v_ice +POSTHOOK: query: create or replace view v_ice tblproperties ('view-format'='iceberg') +as select first_name || '-' || dept_id from src_ice where dept_id = 1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_ice +PREHOOK: query: select * from v_ice +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_ice +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from v_ice +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_ice +POSTHOOK: Output: hdfs://### HDFS PATH ### +fn1-1 +fn2-1 +fn3-1 +fn4-1 +PREHOOK: query: desc formatted v_ice +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_native_view_db@v_ice +POSTHOOK: query: desc formatted v_ice +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_native_view_db@v_ice +# col_name data_type comment +_c0 string + +# Detailed Table Information +Database: ice_native_view_db +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":1,\"fields\":[{\"id\":1,\"name\":\"_c0\",\"required\":false,\"type\":\"string\"}]} + metadata_location hdfs://### HDFS PATH ### + previous_metadata_location hdfs://### HDFS PATH ### + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG-VIEW +#### A masked pattern was here #### + uuid #Masked# + view-format iceberg + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Sort Columns: [] + +# View Information +Original Query: select `src_ice`.`first_name` || '-' || `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` where `src_ice`.`dept_id` = 1 +Expanded Query: select `src_ice`.`first_name` || '-' || `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` where `src_ice`.`dept_id` = 1 +PREHOOK: query: create view if not exists v_def +as select first_name, last_name, dept_id from src_ice where dept_id = 2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@v_def +POSTHOOK: query: create view if not exists v_def +as select first_name, last_name, dept_id from src_ice where dept_id = 2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_def +POSTHOOK: Lineage: v_def.dept_id SIMPLE [] +POSTHOOK: Lineage: v_def.first_name SIMPLE [(src_ice)src_ice.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: v_def.last_name SIMPLE [(src_ice)src_ice.FieldSchema(name:last_name, type:string, comment:null), ] +PREHOOK: query: select * from v_def +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_def +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from v_def +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_def +POSTHOOK: Output: hdfs://### HDFS PATH ### +fn5 ln5 2 +fn6 ln6 2 +fn7 ln7 2 +PREHOOK: query: create view if not exists v_def +as select first_name, last_name, dept_id from src_ice +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@v_def +POSTHOOK: query: create view if not exists v_def +as select first_name, last_name, dept_id from src_ice +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_def +PREHOOK: query: select * from v_def +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_def +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from v_def +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_def +POSTHOOK: Output: hdfs://### HDFS PATH ### +fn5 ln5 2 +fn6 ln6 2 +fn7 ln7 2 +PREHOOK: query: desc formatted v_def +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_native_view_db@v_def +POSTHOOK: query: desc formatted v_def +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_native_view_db@v_def +# col_name data_type comment +first_name string +last_name string +dept_id bigint + +# Detailed Table Information +Database: ice_native_view_db +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} + metadata_location hdfs://### HDFS PATH ### + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG-VIEW +#### A masked pattern was here #### + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Sort Columns: [] + +# View Information +Original Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` where `src_ice`.`dept_id` = 2 +Expanded Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` where `src_ice`.`dept_id` = 2 +PREHOOK: query: drop view v_def +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_native_view_db@v_def +PREHOOK: Output: ice_native_view_db@v_def +POSTHOOK: query: drop view v_def +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_native_view_db@v_def +POSTHOOK: Output: ice_native_view_db@v_def +PREHOOK: query: create view v_hive as select * from src_ice +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@v_hive +POSTHOOK: query: create view v_hive as select * from src_ice +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_hive +POSTHOOK: Lineage: v_hive.dept_id SIMPLE [(src_ice)src_ice.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: v_hive.first_name SIMPLE [(src_ice)src_ice.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: v_hive.last_name SIMPLE [(src_ice)src_ice.FieldSchema(name:last_name, type:string, comment:null), ] +PREHOOK: query: select * from v_hive +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_hive +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from v_hive +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_hive +POSTHOOK: Output: hdfs://### HDFS PATH ### +fn1 ln1 1 +fn2 ln2 1 +fn3 ln3 1 +fn4 ln4 1 +fn5 ln5 2 +fn6 ln6 2 +fn7 ln7 2 +PREHOOK: query: desc formatted v_hive +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_native_view_db@v_hive +POSTHOOK: query: desc formatted v_hive +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_native_view_db@v_hive +# col_name data_type comment +first_name string +last_name string +dept_id bigint + +# Detailed Table Information +Database: ice_native_view_db +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select * from src_ice +Expanded Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` +PREHOOK: query: drop view v_hive +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_native_view_db@v_hive +PREHOOK: Output: ice_native_view_db@v_hive +POSTHOOK: query: drop view v_hive +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_native_view_db@v_hive +POSTHOOK: Output: ice_native_view_db@v_hive +PREHOOK: query: create or replace view v_ice +as select first_name from src_ice where dept_id = 2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@v_ice +POSTHOOK: query: create or replace view v_ice +as select first_name from src_ice where dept_id = 2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_ice +PREHOOK: query: select * from v_ice +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_ice +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from v_ice +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_ice +POSTHOOK: Output: hdfs://### HDFS PATH ### +fn5 +fn6 +fn7 +PREHOOK: query: desc formatted v_ice +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_native_view_db@v_ice +POSTHOOK: query: desc formatted v_ice +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_native_view_db@v_ice +# col_name data_type comment +first_name string + +# Detailed Table Information +Database: ice_native_view_db +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":1,\"fields\":[{\"id\":1,\"name\":\"_c0\",\"required\":false,\"type\":\"string\"}]} + metadata_location hdfs://### HDFS PATH ### + previous_metadata_location hdfs://### HDFS PATH ### + table_type ICEBERG-VIEW +#### A masked pattern was here #### + uuid #Masked# + view-format iceberg + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select first_name from src_ice where dept_id = 2 +Expanded Query: select `src_ice`.`first_name` from `ice_native_view_db`.`src_ice` where `src_ice`.`dept_id` = 2 +PREHOOK: query: drop view v_ice +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_native_view_db@v_ice +PREHOOK: Output: ice_native_view_db@v_ice +POSTHOOK: query: drop view v_ice +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_native_view_db@v_ice +POSTHOOK: Output: ice_native_view_db@v_ice diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out index 8bba659e8fd1..05c7e7bcfb4b 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out @@ -68,8 +68,6 @@ CREATE EXTERNAL TABLE `ice_orc2`( `dept_id` bigint, `team_id` bigint, `company_id` bigint) -PARTITIONED BY ( - `company_id` bigint COMMENT 'Transform: identity') PARTITIONED BY SPEC ( `company_id`) ROW FORMAT SERDE @@ -180,6 +178,259 @@ POSTHOOK: Input: ice_rest@ice_orc2 fn1 ln1 1 10 100 fn2 ln2 2 20 100 fn3 ln3 3 30 100 +PREHOOK: query: create view ice_v1 tblproperties ('view-format'='iceberg') +as select first_name, last_name from ice_orc2 where dept_id in (1, 3) +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: create view ice_v1 tblproperties ('view-format'='iceberg') +as select first_name, last_name from ice_orc2 where dept_id in (1, 3) +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v1 +POSTHOOK: Lineage: ice_v1.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v1.last_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), ] +PREHOOK: query: select * from ice_v1 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +fn1 ln1 +fn3 ln3 +PREHOOK: query: desc formatted ice_v1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v1 +POSTHOOK: query: desc formatted ice_v1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v1 +# col_name data_type comment +first_name string +last_name string + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]} +#### A masked pattern was here #### + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG-VIEW + type rest + uuid #Masked# + view-format iceberg + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Sort Columns: [] + +# View Information +Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name` from `ice_rest`.`ice_orc2` where `ice_orc2`.`dept_id` in (1, 3) +Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name` from `ice_rest`.`ice_orc2` where `ice_orc2`.`dept_id` in (1, 3) +PREHOOK: query: create view if not exists ice_v1 tblproperties ('view-format'='iceberg') +as select * from ice_orc2 where dept_id = 10000 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: create view if not exists ice_v1 tblproperties ('view-format'='iceberg') +as select * from ice_orc2 where dept_id = 10000 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v1 +PREHOOK: query: select * from ice_v1 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +fn1 ln1 +fn3 ln3 +PREHOOK: query: desc formatted ice_v1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v1 +POSTHOOK: query: desc formatted ice_v1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v1 +# col_name data_type comment +first_name string +last_name string + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]} +#### A masked pattern was here #### + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG-VIEW + type rest + uuid #Masked# + view-format iceberg + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Sort Columns: [] + +# View Information +Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name` from `ice_rest`.`ice_orc2` where `ice_orc2`.`dept_id` in (1, 3) +Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name` from `ice_rest`.`ice_orc2` where `ice_orc2`.`dept_id` in (1, 3) +PREHOOK: query: create or replace view ice_v1 tblproperties ('view-format'='iceberg') +as select first_name || '-' || dept_id from ice_orc2 where dept_id = 2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: create or replace view ice_v1 tblproperties ('view-format'='iceberg') +as select first_name || '-' || dept_id from ice_orc2 where dept_id = 2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v1 +PREHOOK: query: select * from ice_v1 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +fn2-2 +PREHOOK: query: desc formatted ice_v1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v1 +POSTHOOK: query: desc formatted ice_v1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v1 +# col_name data_type comment +_c0 string + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":1,\"fields\":[{\"id\":1,\"name\":\"_c0\",\"required\":false,\"type\":\"string\"}]} +#### A masked pattern was here #### + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG-VIEW + type rest + uuid #Masked# + view-format iceberg + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Sort Columns: [] + +# View Information +Original Query: select `ice_orc2`.`first_name` || '-' || `ice_orc2`.`dept_id` from `ice_rest`.`ice_orc2` where `ice_orc2`.`dept_id` = 2 +Expanded Query: select `ice_orc2`.`first_name` || '-' || `ice_orc2`.`dept_id` from `ice_rest`.`ice_orc2` where `ice_orc2`.`dept_id` = 2 +PREHOOK: query: drop view ice_v1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v1 +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: drop view ice_v1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v1 +POSTHOOK: Output: ice_rest@ice_v1 +PREHOOK: query: create view ice_v2 +as select first_name, last_name || '-' || dept_id from ice_orc2 where team_id in (20, 30) +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v2 +POSTHOOK: query: create view ice_v2 +as select first_name, last_name || '-' || dept_id from ice_orc2 where team_id in (20, 30) +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v2 +POSTHOOK: Lineage: ice_v2._c1 EXPRESSION [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), (ice_orc2)ice_orc2.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v2.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] +PREHOOK: query: select * from ice_v2 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Input: ice_rest@ice_v2 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v2 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Input: ice_rest@ice_v2 +#### A masked pattern was here #### +fn2 ln2-2 +fn3 ln3-3 +PREHOOK: query: desc formatted ice_v2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v2 +POSTHOOK: query: desc formatted ice_v2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v2 +# col_name data_type comment +first_name string +_c1 string + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"_c1\",\"required\":false,\"type\":\"string\"}]} +#### A masked pattern was here #### + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Sort Columns: [] + +# View Information +Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name` || '-' || `ice_orc2`.`dept_id` from `ice_rest`.`ice_orc2` where `ice_orc2`.`team_id` in (20, 30) +Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name` || '-' || `ice_orc2`.`dept_id` from `ice_rest`.`ice_orc2` where `ice_orc2`.`team_id` in (20, 30) +PREHOOK: query: drop view ice_v2 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v2 +PREHOOK: Output: ice_rest@ice_v2 +POSTHOOK: query: drop view ice_v2 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v2 +POSTHOOK: Output: ice_rest@ice_v2 PREHOOK: query: show tables PREHOOK: type: SHOWTABLES PREHOOK: Input: database:ice_rest diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out index 409eb484480b..a92beb820d1c 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out @@ -68,8 +68,6 @@ CREATE EXTERNAL TABLE `ice_orc2`( `dept_id` bigint, `team_id` bigint, `company_id` bigint) -PARTITIONED BY ( - `company_id` bigint COMMENT 'Transform: identity') PARTITIONED BY SPEC ( `company_id`) ROW FORMAT SERDE @@ -180,6 +178,144 @@ POSTHOOK: Input: ice_rest@ice_orc2 fn1 ln1 1 10 100 fn2 ln2 2 20 100 fn3 ln3 3 30 100 +PREHOOK: query: create view ice_v1 tblproperties ('view-format'='iceberg') +as select first_name, last_name from ice_orc2 where dept_id in (1,2) +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: create view ice_v1 tblproperties ('view-format'='iceberg') +as select first_name, last_name from ice_orc2 where dept_id in (1,2) +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v1 +POSTHOOK: Lineage: ice_v1.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v1.last_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), ] +PREHOOK: query: select * from ice_v1 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +fn1 ln1 +fn2 ln2 +PREHOOK: query: desc formatted ice_v1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v1 +POSTHOOK: query: desc formatted ice_v1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v1 +# col_name data_type comment +first_name string +last_name string + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]} +#### A masked pattern was here #### + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG-VIEW + type rest + uuid #Masked# + view-format iceberg + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Sort Columns: [] + +# View Information +Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name` from `ice_rest`.`ice_orc2` where `ice_orc2`.`dept_id` in (1,2) +Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name` from `ice_rest`.`ice_orc2` where `ice_orc2`.`dept_id` in (1,2) +PREHOOK: query: drop view ice_v1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v1 +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: drop view ice_v1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v1 +POSTHOOK: Output: ice_rest@ice_v1 +PREHOOK: query: create view ice_v2 +as select dept_id, team_id from ice_orc2 where company_id = 100 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v2 +POSTHOOK: query: create view ice_v2 +as select dept_id, team_id from ice_orc2 where company_id = 100 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v2 +POSTHOOK: Lineage: ice_v2.dept_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v2.team_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v2 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Input: ice_rest@ice_v2 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v2 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Input: ice_rest@ice_v2 +#### A masked pattern was here #### +1 10 +2 20 +3 30 +PREHOOK: query: desc formatted ice_v2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v2 +POSTHOOK: query: desc formatted ice_v2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v2 +# col_name data_type comment +dept_id bigint +team_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":2,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"}]} +#### A masked pattern was here #### + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Sort Columns: [] + +# View Information +Original Query: select `ice_orc2`.`dept_id`, `ice_orc2`.`team_id` from `ice_rest`.`ice_orc2` where `ice_orc2`.`company_id` = 100 +Expanded Query: select `ice_orc2`.`dept_id`, `ice_orc2`.`team_id` from `ice_rest`.`ice_orc2` where `ice_orc2`.`company_id` = 100 +PREHOOK: query: drop view ice_v2 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v2 +PREHOOK: Output: ice_rest@ice_v2 +POSTHOOK: query: drop view ice_v2 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v2 +POSTHOOK: Output: ice_rest@ice_v2 PREHOOK: query: show tables PREHOOK: type: SHOWTABLES PREHOOK: Input: database:ice_rest diff --git a/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java index fd30223c9934..ec5f801c8aaa 100644 --- a/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java +++ b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java @@ -30,7 +30,9 @@ import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -38,13 +40,15 @@ import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.mapred.TextInputFormat; +import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.TableProperties; -import org.apache.iceberg.hive.IcebergCatalogProperties; +import org.apache.iceberg.hive.HiveOperationsBase; import org.apache.iceberg.hive.HiveSchemaUtil; +import org.apache.iceberg.hive.IcebergCatalogProperties; import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -67,6 +71,8 @@ public abstract class TestHiveRESTCatalogClientITBase { static final String DB_NAME = "ice_db"; + static final String VIEW_DB_NAME = "ice_db_view"; + static final String NATIVE_VIEW_NAME = "native_rest_v"; static final String TABLE_NAME = "ice_tbl"; static final String CATALOG_NAME = "ice01"; static final String HIVE_ICEBERG_STORAGE_HANDLER = "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler"; @@ -109,6 +115,20 @@ void setup() throws Exception { msClient = new HiveMetaStoreClient(conf, hookLoader); hiveConf = new HiveConf(conf, HiveConf.class); hive = Hive.get(hiveConf); + dropDatabaseIfExists(VIEW_DB_NAME); + } + + private void dropDatabaseIfExists(String dbName) { + try { + msClient.dropTable(CATALOG_NAME, dbName, NATIVE_VIEW_NAME); + } catch (Exception ignored) { + // view may not exist + } + try { + msClient.dropDatabase(dbName); + } catch (Exception ignored) { + // database may not exist + } } @AfterEach @@ -173,8 +193,14 @@ public void testIceberg() throws Exception { Assertions.assertEquals(TABLE_NAME, table.getTableName()); Assertions.assertEquals(HIVE_ICEBERG_STORAGE_HANDLER, table.getParameters().get("storage_handler")); Assertions.assertNotNull(table.getParameters().get(TableProperties.DEFAULT_PARTITION_SPEC)); - Assertions.assertEquals(1, table.getPartitionKeys().size()); - Assertions.assertEquals("city", table.getPartitionKeys().getFirst().getName()); + + // TODO: Revert after HIVE-29633 is fixed + // Assertions.assertEquals(1, table.getPartitionKeys().size()); + Assertions.assertTrue(table.getPartitionKeys().isEmpty()); + + List columnNames = + table.getSd().getCols().stream().map(FieldSchema::getName).toList(); + Assertions.assertTrue(columnNames.contains("city")); // --- Get Tables --- List tables = msClient.getTables(CATALOG_NAME, DB_NAME, "ice_*"); @@ -195,7 +221,69 @@ public void testIceberg() throws Exception { Assertions.assertFalse(msClient.getAllDatabases(CATALOG_NAME).contains(DB_NAME)); } - private static Table createPartitionedTable(IMetaStoreClient db, String catName, String dbName, String tableName, + @Test + public void testIcebergLogicalView() throws Exception { + Database db = new Database(); + db.setCatalogName(CATALOG_NAME); + db.setName(VIEW_DB_NAME); + db.setOwnerType(PrincipalType.USER); + db.setOwnerName(System.getProperty("user.name")); + String warehouseDir = MetastoreConf.get(conf, MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL.getVarname()); + db.setLocationUri(warehouseDir + "/" + VIEW_DB_NAME + ".db"); + hive.createDatabase(db, true); + + List cols = Collections.singletonList(new FieldSchema("x", "int", "")); + createIcebergLogicalView(VIEW_DB_NAME, NATIVE_VIEW_NAME, cols, "select 1 as x", "rest-native-view"); + + Assertions.assertTrue(msClient.tableExists(CATALOG_NAME, VIEW_DB_NAME, NATIVE_VIEW_NAME)); + + GetTableRequest getTableRequest = new GetTableRequest(); + getTableRequest.setCatName(CATALOG_NAME); + getTableRequest.setDbName(VIEW_DB_NAME); + getTableRequest.setTblName(NATIVE_VIEW_NAME); + Table view = msClient.getTable(getTableRequest); + + Assertions.assertEquals(TableType.VIRTUAL_VIEW.name(), view.getTableType()); + String tableTypeProp = view.getParameters().get(BaseMetastoreTableOperations.TABLE_TYPE_PROP); + Assertions.assertNotNull(tableTypeProp); + Assertions.assertEquals(HiveOperationsBase.ICEBERG_VIEW_TYPE_VALUE, tableTypeProp.toLowerCase()); + + List names = msClient.getTables(CATALOG_NAME, VIEW_DB_NAME, "*"); + Assertions.assertTrue(names.contains(NATIVE_VIEW_NAME)); + + msClient.dropTable(CATALOG_NAME, VIEW_DB_NAME, NATIVE_VIEW_NAME); + Assertions.assertFalse(msClient.tableExists(CATALOG_NAME, VIEW_DB_NAME, NATIVE_VIEW_NAME)); + + msClient.dropDatabase(VIEW_DB_NAME); + } + + private void createIcebergLogicalView( + String dbName, String viewName, List cols, String viewSql, String comment) throws Exception { + Table view = new Table(); + view.setCatName(CATALOG_NAME); + view.setDbName(dbName); + view.setTableName(viewName); + view.setTableType(TableType.VIRTUAL_VIEW.toString()); + view.setViewOriginalText(viewSql); + view.setViewExpandedText(viewSql); + + StorageDescriptor sd = new StorageDescriptor(); + sd.setCols(cols); + sd.setSerdeInfo(new SerDeInfo()); + sd.getSerdeInfo().setParameters(new java.util.HashMap<>()); + view.setSd(sd); + + view.setParameters(new java.util.HashMap<>()); + view.getParameters().put(hive_metastoreConstants.META_TABLE_STORAGE, HIVE_ICEBERG_STORAGE_HANDLER); + view.getParameters().put("view-format", "iceberg"); + if (comment != null) { + view.getParameters().put("comment", comment); + } + + msClient.createTable(view); + } + + private static Table createPartitionedTable(IMetaStoreClient db, String catName, String dbName, String tableName, Map tableParameters) throws Exception { db.dropTable(catName, dbName, tableName); Table table = new Table(); diff --git a/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseCreateView.java b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseCreateView.java new file mode 100644 index 000000000000..22c31f74d0b3 --- /dev/null +++ b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseCreateView.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.hadoop.hive.ql.parse; + +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +public class TestParseCreateView { + + private final ParseDriver parseDriver = new ParseDriver(); + + @Test + public void testParseCreateViewWithTableProperties() throws Exception { + ASTNode tree = + parseDriver + .parse( + "create view v1 tblproperties ('view-format'='iceberg') as select * from t", null) + .getTree(); + assertTrue(tree.dump(), tree.toStringTree().contains("tok_createview")); + assertTrue(tree.dump(), tree.toStringTree().contains("tok_tableproperties")); + assertTrue(tree.dump(), tree.toStringTree().contains("view-format")); + assertTrue(tree.dump(), tree.toStringTree().contains("iceberg")); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java index 60b123dbb7bd..73b6da2cb28e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java @@ -24,7 +24,9 @@ import java.util.List; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.ErrorMsg; @@ -34,6 +36,7 @@ import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ASTNode; @@ -41,12 +44,16 @@ import org.apache.hadoop.hive.ql.parse.ParseUtils; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.parse.StorageFormat; /** * Analyzer for create view commands. */ @DDLType(types = HiveParser.TOK_CREATEVIEW) public class CreateViewAnalyzer extends AbstractCreateViewAnalyzer { + + private static final String VIEW_FORMAT_TABLE_PROPERTY = "view-format"; + public CreateViewAnalyzer(QueryState queryState) throws SemanticException { super(queryState); } @@ -75,6 +82,14 @@ public void analyzeInternal(ASTNode root) throws SemanticException { List partitionColumnNames = children.containsKey(HiveParser.TOK_VIEWPARTCOLS) ? getColumnNames((ASTNode) children.remove(HiveParser.TOK_VIEWPARTCOLS).getChild(0)) : null; + String storageHandlerClassFromTableProps = getViewStorageHandlerClassFromTableProps(properties); + String storageHandlerClass = resolveViewStorageHandlerClass(storageHandlerClassFromTableProps); + + if (storageHandlerClassFromTableProps != null && storageHandlerClass == null) { + throw new SemanticException( + ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.format(storageHandlerClassFromTableProps)); + } + assert children.isEmpty(); if (ifNotExists && orReplace) { @@ -94,7 +109,7 @@ public void analyzeInternal(ASTNode root) throws SemanticException { List partitionColumns = getPartitionColumns(partitionColumnNames); setColumnAccessInfo(analyzer.getColumnAccessInfo()); CreateViewDesc desc = new CreateViewDesc(fqViewName, schema, comment, properties, partitionColumnNames, - ifNotExists, orReplace, originalText, expandedText, partitionColumns); + ifNotExists, orReplace, originalText, expandedText, partitionColumns, storageHandlerClass); validateCreateView(desc, analyzer); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc))); @@ -191,6 +206,52 @@ private List getPartitionColumns(List partitionColumnNames) return partitionColumnsCopy; } + /** + * Returns the FQCN of the storage handler that should own external logical view metadata, or {@code null} for a + * classic HMS virtual view. Uses {@code storageHandlerClassFromTableProps} when non-null (from the + * {@code view-format} table property); otherwise the Hive config {@code hive.default.storage.handler.class}. + */ + private String resolveViewStorageHandlerClass(String storageHandlerClassFromTableProps) + throws SemanticException { + + String storageHandlerClassFromConfig = + StringUtils.trimToNull(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_DEFAULT_STORAGE_HANDLER)); + + String storageHandlerClass = + storageHandlerClassFromTableProps != null ? storageHandlerClassFromTableProps : storageHandlerClassFromConfig; + + if (StringUtils.isBlank(storageHandlerClass)) { + return null; + } + + try { + HiveStorageHandler storageHandler = HiveUtils.getStorageHandler(conf, storageHandlerClass); + + if (storageHandler != null && storageHandler.supportsExternalLogicalViewCatalog()) { + return storageHandlerClass; + } + } catch (HiveException e) { + throw new SemanticException(e); + } + + return null; + } + + private static String getViewStorageHandlerClassFromTableProps(Map properties) + throws SemanticException { + if (properties == null) { + return null; + } + for (Map.Entry e : properties.entrySet()) { + if (e.getKey() != null + && VIEW_FORMAT_TABLE_PROPERTY.equalsIgnoreCase(e.getKey()) + && StringUtils.isNotBlank(e.getValue())) { + return StorageFormat.resolveStorageHandlerClassName(e.getValue().trim()); + } + } + return null; + } + private void validateCreateView(CreateViewDesc desc, SemanticAnalyzer analyzer) throws SemanticException { try { validateTablesUsed(analyzer); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java index e71cbce773f1..bcbe536bf559 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.parse.ReplicationSpec; @@ -40,13 +41,14 @@ public class CreateViewDesc extends AbstractCreateViewDesc { private final boolean ifNotExists; private final boolean replace; private final List partitionColumns; + private final String storageHandlerClass; private ReplicationSpec replicationSpec = null; private String ownerName = null; public CreateViewDesc(String viewName, List schema, String comment, Map properties, List partitionColumnNames, boolean ifNotExists, boolean replace, String originalText, - String expandedText, List partitionColumns) { + String expandedText, List partitionColumns, String storageHandlerClass) { super(viewName, schema, originalText, expandedText); this.comment = comment; this.properties = properties; @@ -54,6 +56,7 @@ public CreateViewDesc(String viewName, List schema, String comment, this.ifNotExists = ifNotExists; this.replace = replace; this.partitionColumns = partitionColumns; + this.storageHandlerClass = storageHandlerClass; } @Explain(displayName = "partition columns") @@ -89,6 +92,19 @@ public boolean isReplace() { return replace; } + /** + * @return FQCN of the {@link org.apache.hadoop.hive.ql.metadata.HiveStorageHandler} that stores view metadata in an + * external catalog, or {@code null} for a classic HMS-only virtual view. + */ + @Explain(displayName = "external logical view storage handler", displayOnlyOnTrue = true) + public String getStorageHandlerClass() { + return storageHandlerClass; + } + + public boolean usesStorageHandler() { + return !StringUtils.isBlank(storageHandlerClass); + } + /** * @param replicationSpec Sets the replication spec governing this create. * This parameter will have meaningful values only for creates happening as a result of a replication. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java index b6a41a8d4fb8..4001b1a70b44 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java @@ -82,6 +82,10 @@ public int execute() throws HiveException { if (desc.getProperties() != null) { oldview.getTTable().getParameters().putAll(desc.getProperties()); } + if (!desc.usesStorageHandler()) { + // External logical view is replaced with a native Hive view + clearStorageHandlerProp(oldview); + } oldview.setPartCols(desc.getPartitionColumns()); oldview.checkValidity(null); @@ -105,6 +109,18 @@ public int execute() throws HiveException { return 0; } + private void clearStorageHandlerProp(Table oldview) { + Map params = oldview.getParameters(); + if (params == null) { + return; + } + String fqcn = params.get(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE); + if (fqcn == null) { + return; + } + params.remove(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE); + } + private Table createViewObject() throws HiveException { TableName name = HiveTableName.of(desc.getViewName()); Table view = new Table(name.getDb(), name.getTable()); @@ -129,6 +145,13 @@ private Table createViewObject() throws HiveException { StorageFormat storageFormat = new StorageFormat(context.getConf()); storageFormat.fillDefaultStorageFormat(false, false); + if (desc.usesStorageHandler()) { + storageFormat.setStorageHandler(desc.getStorageHandlerClass()); + view.setProperty( + org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE, + desc.getStorageHandlerClass().trim()); + } + view.setInputFormatClass(storageFormat.getInputFormat()); view.setOutputFormatClass(storageFormat.getOutputFormat()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java index 9287fd75e766..787cadd256b1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java @@ -547,8 +547,9 @@ public static Task createViewTask(MetaData metaData, String dbNameToLoadIn, H // texts using new DB name. Currently it refers to the source database name. } - CreateViewDesc desc = new CreateViewDesc(dbDotView, table.getCols(), null, table.getParameters(), - table.getPartColNames(), false, false, viewOriginalText, viewExpandedText, table.getPartCols()); + CreateViewDesc desc = new CreateViewDesc(dbDotView, table.getCols(), null, table.getParameters(), + table.getPartColNames(), false, false, viewOriginalText, viewExpandedText, + table.getPartCols(), null); desc.setReplicationSpec(metaData.getReplicationSpec()); desc.setOwnerName(table.getOwner()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/CreateExternalLogicalViewRequest.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/CreateExternalLogicalViewRequest.java new file mode 100644 index 000000000000..d6b6f2cb4e35 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/CreateExternalLogicalViewRequest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.metadata; + +import java.io.Serializable; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.hive.metastore.api.FieldSchema; + +/** + * Parameters for {@link HiveStorageHandler#createOrReplaceExternalLogicalView}. + */ +public final class CreateExternalLogicalViewRequest implements Serializable { + private static final long serialVersionUID = 1L; + + private final String databaseName; + private final String viewName; + private final List schema; + private final String expandedText; + private final Map properties; + private final String comment; + private final boolean replace; + private final boolean ifNotExists; + + public CreateExternalLogicalViewRequest( + String databaseName, + String viewName, + List schema, + String expandedText, + Map properties, + String comment, + boolean replace, + boolean ifNotExists) { + this.databaseName = databaseName; + this.viewName = viewName; + this.schema = schema; + this.expandedText = expandedText; + this.properties = properties == null ? null : Collections.unmodifiableMap(properties); + this.comment = comment; + this.replace = replace; + this.ifNotExists = ifNotExists; + } + + public String getDatabaseName() { + return databaseName; + } + + public String getViewName() { + return viewName; + } + + public List getSchema() { + return schema; + } + + public String getExpandedText() { + return expandedText; + } + + public Map getProperties() { + return properties; + } + + public String getComment() { + return comment; + } + + public boolean isReplace() { + return replace; + } + + public boolean isIfNotExists() { + return ifNotExists; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java index 520c52a24a8f..31a14688f5b1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java @@ -1028,6 +1028,14 @@ default void setMergeTaskDeleteProperties(TableDesc tableDesc) { throw new UnsupportedOperationException("Storage handler does not support getting custom delete merge schema."); } + /** + * @return {@code true} if this handler may store CREATE VIEW text and column metadata in an external catalog + * rather than only as a classic HMS virtual view. + */ + default boolean supportsExternalLogicalViewCatalog() { + return false; + } + default boolean supportsDefaultColumnValues(Map tblProps) { return false; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java index 2472ad44ad00..7fb5269ec25c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java @@ -88,6 +88,19 @@ public String outputFormat() { return outputFormat; } } + + public static String resolveStorageHandlerClassName(String formatType) throws SemanticException { + if (StringUtils.isBlank(formatType)) { + throw new SemanticException("Format type cannot be empty"); + } + for (StorageHandlerTypes type : StorageHandlerTypes.NON_DEFAULT_TYPES) { + if (type.name().equalsIgnoreCase(formatType.trim())) { + Objects.requireNonNull(type.className()); + return ensureClassExists(BaseSemanticAnalyzer.unescapeSQLString(type.className())); + } + } + return ensureClassExists(BaseSemanticAnalyzer.unescapeSQLString(formatType.trim())); + } public StorageFormat(Configuration conf) { this.conf = conf;