From db655f2d574929fed3bd609b3f3ac15eea068f2c Mon Sep 17 00:00:00 2001 From: Dmitriy Fingerman Date: Thu, 23 Apr 2026 15:49:20 -0400 Subject: [PATCH 1/6] HIVE-29578: Iceberg: support for Iceberg logical views. --- .../apache/hadoop/hive/conf/Constants.java | 3 + .../org/apache/hadoop/hive/ql/ErrorMsg.java | 1 + .../iceberg/hive/HMSTablePropertyHelper.java | 1 + .../iceberg/hive/HiveViewOperations.java | 2 +- .../hive/IcebergNativeLogicalViewSupport.java | 130 +++++++ .../apache/iceberg/hive/MetastoreUtil.java | 77 ++++- .../hive/client/HiveRESTCatalogClient.java | 118 +++++-- .../TestIcebergNativeLogicalViewSupport.java | 153 +++++++++ .../mr/hive/BaseHiveIcebergMetaHook.java | 44 ++- .../iceberg/mr/hive/HiveIcebergMetaHook.java | 54 +++ .../mr/hive/HiveIcebergStorageHandler.java | 25 ++ .../queries/positive/iceberg_native_view.q | 78 +++++ .../positive/iceberg_rest_catalog_gravitino.q | 50 ++- .../positive/iceberg_rest_catalog_hms.q | 50 ++- .../positive/iceberg_native_view.q.out | 316 ++++++++++++++++++ .../llap/iceberg_rest_catalog_gravitino.q.out | 290 +++++++++++++++- .../llap/iceberg_rest_catalog_hms.q.out | 290 +++++++++++++++- .../hive/TestHiveRESTCatalogClientITBase.java | 58 +++- .../hive/ql/parse/TestParseCreateView.java | 40 +++ .../ddl/view/create/CreateViewAnalyzer.java | 67 +++- .../ql/ddl/view/create/CreateViewDesc.java | 18 +- .../ddl/view/create/CreateViewOperation.java | 35 ++ .../hive/ql/exec/repl/ReplLoadTask.java | 5 +- .../CreateExternalLogicalViewRequest.java | 92 +++++ .../hive/ql/metadata/HiveStorageHandler.java | 9 + .../hadoop/hive/ql/parse/StorageFormat.java | 13 + 26 files changed, 1974 insertions(+), 45 deletions(-) create mode 100644 iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergNativeLogicalViewSupport.java create mode 100644 iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergNativeLogicalViewSupport.java create mode 100644 iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q create mode 100644 iceberg/iceberg-handler/src/test/results/positive/iceberg_native_view.q.out create mode 100644 parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseCreateView.java create mode 100644 ql/src/java/org/apache/hadoop/hive/ql/metadata/CreateExternalLogicalViewRequest.java diff --git a/common/src/java/org/apache/hadoop/hive/conf/Constants.java b/common/src/java/org/apache/hadoop/hive/conf/Constants.java index 4b0e75fafb51..0af398ef8768 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/Constants.java +++ b/common/src/java/org/apache/hadoop/hive/conf/Constants.java @@ -111,4 +111,7 @@ public class Constants { public static final String CLUSTER_ID_CLI_OPT_NAME = "hive.cluster.id"; public static final String CLUSTER_ID_HIVE_CONF_PROP = "hive.cluster.id"; public static final String ICEBERG_PARTITION_COLUMNS = "partition,spec_id"; + + public static final String EXTERNAL_LOGICAL_VIEW_DDL_REPLACE = "externalLogicalViewDdlReplace"; + public static final String EXTERNAL_LOGICAL_VIEW_CREATE_IF_NOT_EXISTS = "externalLogicalViewCreateIfNotExists"; } diff --git a/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java index 57f7cadac181..156d7c7ebe76 100644 --- a/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java +++ b/common/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java @@ -445,6 +445,7 @@ public enum ErrorMsg { @Deprecated // kept for backwards reference REPLACE_VIEW_WITH_MATERIALIZED(10400, "Attempt to replace view {0} with materialized view", true), REPLACE_MATERIALIZED_WITH_VIEW(10401, "Attempt to replace materialized view {0} with view", true), + VIEW_STORAGE_HANDLER_UNSUPPORTED(10448, "Storage handler {0} doesn't support external logical views", true), UPDATE_DELETE_VIEW(10402, "You cannot update or delete records in a view"), MATERIALIZED_VIEW_DEF_EMPTY(10403, "Query for the materialized view rebuild could not be retrieved"), MERGE_PREDIACTE_REQUIRED(10404, "MERGE statement with both UPDATE and DELETE clauses " + diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HMSTablePropertyHelper.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HMSTablePropertyHelper.java index f7de4dd07196..43098c9913a5 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HMSTablePropertyHelper.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HMSTablePropertyHelper.java @@ -161,6 +161,7 @@ public static void updateHmsTableForIcebergView( HiveOperationsBase.ICEBERG_VIEW_TYPE_VALUE.toUpperCase(Locale.ENGLISH), metadata.schema(), maxHiveTablePropertySize); + parameters.put(hive_metastoreConstants.META_TABLE_STORAGE, HIVE_ICEBERG_STORAGE_HANDLER); tbl.setParameters(parameters); } diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveViewOperations.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveViewOperations.java index c0c959974a61..10f6736c3538 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveViewOperations.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveViewOperations.java @@ -302,7 +302,7 @@ private Table newHMSView(ViewMetadata metadata) { tableType().name()); } - private String sqlFor(ViewMetadata metadata) { + public static String sqlFor(ViewMetadata metadata) { SQLViewRepresentation closest = null; for (ViewRepresentation representation : metadata.currentVersion().representations()) { if (representation instanceof SQLViewRepresentation) { diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergNativeLogicalViewSupport.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergNativeLogicalViewSupport.java new file mode 100644 index 000000000000..d3d3b1f23591 --- /dev/null +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergNativeLogicalViewSupport.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.hive; + +import java.io.Closeable; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.catalog.ViewCatalog; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.view.ViewBuilder; + +/** + * Commits a native Iceberg view through the configured default Iceberg catalog (HiveCatalog or REST + * catalog, etc.) when {@code Catalog} also implements {@link ViewCatalog}. + * + *

CREATE VIEW session hints and metastore {@code EnvironmentContext} keys use + * {@link org.apache.hadoop.hive.conf.Constants#EXTERNAL_LOGICAL_VIEW_DDL_REPLACE} and + * {@link org.apache.hadoop.hive.conf.Constants#EXTERNAL_LOGICAL_VIEW_CREATE_IF_NOT_EXISTS}. + */ +public final class IcebergNativeLogicalViewSupport { + + /** Value for HMS {@code table_type} on native Iceberg logical views (uppercase, HMS convention). */ + public static final String ICEBERG_VIEW_HMS_TABLE_TYPE_VALUE = + HiveOperationsBase.ICEBERG_VIEW_TYPE_VALUE.toUpperCase(java.util.Locale.ENGLISH); + + private IcebergNativeLogicalViewSupport() { + } + + /** + * Creates or replaces a view in the Iceberg catalog. + * + * @return {@code false} if skipped because {@code ifNotExists} is true and the view already exists + */ + public static boolean createOrReplaceNativeView( + Configuration conf, + String databaseName, + String viewName, + List fieldSchemas, + String viewSql, + Map tblProperties, + String comment, + boolean replace, + boolean ifNotExists) { + + TableIdentifier identifier = TableIdentifier.of(databaseName, viewName); + String catalogName = IcebergCatalogProperties.getCatalogName(conf); + Map catalogProps = IcebergCatalogProperties.getCatalogProperties(conf, catalogName); + Catalog catalog = CatalogUtil.buildIcebergCatalog(catalogName, catalogProps, conf); + + try { + ViewCatalog viewCatalog = asViewCatalog(catalog, catalogName); + if (!replace && ifNotExists && viewCatalog.viewExists(identifier)) { + return false; + } + + ViewBuilder builder = + viewCatalog + .buildView(identifier) + .withSchema(HiveSchemaUtil.convert(fieldSchemas, Collections.emptyMap(), true)) + .withDefaultNamespace(Namespace.of(identifier.namespace().level(0))) + .withQuery("hive", viewSql); + + if (StringUtils.isNotBlank(comment)) { + builder = builder.withProperty("comment", comment); + } + + Map tblProps = + tblProperties == null ? Maps.newHashMap() : Maps.newHashMap(tblProperties); + + for (Map.Entry e : tblProps.entrySet()) { + if (e.getKey() != null && e.getValue() != null) { + builder = builder.withProperty(e.getKey(), e.getValue()); + } + } + + if (replace) { + builder.createOrReplace(); + } else { + builder.create(); + } + return true; + } finally { + if (catalog instanceof Closeable closeable) { + try { + closeable.close(); + } catch (IOException e) { + throw new UncheckedIOException("Failed to close Iceberg catalog", e); + } + } + } + } + + private static ViewCatalog asViewCatalog(Catalog catalog, String catalogName) { + if (catalog instanceof ViewCatalog viewCatalog) { + return viewCatalog; + } + throw new UnsupportedOperationException( + String.format( + "Iceberg catalog '%s' does not implement ViewCatalog.", + catalogName) + + " Native views require a catalog that implements ViewCatalog (e.g. HiveCatalog or REST)."); + } +} diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java index 95e1e5b36623..6eb9dff8ba3b 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java @@ -46,6 +46,11 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.util.PropertyUtil; +import org.apache.iceberg.view.BaseView; +import org.apache.iceberg.view.SQLViewRepresentation; +import org.apache.iceberg.view.View; +import org.apache.iceberg.view.ViewMetadata; import org.apache.thrift.TException; public class MetastoreUtil { @@ -134,7 +139,7 @@ public static Table toHiveTable(org.apache.iceberg.Table table, Configuration co result.setDbName(tableName.getDb()); result.setTableName(tableName.getTable()); result.setTableType(TableType.EXTERNAL_TABLE.toString()); - result.setPartitionKeys(getPartitionKeys(table, table.spec().specId())); + result.setPartitionKeys(Lists.newArrayList()); TableMetadata metadata = ((BaseTable) table).operations().current(); long maxHiveTablePropertySize = conf.getLong(HiveOperationsBase.HIVE_TABLE_PROPERTY_MAX_SIZE, HiveOperationsBase.HIVE_TABLE_PROPERTY_MAX_SIZE_DEFAULT); @@ -142,12 +147,80 @@ public static Table toHiveTable(org.apache.iceberg.Table table, Configuration co null, true, maxHiveTablePropertySize, null); String catalogType = IcebergCatalogProperties.getCatalogType(conf); if (!StringUtils.isEmpty(catalogType) && !IcebergCatalogProperties.NO_CATALOG_TYPE.equals(catalogType)) { - result.getParameters().put(CatalogUtil.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.getCatalogType(conf)); + result.getParameters().put(CatalogUtil.ICEBERG_CATALOG_TYPE, catalogType); } result.setSd(getHiveStorageDescriptor(table)); return result; } + /** + * Builds a Hive metastore {@link Table} representation for an Iceberg {@link View}, for clients + * (e.g. {@code HiveRESTCatalogClient}) that bridge Iceberg catalog metadata into the HMS API. + */ + public static Table toHiveView(View view, Configuration conf) { + Table result = new Table(); + TableName tableName = + TableName.fromString( + view.name(), MetaStoreUtils.getDefaultCatalog(conf), Warehouse.DEFAULT_DATABASE_NAME); + result.setCatName(tableName.getCat()); + result.setDbName(tableName.getDb()); + result.setTableName(tableName.getTable()); + result.setTableType(TableType.VIRTUAL_VIEW.toString()); + + ViewMetadata metadata = ((BaseView) view).operations().current(); + String sqlText = viewSqlText(view, metadata); + result.setViewOriginalText(sqlText); + result.setViewExpandedText(sqlText); + + long nowMillis = System.currentTimeMillis(); + int nowSec = (int) (nowMillis / 1000); + String owner = + PropertyUtil.propertyAsString( + metadata.properties(), HiveCatalog.HMS_TABLE_OWNER, System.getProperty("user.name")); + result.setOwner(owner); + result.setCreateTime(nowSec); + result.setLastAccessTime(nowSec); + result.setRetention(Integer.MAX_VALUE); + + boolean hiveEngineEnabled = false; + result.setSd(HiveOperationsBase.storageDescriptor(metadata.schema(), metadata.location(), hiveEngineEnabled)); + StorageDescriptor sd = result.getSd(); + + if (sd.getBucketCols() == null) { + sd.setBucketCols(Lists.newArrayList()); + } + + if (sd.getSortCols() == null) { + sd.setSortCols(Lists.newArrayList()); + } + + long maxHiveTablePropertySize = + conf.getLong( + HiveOperationsBase.HIVE_TABLE_PROPERTY_MAX_SIZE, + HiveOperationsBase.HIVE_TABLE_PROPERTY_MAX_SIZE_DEFAULT); + HMSTablePropertyHelper.updateHmsTableForIcebergView( + metadata.metadataFileLocation(), + result, + metadata, + Collections.emptySet(), + maxHiveTablePropertySize, + null); + + String catalogType = IcebergCatalogProperties.getCatalogType(conf); + if (!StringUtils.isEmpty(catalogType) && !IcebergCatalogProperties.NO_CATALOG_TYPE.equals(catalogType)) { + result.getParameters().put(CatalogUtil.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.getCatalogType(conf)); + } + return result; + } + + private static String viewSqlText(View view, ViewMetadata metadata) { + SQLViewRepresentation hiveRepr = view.sqlFor("hive"); + if (hiveRepr != null) { + return hiveRepr.sql(); + } + return HiveViewOperations.sqlFor(metadata); + } + private static StorageDescriptor getHiveStorageDescriptor(org.apache.iceberg.Table table) { var result = new StorageDescriptor(); result.setCols(HiveSchemaUtil.convert(table.schema())); diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java index 4390d5a0bca1..86d51cf01bc3 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java @@ -21,12 +21,16 @@ import java.io.IOException; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Properties; +import java.util.Set; import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.Constants; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.CreateTableRequest; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.DropDatabaseRequest; @@ -38,15 +42,19 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.client.BaseMetaStoreClient; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; +import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.Schema; import org.apache.iceberg.SortOrder; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.catalog.ViewCatalog; import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.exceptions.NoSuchViewException; import org.apache.iceberg.hive.HMSTablePropertyHelper; import org.apache.iceberg.hive.HiveSchemaUtil; import org.apache.iceberg.hive.IcebergCatalogProperties; +import org.apache.iceberg.hive.IcebergNativeLogicalViewSupport; import org.apache.iceberg.hive.IcebergTableProperties; import org.apache.iceberg.hive.MetastoreUtil; import org.apache.iceberg.hive.RuntimeMetaException; @@ -54,6 +62,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.rest.RESTCatalog; +import org.apache.iceberg.view.View; import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -123,10 +132,20 @@ public List getTables(String catName, String dbName, String tablePattern Pattern pattern = Pattern.compile(regex); // List tables from the specific database (namespace) and filter them. - return restCatalog.listTables(Namespace.of(dbName)).stream() + Set names = new LinkedHashSet<>(); + restCatalog.listTables(Namespace.of(dbName)).stream() .map(TableIdentifier::name) .filter(pattern.asPredicate()) - .toList(); + .forEach(names::add); + + if (restCatalog instanceof ViewCatalog viewCatalog) { + viewCatalog + .listViews(Namespace.of(dbName)).stream() + .map(TableIdentifier::name) + .filter(pattern.asPredicate()) + .forEach(names::add); + } + return Lists.newArrayList(names); } @Override @@ -136,7 +155,12 @@ public List getAllTables(String catName, String dbName) { @Override public void dropTable(Table table, boolean deleteData, boolean ignoreUnknownTab, boolean ifPurge) throws TException { - restCatalog.dropTable(TableIdentifier.of(table.getDbName(), table.getTableName())); + TableIdentifier id = TableIdentifier.of(table.getDbName(), table.getTableName()); + if (restCatalog instanceof ViewCatalog viewCatalog && viewCatalog.viewExists(id)) { + viewCatalog.dropView(id); + } else { + restCatalog.dropTable(id); + } } private void validateCurrentCatalog(String catName) { @@ -149,7 +173,11 @@ private void validateCurrentCatalog(String catName) { @Override public boolean tableExists(String catName, String dbName, String tableName) { validateCurrentCatalog(catName); - return restCatalog.tableExists(TableIdentifier.of(dbName, tableName)); + TableIdentifier id = TableIdentifier.of(dbName, tableName); + if (restCatalog.tableExists(id)) { + return true; + } + return restCatalog instanceof ViewCatalog viewCatalog && viewCatalog.viewExists(id); } @Override @@ -178,38 +206,80 @@ public Database getDatabase(String catName, String dbName) throws NoSuchObjectEx @Override public Table getTable(GetTableRequest tableRequest) throws TException { validateCurrentCatalog(tableRequest.getCatName()); - org.apache.iceberg.Table icebergTable; + TableIdentifier id = + TableIdentifier.of(tableRequest.getDbName(), tableRequest.getTblName()); try { - icebergTable = restCatalog.loadTable(TableIdentifier.of(tableRequest.getDbName(), - tableRequest.getTblName())); - } catch (NoSuchTableException exception) { + org.apache.iceberg.Table icebergTable = restCatalog.loadTable(id); + return MetastoreUtil.toHiveTable(icebergTable, conf); + } catch (NoSuchTableException tableMissing) { + if (restCatalog instanceof ViewCatalog viewCatalog) { + try { + View icebergView = viewCatalog.loadView(id); + return MetastoreUtil.toHiveView(icebergView, conf); + } catch (NoSuchViewException viewMissing) { + throw new NoSuchObjectException(); + } + } throw new NoSuchObjectException(); } - return MetastoreUtil.toHiveTable(icebergTable, conf); + } + + private static boolean hasIcebergNativeViewTableType(Table table) { + if (!TableType.VIRTUAL_VIEW.toString().equals(table.getTableType())) { + return false; + } + Map params = table.getParameters(); + if (params == null) { + return false; + } + return IcebergNativeLogicalViewSupport.ICEBERG_VIEW_HMS_TABLE_TYPE_VALUE.equals( + params.get(BaseMetastoreTableOperations.TABLE_TYPE_PROP)); } @Override public void createTable(CreateTableRequest request) throws TException { Table table = request.getTable(); List cols = Lists.newArrayList(table.getSd().getCols()); + if (table.isSetPartitionKeys() && !table.getPartitionKeys().isEmpty()) { cols.addAll(table.getPartitionKeys()); } - Properties tableProperties = IcebergTableProperties.getTableProperties(table, conf); - Schema schema = HiveSchemaUtil.convert(cols, Collections.emptyMap(), true); - Map envCtxProps = Optional.ofNullable(request.getEnvContext()) - .map(EnvironmentContext::getProperties) - .orElse(Collections.emptyMap()); - org.apache.iceberg.PartitionSpec partitionSpec = - HMSTablePropertyHelper.getPartitionSpec(envCtxProps, schema); - SortOrder sortOrder = HMSTablePropertyHelper.getSortOrder(tableProperties, schema); - - restCatalog.buildTable(TableIdentifier.of(table.getDbName(), table.getTableName()), schema) - .withPartitionSpec(partitionSpec) - .withLocation(tableProperties.getProperty(IcebergTableProperties.LOCATION)) - .withSortOrder(sortOrder) - .withProperties(Maps.fromProperties(tableProperties)) - .create(); + + if (hasIcebergNativeViewTableType(table) && restCatalog instanceof ViewCatalog) { + Map envProps = + Optional.ofNullable(request.getEnvContext()) + .map(EnvironmentContext::getProperties) + .orElse(Collections.emptyMap()); + boolean replace = + Boolean.parseBoolean( + envProps.getOrDefault(Constants.EXTERNAL_LOGICAL_VIEW_DDL_REPLACE, "false")); + boolean ifNotExists = + Boolean.parseBoolean( + envProps.getOrDefault(Constants.EXTERNAL_LOGICAL_VIEW_CREATE_IF_NOT_EXISTS, "false")); + Map tblProps = + table.getParameters() == null ? Maps.newHashMap() : Maps.newHashMap(table.getParameters()); + String comment = tblProps.get("comment"); + + IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + conf, table.getDbName(), table.getTableName(), cols, table.getViewExpandedText(), tblProps, comment, replace, + ifNotExists); + } else { + Properties tableProperties = IcebergTableProperties.getTableProperties(table, conf); + Schema schema = HiveSchemaUtil.convert(cols, Collections.emptyMap(), true); + Map envCtxProps = Optional.ofNullable(request.getEnvContext()) + .map(EnvironmentContext::getProperties) + .orElse(Collections.emptyMap()); + org.apache.iceberg.PartitionSpec partitionSpec = + HMSTablePropertyHelper.getPartitionSpec(envCtxProps, schema); + SortOrder sortOrder = HMSTablePropertyHelper.getSortOrder(tableProperties, schema); + + restCatalog.buildTable(TableIdentifier.of(table.getDbName(), table.getTableName()), schema) + .withPartitionSpec(partitionSpec) + .withLocation(tableProperties.getProperty(IcebergTableProperties.LOCATION)) + .withSortOrder(sortOrder) + .withProperties(Maps.fromProperties(tableProperties)) + .create(); + } } @Override diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergNativeLogicalViewSupport.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergNativeLogicalViewSupport.java new file mode 100644 index 000000000000..449f2ac2aceb --- /dev/null +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergNativeLogicalViewSupport.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iceberg.hive; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; +import org.apache.iceberg.view.BaseView; +import org.apache.iceberg.view.View; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE; +import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE; +import static org.assertj.core.api.Assertions.assertThat; + +class TestIcebergNativeLogicalViewSupport { + + private static final String DB = "native_vw_db"; + private static final String VIEW = "native_vw"; + + @RegisterExtension + private static final HiveMetastoreExtension HIVE_METASTORE_EXTENSION = + HiveMetastoreExtension.builder().withDatabase(DB).build(); + + @AfterEach + void dropView() { + HiveCatalog cat = loadCatalog(); + TableIdentifier id = TableIdentifier.of(DB, VIEW); + cat.dropView(id); + } + + private HiveConf nativeViewConf() { + HiveConf conf = new HiveConf(HIVE_METASTORE_EXTENSION.hiveConf()); + MetastoreConf.setVar(conf, MetastoreConf.ConfVars.CATALOG_DEFAULT, "hive"); + conf.set( + IcebergCatalogProperties.catalogPropertyConfigKey("hive", ICEBERG_CATALOG_TYPE), + ICEBERG_CATALOG_TYPE_HIVE); + return conf; + } + + private HiveCatalog loadCatalog() { + return (HiveCatalog) + CatalogUtil.loadCatalog( + HiveCatalog.class.getName(), + ICEBERG_CATALOG_TYPE_HIVE, + ImmutableMap.of( + CatalogProperties.CLIENT_POOL_CACHE_EVICTION_INTERVAL_MS, + String.valueOf(TimeUnit.SECONDS.toMillis(10))), + HIVE_METASTORE_EXTENSION.hiveConf()); + } + + @Test + void testCreateCommitsNativeViewWithUserProperties() throws Exception { + HiveConf conf = nativeViewConf(); + List cols = + Arrays.asList(new FieldSchema("id", "int", null), new FieldSchema("name", "string", null)); + String sql = String.format("select id, name from %s.src_tbl", DB); + Map props = Collections.singletonMap("k1", "v1"); + + boolean created = + IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + conf, DB, VIEW, cols, sql, props, "hello-view", false, false); + assertThat(created).isTrue(); + + HiveCatalog cat = loadCatalog(); + TableIdentifier id = TableIdentifier.of(DB, VIEW); + assertThat(cat.viewExists(id)).isTrue(); + View view = cat.loadView(id); + assertThat(view.properties()) + .containsEntry("comment", "hello-view") + .containsEntry("k1", "v1") + .doesNotContainKey("hive.storage.external.logical.view.handler"); + HiveViewOperations ops = (HiveViewOperations) ((BaseView) view).operations(); + assertThat(ops.current().currentVersion().representations()).isNotEmpty(); + } + + @Test + void testCreateOrReplaceNativeViewSkipsWhenViewExistsAndIfNotExistsFlagTrue() throws Exception { + HiveConf conf = nativeViewConf(); + List cols = Collections.singletonList(new FieldSchema("id", "int", null)); + + assertThat( + IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + conf, DB, VIEW, cols, "select 1 as id", null, null, false, false)) + .isTrue(); + assertThat( + IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + conf, DB, VIEW, cols, "select 2 as id", null, null, false, true)) + .isFalse(); + } + + @Test + void testCreateOrReplaceNativeViewOrThrowDoesNotThrowWhenIfNotExistsSkips() { + HiveConf conf = nativeViewConf(); + List cols = Collections.singletonList(new FieldSchema("id", "int", null)); + TableIdentifier id = TableIdentifier.of(DB, VIEW); + + IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + conf, DB, VIEW, cols, "select 1 as id", null, null, false, false); + IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + conf, DB, VIEW, cols, "select 2 as id", null, null, false, true); + assertThat(loadCatalog().loadView(id).sqlFor("hive").sql().trim()).isEqualTo("select 1 as id"); + } + + @Test + void testCreateOrReplaceNativeViewReplacesExistingWhenReplaceFlagTrue() throws Exception { + HiveConf conf = nativeViewConf(); + List cols = Collections.singletonList(new FieldSchema("id", "int", null)); + TableIdentifier id = TableIdentifier.of(DB, VIEW); + + IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + conf, DB, VIEW, cols, "select 1 as id", null, null, false, false); + View afterCreate = loadCatalog().loadView(id); + assertThat(afterCreate.sqlFor("hive").sql().trim()).isEqualTo("select 1 as id"); + + assertThat( + IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + conf, DB, VIEW, cols, "select 2 as id", null, null, true, false)) + .isTrue(); + + assertThat(loadCatalog().viewExists(id)).isTrue(); + View afterReplace = loadCatalog().loadView(id); + assertThat(afterReplace.sqlFor("hive").sql().trim()).isEqualTo("select 2 as id"); + } +} diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java index 69fbe5bf99c2..a9492170a2e4 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java @@ -33,10 +33,13 @@ import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.metastore.HiveMetaHook; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.CreateTableRequest; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint; import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; @@ -44,6 +47,7 @@ import org.apache.hadoop.hive.ql.ddl.misc.sortoder.SortFields; import org.apache.hadoop.hive.ql.ddl.misc.sortoder.ZOrderFieldDesc; import org.apache.hadoop.hive.ql.ddl.misc.sortoder.ZOrderFields; +import org.apache.hadoop.hive.ql.session.SessionStateUtil; import org.apache.hadoop.hive.ql.util.NullOrdering; import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.NullOrder; @@ -62,6 +66,7 @@ import org.apache.iceberg.hive.HMSTablePropertyHelper; import org.apache.iceberg.hive.HiveSchemaUtil; import org.apache.iceberg.hive.IcebergCatalogProperties; +import org.apache.iceberg.hive.IcebergNativeLogicalViewSupport; import org.apache.iceberg.hive.IcebergTableProperties; import org.apache.iceberg.mr.Catalogs; import org.apache.iceberg.mr.InputFormatConfig; @@ -115,6 +120,16 @@ public BaseHiveIcebergMetaHook(Configuration conf) { this.conf = conf; } + public static boolean isNativeIcebergLogicalView(org.apache.hadoop.hive.metastore.api.Table hmsTable) { + if (hmsTable == null || + hmsTable.getParameters() == null || + !TableType.VIRTUAL_VIEW.toString().equals(hmsTable.getTableType())) { + return false; + } + String storageHandler = hmsTable.getParameters().get(hive_metastoreConstants.META_TABLE_STORAGE); + return HiveMetaHook.HIVE_ICEBERG_STORAGE_HANDLER.equals(storageHandler); + } + @Override public void preCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) { CreateTableRequest request = new CreateTableRequest(hmsTable); @@ -127,6 +142,10 @@ public void preCreateTable(CreateTableRequest request) { if (hmsTable.isTemporary()) { throw new UnsupportedOperationException("Creation of temporary iceberg tables is not supported."); } + if (isNativeIcebergLogicalView(hmsTable)) { + preCreateNativeIcebergLogicalView(request); + return; + } this.tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); // Set the table type even for non HiveCatalog based tables @@ -204,6 +223,26 @@ public void preCreateTable(CreateTableRequest request) { setSortOrder(hmsTable, schema, tableProperties); } + private void preCreateNativeIcebergLogicalView(CreateTableRequest request) { + org.apache.hadoop.hive.metastore.api.Table hmsTable = request.getTable(); + this.tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); + + if (request.getEnvContext() == null) { + request.setEnvContext(new EnvironmentContext()); + } + final EnvironmentContext env = request.getEnvContext(); + SessionStateUtil.getProperty(conf, Constants.EXTERNAL_LOGICAL_VIEW_DDL_REPLACE) + .ifPresent(v -> env.putToProperties(Constants.EXTERNAL_LOGICAL_VIEW_DDL_REPLACE, v)); + SessionStateUtil.getProperty(conf, Constants.EXTERNAL_LOGICAL_VIEW_CREATE_IF_NOT_EXISTS) + .ifPresent(v -> env.putToProperties(Constants.EXTERNAL_LOGICAL_VIEW_CREATE_IF_NOT_EXISTS, v)); + + hmsTable + .getParameters() + .put( + BaseMetastoreTableOperations.TABLE_TYPE_PROP, + IcebergNativeLogicalViewSupport.ICEBERG_VIEW_HMS_TABLE_TYPE_VALUE); + } + /** * Method for verification that necessary catalog configs are defined in Session Conf. * @@ -342,7 +381,7 @@ public void rollbackCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTa } @Override - public void commitCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) { + public void commitCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) throws MetaException { // do nothing } @@ -504,6 +543,9 @@ protected void setWriteModeDefaults(Table icebergTbl, Map newPro public void postGetTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) { if (hmsTable != null) { try { + if (isNativeIcebergLogicalView(hmsTable)) { + return; + } Table tbl = IcebergTableUtil.getTable(conf, hmsTable); String formatVersion = String.valueOf(TableUtil.formatVersion(tbl)); hmsTable.getParameters().put(TableProperties.FORMAT_VERSION, formatVersion); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index 58c2d19373dd..cfd64df238ed 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -37,6 +37,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.metastore.HiveMetaHook; import org.apache.hadoop.hive.metastore.PartitionDropOptions; import org.apache.hadoop.hive.metastore.Warehouse; @@ -117,6 +118,7 @@ import org.apache.iceberg.hive.HiveLock; import org.apache.iceberg.hive.HiveSchemaUtil; import org.apache.iceberg.hive.HiveTableOperations; +import org.apache.iceberg.hive.IcebergNativeLogicalViewSupport; import org.apache.iceberg.hive.IcebergTableProperties; import org.apache.iceberg.hive.MetastoreLock; import org.apache.iceberg.hive.NoLock; @@ -183,6 +185,32 @@ public void rollbackCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTa @Override public void commitCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) { + if (BaseHiveIcebergMetaHook.isNativeIcebergLogicalView(hmsTable)) { + tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); + if (Catalogs.hiveCatalog(conf, tableProperties)) { + boolean replace = + Boolean.parseBoolean( + SessionStateUtil.getProperty(conf, Constants.EXTERNAL_LOGICAL_VIEW_DDL_REPLACE).orElse("false")); + boolean ifNotExists = + Boolean.parseBoolean( + SessionStateUtil.getProperty(conf, Constants.EXTERNAL_LOGICAL_VIEW_CREATE_IF_NOT_EXISTS) + .orElse("false")); + Map tblProps = + hmsTable.getParameters() == null ? Maps.newHashMap() : Maps.newHashMap(hmsTable.getParameters()); + String comment = tblProps.get("comment"); + IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + conf, + hmsTable.getDbName(), + hmsTable.getTableName(), + hmsTable.getSd().getCols(), + hmsTable.getViewExpandedText(), + tblProps, + comment, + replace, + ifNotExists); + } + return; + } if (icebergTable == null) { setFileFormat(tableProperties.getProperty(TableProperties.DEFAULT_FILE_FORMAT)); @@ -265,6 +293,15 @@ public void commitDropTable(org.apache.hadoop.hive.metastore.api.Table hmsTable, @Override public void preAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable, EnvironmentContext context) throws MetaException { + if (BaseHiveIcebergMetaHook.isNativeIcebergLogicalView(hmsTable)) { + currentAlterTableOp = null; + if (commitLock == null) { + commitLock = new NoLock(); + } + commitLock.lock(); + tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); + return; + } tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); setupAlterOperationType(hmsTable, context); if (AlterTableType.RENAME.equals(currentAlterTableOp)) { @@ -494,6 +531,23 @@ public void commitAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable throw new IllegalStateException("Hive commit lock should already be set"); } commitLock.unlock(); + if (BaseHiveIcebergMetaHook.isNativeIcebergLogicalView(hmsTable)) { + tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); + Map tblProps = + hmsTable.getParameters() == null ? Maps.newHashMap() : Maps.newHashMap(hmsTable.getParameters()); + String comment = tblProps.get("comment"); + IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + conf, + hmsTable.getDbName(), + hmsTable.getTableName(), + hmsTable.getSd().getCols(), + hmsTable.getViewExpandedText(), + tblProps, + comment, + true, + false); + return; + } if (isTableMigration) { tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); tableProperties.put(InputFormatConfig.TABLE_SCHEMA, SchemaParser.toJson(preAlterTableProperties.schema)); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 65852f1a8553..4a4c66fe3f10 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -383,6 +383,11 @@ public boolean supportsPartitioning() { return true; } + @Override + public boolean supportsExternalLogicalViewCatalog() { + return true; + } + /** * @param jobConf Job configuration for InputFormat to access * @param deserializer Deserializer @@ -428,6 +433,9 @@ public boolean canProvidePartitionStatistics(org.apache.hadoop.hive.ql.metadata. if (!getStatsSource().equals(HiveMetaHook.ICEBERG)) { return false; } + if (BaseHiveIcebergMetaHook.isNativeIcebergLogicalView(hmsTable.getTTable())) { + return false; + } Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); Snapshot snapshot = IcebergTableUtil.getTableSnapshot(table, hmsTable); if (snapshot != null) { @@ -891,6 +899,9 @@ public boolean supportsPartitionTransform() { @Override public List getPartitionTransformSpec(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { + if (BaseHiveIcebergMetaHook.isNativeIcebergLogicalView(hmsTable.getTTable())) { + return Collections.emptyList(); + } Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); return table.spec().fields().stream() .filter(f -> !f.transform().isVoid()) @@ -905,6 +916,9 @@ public List getPartitionTransformSpec(org.apache.hadoop.hive.ql.m @Override public Map> getPartitionTransformSpecs( org.apache.hadoop.hive.ql.metadata.Table hmsTable) { + if (BaseHiveIcebergMetaHook.isNativeIcebergLogicalView(hmsTable.getTTable())) { + return Collections.emptyMap(); + } Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); return table.specs().entrySet().stream().flatMap(e -> e.getValue().fields().stream() @@ -1570,6 +1584,9 @@ public boolean supportsSortColumns() { @Override public List sortColumns(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { + if (BaseHiveIcebergMetaHook.isNativeIcebergLogicalView(hmsTable.getTTable())) { + return Collections.emptyList(); + } TableDesc tableDesc = Utilities.getTableDesc(hmsTable); Table table = IcebergTableUtil.getTable(conf, tableDesc.getProperties()); if (table.sortOrder().isUnsorted()) { @@ -2133,6 +2150,10 @@ public List getPartitions(org.apache.hadoop.hive.ql.metadata.Table hm } public boolean isPartitioned(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { + if (BaseHiveIcebergMetaHook.isNativeIcebergLogicalView(hmsTable.getTTable())) { + List partCols = hmsTable.getPartCols(); + return partCols != null && !partCols.isEmpty(); + } if (!hmsTable.getTTable().isSetId()) { return false; } @@ -2278,6 +2299,10 @@ public boolean canPerformMetadataDelete(org.apache.hadoop.hive.ql.metadata.Table @Override public List getPartitionKeys(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { + if (BaseHiveIcebergMetaHook.isNativeIcebergLogicalView(hmsTable.getTTable())) { + List partCols = hmsTable.getPartCols(); + return partCols != null ? partCols : Collections.emptyList(); + } if (!hmsTable.getTTable().isSetId()) { return Collections.emptyList(); } diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q new file mode 100644 index 000000000000..116248ed623e --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q @@ -0,0 +1,78 @@ +-- SORT_QUERY_RESULTS +-- Mask random uuid +--! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/ + +create database ice_native_view_db; +use ice_native_view_db; + +create table src_ice ( + first_name string, + last_name string + ) +partitioned by (dept_id bigint) +stored by iceberg stored as orc; + +INSERT INTO src_ice VALUES + ('fn1','ln1', 1), + ('fn2','ln2', 1), + ('fn3','ln3', 1), + ('fn4','ln4', 1), + ('fn5','ln5', 2), + ('fn6','ln6', 2), + ('fn7','ln7', 2); + +------------------------------------------------------------------------------- +-- Native Iceberg view via TBLPROPERTIES +------------------------------------------------------------------------------- + +-- TEST VIEW CREATION -- + +create view v_ice tblproperties ('view-format'='iceberg') +as select * from src_ice; + +select * from v_ice; + +-- TEST VIEW REPLACEMENT -- + +create or replace view v_ice tblproperties ('view-format'='iceberg') +as select first_name || '-' || dept_id from src_ice where dept_id = 1; + +select * from v_ice; + +desc formatted v_ice; +drop view v_ice; + +------------------------------------------------------------------------------- +-- Native Iceberg view when default storage handler is Iceberg +-- and no 'view-format' property in TBLPROPERTIES +------------------------------------------------------------------------------- + +set hive.default.storage.handler.class=org.apache.iceberg.mr.hive.HiveIcebergStorageHandler; + +-- TEST VIEW CREATION WITH IF EXISTS -- + +create view if not exists v_def +as select first_name, last_name, dept_id from src_ice where dept_id = 2; + +select * from v_def; + +-- TEST VIEW IS NOT CREATED BECAUSE IT ALREADY EXISTS -- + +create view if not exists v_def +as select first_name, last_name, dept_id from src_ice; + +select * from v_def; + +desc formatted v_def; +drop view v_def; + +----------------------------------------------------------------------------------------- +-- Classic Hive view when the base table is Iceberg and default storage handler is unset +----------------------------------------------------------------------------------------- + +set hive.default.storage.handler.class=; + +create view v_hive as select * from src_ice; +select * from v_hive; +desc formatted v_hive; +drop view v_hive; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q index 81982ca44d98..2a029fd0432a 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q @@ -15,8 +15,6 @@ --! qt:replace:/(\s+current-snapshot-timestamp-ms\s+)\S+(\s*)/$1#Masked#$2/ --! qt:replace:/(MAJOR\s+succeeded\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/ --! qt:replace:/(MAJOR\s+refused\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/ --- Mask compaction id as they will be allocated in parallel threads ---! qt:replace:/^[0-9]/#Masked#/ -- Mask removed file size --! qt:replace:/(\S\"removed-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/ -- Mask iceberg version @@ -47,7 +45,7 @@ partitioned by (company_id bigint) stored by iceberg stored as orc; ----------------------------------------------------------------------------- ---! Creating table with a valid catalog name in table properties +--! Creating a table with a valid catalog name in table properties ----------------------------------------------------------------------------- create table ice_orc2 ( @@ -74,6 +72,52 @@ VALUES ('fn1','ln1', 1, 10), ('fn2','ln2', 2, 20), ('fn3','ln3', 3, 30); describe formatted ice_orc2; select * from ice_orc2; +--------------------------------------------------------------------------------------------------------------------- +--! Iceberg Native View tests +--------------------------------------------------------------------------------------------------------------------- + +----------------------------------------------------------------------------------------------- +--! Native Iceberg view with TBLPROPERTIES ('view-format'='iceberg') on a REST catalog table +--! without a catalog name in the base table properties +----------------------------------------------------------------------------------------------- + +create view ice_v1 tblproperties ('view-format'='iceberg') as select * from ice_orc1; +select * from ice_v1; +desc formatted ice_v1; +drop view ice_v1; + +----------------------------------------------------------------------------------------------- +--! Iceberg native view with TBLPROPERTIES ('view-format'='iceberg') on a REST catalog table +--! with a catalog name in the base table properties +----------------------------------------------------------------------------------------------- + +create view ice_v2 tblproperties ('view-format'='iceberg') as select * from ice_orc2; +select * from ice_v2; +desc formatted ice_v2; +drop view ice_v2; + +----------------------------------------------------------------------------------------------- +--! Native Iceberg view: 'view-format' table properly omitted, Hive config 'hive.default.storage.handler.class' +--! set to 'HiveIcebergStorageHandler' and no catalog name in the base table properties +----------------------------------------------------------------------------------------------- + +set hive.default.storage.handler.class=org.apache.iceberg.mr.hive.HiveIcebergStorageHandler; + +create view ice_v3 as select * from ice_orc1; +select * from ice_v3; +desc formatted ice_v3; +drop view ice_v3; + +----------------------------------------------------------------------------------------------- +--! Native Iceberg view with default Iceberg storage handler and REST catalog table +--! with catalog name in the base table properties +----------------------------------------------------------------------------------------------- + +create view ice_v4 as select * from ice_orc2; +select * from ice_v4; +desc formatted ice_v4; +drop view ice_v4; + ----------------------------------------------------------------------------- show tables; diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q index 27f23122240b..2ad2fd628028 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q @@ -15,8 +15,6 @@ --! qt:replace:/(\s+current-snapshot-timestamp-ms\s+)\S+(\s*)/$1#Masked#$2/ --! qt:replace:/(MAJOR\s+succeeded\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/ --! qt:replace:/(MAJOR\s+refused\s+)[a-zA-Z0-9\-\.\s+]+(\s+manual)/$1#Masked#$2/ --- Mask compaction id as they will be allocated in parallel threads ---! qt:replace:/^[0-9]/#Masked#/ -- Mask removed file size --! qt:replace:/(\S\"removed-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/ -- Mask iceberg version @@ -47,7 +45,7 @@ partitioned by (company_id bigint) stored by iceberg stored as orc; ----------------------------------------------------------------------------- ---! Creating table with a valid catalog name in table properties +--! Creating a table with a valid catalog name in table properties ----------------------------------------------------------------------------- create table ice_orc2 ( @@ -69,6 +67,52 @@ VALUES ('fn1','ln1', 1, 10), ('fn2','ln2', 2, 20), ('fn3','ln3', 3, 30); describe formatted ice_orc2; select * from ice_orc2; +--------------------------------------------------------------------------------------------------------------------- +--! Iceberg Native View tests +--------------------------------------------------------------------------------------------------------------------- + +----------------------------------------------------------------------------------------------- +--! Native Iceberg view with TBLPROPERTIES ('view-format'='iceberg') on a REST catalog table +--! without a catalog name in the base table properties +----------------------------------------------------------------------------------------------- + +create view ice_v1 tblproperties ('view-format'='iceberg') as select * from ice_orc1; +select * from ice_v1; +desc formatted ice_v1; +drop view ice_v1; + +----------------------------------------------------------------------------------------------- +--! Iceberg native view with TBLPROPERTIES ('view-format'='iceberg') on a REST catalog table +--! with a catalog name in the base table properties +----------------------------------------------------------------------------------------------- + +create view ice_v2 tblproperties ('view-format'='iceberg') as select * from ice_orc2; +select * from ice_v2; +desc formatted ice_v2; +drop view ice_v2; + +----------------------------------------------------------------------------------------------- +--! Native Iceberg view: 'view-format' table properly omitted, Hive config 'hive.default.storage.handler.class' +--! set to 'HiveIcebergStorageHandler' and no catalog name in the base table properties +----------------------------------------------------------------------------------------------- + +set hive.default.storage.handler.class=org.apache.iceberg.mr.hive.HiveIcebergStorageHandler; + +create view ice_v3 as select * from ice_orc1; +select * from ice_v3; +desc formatted ice_v3; +drop view ice_v3; + +----------------------------------------------------------------------------------------------- +--! Native Iceberg view with default Iceberg storage handler and REST catalog table +--! with catalog name in the base table properties +----------------------------------------------------------------------------------------------- + +create view ice_v4 as select * from ice_orc2; +select * from ice_v4; +desc formatted ice_v4; +drop view ice_v4; + ----------------------------------------------------------------------------- show tables; diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_native_view.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_native_view.q.out new file mode 100644 index 000000000000..d9082ecb068e --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_native_view.q.out @@ -0,0 +1,316 @@ +PREHOOK: query: create database ice_native_view_db +PREHOOK: type: CREATEDATABASE +PREHOOK: Output: database:ice_native_view_db +POSTHOOK: query: create database ice_native_view_db +POSTHOOK: type: CREATEDATABASE +POSTHOOK: Output: database:ice_native_view_db +PREHOOK: query: use ice_native_view_db +PREHOOK: type: SWITCHDATABASE +PREHOOK: Input: database:ice_native_view_db +POSTHOOK: query: use ice_native_view_db +POSTHOOK: type: SWITCHDATABASE +POSTHOOK: Input: database:ice_native_view_db +PREHOOK: query: create table src_ice ( + first_name string, + last_name string + ) +partitioned by (dept_id bigint) +stored by iceberg stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: create table src_ice ( + first_name string, + last_name string + ) +partitioned by (dept_id bigint) +stored by iceberg stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: INSERT INTO src_ice VALUES + ('fn1','ln1', 1), + ('fn2','ln2', 1), + ('fn3','ln3', 1), + ('fn4','ln4', 1), + ('fn5','ln5', 2), + ('fn6','ln6', 2), + ('fn7','ln7', 2) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: ice_native_view_db@src_ice +POSTHOOK: query: INSERT INTO src_ice VALUES + ('fn1','ln1', 1), + ('fn2','ln2', 1), + ('fn3','ln3', 1), + ('fn4','ln4', 1), + ('fn5','ln5', 2), + ('fn6','ln6', 2), + ('fn7','ln7', 2) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: ice_native_view_db@src_ice +PREHOOK: query: create view v_ice tblproperties ('view-format'='iceberg') +as select * from src_ice +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@v_ice +POSTHOOK: query: create view v_ice tblproperties ('view-format'='iceberg') +as select * from src_ice +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_ice +POSTHOOK: Lineage: v_ice.dept_id SIMPLE [(src_ice)src_ice.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: v_ice.first_name SIMPLE [(src_ice)src_ice.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: v_ice.last_name SIMPLE [(src_ice)src_ice.FieldSchema(name:last_name, type:string, comment:null), ] +PREHOOK: query: select * from v_ice +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_ice +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from v_ice +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_ice +POSTHOOK: Output: hdfs://### HDFS PATH ### +fn1 ln1 1 +fn2 ln2 1 +fn3 ln3 1 +fn4 ln4 1 +fn5 ln5 2 +fn6 ln6 2 +fn7 ln7 2 +PREHOOK: query: create or replace view v_ice tblproperties ('view-format'='iceberg') +as select first_name || '-' || dept_id from src_ice where dept_id = 1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@v_ice +POSTHOOK: query: create or replace view v_ice tblproperties ('view-format'='iceberg') +as select first_name || '-' || dept_id from src_ice where dept_id = 1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_ice +PREHOOK: query: select * from v_ice +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_ice +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from v_ice +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_ice +POSTHOOK: Output: hdfs://### HDFS PATH ### +fn1-1 +fn2-1 +fn3-1 +fn4-1 +PREHOOK: query: desc formatted v_ice +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_native_view_db@v_ice +POSTHOOK: query: desc formatted v_ice +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_native_view_db@v_ice +# col_name data_type comment +_c0 string + +# Detailed Table Information +Database: ice_native_view_db +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":1,\"fields\":[{\"id\":1,\"name\":\"_c0\",\"required\":false,\"type\":\"string\"}]} + metadata_location hdfs://### HDFS PATH ### + previous_metadata_location hdfs://### HDFS PATH ### + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG-VIEW +#### A masked pattern was here #### + uuid #Masked# + view-format iceberg + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Sort Columns: [] + +# View Information +Original Query: select first_name || '-' || dept_id from src_ice where dept_id = 1 +Expanded Query: select `src_ice`.`first_name` || '-' || `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` where `src_ice`.`dept_id` = 1 +PREHOOK: query: drop view v_ice +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_native_view_db@v_ice +PREHOOK: Output: ice_native_view_db@v_ice +POSTHOOK: query: drop view v_ice +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_native_view_db@v_ice +POSTHOOK: Output: ice_native_view_db@v_ice +PREHOOK: query: create view if not exists v_def +as select first_name, last_name, dept_id from src_ice where dept_id = 2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@v_def +POSTHOOK: query: create view if not exists v_def +as select first_name, last_name, dept_id from src_ice where dept_id = 2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_def +POSTHOOK: Lineage: v_def.dept_id SIMPLE [] +POSTHOOK: Lineage: v_def.first_name SIMPLE [(src_ice)src_ice.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: v_def.last_name SIMPLE [(src_ice)src_ice.FieldSchema(name:last_name, type:string, comment:null), ] +PREHOOK: query: select * from v_def +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_def +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from v_def +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_def +POSTHOOK: Output: hdfs://### HDFS PATH ### +fn5 ln5 2 +fn6 ln6 2 +fn7 ln7 2 +PREHOOK: query: create view if not exists v_def +as select first_name, last_name, dept_id from src_ice +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@v_def +POSTHOOK: query: create view if not exists v_def +as select first_name, last_name, dept_id from src_ice +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_def +PREHOOK: query: select * from v_def +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_def +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from v_def +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_def +POSTHOOK: Output: hdfs://### HDFS PATH ### +fn5 ln5 2 +fn6 ln6 2 +fn7 ln7 2 +PREHOOK: query: desc formatted v_def +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_native_view_db@v_def +POSTHOOK: query: desc formatted v_def +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_native_view_db@v_def +# col_name data_type comment +first_name string +last_name string +dept_id bigint + +# Detailed Table Information +Database: ice_native_view_db +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG-VIEW +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Sort Columns: [] + +# View Information +Original Query: select first_name, last_name, dept_id from src_ice where dept_id = 2 +Expanded Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` where `src_ice`.`dept_id` = 2 +PREHOOK: query: drop view v_def +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_native_view_db@v_def +PREHOOK: Output: ice_native_view_db@v_def +POSTHOOK: query: drop view v_def +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_native_view_db@v_def +POSTHOOK: Output: ice_native_view_db@v_def +PREHOOK: query: create view v_hive as select * from src_ice +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@v_hive +POSTHOOK: query: create view v_hive as select * from src_ice +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_hive +POSTHOOK: Lineage: v_hive.dept_id SIMPLE [(src_ice)src_ice.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: v_hive.first_name SIMPLE [(src_ice)src_ice.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: v_hive.last_name SIMPLE [(src_ice)src_ice.FieldSchema(name:last_name, type:string, comment:null), ] +PREHOOK: query: select * from v_hive +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_hive +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from v_hive +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_hive +POSTHOOK: Output: hdfs://### HDFS PATH ### +fn1 ln1 1 +fn2 ln2 1 +fn3 ln3 1 +fn4 ln4 1 +fn5 ln5 2 +fn6 ln6 2 +fn7 ln7 2 +PREHOOK: query: desc formatted v_hive +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_native_view_db@v_hive +POSTHOOK: query: desc formatted v_hive +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_native_view_db@v_hive +# col_name data_type comment +first_name string +last_name string +dept_id bigint + +# Detailed Table Information +Database: ice_native_view_db +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: null +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select * from src_ice +Expanded Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` +PREHOOK: query: drop view v_hive +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_native_view_db@v_hive +PREHOOK: Output: ice_native_view_db@v_hive +POSTHOOK: query: drop view v_hive +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_native_view_db@v_hive +POSTHOOK: Output: ice_native_view_db@v_hive diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out index 8bba659e8fd1..4125fef590c9 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out @@ -68,8 +68,6 @@ CREATE EXTERNAL TABLE `ice_orc2`( `dept_id` bigint, `team_id` bigint, `company_id` bigint) -PARTITIONED BY ( - `company_id` bigint COMMENT 'Transform: identity') PARTITIONED BY SPEC ( `company_id`) ROW FORMAT SERDE @@ -180,6 +178,294 @@ POSTHOOK: Input: ice_rest@ice_orc2 fn1 ln1 1 10 100 fn2 ln2 2 20 100 fn3 ln3 3 30 100 +PREHOOK: query: create view ice_v1 tblproperties ('view-format'='iceberg') as select * from ice_orc1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: create view ice_v1 tblproperties ('view-format'='iceberg') as select * from ice_orc1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v1 +POSTHOOK: Lineage: ice_v1.company_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:company_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v1.dept_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v1.first_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v1.last_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v1.team_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v1 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +PREHOOK: query: desc formatted ice_v1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v1 +POSTHOOK: query: desc formatted ice_v1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v1 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} +#### A masked pattern was here #### + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG-VIEW + type rest + uuid #Masked# + view-format iceberg + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Sort Columns: [] + +# View Information +Original Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +Expanded Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +PREHOOK: query: drop view ice_v1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v1 +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: drop view ice_v1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v1 +POSTHOOK: Output: ice_rest@ice_v1 +PREHOOK: query: create view ice_v2 tblproperties ('view-format'='iceberg') as select * from ice_orc2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v2 +POSTHOOK: query: create view ice_v2 tblproperties ('view-format'='iceberg') as select * from ice_orc2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v2 +POSTHOOK: Lineage: ice_v2.company_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:company_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v2.dept_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v2.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v2.last_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v2.team_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v2 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Input: ice_rest@ice_v2 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v2 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Input: ice_rest@ice_v2 +#### A masked pattern was here #### +fn1 ln1 1 10 100 +fn2 ln2 2 20 100 +fn3 ln3 3 30 100 +PREHOOK: query: desc formatted ice_v2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v2 +POSTHOOK: query: desc formatted ice_v2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v2 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} +#### A masked pattern was here #### + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG-VIEW + type rest + uuid #Masked# + view-format iceberg + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Sort Columns: [] + +# View Information +Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +PREHOOK: query: drop view ice_v2 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v2 +PREHOOK: Output: ice_rest@ice_v2 +POSTHOOK: query: drop view ice_v2 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v2 +POSTHOOK: Output: ice_rest@ice_v2 +PREHOOK: query: create view ice_v3 as select * from ice_orc1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v3 +POSTHOOK: query: create view ice_v3 as select * from ice_orc1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v3 +POSTHOOK: Lineage: ice_v3.company_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:company_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v3.dept_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v3.first_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v3.last_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v3.team_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v3 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Input: ice_rest@ice_v3 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v3 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Input: ice_rest@ice_v3 +#### A masked pattern was here #### +PREHOOK: query: desc formatted ice_v3 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v3 +POSTHOOK: query: desc formatted ice_v3 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v3 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} +#### A masked pattern was here #### + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Sort Columns: [] + +# View Information +Original Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +Expanded Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +PREHOOK: query: drop view ice_v3 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v3 +PREHOOK: Output: ice_rest@ice_v3 +POSTHOOK: query: drop view ice_v3 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v3 +POSTHOOK: Output: ice_rest@ice_v3 +PREHOOK: query: create view ice_v4 as select * from ice_orc2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v4 +POSTHOOK: query: create view ice_v4 as select * from ice_orc2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v4 +POSTHOOK: Lineage: ice_v4.company_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:company_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v4.dept_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v4.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v4.last_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v4.team_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v4 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Input: ice_rest@ice_v4 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v4 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Input: ice_rest@ice_v4 +#### A masked pattern was here #### +fn1 ln1 1 10 100 +fn2 ln2 2 20 100 +fn3 ln3 3 30 100 +PREHOOK: query: desc formatted ice_v4 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v4 +POSTHOOK: query: desc formatted ice_v4 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v4 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} +#### A masked pattern was here #### + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Sort Columns: [] + +# View Information +Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +PREHOOK: query: drop view ice_v4 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v4 +PREHOOK: Output: ice_rest@ice_v4 +POSTHOOK: query: drop view ice_v4 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v4 +POSTHOOK: Output: ice_rest@ice_v4 PREHOOK: query: show tables PREHOOK: type: SHOWTABLES PREHOOK: Input: database:ice_rest diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out index 409eb484480b..5b6b321c6d8b 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out @@ -68,8 +68,6 @@ CREATE EXTERNAL TABLE `ice_orc2`( `dept_id` bigint, `team_id` bigint, `company_id` bigint) -PARTITIONED BY ( - `company_id` bigint COMMENT 'Transform: identity') PARTITIONED BY SPEC ( `company_id`) ROW FORMAT SERDE @@ -180,6 +178,294 @@ POSTHOOK: Input: ice_rest@ice_orc2 fn1 ln1 1 10 100 fn2 ln2 2 20 100 fn3 ln3 3 30 100 +PREHOOK: query: create view ice_v1 tblproperties ('view-format'='iceberg') as select * from ice_orc1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: create view ice_v1 tblproperties ('view-format'='iceberg') as select * from ice_orc1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v1 +POSTHOOK: Lineage: ice_v1.company_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:company_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v1.dept_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v1.first_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v1.last_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v1.team_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v1 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +PREHOOK: query: desc formatted ice_v1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v1 +POSTHOOK: query: desc formatted ice_v1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v1 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} +#### A masked pattern was here #### + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG-VIEW + type rest + uuid #Masked# + view-format iceberg + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Sort Columns: [] + +# View Information +Original Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +Expanded Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +PREHOOK: query: drop view ice_v1 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v1 +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: drop view ice_v1 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v1 +POSTHOOK: Output: ice_rest@ice_v1 +PREHOOK: query: create view ice_v2 tblproperties ('view-format'='iceberg') as select * from ice_orc2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v2 +POSTHOOK: query: create view ice_v2 tblproperties ('view-format'='iceberg') as select * from ice_orc2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v2 +POSTHOOK: Lineage: ice_v2.company_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:company_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v2.dept_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v2.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v2.last_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v2.team_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v2 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Input: ice_rest@ice_v2 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v2 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Input: ice_rest@ice_v2 +#### A masked pattern was here #### +fn1 ln1 1 10 100 +fn2 ln2 2 20 100 +fn3 ln3 3 30 100 +PREHOOK: query: desc formatted ice_v2 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v2 +POSTHOOK: query: desc formatted ice_v2 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v2 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} +#### A masked pattern was here #### + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG-VIEW + type rest + uuid #Masked# + view-format iceberg + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Sort Columns: [] + +# View Information +Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +PREHOOK: query: drop view ice_v2 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v2 +PREHOOK: Output: ice_rest@ice_v2 +POSTHOOK: query: drop view ice_v2 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v2 +POSTHOOK: Output: ice_rest@ice_v2 +PREHOOK: query: create view ice_v3 as select * from ice_orc1 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v3 +POSTHOOK: query: create view ice_v3 as select * from ice_orc1 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v3 +POSTHOOK: Lineage: ice_v3.company_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:company_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v3.dept_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v3.first_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v3.last_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v3.team_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v3 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Input: ice_rest@ice_v3 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v3 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Input: ice_rest@ice_v3 +#### A masked pattern was here #### +PREHOOK: query: desc formatted ice_v3 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v3 +POSTHOOK: query: desc formatted ice_v3 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v3 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} +#### A masked pattern was here #### + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Sort Columns: [] + +# View Information +Original Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +Expanded Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +PREHOOK: query: drop view ice_v3 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v3 +PREHOOK: Output: ice_rest@ice_v3 +POSTHOOK: query: drop view ice_v3 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v3 +POSTHOOK: Output: ice_rest@ice_v3 +PREHOOK: query: create view ice_v4 as select * from ice_orc2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v4 +POSTHOOK: query: create view ice_v4 as select * from ice_orc2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v4 +POSTHOOK: Lineage: ice_v4.company_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:company_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v4.dept_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v4.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v4.last_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v4.team_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:team_id, type:bigint, comment:null), ] +PREHOOK: query: select * from ice_v4 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Input: ice_rest@ice_v4 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v4 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Input: ice_rest@ice_v4 +#### A masked pattern was here #### +fn1 ln1 1 10 100 +fn2 ln2 2 20 100 +fn3 ln3 3 30 100 +PREHOOK: query: desc formatted ice_v4 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v4 +POSTHOOK: query: desc formatted ice_v4 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v4 +# col_name data_type comment +first_name string +last_name string +dept_id bigint +team_id bigint +company_id bigint + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} +#### A masked pattern was here #### + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG-VIEW + type rest + uuid #Masked# + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Sort Columns: [] + +# View Information +Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +PREHOOK: query: drop view ice_v4 +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_rest@ice_v4 +PREHOOK: Output: ice_rest@ice_v4 +POSTHOOK: query: drop view ice_v4 +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_rest@ice_v4 +POSTHOOK: Output: ice_rest@ice_v4 PREHOOK: query: show tables PREHOOK: type: SHOWTABLES PREHOOK: Input: database:ice_rest diff --git a/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java index fd30223c9934..66cd4504b271 100644 --- a/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java +++ b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; @@ -38,13 +39,15 @@ import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.mapred.TextInputFormat; +import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.TableProperties; -import org.apache.iceberg.hive.IcebergCatalogProperties; import org.apache.iceberg.hive.HiveSchemaUtil; +import org.apache.iceberg.hive.IcebergCatalogProperties; +import org.apache.iceberg.hive.IcebergNativeLogicalViewSupport; import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -67,11 +70,14 @@ public abstract class TestHiveRESTCatalogClientITBase { static final String DB_NAME = "ice_db"; + static final String VIEW_DB_NAME = "ice_db_view"; + static final String NATIVE_VIEW_NAME = "native_rest_v"; static final String TABLE_NAME = "ice_tbl"; static final String CATALOG_NAME = "ice01"; static final String HIVE_ICEBERG_STORAGE_HANDLER = "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler"; static final String REST_CATALOG_PREFIX = String.format("%s%s.", IcebergCatalogProperties.CATALOG_CONFIG_PREFIX, CATALOG_NAME); + static final String ICEBERG_VIEW_TYPE_VALUE = "iceberg-view"; HiveConf hiveConf; Configuration conf; @@ -173,8 +179,10 @@ public void testIceberg() throws Exception { Assertions.assertEquals(TABLE_NAME, table.getTableName()); Assertions.assertEquals(HIVE_ICEBERG_STORAGE_HANDLER, table.getParameters().get("storage_handler")); Assertions.assertNotNull(table.getParameters().get(TableProperties.DEFAULT_PARTITION_SPEC)); - Assertions.assertEquals(1, table.getPartitionKeys().size()); - Assertions.assertEquals("city", table.getPartitionKeys().getFirst().getName()); + Assertions.assertTrue(table.getPartitionKeys().isEmpty()); + List columnNames = + table.getSd().getCols().stream().map(FieldSchema::getName).toList(); + Assertions.assertTrue(columnNames.contains("city")); // --- Get Tables --- List tables = msClient.getTables(CATALOG_NAME, DB_NAME, "ice_*"); @@ -195,6 +203,50 @@ public void testIceberg() throws Exception { Assertions.assertFalse(msClient.getAllDatabases(CATALOG_NAME).contains(DB_NAME)); } + @Test + public void testNativeIcebergLogicalView() throws Exception { + Database db = new Database(); + db.setCatalogName(CATALOG_NAME); + db.setName(VIEW_DB_NAME); + db.setOwnerType(PrincipalType.USER); + db.setOwnerName(System.getProperty("user.name")); + String warehouseDir = MetastoreConf.get(conf, MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL.getVarname()); + db.setLocationUri(warehouseDir + "/" + VIEW_DB_NAME + ".db"); + hive.createDatabase(db, true); + + List cols = Collections.singletonList(new FieldSchema("x", "int", "")); + IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + hiveConf, + VIEW_DB_NAME, + NATIVE_VIEW_NAME, + cols, + "select 1 as x", + null, + "rest-native-view", + false, + false); + + GetTableRequest getTableRequest = new GetTableRequest(); + getTableRequest.setCatName(CATALOG_NAME); + getTableRequest.setDbName(VIEW_DB_NAME); + getTableRequest.setTblName(NATIVE_VIEW_NAME); + Table view = msClient.getTable(getTableRequest); + Assertions.assertEquals(TableType.VIRTUAL_VIEW.name(), view.getTableType()); + String tableTypeProp = view.getParameters().get(BaseMetastoreTableOperations.TABLE_TYPE_PROP); + Assertions.assertNotNull(tableTypeProp); + Assertions.assertEquals(ICEBERG_VIEW_TYPE_VALUE, tableTypeProp.toLowerCase()); + + Assertions.assertTrue(msClient.tableExists(CATALOG_NAME, VIEW_DB_NAME, NATIVE_VIEW_NAME)); + + List names = msClient.getTables(CATALOG_NAME, VIEW_DB_NAME, "*"); + Assertions.assertTrue(names.contains(NATIVE_VIEW_NAME)); + + msClient.dropTable(CATALOG_NAME, VIEW_DB_NAME, NATIVE_VIEW_NAME); + Assertions.assertFalse(msClient.tableExists(CATALOG_NAME, VIEW_DB_NAME, NATIVE_VIEW_NAME)); + + msClient.dropDatabase(VIEW_DB_NAME); + } + private static Table createPartitionedTable(IMetaStoreClient db, String catName, String dbName, String tableName, Map tableParameters) throws Exception { db.dropTable(catName, dbName, tableName); diff --git a/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseCreateView.java b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseCreateView.java new file mode 100644 index 000000000000..cab826e5963b --- /dev/null +++ b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseCreateView.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hive.ql.parse; + +import org.junit.Test; + +import static org.junit.Assert.assertTrue; + +public class TestParseCreateView { + + private final ParseDriver parseDriver = new ParseDriver(); + + @Test + public void testParseCreateViewWithTableProperties() throws Exception { + ASTNode tree = + parseDriver + .parse( + "create view v1 tblproperties ('view-format'='iceberg') as select * from t", null) + .getTree(); + assertTrue(tree.dump(), tree.toStringTree().contains("tok_createview")); + assertTrue(tree.dump(), tree.toStringTree().contains("tok_tableproperties")); + assertTrue(tree.dump(), tree.toStringTree().contains("view-format")); + assertTrue(tree.dump(), tree.toStringTree().contains("iceberg")); + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java index 60b123dbb7bd..62d7a876f143 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java @@ -24,7 +24,9 @@ import java.util.List; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.ErrorMsg; @@ -34,6 +36,7 @@ import org.apache.hadoop.hive.ql.ddl.DDLUtils; import org.apache.hadoop.hive.ql.exec.TaskFactory; import org.apache.hadoop.hive.ql.metadata.HiveException; +import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler; import org.apache.hadoop.hive.ql.metadata.HiveUtils; import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.ASTNode; @@ -41,12 +44,16 @@ import org.apache.hadoop.hive.ql.parse.ParseUtils; import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer; import org.apache.hadoop.hive.ql.parse.SemanticException; +import org.apache.hadoop.hive.ql.parse.StorageFormat; /** * Analyzer for create view commands. */ @DDLType(types = HiveParser.TOK_CREATEVIEW) public class CreateViewAnalyzer extends AbstractCreateViewAnalyzer { + + private static final String VIEW_FORMAT_TABLE_PROPERTY = "view-format"; + public CreateViewAnalyzer(QueryState queryState) throws SemanticException { super(queryState); } @@ -75,6 +82,14 @@ public void analyzeInternal(ASTNode root) throws SemanticException { List partitionColumnNames = children.containsKey(HiveParser.TOK_VIEWPARTCOLS) ? getColumnNames((ASTNode) children.remove(HiveParser.TOK_VIEWPARTCOLS).getChild(0)) : null; + String storageHandlerClassFromTableProps = getViewStorageHandlerClassFromTableProps(properties); + String storageHandlerClass = resolveViewStorageHandlerClass(storageHandlerClassFromTableProps); + + if (storageHandlerClassFromTableProps != null && storageHandlerClass == null) { + throw new SemanticException( + ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.format(storageHandlerClassFromTableProps)); + } + assert children.isEmpty(); if (ifNotExists && orReplace) { @@ -94,7 +109,7 @@ public void analyzeInternal(ASTNode root) throws SemanticException { List partitionColumns = getPartitionColumns(partitionColumnNames); setColumnAccessInfo(analyzer.getColumnAccessInfo()); CreateViewDesc desc = new CreateViewDesc(fqViewName, schema, comment, properties, partitionColumnNames, - ifNotExists, orReplace, originalText, expandedText, partitionColumns); + ifNotExists, orReplace, originalText, expandedText, partitionColumns, storageHandlerClass); validateCreateView(desc, analyzer); rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc))); @@ -191,6 +206,56 @@ private List getPartitionColumns(List partitionColumnNames) return partitionColumnsCopy; } + /** + * Returns the FQCN of the storage handler that should own external logical view metadata, or {@code null} for a + * classic HMS virtual view. Uses {@code storageHandlerClassFromTableProps} when non-null (from the + * {@code view-format} table property); otherwise the Hive config {@code hive.default.storage.handler.class}. + */ + private String resolveViewStorageHandlerClass(String storageHandlerClassFromTableProps) + throws SemanticException { + + String storageHandlerClassFromConfig = + StringUtils.trimToNull(HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_DEFAULT_STORAGE_HANDLER)); + + String storageHandlerClass = + storageHandlerClassFromTableProps != null ? storageHandlerClassFromTableProps : storageHandlerClassFromConfig; + + if (StringUtils.isBlank(storageHandlerClass)) { + return null; + } + + boolean explicitViewFormat = storageHandlerClassFromTableProps != null; + try { + HiveStorageHandler storageHandler = HiveUtils.getStorageHandler(conf, storageHandlerClass); + + if (storageHandler != null && storageHandler.supportsExternalLogicalViewCatalog()) { + return storageHandlerClass; + } + } catch (HiveException e) { + + if (explicitViewFormat) { + throw new SemanticException(ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.format(storageHandlerClass), e); + } + } + + return null; + } + + private static String getViewStorageHandlerClassFromTableProps(Map properties) + throws SemanticException { + if (properties == null) { + return null; + } + for (Map.Entry e : properties.entrySet()) { + if (e.getKey() != null + && VIEW_FORMAT_TABLE_PROPERTY.equalsIgnoreCase(e.getKey()) + && StringUtils.isNotBlank(e.getValue())) { + return StorageFormat.resolveStorageHandlerClassName(e.getValue().trim()); + } + } + return null; + } + private void validateCreateView(CreateViewDesc desc, SemanticAnalyzer analyzer) throws SemanticException { try { validateTablesUsed(analyzer); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java index e71cbce773f1..bcbe536bf559 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewDesc.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; +import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.ql.exec.Utilities; import org.apache.hadoop.hive.ql.parse.ReplicationSpec; @@ -40,13 +41,14 @@ public class CreateViewDesc extends AbstractCreateViewDesc { private final boolean ifNotExists; private final boolean replace; private final List partitionColumns; + private final String storageHandlerClass; private ReplicationSpec replicationSpec = null; private String ownerName = null; public CreateViewDesc(String viewName, List schema, String comment, Map properties, List partitionColumnNames, boolean ifNotExists, boolean replace, String originalText, - String expandedText, List partitionColumns) { + String expandedText, List partitionColumns, String storageHandlerClass) { super(viewName, schema, originalText, expandedText); this.comment = comment; this.properties = properties; @@ -54,6 +56,7 @@ public CreateViewDesc(String viewName, List schema, String comment, this.ifNotExists = ifNotExists; this.replace = replace; this.partitionColumns = partitionColumns; + this.storageHandlerClass = storageHandlerClass; } @Explain(displayName = "partition columns") @@ -89,6 +92,19 @@ public boolean isReplace() { return replace; } + /** + * @return FQCN of the {@link org.apache.hadoop.hive.ql.metadata.HiveStorageHandler} that stores view metadata in an + * external catalog, or {@code null} for a classic HMS-only virtual view. + */ + @Explain(displayName = "external logical view storage handler", displayOnlyOnTrue = true) + public String getStorageHandlerClass() { + return storageHandlerClass; + } + + public boolean usesStorageHandler() { + return !StringUtils.isBlank(storageHandlerClass); + } + /** * @param replicationSpec Sets the replication spec governing this create. * This parameter will have meaningful values only for creates happening as a result of a replication. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java index b6a41a8d4fb8..ad27f1dae281 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java @@ -22,6 +22,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.TableName; +import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.ddl.DDLOperation; @@ -33,6 +34,7 @@ import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.HiveTableName; import org.apache.hadoop.hive.ql.parse.StorageFormat; +import org.apache.hadoop.hive.ql.session.SessionStateUtil; import java.util.Map; @@ -82,6 +84,10 @@ public int execute() throws HiveException { if (desc.getProperties() != null) { oldview.getTTable().getParameters().putAll(desc.getProperties()); } + if (!desc.usesStorageHandler()) { + // External logical view is replaced with a native Hive view + clearStorageHandlerProp(oldview); + } oldview.setPartCols(desc.getPartitionColumns()); oldview.checkValidity(null); @@ -94,6 +100,9 @@ public int execute() throws HiveException { } else { // We create new view Table view = createViewObject(); + if (desc.usesStorageHandler()) { + pushExternalLogicalViewSessionHints(desc.isReplace(), desc.getIfNotExists()); + } context.getDb().createTable(view, desc.getIfNotExists()); DDLUtils.addIfAbsentByName(new WriteEntity(view, WriteEntity.WriteType.DDL_NO_LOCK), context.getWork().getOutputs()); @@ -105,6 +114,25 @@ public int execute() throws HiveException { return 0; } + private void pushExternalLogicalViewSessionHints(boolean replace, boolean ifNotExists) { + SessionStateUtil.addResource(context.getConf(), Constants.EXTERNAL_LOGICAL_VIEW_DDL_REPLACE, + Boolean.toString(replace)); + SessionStateUtil.addResource(context.getConf(), Constants.EXTERNAL_LOGICAL_VIEW_CREATE_IF_NOT_EXISTS, + Boolean.toString(ifNotExists)); + } + + private void clearStorageHandlerProp(Table oldview) { + Map params = oldview.getParameters(); + if (params == null) { + return; + } + String fqcn = params.get(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE); + if (fqcn == null) { + return; + } + params.remove(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE); + } + private Table createViewObject() throws HiveException { TableName name = HiveTableName.of(desc.getViewName()); Table view = new Table(name.getDb(), name.getTable()); @@ -129,6 +157,13 @@ private Table createViewObject() throws HiveException { StorageFormat storageFormat = new StorageFormat(context.getConf()); storageFormat.fillDefaultStorageFormat(false, false); + if (desc.usesStorageHandler()) { + storageFormat.setStorageHandler(desc.getStorageHandlerClass()); + view.setProperty( + org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE, + desc.getStorageHandlerClass().trim()); + } + view.setInputFormatClass(storageFormat.getInputFormat()); view.setOutputFormatClass(storageFormat.getOutputFormat()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java index 9287fd75e766..787cadd256b1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplLoadTask.java @@ -547,8 +547,9 @@ public static Task createViewTask(MetaData metaData, String dbNameToLoadIn, H // texts using new DB name. Currently it refers to the source database name. } - CreateViewDesc desc = new CreateViewDesc(dbDotView, table.getCols(), null, table.getParameters(), - table.getPartColNames(), false, false, viewOriginalText, viewExpandedText, table.getPartCols()); + CreateViewDesc desc = new CreateViewDesc(dbDotView, table.getCols(), null, table.getParameters(), + table.getPartColNames(), false, false, viewOriginalText, viewExpandedText, + table.getPartCols(), null); desc.setReplicationSpec(metaData.getReplicationSpec()); desc.setOwnerName(table.getOwner()); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/CreateExternalLogicalViewRequest.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/CreateExternalLogicalViewRequest.java new file mode 100644 index 000000000000..aad1e2c3f7dd --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/CreateExternalLogicalViewRequest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.metadata; + +import java.io.Serializable; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import org.apache.hadoop.hive.metastore.api.FieldSchema; + +/** + * Parameters for {@link HiveStorageHandler#createOrReplaceExternalLogicalView} + */ +public final class CreateExternalLogicalViewRequest implements Serializable { + private static final long serialVersionUID = 1L; + + private final String databaseName; + private final String viewName; + private final List schema; + private final String expandedText; + private final Map properties; + private final String comment; + private final boolean replace; + private final boolean ifNotExists; + + public CreateExternalLogicalViewRequest( + String databaseName, + String viewName, + List schema, + String expandedText, + Map properties, + String comment, + boolean replace, + boolean ifNotExists) { + this.databaseName = databaseName; + this.viewName = viewName; + this.schema = schema; + this.expandedText = expandedText; + this.properties = properties == null ? null : Collections.unmodifiableMap(properties); + this.comment = comment; + this.replace = replace; + this.ifNotExists = ifNotExists; + } + + public String getDatabaseName() { + return databaseName; + } + + public String getViewName() { + return viewName; + } + + public List getSchema() { + return schema; + } + + public String getExpandedText() { + return expandedText; + } + + public Map getProperties() { + return properties; + } + + public String getComment() { + return comment; + } + + public boolean isReplace() { + return replace; + } + + public boolean isIfNotExists() { + return ifNotExists; + } +} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java index 520c52a24a8f..af2699ebe5f3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java @@ -27,6 +27,7 @@ import java.util.concurrent.ExecutorService; import com.google.common.collect.Maps; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.hive.common.classification.InterfaceAudience; @@ -1028,6 +1029,14 @@ default void setMergeTaskDeleteProperties(TableDesc tableDesc) { throw new UnsupportedOperationException("Storage handler does not support getting custom delete merge schema."); } + /** + * @return {@code true} if this handler may store CREATE VIEW text and column metadata in an external catalog + * rather than only as a classic HMS virtual view. + */ + default boolean supportsExternalLogicalViewCatalog() { + return false; + } + default boolean supportsDefaultColumnValues(Map tblProps) { return false; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java index 2472ad44ad00..7fb5269ec25c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java @@ -88,6 +88,19 @@ public String outputFormat() { return outputFormat; } } + + public static String resolveStorageHandlerClassName(String formatType) throws SemanticException { + if (StringUtils.isBlank(formatType)) { + throw new SemanticException("Format type cannot be empty"); + } + for (StorageHandlerTypes type : StorageHandlerTypes.NON_DEFAULT_TYPES) { + if (type.name().equalsIgnoreCase(formatType.trim())) { + Objects.requireNonNull(type.className()); + return ensureClassExists(BaseSemanticAnalyzer.unescapeSQLString(type.className())); + } + } + return ensureClassExists(BaseSemanticAnalyzer.unescapeSQLString(formatType.trim())); + } public StorageFormat(Configuration conf) { this.conf = conf; From f3ddf975a02b1c201e727354fbc93542c31b82e7 Mon Sep 17 00:00:00 2001 From: Dmitriy Fingerman Date: Wed, 27 May 2026 18:03:47 -0400 Subject: [PATCH 2/6] code review comments May 27 --- .../TestIcebergNativeLogicalViewSupport.java | 2 + .../iceberg/mr/hive/HiveIcebergMetaHook.java | 20 +---- ...e_view.q => iceberg_native_logical_view.q} | 13 +++- ....out => iceberg_native_logical_view.q.out} | 77 +++++++++++++++++-- 4 files changed, 85 insertions(+), 27 deletions(-) rename iceberg/iceberg-handler/src/test/queries/positive/{iceberg_native_view.q => iceberg_native_logical_view.q} (84%) rename iceberg/iceberg-handler/src/test/results/positive/{iceberg_native_view.q.out => iceberg_native_logical_view.q.out} (83%) diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergNativeLogicalViewSupport.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergNativeLogicalViewSupport.java index 449f2ac2aceb..f40fc2cd86a5 100644 --- a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergNativeLogicalViewSupport.java +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergNativeLogicalViewSupport.java @@ -115,6 +115,8 @@ void testCreateOrReplaceNativeViewSkipsWhenViewExistsAndIfNotExistsFlagTrue() th IcebergNativeLogicalViewSupport.createOrReplaceNativeView( conf, DB, VIEW, cols, "select 2 as id", null, null, false, true)) .isFalse(); + assertThat(loadCatalog().loadView(TableIdentifier.of(DB, VIEW)).sqlFor("hive").sql().trim()) + .isEqualTo("select 1 as id"); } @Test diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index cfd64df238ed..152e8141e7ab 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -178,19 +178,15 @@ public HiveIcebergMetaHook(Configuration conf) { super(conf); } - @Override - public void rollbackCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) { - // do nothing - } - @Override public void commitCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) { - if (BaseHiveIcebergMetaHook.isNativeIcebergLogicalView(hmsTable)) { + if (isNativeIcebergLogicalView(hmsTable)) { tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); if (Catalogs.hiveCatalog(conf, tableProperties)) { boolean replace = Boolean.parseBoolean( - SessionStateUtil.getProperty(conf, Constants.EXTERNAL_LOGICAL_VIEW_DDL_REPLACE).orElse("false")); + SessionStateUtil.getProperty(conf, Constants.EXTERNAL_LOGICAL_VIEW_DDL_REPLACE) + .orElse("false")); boolean ifNotExists = Boolean.parseBoolean( SessionStateUtil.getProperty(conf, Constants.EXTERNAL_LOGICAL_VIEW_CREATE_IF_NOT_EXISTS) @@ -236,11 +232,6 @@ public void commitCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTabl } } - @Override - public void preDropTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) { - // do nothing - } - @Override public void preDropTable(org.apache.hadoop.hive.metastore.api.Table hmsTable, boolean deleteData) { this.tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); @@ -263,11 +254,6 @@ public void preDropTable(org.apache.hadoop.hive.metastore.api.Table hmsTable, bo } } - @Override - public void rollbackDropTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) { - // do nothing - } - @Override public void commitDropTable(org.apache.hadoop.hive.metastore.api.Table hmsTable, boolean deleteData) { if (deleteData && deleteIcebergTable) { diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_logical_view.q similarity index 84% rename from iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q rename to iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_logical_view.q index 116248ed623e..037b67371b13 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_view.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_native_logical_view.q @@ -38,9 +38,7 @@ create or replace view v_ice tblproperties ('view-format'='iceberg') as select first_name || '-' || dept_id from src_ice where dept_id = 1; select * from v_ice; - desc formatted v_ice; -drop view v_ice; ------------------------------------------------------------------------------- -- Native Iceberg view when default storage handler is Iceberg @@ -76,3 +74,14 @@ create view v_hive as select * from src_ice; select * from v_hive; desc formatted v_hive; drop view v_hive; + +----------------------------------------------------------------------------------------- +-- Replace Iceberg logical view with a Hive-native logical view +----------------------------------------------------------------------------------------- + +create or replace view v_ice +as select first_name from src_ice where dept_id = 2; + +select * from v_ice; +desc formatted v_ice; +drop view v_ice; \ No newline at end of file diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_native_view.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_native_logical_view.q.out similarity index 83% rename from iceberg/iceberg-handler/src/test/results/positive/iceberg_native_view.q.out rename to iceberg/iceberg-handler/src/test/results/positive/iceberg_native_logical_view.q.out index d9082ecb068e..733a0b6134e9 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_native_view.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_native_logical_view.q.out @@ -143,14 +143,6 @@ Sort Columns: [] # View Information Original Query: select first_name || '-' || dept_id from src_ice where dept_id = 1 Expanded Query: select `src_ice`.`first_name` || '-' || `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` where `src_ice`.`dept_id` = 1 -PREHOOK: query: drop view v_ice -PREHOOK: type: DROPVIEW -PREHOOK: Input: ice_native_view_db@v_ice -PREHOOK: Output: ice_native_view_db@v_ice -POSTHOOK: query: drop view v_ice -POSTHOOK: type: DROPVIEW -POSTHOOK: Input: ice_native_view_db@v_ice -POSTHOOK: Output: ice_native_view_db@v_ice PREHOOK: query: create view if not exists v_def as select first_name, last_name, dept_id from src_ice where dept_id = 2 PREHOOK: type: CREATEVIEW @@ -314,3 +306,72 @@ POSTHOOK: query: drop view v_hive POSTHOOK: type: DROPVIEW POSTHOOK: Input: ice_native_view_db@v_hive POSTHOOK: Output: ice_native_view_db@v_hive +PREHOOK: query: create or replace view v_ice +as select first_name from src_ice where dept_id = 2 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Output: database:ice_native_view_db +PREHOOK: Output: ice_native_view_db@v_ice +POSTHOOK: query: create or replace view v_ice +as select first_name from src_ice where dept_id = 2 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Output: database:ice_native_view_db +POSTHOOK: Output: ice_native_view_db@v_ice +PREHOOK: query: select * from v_ice +PREHOOK: type: QUERY +PREHOOK: Input: ice_native_view_db@src_ice +PREHOOK: Input: ice_native_view_db@v_ice +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: select * from v_ice +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_native_view_db@src_ice +POSTHOOK: Input: ice_native_view_db@v_ice +POSTHOOK: Output: hdfs://### HDFS PATH ### +fn5 +fn6 +fn7 +PREHOOK: query: desc formatted v_ice +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_native_view_db@v_ice +POSTHOOK: query: desc formatted v_ice +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_native_view_db@v_ice +# col_name data_type comment +first_name string + +# Detailed Table Information +Database: ice_native_view_db +#### A masked pattern was here #### +Retention: 0 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":1,\"fields\":[{\"id\":1,\"name\":\"_c0\",\"required\":false,\"type\":\"string\"}]} + metadata_location hdfs://### HDFS PATH ### + previous_metadata_location hdfs://### HDFS PATH ### + table_type ICEBERG-VIEW +#### A masked pattern was here #### + uuid #Masked# + view-format iceberg + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Num Buckets: 0 +Bucket Columns: [] +Sort Columns: [] + +# View Information +Original Query: select first_name from src_ice where dept_id = 2 +Expanded Query: select `src_ice`.`first_name` from `ice_native_view_db`.`src_ice` where `src_ice`.`dept_id` = 2 +PREHOOK: query: drop view v_ice +PREHOOK: type: DROPVIEW +PREHOOK: Input: ice_native_view_db@v_ice +PREHOOK: Output: ice_native_view_db@v_ice +POSTHOOK: query: drop view v_ice +POSTHOOK: type: DROPVIEW +POSTHOOK: Input: ice_native_view_db@v_ice +POSTHOOK: Output: ice_native_view_db@v_ice From 14842781ff5d85454939f246b1f783ab1fbcf363 Mon Sep 17 00:00:00 2001 From: Dmitriy Fingerman Date: Thu, 28 May 2026 11:57:51 -0400 Subject: [PATCH 3/6] Disabling setting partition keys in REST Catalog client with a TODO comment to revert it after HIVE-29633 is fixed --- .../src/main/java/org/apache/iceberg/hive/MetastoreUtil.java | 4 ++++ .../java/org/apache/hive/TestHiveRESTCatalogClientITBase.java | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java index 6eb9dff8ba3b..a7f3a96d4cc7 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java @@ -139,7 +139,11 @@ public static Table toHiveTable(org.apache.iceberg.Table table, Configuration co result.setDbName(tableName.getDb()); result.setTableName(tableName.getTable()); result.setTableType(TableType.EXTERNAL_TABLE.toString()); + + // TODO: Revert after HIVE-29633 is fixed + // result.setPartitionKeys(getPartitionKeys(table, table.spec().specId())); result.setPartitionKeys(Lists.newArrayList()); + TableMetadata metadata = ((BaseTable) table).operations().current(); long maxHiveTablePropertySize = conf.getLong(HiveOperationsBase.HIVE_TABLE_PROPERTY_MAX_SIZE, HiveOperationsBase.HIVE_TABLE_PROPERTY_MAX_SIZE_DEFAULT); diff --git a/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java index 66cd4504b271..14169c072474 100644 --- a/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java +++ b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java @@ -179,7 +179,11 @@ public void testIceberg() throws Exception { Assertions.assertEquals(TABLE_NAME, table.getTableName()); Assertions.assertEquals(HIVE_ICEBERG_STORAGE_HANDLER, table.getParameters().get("storage_handler")); Assertions.assertNotNull(table.getParameters().get(TableProperties.DEFAULT_PARTITION_SPEC)); + + // TODO: Revert after HIVE-29633 is fixed + // Assertions.assertEquals(1, table.getPartitionKeys().size()); Assertions.assertTrue(table.getPartitionKeys().isEmpty()); + List columnNames = table.getSd().getCols().stream().map(FieldSchema::getName).toList(); Assertions.assertTrue(columnNames.contains("city")); From b90b76911597883d70be85bf38a50c3018855836 Mon Sep 17 00:00:00 2001 From: Dmitriy Fingerman Date: Fri, 29 May 2026 15:32:36 -0400 Subject: [PATCH 4/6] code review comments May 29. --- .../apache/hadoop/hive/conf/Constants.java | 3 - .../hive/IcebergNativeLogicalViewSupport.java | 87 ++++--- .../hive/client/HiveRESTCatalogClient.java | 55 +++-- .../TestIcebergNativeLogicalViewSupport.java | 40 +-- .../client/TestHiveRESTCatalogClient.java | 51 ++++ .../mr/hive/BaseHiveIcebergMetaHook.java | 17 +- .../iceberg/mr/hive/HiveIcebergMetaHook.java | 15 +- .../positive/iceberg_rest_catalog_gravitino.q | 47 ++-- .../positive/iceberg_rest_catalog_hms.q | 43 +--- .../iceberg_native_logical_view.q.out | 6 +- .../llap/iceberg_rest_catalog_gravitino.q.out | 230 ++++++------------ .../llap/iceberg_rest_catalog_hms.q.out | 200 ++------------- .../hive/TestHiveRESTCatalogClientITBase.java | 1 - .../ddl/view/create/CreateViewOperation.java | 10 - 14 files changed, 263 insertions(+), 542 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/conf/Constants.java b/common/src/java/org/apache/hadoop/hive/conf/Constants.java index 0af398ef8768..4b0e75fafb51 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/Constants.java +++ b/common/src/java/org/apache/hadoop/hive/conf/Constants.java @@ -111,7 +111,4 @@ public class Constants { public static final String CLUSTER_ID_CLI_OPT_NAME = "hive.cluster.id"; public static final String CLUSTER_ID_HIVE_CONF_PROP = "hive.cluster.id"; public static final String ICEBERG_PARTITION_COLUMNS = "partition,spec_id"; - - public static final String EXTERNAL_LOGICAL_VIEW_DDL_REPLACE = "externalLogicalViewDdlReplace"; - public static final String EXTERNAL_LOGICAL_VIEW_CREATE_IF_NOT_EXISTS = "externalLogicalViewCreateIfNotExists"; } diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergNativeLogicalViewSupport.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergNativeLogicalViewSupport.java index d3d3b1f23591..58c1f20ba565 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergNativeLogicalViewSupport.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergNativeLogicalViewSupport.java @@ -39,10 +39,6 @@ /** * Commits a native Iceberg view through the configured default Iceberg catalog (HiveCatalog or REST * catalog, etc.) when {@code Catalog} also implements {@link ViewCatalog}. - * - *

CREATE VIEW session hints and metastore {@code EnvironmentContext} keys use - * {@link org.apache.hadoop.hive.conf.Constants#EXTERNAL_LOGICAL_VIEW_DDL_REPLACE} and - * {@link org.apache.hadoop.hive.conf.Constants#EXTERNAL_LOGICAL_VIEW_CREATE_IF_NOT_EXISTS}. */ public final class IcebergNativeLogicalViewSupport { @@ -56,7 +52,7 @@ private IcebergNativeLogicalViewSupport() { /** * Creates or replaces a view in the Iceberg catalog. * - * @return {@code false} if skipped because {@code ifNotExists} is true and the view already exists + * @return {@code false} if skipped because the view already exists and {@code replace} is false */ public static boolean createOrReplaceNativeView( Configuration conf, @@ -66,55 +62,66 @@ public static boolean createOrReplaceNativeView( String viewSql, Map tblProperties, String comment, - boolean replace, - boolean ifNotExists) { + boolean replace) { TableIdentifier identifier = TableIdentifier.of(databaseName, viewName); String catalogName = IcebergCatalogProperties.getCatalogName(conf); Map catalogProps = IcebergCatalogProperties.getCatalogProperties(conf, catalogName); Catalog catalog = CatalogUtil.buildIcebergCatalog(catalogName, catalogProps, conf); - try { - ViewCatalog viewCatalog = asViewCatalog(catalog, catalogName); - if (!replace && ifNotExists && viewCatalog.viewExists(identifier)) { - return false; + if (catalog instanceof Closeable closeable) { + try (Closeable ignored = closeable) { + return commitNativeView( + catalog, catalogName, identifier, fieldSchemas, viewSql, tblProperties, comment, replace); + } catch (IOException e) { + throw new UncheckedIOException("Failed to close Iceberg catalog", e); } + } - ViewBuilder builder = - viewCatalog - .buildView(identifier) - .withSchema(HiveSchemaUtil.convert(fieldSchemas, Collections.emptyMap(), true)) - .withDefaultNamespace(Namespace.of(identifier.namespace().level(0))) - .withQuery("hive", viewSql); + return commitNativeView( + catalog, catalogName, identifier, fieldSchemas, viewSql, tblProperties, comment, replace); + } - if (StringUtils.isNotBlank(comment)) { - builder = builder.withProperty("comment", comment); - } + private static boolean commitNativeView( + Catalog catalog, + String catalogName, + TableIdentifier identifier, + List fieldSchemas, + String viewSql, + Map tblProperties, + String comment, + boolean replace) { + ViewCatalog viewCatalog = asViewCatalog(catalog, catalogName); + if (!replace && viewCatalog.viewExists(identifier)) { + return false; + } - Map tblProps = - tblProperties == null ? Maps.newHashMap() : Maps.newHashMap(tblProperties); + ViewBuilder builder = + viewCatalog + .buildView(identifier) + .withSchema(HiveSchemaUtil.convert(fieldSchemas, Collections.emptyMap(), true)) + .withDefaultNamespace(Namespace.of(identifier.namespace().level(0))) + .withQuery("hive", viewSql); - for (Map.Entry e : tblProps.entrySet()) { - if (e.getKey() != null && e.getValue() != null) { - builder = builder.withProperty(e.getKey(), e.getValue()); - } - } + if (StringUtils.isNotBlank(comment)) { + builder = builder.withProperty("comment", comment); + } - if (replace) { - builder.createOrReplace(); - } else { - builder.create(); - } - return true; - } finally { - if (catalog instanceof Closeable closeable) { - try { - closeable.close(); - } catch (IOException e) { - throw new UncheckedIOException("Failed to close Iceberg catalog", e); - } + Map tblProps = + tblProperties == null ? Maps.newHashMap() : Maps.newHashMap(tblProperties); + + for (Map.Entry e : tblProps.entrySet()) { + if (e.getKey() != null && e.getValue() != null) { + builder = builder.withProperty(e.getKey(), e.getValue()); } } + + if (replace) { + builder.createOrReplace(); + } else { + builder.create(); + } + return true; } private static ViewCatalog asViewCatalog(Catalog catalog, String catalogName) { diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java index 86d51cf01bc3..27c7efd662fa 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java @@ -29,7 +29,6 @@ import java.util.Set; import java.util.regex.Pattern; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.CreateTableRequest; import org.apache.hadoop.hive.metastore.api.Database; @@ -237,33 +236,25 @@ private static boolean hasIcebergNativeViewTableType(Table table) { } @Override - public void createTable(CreateTableRequest request) throws TException { - Table table = request.getTable(); - List cols = Lists.newArrayList(table.getSd().getCols()); - - if (table.isSetPartitionKeys() && !table.getPartitionKeys().isEmpty()) { - cols.addAll(table.getPartitionKeys()); + public void alter_table(String catName, String dbName, String tblName, Table newTable, + EnvironmentContext envContext, String validWriteIdList) throws TException { + validateCurrentCatalog(catName); + if (hasIcebergNativeViewTableType(newTable) && restCatalog instanceof ViewCatalog) { + createOrReplaceLogicalView(newTable, dbName, tblName, true); } + } + @Override + public void createTable(CreateTableRequest request) throws TException { + Table table = request.getTable(); if (hasIcebergNativeViewTableType(table) && restCatalog instanceof ViewCatalog) { - Map envProps = - Optional.ofNullable(request.getEnvContext()) - .map(EnvironmentContext::getProperties) - .orElse(Collections.emptyMap()); - boolean replace = - Boolean.parseBoolean( - envProps.getOrDefault(Constants.EXTERNAL_LOGICAL_VIEW_DDL_REPLACE, "false")); - boolean ifNotExists = - Boolean.parseBoolean( - envProps.getOrDefault(Constants.EXTERNAL_LOGICAL_VIEW_CREATE_IF_NOT_EXISTS, "false")); - Map tblProps = - table.getParameters() == null ? Maps.newHashMap() : Maps.newHashMap(table.getParameters()); - String comment = tblProps.get("comment"); - - IcebergNativeLogicalViewSupport.createOrReplaceNativeView( - conf, table.getDbName(), table.getTableName(), cols, table.getViewExpandedText(), tblProps, comment, replace, - ifNotExists); + createOrReplaceLogicalView( + table, table.getDbName(), table.getTableName(), false); } else { + List cols = Lists.newArrayList(table.getSd().getCols()); + if (table.isSetPartitionKeys() && !table.getPartitionKeys().isEmpty()) { + cols.addAll(table.getPartitionKeys()); + } Properties tableProperties = IcebergTableProperties.getTableProperties(table, conf); Schema schema = HiveSchemaUtil.convert(cols, Collections.emptyMap(), true); Map envCtxProps = Optional.ofNullable(request.getEnvContext()) @@ -282,6 +273,22 @@ public void createTable(CreateTableRequest request) throws TException { } } + private void createOrReplaceLogicalView( + Table table, String dbName, String tableName, boolean replace) { + + List cols = Lists.newArrayList(table.getSd().getCols()); + if (table.isSetPartitionKeys() && !table.getPartitionKeys().isEmpty()) { + cols.addAll(table.getPartitionKeys()); + } + + Map tblProps = + table.getParameters() == null ? Maps.newHashMap() : Maps.newHashMap(table.getParameters()); + + String comment = tblProps.get("comment"); + IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + conf, dbName, tableName, cols, table.getViewExpandedText(), tblProps, comment, replace); + } + @Override public void createDatabase(Database db) { validateCurrentCatalog(db.getCatalogName()); diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergNativeLogicalViewSupport.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergNativeLogicalViewSupport.java index f40fc2cd86a5..88c88645ff5e 100644 --- a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergNativeLogicalViewSupport.java +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergNativeLogicalViewSupport.java @@ -78,7 +78,7 @@ private HiveCatalog loadCatalog() { } @Test - void testCreateCommitsNativeViewWithUserProperties() throws Exception { + void testCreateCommitsNativeViewWithUserProperties() { HiveConf conf = nativeViewConf(); List cols = Arrays.asList(new FieldSchema("id", "int", null), new FieldSchema("name", "string", null)); @@ -87,7 +87,7 @@ void testCreateCommitsNativeViewWithUserProperties() throws Exception { boolean created = IcebergNativeLogicalViewSupport.createOrReplaceNativeView( - conf, DB, VIEW, cols, sql, props, "hello-view", false, false); + conf, DB, VIEW, cols, sql, props, "hello-view", false); assertThat(created).isTrue(); HiveCatalog cat = loadCatalog(); @@ -103,49 +103,19 @@ void testCreateCommitsNativeViewWithUserProperties() throws Exception { } @Test - void testCreateOrReplaceNativeViewSkipsWhenViewExistsAndIfNotExistsFlagTrue() throws Exception { - HiveConf conf = nativeViewConf(); - List cols = Collections.singletonList(new FieldSchema("id", "int", null)); - - assertThat( - IcebergNativeLogicalViewSupport.createOrReplaceNativeView( - conf, DB, VIEW, cols, "select 1 as id", null, null, false, false)) - .isTrue(); - assertThat( - IcebergNativeLogicalViewSupport.createOrReplaceNativeView( - conf, DB, VIEW, cols, "select 2 as id", null, null, false, true)) - .isFalse(); - assertThat(loadCatalog().loadView(TableIdentifier.of(DB, VIEW)).sqlFor("hive").sql().trim()) - .isEqualTo("select 1 as id"); - } - - @Test - void testCreateOrReplaceNativeViewOrThrowDoesNotThrowWhenIfNotExistsSkips() { - HiveConf conf = nativeViewConf(); - List cols = Collections.singletonList(new FieldSchema("id", "int", null)); - TableIdentifier id = TableIdentifier.of(DB, VIEW); - - IcebergNativeLogicalViewSupport.createOrReplaceNativeView( - conf, DB, VIEW, cols, "select 1 as id", null, null, false, false); - IcebergNativeLogicalViewSupport.createOrReplaceNativeView( - conf, DB, VIEW, cols, "select 2 as id", null, null, false, true); - assertThat(loadCatalog().loadView(id).sqlFor("hive").sql().trim()).isEqualTo("select 1 as id"); - } - - @Test - void testCreateOrReplaceNativeViewReplacesExistingWhenReplaceFlagTrue() throws Exception { + void testCreateOrReplaceNativeViewReplacesExistingWhenReplaceFlagTrue() { HiveConf conf = nativeViewConf(); List cols = Collections.singletonList(new FieldSchema("id", "int", null)); TableIdentifier id = TableIdentifier.of(DB, VIEW); IcebergNativeLogicalViewSupport.createOrReplaceNativeView( - conf, DB, VIEW, cols, "select 1 as id", null, null, false, false); + conf, DB, VIEW, cols, "select 1 as id", null, null, false); View afterCreate = loadCatalog().loadView(id); assertThat(afterCreate.sqlFor("hive").sql().trim()).isEqualTo("select 1 as id"); assertThat( IcebergNativeLogicalViewSupport.createOrReplaceNativeView( - conf, DB, VIEW, cols, "select 2 as id", null, null, true, false)) + conf, DB, VIEW, cols, "select 2 as id", null, null, true)) .isTrue(); assertThat(loadCatalog().viewExists(id)).isTrue(); diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/client/TestHiveRESTCatalogClient.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/client/TestHiveRESTCatalogClient.java index 1ae7e742774c..3bcbc77419ee 100644 --- a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/client/TestHiveRESTCatalogClient.java +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/client/TestHiveRESTCatalogClient.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.CreateTableRequest; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; @@ -32,6 +33,7 @@ import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.BaseTable; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.PartitionSpec; @@ -43,7 +45,9 @@ import org.apache.iceberg.catalog.Catalog; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.catalog.ViewCatalog; import org.apache.iceberg.hive.HiveSchemaUtil; +import org.apache.iceberg.hive.IcebergNativeLogicalViewSupport; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.LocationProvider; import org.apache.iceberg.relocated.com.google.common.collect.Maps; @@ -59,6 +63,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; public class TestHiveRESTCatalogClient { @@ -197,4 +202,50 @@ public void testGetDatabase() throws TException { assertThat(aDefault.getName()).isEqualTo("default"); Mockito.verify(mockRestCatalog).listNamespaces(Namespace.empty()); } + + @Test + public void testAlterNativeIcebergView() throws TException { + RESTCatalog viewCapableCatalog = + Mockito.mock(RESTCatalog.class, Mockito.withSettings().extraInterfaces(ViewCatalog.class)); + Mockito.doReturn("hive").when(viewCapableCatalog).name(); + mockCatalogUtil.when(() -> CatalogUtil.buildIcebergCatalog(any(), any(), any())) + .thenReturn(viewCapableCatalog); + + Configuration configuration = new Configuration(); + configuration.set("iceberg.catalog", "ice01"); + configuration.set("iceberg.catalog.ice01.uri", "http://localhost"); + HiveRESTCatalogClient client = new HiveRESTCatalogClient(configuration); + + try (MockedStatic viewSupport = + Mockito.mockStatic(IcebergNativeLogicalViewSupport.class)) { + client.alter_table("hive", "ice_db", "ice_v1", createLogicalView(), null, null); + viewSupport.verify( + () -> + IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + any(), + eq("ice_db"), + eq("ice_v1"), + any(), + eq("select 1"), + any(), + eq(null), + eq(true))); + } + } + + private static Table createLogicalView() { + Table view = new Table(); + view.setTableName("ice_v1"); + view.setDbName("ice_db"); + view.setTableType(TableType.VIRTUAL_VIEW.toString()); + view.setViewExpandedText("select 1"); + view.setSd(new StorageDescriptor()); + view.getSd().setCols(Collections.singletonList(new FieldSchema("x", "int", ""))); + view.setParameters( + Maps.newHashMap( + Map.of( + BaseMetastoreTableOperations.TABLE_TYPE_PROP, + IcebergNativeLogicalViewSupport.ICEBERG_VIEW_HMS_TABLE_TYPE_VALUE))); + return view; + } } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java index a9492170a2e4..708c48294e23 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java @@ -33,13 +33,11 @@ import java.util.stream.Stream; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.metastore.HiveMetaHook; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.CreateTableRequest; import org.apache.hadoop.hive.metastore.api.EnvironmentContext; import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint; import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; @@ -47,7 +45,6 @@ import org.apache.hadoop.hive.ql.ddl.misc.sortoder.SortFields; import org.apache.hadoop.hive.ql.ddl.misc.sortoder.ZOrderFieldDesc; import org.apache.hadoop.hive.ql.ddl.misc.sortoder.ZOrderFields; -import org.apache.hadoop.hive.ql.session.SessionStateUtil; import org.apache.hadoop.hive.ql.util.NullOrdering; import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.NullOrder; @@ -224,17 +221,9 @@ public void preCreateTable(CreateTableRequest request) { } private void preCreateNativeIcebergLogicalView(CreateTableRequest request) { - org.apache.hadoop.hive.metastore.api.Table hmsTable = request.getTable(); - this.tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); - if (request.getEnvContext() == null) { - request.setEnvContext(new EnvironmentContext()); - } - final EnvironmentContext env = request.getEnvContext(); - SessionStateUtil.getProperty(conf, Constants.EXTERNAL_LOGICAL_VIEW_DDL_REPLACE) - .ifPresent(v -> env.putToProperties(Constants.EXTERNAL_LOGICAL_VIEW_DDL_REPLACE, v)); - SessionStateUtil.getProperty(conf, Constants.EXTERNAL_LOGICAL_VIEW_CREATE_IF_NOT_EXISTS) - .ifPresent(v -> env.putToProperties(Constants.EXTERNAL_LOGICAL_VIEW_CREATE_IF_NOT_EXISTS, v)); + org.apache.hadoop.hive.metastore.api.Table hmsTable = request.getTable(); + tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); hmsTable .getParameters() @@ -381,7 +370,7 @@ public void rollbackCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTa } @Override - public void commitCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) throws MetaException { + public void commitCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) { // do nothing } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index 152e8141e7ab..7dcfd4e97471 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -37,7 +37,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.TableName; -import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.metastore.HiveMetaHook; import org.apache.hadoop.hive.metastore.PartitionDropOptions; import org.apache.hadoop.hive.metastore.Warehouse; @@ -183,14 +182,6 @@ public void commitCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTabl if (isNativeIcebergLogicalView(hmsTable)) { tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); if (Catalogs.hiveCatalog(conf, tableProperties)) { - boolean replace = - Boolean.parseBoolean( - SessionStateUtil.getProperty(conf, Constants.EXTERNAL_LOGICAL_VIEW_DDL_REPLACE) - .orElse("false")); - boolean ifNotExists = - Boolean.parseBoolean( - SessionStateUtil.getProperty(conf, Constants.EXTERNAL_LOGICAL_VIEW_CREATE_IF_NOT_EXISTS) - .orElse("false")); Map tblProps = hmsTable.getParameters() == null ? Maps.newHashMap() : Maps.newHashMap(hmsTable.getParameters()); String comment = tblProps.get("comment"); @@ -202,8 +193,7 @@ public void commitCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTabl hmsTable.getViewExpandedText(), tblProps, comment, - replace, - ifNotExists); + false); } return; } @@ -530,8 +520,7 @@ public void commitAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable hmsTable.getViewExpandedText(), tblProps, comment, - true, - false); + true); return; } if (isTableMigration) { diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q index 2a029fd0432a..16f99a010a6f 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q @@ -76,47 +76,36 @@ select * from ice_orc2; --! Iceberg Native View tests --------------------------------------------------------------------------------------------------------------------- ------------------------------------------------------------------------------------------------ ---! Native Iceberg view with TBLPROPERTIES ('view-format'='iceberg') on a REST catalog table ---! without a catalog name in the base table properties ------------------------------------------------------------------------------------------------ +----------------------------------------------------------------------------------------------------- +--! Native Iceberg logical view with TBLPROPERTIES ('view-format'='iceberg') on a REST catalog table +----------------------------------------------------------------------------------------------------- + +create view if not exists ice_v1 tblproperties ('view-format'='iceberg') +as select first_name, last_name from ice_orc2 where dept_id in (1, 3); -create view ice_v1 tblproperties ('view-format'='iceberg') as select * from ice_orc1; select * from ice_v1; desc formatted ice_v1; -drop view ice_v1; ------------------------------------------------------------------------------------------------ ---! Iceberg native view with TBLPROPERTIES ('view-format'='iceberg') on a REST catalog table ---! with a catalog name in the base table properties ------------------------------------------------------------------------------------------------ +create or replace view ice_v1 tblproperties ('view-format'='iceberg') +as select first_name || '-' || dept_id from ice_orc2 where dept_id = 2; -create view ice_v2 tblproperties ('view-format'='iceberg') as select * from ice_orc2; -select * from ice_v2; -desc formatted ice_v2; -drop view ice_v2; +select * from ice_v1; +desc formatted ice_v1; + +drop view ice_v1; ----------------------------------------------------------------------------------------------- ---! Native Iceberg view: 'view-format' table properly omitted, Hive config 'hive.default.storage.handler.class' ---! set to 'HiveIcebergStorageHandler' and no catalog name in the base table properties +--! Native Iceberg logical view with default Iceberg storage handler and REST catalog table ----------------------------------------------------------------------------------------------- set hive.default.storage.handler.class=org.apache.iceberg.mr.hive.HiveIcebergStorageHandler; -create view ice_v3 as select * from ice_orc1; -select * from ice_v3; -desc formatted ice_v3; -drop view ice_v3; +create view ice_v2 +as select first_name, last_name || '-' || dept_id from ice_orc2 where team_id in (20, 30); ------------------------------------------------------------------------------------------------ ---! Native Iceberg view with default Iceberg storage handler and REST catalog table ---! with catalog name in the base table properties ------------------------------------------------------------------------------------------------ - -create view ice_v4 as select * from ice_orc2; -select * from ice_v4; -desc formatted ice_v4; -drop view ice_v4; +select * from ice_v2; +desc formatted ice_v2; +drop view ice_v2; ----------------------------------------------------------------------------- diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q index 2ad2fd628028..47e080ec8881 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_hms.q @@ -67,51 +67,30 @@ VALUES ('fn1','ln1', 1, 10), ('fn2','ln2', 2, 20), ('fn3','ln3', 3, 30); describe formatted ice_orc2; select * from ice_orc2; ---------------------------------------------------------------------------------------------------------------------- ---! Iceberg Native View tests ---------------------------------------------------------------------------------------------------------------------- - ----------------------------------------------------------------------------------------------- ---! Native Iceberg view with TBLPROPERTIES ('view-format'='iceberg') on a REST catalog table ---! without a catalog name in the base table properties +--! Native Iceberg logical view with TBLPROPERTIES ('view-format'='iceberg') on a REST catalog table ----------------------------------------------------------------------------------------------- -create view ice_v1 tblproperties ('view-format'='iceberg') as select * from ice_orc1; +create view ice_v1 tblproperties ('view-format'='iceberg') +as select first_name, last_name from ice_orc2 where dept_id in (1,2); + select * from ice_v1; desc formatted ice_v1; drop view ice_v1; ----------------------------------------------------------------------------------------------- ---! Iceberg native view with TBLPROPERTIES ('view-format'='iceberg') on a REST catalog table ---! with a catalog name in the base table properties ------------------------------------------------------------------------------------------------ - -create view ice_v2 tblproperties ('view-format'='iceberg') as select * from ice_orc2; -select * from ice_v2; -desc formatted ice_v2; -drop view ice_v2; - ------------------------------------------------------------------------------------------------ ---! Native Iceberg view: 'view-format' table properly omitted, Hive config 'hive.default.storage.handler.class' ---! set to 'HiveIcebergStorageHandler' and no catalog name in the base table properties +--! Native Iceberg logical view: 'view-format' table properly omitted, Hive config 'hive.default.storage.handler.class' +--! set to 'HiveIcebergStorageHandler' ----------------------------------------------------------------------------------------------- set hive.default.storage.handler.class=org.apache.iceberg.mr.hive.HiveIcebergStorageHandler; -create view ice_v3 as select * from ice_orc1; -select * from ice_v3; -desc formatted ice_v3; -drop view ice_v3; - ------------------------------------------------------------------------------------------------ ---! Native Iceberg view with default Iceberg storage handler and REST catalog table ---! with catalog name in the base table properties ------------------------------------------------------------------------------------------------ +create view ice_v2 +as select dept_id, team_id from ice_orc2 where company_id = 100; -create view ice_v4 as select * from ice_orc2; -select * from ice_v4; -desc formatted ice_v4; -drop view ice_v4; +select * from ice_v2; +desc formatted ice_v2; +drop view ice_v2; ----------------------------------------------------------------------------- diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_native_logical_view.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_native_logical_view.q.out index 733a0b6134e9..82cbb661dc49 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_native_logical_view.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_native_logical_view.q.out @@ -124,9 +124,8 @@ Retention: 0 Table Type: VIRTUAL_VIEW Table Parameters: bucketing_version 2 - current-schema {\"type\":\"struct\",\"schema-id\":1,\"fields\":[{\"id\":1,\"name\":\"_c0\",\"required\":false,\"type\":\"string\"}]} + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"_c0\",\"required\":false,\"type\":\"string\"}]} metadata_location hdfs://### HDFS PATH ### - previous_metadata_location hdfs://### HDFS PATH ### storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG-VIEW #### A masked pattern was here #### @@ -347,9 +346,8 @@ Retention: 0 Table Type: VIRTUAL_VIEW Table Parameters: bucketing_version 2 - current-schema {\"type\":\"struct\",\"schema-id\":1,\"fields\":[{\"id\":1,\"name\":\"_c0\",\"required\":false,\"type\":\"string\"}]} + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"_c0\",\"required\":false,\"type\":\"string\"}]} metadata_location hdfs://### HDFS PATH ### - previous_metadata_location hdfs://### HDFS PATH ### table_type ICEBERG-VIEW #### A masked pattern was here #### uuid #Masked# diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out index 4125fef590c9..aadc224194cb 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out @@ -178,31 +178,32 @@ POSTHOOK: Input: ice_rest@ice_orc2 fn1 ln1 1 10 100 fn2 ln2 2 20 100 fn3 ln3 3 30 100 -PREHOOK: query: create view ice_v1 tblproperties ('view-format'='iceberg') as select * from ice_orc1 +PREHOOK: query: create view if not exists ice_v1 tblproperties ('view-format'='iceberg') +as select first_name, last_name from ice_orc2 where dept_id in (1, 3) PREHOOK: type: CREATEVIEW -PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Input: ice_rest@ice_orc2 PREHOOK: Output: database:ice_rest PREHOOK: Output: ice_rest@ice_v1 -POSTHOOK: query: create view ice_v1 tblproperties ('view-format'='iceberg') as select * from ice_orc1 +POSTHOOK: query: create view if not exists ice_v1 tblproperties ('view-format'='iceberg') +as select first_name, last_name from ice_orc2 where dept_id in (1, 3) POSTHOOK: type: CREATEVIEW -POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Input: ice_rest@ice_orc2 POSTHOOK: Output: database:ice_rest POSTHOOK: Output: ice_rest@ice_v1 -POSTHOOK: Lineage: ice_v1.company_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:company_id, type:bigint, comment:null), ] -POSTHOOK: Lineage: ice_v1.dept_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:dept_id, type:bigint, comment:null), ] -POSTHOOK: Lineage: ice_v1.first_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:first_name, type:string, comment:null), ] -POSTHOOK: Lineage: ice_v1.last_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:last_name, type:string, comment:null), ] -POSTHOOK: Lineage: ice_v1.team_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:team_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v1.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v1.last_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), ] PREHOOK: query: select * from ice_v1 PREHOOK: type: QUERY -PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Input: ice_rest@ice_orc2 PREHOOK: Input: ice_rest@ice_v1 #### A masked pattern was here #### POSTHOOK: query: select * from ice_v1 POSTHOOK: type: QUERY -POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Input: ice_rest@ice_orc2 POSTHOOK: Input: ice_rest@ice_v1 #### A masked pattern was here #### +fn1 ln1 +fn3 ln3 PREHOOK: query: desc formatted ice_v1 PREHOOK: type: DESCTABLE PREHOOK: Input: ice_rest@ice_v1 @@ -212,9 +213,6 @@ POSTHOOK: Input: ice_rest@ice_v1 # col_name data_type comment first_name string last_name string -dept_id bigint -team_id bigint -company_id bigint # Detailed Table Information Database: ice_rest @@ -223,7 +221,7 @@ Retention: 2147483647 Table Type: VIRTUAL_VIEW Table Parameters: bucketing_version 2 - current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]} #### A masked pattern was here #### storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG-VIEW @@ -239,56 +237,39 @@ Compressed: No Sort Columns: [] # View Information -Original Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` -Expanded Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` -PREHOOK: query: drop view ice_v1 -PREHOOK: type: DROPVIEW -PREHOOK: Input: ice_rest@ice_v1 -PREHOOK: Output: ice_rest@ice_v1 -POSTHOOK: query: drop view ice_v1 -POSTHOOK: type: DROPVIEW -POSTHOOK: Input: ice_rest@ice_v1 -POSTHOOK: Output: ice_rest@ice_v1 -PREHOOK: query: create view ice_v2 tblproperties ('view-format'='iceberg') as select * from ice_orc2 +Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name` from `ice_rest`.`ice_orc2` where `ice_orc2`.`dept_id` in (1, 3) +Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name` from `ice_rest`.`ice_orc2` where `ice_orc2`.`dept_id` in (1, 3) +PREHOOK: query: create or replace view ice_v1 tblproperties ('view-format'='iceberg') +as select first_name || '-' || dept_id from ice_orc2 where dept_id = 2 PREHOOK: type: CREATEVIEW PREHOOK: Input: ice_rest@ice_orc2 PREHOOK: Output: database:ice_rest -PREHOOK: Output: ice_rest@ice_v2 -POSTHOOK: query: create view ice_v2 tblproperties ('view-format'='iceberg') as select * from ice_orc2 +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: create or replace view ice_v1 tblproperties ('view-format'='iceberg') +as select first_name || '-' || dept_id from ice_orc2 where dept_id = 2 POSTHOOK: type: CREATEVIEW POSTHOOK: Input: ice_rest@ice_orc2 POSTHOOK: Output: database:ice_rest -POSTHOOK: Output: ice_rest@ice_v2 -POSTHOOK: Lineage: ice_v2.company_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:company_id, type:bigint, comment:null), ] -POSTHOOK: Lineage: ice_v2.dept_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:dept_id, type:bigint, comment:null), ] -POSTHOOK: Lineage: ice_v2.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] -POSTHOOK: Lineage: ice_v2.last_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), ] -POSTHOOK: Lineage: ice_v2.team_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:team_id, type:bigint, comment:null), ] -PREHOOK: query: select * from ice_v2 +POSTHOOK: Output: ice_rest@ice_v1 +PREHOOK: query: select * from ice_v1 PREHOOK: type: QUERY PREHOOK: Input: ice_rest@ice_orc2 -PREHOOK: Input: ice_rest@ice_v2 +PREHOOK: Input: ice_rest@ice_v1 #### A masked pattern was here #### -POSTHOOK: query: select * from ice_v2 +POSTHOOK: query: select * from ice_v1 POSTHOOK: type: QUERY POSTHOOK: Input: ice_rest@ice_orc2 -POSTHOOK: Input: ice_rest@ice_v2 +POSTHOOK: Input: ice_rest@ice_v1 #### A masked pattern was here #### -fn1 ln1 1 10 100 -fn2 ln2 2 20 100 -fn3 ln3 3 30 100 -PREHOOK: query: desc formatted ice_v2 +fn2-2 +PREHOOK: query: desc formatted ice_v1 PREHOOK: type: DESCTABLE -PREHOOK: Input: ice_rest@ice_v2 -POSTHOOK: query: desc formatted ice_v2 +PREHOOK: Input: ice_rest@ice_v1 +POSTHOOK: query: desc formatted ice_v1 POSTHOOK: type: DESCTABLE -POSTHOOK: Input: ice_rest@ice_v2 +POSTHOOK: Input: ice_rest@ice_v1 # col_name data_type comment -first_name string -last_name string -dept_id bigint -team_id bigint -company_id bigint +_c0 string # Detailed Table Information Database: ice_rest @@ -297,7 +278,7 @@ Retention: 2147483647 Table Type: VIRTUAL_VIEW Table Parameters: bucketing_version 2 - current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + current-schema {\"type\":\"struct\",\"schema-id\":1,\"fields\":[{\"id\":1,\"name\":\"_c0\",\"required\":false,\"type\":\"string\"}]} #### A masked pattern was here #### storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG-VIEW @@ -313,126 +294,51 @@ Compressed: No Sort Columns: [] # View Information -Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` -Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` -PREHOOK: query: drop view ice_v2 -PREHOOK: type: DROPVIEW -PREHOOK: Input: ice_rest@ice_v2 -PREHOOK: Output: ice_rest@ice_v2 -POSTHOOK: query: drop view ice_v2 -POSTHOOK: type: DROPVIEW -POSTHOOK: Input: ice_rest@ice_v2 -POSTHOOK: Output: ice_rest@ice_v2 -PREHOOK: query: create view ice_v3 as select * from ice_orc1 -PREHOOK: type: CREATEVIEW -PREHOOK: Input: ice_rest@ice_orc1 -PREHOOK: Output: database:ice_rest -PREHOOK: Output: ice_rest@ice_v3 -POSTHOOK: query: create view ice_v3 as select * from ice_orc1 -POSTHOOK: type: CREATEVIEW -POSTHOOK: Input: ice_rest@ice_orc1 -POSTHOOK: Output: database:ice_rest -POSTHOOK: Output: ice_rest@ice_v3 -POSTHOOK: Lineage: ice_v3.company_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:company_id, type:bigint, comment:null), ] -POSTHOOK: Lineage: ice_v3.dept_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:dept_id, type:bigint, comment:null), ] -POSTHOOK: Lineage: ice_v3.first_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:first_name, type:string, comment:null), ] -POSTHOOK: Lineage: ice_v3.last_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:last_name, type:string, comment:null), ] -POSTHOOK: Lineage: ice_v3.team_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:team_id, type:bigint, comment:null), ] -PREHOOK: query: select * from ice_v3 -PREHOOK: type: QUERY -PREHOOK: Input: ice_rest@ice_orc1 -PREHOOK: Input: ice_rest@ice_v3 -#### A masked pattern was here #### -POSTHOOK: query: select * from ice_v3 -POSTHOOK: type: QUERY -POSTHOOK: Input: ice_rest@ice_orc1 -POSTHOOK: Input: ice_rest@ice_v3 -#### A masked pattern was here #### -PREHOOK: query: desc formatted ice_v3 -PREHOOK: type: DESCTABLE -PREHOOK: Input: ice_rest@ice_v3 -POSTHOOK: query: desc formatted ice_v3 -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: ice_rest@ice_v3 -# col_name data_type comment -first_name string -last_name string -dept_id bigint -team_id bigint -company_id bigint - -# Detailed Table Information -Database: ice_rest -#### A masked pattern was here #### -Retention: 2147483647 -Table Type: VIRTUAL_VIEW -Table Parameters: - bucketing_version 2 - current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} -#### A masked pattern was here #### - storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler - table_type ICEBERG-VIEW - type rest - uuid #Masked# - -# Storage Information -SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -InputFormat: org.apache.hadoop.mapred.FileInputFormat -OutputFormat: org.apache.hadoop.mapred.FileOutputFormat -Compressed: No -Sort Columns: [] - -# View Information -Original Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` -Expanded Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` -PREHOOK: query: drop view ice_v3 +Original Query: select `ice_orc2`.`first_name` || '-' || `ice_orc2`.`dept_id` from `ice_rest`.`ice_orc2` where `ice_orc2`.`dept_id` = 2 +Expanded Query: select `ice_orc2`.`first_name` || '-' || `ice_orc2`.`dept_id` from `ice_rest`.`ice_orc2` where `ice_orc2`.`dept_id` = 2 +PREHOOK: query: drop view ice_v1 PREHOOK: type: DROPVIEW -PREHOOK: Input: ice_rest@ice_v3 -PREHOOK: Output: ice_rest@ice_v3 -POSTHOOK: query: drop view ice_v3 +PREHOOK: Input: ice_rest@ice_v1 +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: drop view ice_v1 POSTHOOK: type: DROPVIEW -POSTHOOK: Input: ice_rest@ice_v3 -POSTHOOK: Output: ice_rest@ice_v3 -PREHOOK: query: create view ice_v4 as select * from ice_orc2 +POSTHOOK: Input: ice_rest@ice_v1 +POSTHOOK: Output: ice_rest@ice_v1 +PREHOOK: query: create view ice_v2 +as select first_name, last_name || '-' || dept_id from ice_orc2 where team_id in (20, 30) PREHOOK: type: CREATEVIEW PREHOOK: Input: ice_rest@ice_orc2 PREHOOK: Output: database:ice_rest -PREHOOK: Output: ice_rest@ice_v4 -POSTHOOK: query: create view ice_v4 as select * from ice_orc2 +PREHOOK: Output: ice_rest@ice_v2 +POSTHOOK: query: create view ice_v2 +as select first_name, last_name || '-' || dept_id from ice_orc2 where team_id in (20, 30) POSTHOOK: type: CREATEVIEW POSTHOOK: Input: ice_rest@ice_orc2 POSTHOOK: Output: database:ice_rest -POSTHOOK: Output: ice_rest@ice_v4 -POSTHOOK: Lineage: ice_v4.company_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:company_id, type:bigint, comment:null), ] -POSTHOOK: Lineage: ice_v4.dept_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:dept_id, type:bigint, comment:null), ] -POSTHOOK: Lineage: ice_v4.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] -POSTHOOK: Lineage: ice_v4.last_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), ] -POSTHOOK: Lineage: ice_v4.team_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:team_id, type:bigint, comment:null), ] -PREHOOK: query: select * from ice_v4 +POSTHOOK: Output: ice_rest@ice_v2 +POSTHOOK: Lineage: ice_v2._c1 EXPRESSION [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), (ice_orc2)ice_orc2.FieldSchema(name:dept_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v2.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] +PREHOOK: query: select * from ice_v2 PREHOOK: type: QUERY PREHOOK: Input: ice_rest@ice_orc2 -PREHOOK: Input: ice_rest@ice_v4 +PREHOOK: Input: ice_rest@ice_v2 #### A masked pattern was here #### -POSTHOOK: query: select * from ice_v4 +POSTHOOK: query: select * from ice_v2 POSTHOOK: type: QUERY POSTHOOK: Input: ice_rest@ice_orc2 -POSTHOOK: Input: ice_rest@ice_v4 +POSTHOOK: Input: ice_rest@ice_v2 #### A masked pattern was here #### -fn1 ln1 1 10 100 -fn2 ln2 2 20 100 -fn3 ln3 3 30 100 -PREHOOK: query: desc formatted ice_v4 +fn2 ln2-2 +fn3 ln3-3 +PREHOOK: query: desc formatted ice_v2 PREHOOK: type: DESCTABLE -PREHOOK: Input: ice_rest@ice_v4 -POSTHOOK: query: desc formatted ice_v4 +PREHOOK: Input: ice_rest@ice_v2 +POSTHOOK: query: desc formatted ice_v2 POSTHOOK: type: DESCTABLE -POSTHOOK: Input: ice_rest@ice_v4 +POSTHOOK: Input: ice_rest@ice_v2 # col_name data_type comment first_name string -last_name string -dept_id bigint -team_id bigint -company_id bigint +_c1 string # Detailed Table Information Database: ice_rest @@ -441,7 +347,7 @@ Retention: 2147483647 Table Type: VIRTUAL_VIEW Table Parameters: bucketing_version 2 - current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"_c1\",\"required\":false,\"type\":\"string\"}]} #### A masked pattern was here #### storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG-VIEW @@ -456,16 +362,16 @@ Compressed: No Sort Columns: [] # View Information -Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` -Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` -PREHOOK: query: drop view ice_v4 +Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name` || '-' || `ice_orc2`.`dept_id` from `ice_rest`.`ice_orc2` where `ice_orc2`.`team_id` in (20, 30) +Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name` || '-' || `ice_orc2`.`dept_id` from `ice_rest`.`ice_orc2` where `ice_orc2`.`team_id` in (20, 30) +PREHOOK: query: drop view ice_v2 PREHOOK: type: DROPVIEW -PREHOOK: Input: ice_rest@ice_v4 -PREHOOK: Output: ice_rest@ice_v4 -POSTHOOK: query: drop view ice_v4 +PREHOOK: Input: ice_rest@ice_v2 +PREHOOK: Output: ice_rest@ice_v2 +POSTHOOK: query: drop view ice_v2 POSTHOOK: type: DROPVIEW -POSTHOOK: Input: ice_rest@ice_v4 -POSTHOOK: Output: ice_rest@ice_v4 +POSTHOOK: Input: ice_rest@ice_v2 +POSTHOOK: Output: ice_rest@ice_v2 PREHOOK: query: show tables PREHOOK: type: SHOWTABLES PREHOOK: Input: database:ice_rest diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out index 5b6b321c6d8b..09dfb18b6e3e 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out @@ -178,31 +178,32 @@ POSTHOOK: Input: ice_rest@ice_orc2 fn1 ln1 1 10 100 fn2 ln2 2 20 100 fn3 ln3 3 30 100 -PREHOOK: query: create view ice_v1 tblproperties ('view-format'='iceberg') as select * from ice_orc1 +PREHOOK: query: create view ice_v1 tblproperties ('view-format'='iceberg') +as select first_name, last_name from ice_orc2 where dept_id in (1,2) PREHOOK: type: CREATEVIEW -PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Input: ice_rest@ice_orc2 PREHOOK: Output: database:ice_rest PREHOOK: Output: ice_rest@ice_v1 -POSTHOOK: query: create view ice_v1 tblproperties ('view-format'='iceberg') as select * from ice_orc1 +POSTHOOK: query: create view ice_v1 tblproperties ('view-format'='iceberg') +as select first_name, last_name from ice_orc2 where dept_id in (1,2) POSTHOOK: type: CREATEVIEW -POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Input: ice_rest@ice_orc2 POSTHOOK: Output: database:ice_rest POSTHOOK: Output: ice_rest@ice_v1 -POSTHOOK: Lineage: ice_v1.company_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:company_id, type:bigint, comment:null), ] -POSTHOOK: Lineage: ice_v1.dept_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:dept_id, type:bigint, comment:null), ] -POSTHOOK: Lineage: ice_v1.first_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:first_name, type:string, comment:null), ] -POSTHOOK: Lineage: ice_v1.last_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:last_name, type:string, comment:null), ] -POSTHOOK: Lineage: ice_v1.team_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:team_id, type:bigint, comment:null), ] +POSTHOOK: Lineage: ice_v1.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] +POSTHOOK: Lineage: ice_v1.last_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), ] PREHOOK: query: select * from ice_v1 PREHOOK: type: QUERY -PREHOOK: Input: ice_rest@ice_orc1 +PREHOOK: Input: ice_rest@ice_orc2 PREHOOK: Input: ice_rest@ice_v1 #### A masked pattern was here #### POSTHOOK: query: select * from ice_v1 POSTHOOK: type: QUERY -POSTHOOK: Input: ice_rest@ice_orc1 +POSTHOOK: Input: ice_rest@ice_orc2 POSTHOOK: Input: ice_rest@ice_v1 #### A masked pattern was here #### +fn1 ln1 +fn2 ln2 PREHOOK: query: desc formatted ice_v1 PREHOOK: type: DESCTABLE PREHOOK: Input: ice_rest@ice_v1 @@ -212,9 +213,6 @@ POSTHOOK: Input: ice_rest@ice_v1 # col_name data_type comment first_name string last_name string -dept_id bigint -team_id bigint -company_id bigint # Detailed Table Information Database: ice_rest @@ -223,7 +221,7 @@ Retention: 2147483647 Table Type: VIRTUAL_VIEW Table Parameters: bucketing_version 2 - current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]} #### A masked pattern was here #### storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG-VIEW @@ -239,8 +237,8 @@ Compressed: No Sort Columns: [] # View Information -Original Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` -Expanded Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` +Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name` from `ice_rest`.`ice_orc2` where `ice_orc2`.`dept_id` in (1,2) +Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name` from `ice_rest`.`ice_orc2` where `ice_orc2`.`dept_id` in (1,2) PREHOOK: query: drop view ice_v1 PREHOOK: type: DROPVIEW PREHOOK: Input: ice_rest@ice_v1 @@ -249,20 +247,19 @@ POSTHOOK: query: drop view ice_v1 POSTHOOK: type: DROPVIEW POSTHOOK: Input: ice_rest@ice_v1 POSTHOOK: Output: ice_rest@ice_v1 -PREHOOK: query: create view ice_v2 tblproperties ('view-format'='iceberg') as select * from ice_orc2 +PREHOOK: query: create view ice_v2 +as select dept_id, team_id from ice_orc2 where company_id = 100 PREHOOK: type: CREATEVIEW PREHOOK: Input: ice_rest@ice_orc2 PREHOOK: Output: database:ice_rest PREHOOK: Output: ice_rest@ice_v2 -POSTHOOK: query: create view ice_v2 tblproperties ('view-format'='iceberg') as select * from ice_orc2 +POSTHOOK: query: create view ice_v2 +as select dept_id, team_id from ice_orc2 where company_id = 100 POSTHOOK: type: CREATEVIEW POSTHOOK: Input: ice_rest@ice_orc2 POSTHOOK: Output: database:ice_rest POSTHOOK: Output: ice_rest@ice_v2 -POSTHOOK: Lineage: ice_v2.company_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:company_id, type:bigint, comment:null), ] POSTHOOK: Lineage: ice_v2.dept_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:dept_id, type:bigint, comment:null), ] -POSTHOOK: Lineage: ice_v2.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] -POSTHOOK: Lineage: ice_v2.last_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), ] POSTHOOK: Lineage: ice_v2.team_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:team_id, type:bigint, comment:null), ] PREHOOK: query: select * from ice_v2 PREHOOK: type: QUERY @@ -274,9 +271,9 @@ POSTHOOK: type: QUERY POSTHOOK: Input: ice_rest@ice_orc2 POSTHOOK: Input: ice_rest@ice_v2 #### A masked pattern was here #### -fn1 ln1 1 10 100 -fn2 ln2 2 20 100 -fn3 ln3 3 30 100 +1 10 +2 20 +3 30 PREHOOK: query: desc formatted ice_v2 PREHOOK: type: DESCTABLE PREHOOK: Input: ice_rest@ice_v2 @@ -284,11 +281,8 @@ POSTHOOK: query: desc formatted ice_v2 POSTHOOK: type: DESCTABLE POSTHOOK: Input: ice_rest@ice_v2 # col_name data_type comment -first_name string -last_name string dept_id bigint team_id bigint -company_id bigint # Detailed Table Information Database: ice_rest @@ -297,13 +291,12 @@ Retention: 2147483647 Table Type: VIRTUAL_VIEW Table Parameters: bucketing_version 2 - current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":2,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"}]} #### A masked pattern was here #### storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG-VIEW type rest uuid #Masked# - view-format iceberg # Storage Information SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe @@ -313,8 +306,8 @@ Compressed: No Sort Columns: [] # View Information -Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` -Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` +Original Query: select `ice_orc2`.`dept_id`, `ice_orc2`.`team_id` from `ice_rest`.`ice_orc2` where `ice_orc2`.`company_id` = 100 +Expanded Query: select `ice_orc2`.`dept_id`, `ice_orc2`.`team_id` from `ice_rest`.`ice_orc2` where `ice_orc2`.`company_id` = 100 PREHOOK: query: drop view ice_v2 PREHOOK: type: DROPVIEW PREHOOK: Input: ice_rest@ice_v2 @@ -323,149 +316,6 @@ POSTHOOK: query: drop view ice_v2 POSTHOOK: type: DROPVIEW POSTHOOK: Input: ice_rest@ice_v2 POSTHOOK: Output: ice_rest@ice_v2 -PREHOOK: query: create view ice_v3 as select * from ice_orc1 -PREHOOK: type: CREATEVIEW -PREHOOK: Input: ice_rest@ice_orc1 -PREHOOK: Output: database:ice_rest -PREHOOK: Output: ice_rest@ice_v3 -POSTHOOK: query: create view ice_v3 as select * from ice_orc1 -POSTHOOK: type: CREATEVIEW -POSTHOOK: Input: ice_rest@ice_orc1 -POSTHOOK: Output: database:ice_rest -POSTHOOK: Output: ice_rest@ice_v3 -POSTHOOK: Lineage: ice_v3.company_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:company_id, type:bigint, comment:null), ] -POSTHOOK: Lineage: ice_v3.dept_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:dept_id, type:bigint, comment:null), ] -POSTHOOK: Lineage: ice_v3.first_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:first_name, type:string, comment:null), ] -POSTHOOK: Lineage: ice_v3.last_name SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:last_name, type:string, comment:null), ] -POSTHOOK: Lineage: ice_v3.team_id SIMPLE [(ice_orc1)ice_orc1.FieldSchema(name:team_id, type:bigint, comment:null), ] -PREHOOK: query: select * from ice_v3 -PREHOOK: type: QUERY -PREHOOK: Input: ice_rest@ice_orc1 -PREHOOK: Input: ice_rest@ice_v3 -#### A masked pattern was here #### -POSTHOOK: query: select * from ice_v3 -POSTHOOK: type: QUERY -POSTHOOK: Input: ice_rest@ice_orc1 -POSTHOOK: Input: ice_rest@ice_v3 -#### A masked pattern was here #### -PREHOOK: query: desc formatted ice_v3 -PREHOOK: type: DESCTABLE -PREHOOK: Input: ice_rest@ice_v3 -POSTHOOK: query: desc formatted ice_v3 -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: ice_rest@ice_v3 -# col_name data_type comment -first_name string -last_name string -dept_id bigint -team_id bigint -company_id bigint - -# Detailed Table Information -Database: ice_rest -#### A masked pattern was here #### -Retention: 2147483647 -Table Type: VIRTUAL_VIEW -Table Parameters: - bucketing_version 2 - current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} -#### A masked pattern was here #### - storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler - table_type ICEBERG-VIEW - type rest - uuid #Masked# - -# Storage Information -SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -InputFormat: org.apache.hadoop.mapred.FileInputFormat -OutputFormat: org.apache.hadoop.mapred.FileOutputFormat -Compressed: No -Sort Columns: [] - -# View Information -Original Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` -Expanded Query: select `ice_orc1`.`first_name`, `ice_orc1`.`last_name`, `ice_orc1`.`dept_id`, `ice_orc1`.`team_id`, `ice_orc1`.`company_id` from `ice_rest`.`ice_orc1` -PREHOOK: query: drop view ice_v3 -PREHOOK: type: DROPVIEW -PREHOOK: Input: ice_rest@ice_v3 -PREHOOK: Output: ice_rest@ice_v3 -POSTHOOK: query: drop view ice_v3 -POSTHOOK: type: DROPVIEW -POSTHOOK: Input: ice_rest@ice_v3 -POSTHOOK: Output: ice_rest@ice_v3 -PREHOOK: query: create view ice_v4 as select * from ice_orc2 -PREHOOK: type: CREATEVIEW -PREHOOK: Input: ice_rest@ice_orc2 -PREHOOK: Output: database:ice_rest -PREHOOK: Output: ice_rest@ice_v4 -POSTHOOK: query: create view ice_v4 as select * from ice_orc2 -POSTHOOK: type: CREATEVIEW -POSTHOOK: Input: ice_rest@ice_orc2 -POSTHOOK: Output: database:ice_rest -POSTHOOK: Output: ice_rest@ice_v4 -POSTHOOK: Lineage: ice_v4.company_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:company_id, type:bigint, comment:null), ] -POSTHOOK: Lineage: ice_v4.dept_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:dept_id, type:bigint, comment:null), ] -POSTHOOK: Lineage: ice_v4.first_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:first_name, type:string, comment:null), ] -POSTHOOK: Lineage: ice_v4.last_name SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:last_name, type:string, comment:null), ] -POSTHOOK: Lineage: ice_v4.team_id SIMPLE [(ice_orc2)ice_orc2.FieldSchema(name:team_id, type:bigint, comment:null), ] -PREHOOK: query: select * from ice_v4 -PREHOOK: type: QUERY -PREHOOK: Input: ice_rest@ice_orc2 -PREHOOK: Input: ice_rest@ice_v4 -#### A masked pattern was here #### -POSTHOOK: query: select * from ice_v4 -POSTHOOK: type: QUERY -POSTHOOK: Input: ice_rest@ice_orc2 -POSTHOOK: Input: ice_rest@ice_v4 -#### A masked pattern was here #### -fn1 ln1 1 10 100 -fn2 ln2 2 20 100 -fn3 ln3 3 30 100 -PREHOOK: query: desc formatted ice_v4 -PREHOOK: type: DESCTABLE -PREHOOK: Input: ice_rest@ice_v4 -POSTHOOK: query: desc formatted ice_v4 -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: ice_rest@ice_v4 -# col_name data_type comment -first_name string -last_name string -dept_id bigint -team_id bigint -company_id bigint - -# Detailed Table Information -Database: ice_rest -#### A masked pattern was here #### -Retention: 2147483647 -Table Type: VIRTUAL_VIEW -Table Parameters: - bucketing_version 2 - current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"},{\"id\":4,\"name\":\"team_id\",\"required\":false,\"type\":\"long\"},{\"id\":5,\"name\":\"company_id\",\"required\":false,\"type\":\"long\"}]} -#### A masked pattern was here #### - storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler - table_type ICEBERG-VIEW - type rest - uuid #Masked# - -# Storage Information -SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe -InputFormat: org.apache.hadoop.mapred.FileInputFormat -OutputFormat: org.apache.hadoop.mapred.FileOutputFormat -Compressed: No -Sort Columns: [] - -# View Information -Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` -Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name`, `ice_orc2`.`dept_id`, `ice_orc2`.`team_id`, `ice_orc2`.`company_id` from `ice_rest`.`ice_orc2` -PREHOOK: query: drop view ice_v4 -PREHOOK: type: DROPVIEW -PREHOOK: Input: ice_rest@ice_v4 -PREHOOK: Output: ice_rest@ice_v4 -POSTHOOK: query: drop view ice_v4 -POSTHOOK: type: DROPVIEW -POSTHOOK: Input: ice_rest@ice_v4 -POSTHOOK: Output: ice_rest@ice_v4 PREHOOK: query: show tables PREHOOK: type: SHOWTABLES PREHOOK: Input: database:ice_rest diff --git a/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java index 14169c072474..e4ff29376a0d 100644 --- a/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java +++ b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java @@ -227,7 +227,6 @@ public void testNativeIcebergLogicalView() throws Exception { "select 1 as x", null, "rest-native-view", - false, false); GetTableRequest getTableRequest = new GetTableRequest(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java index ad27f1dae281..17638a0fe804 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java @@ -100,9 +100,6 @@ public int execute() throws HiveException { } else { // We create new view Table view = createViewObject(); - if (desc.usesStorageHandler()) { - pushExternalLogicalViewSessionHints(desc.isReplace(), desc.getIfNotExists()); - } context.getDb().createTable(view, desc.getIfNotExists()); DDLUtils.addIfAbsentByName(new WriteEntity(view, WriteEntity.WriteType.DDL_NO_LOCK), context.getWork().getOutputs()); @@ -114,13 +111,6 @@ public int execute() throws HiveException { return 0; } - private void pushExternalLogicalViewSessionHints(boolean replace, boolean ifNotExists) { - SessionStateUtil.addResource(context.getConf(), Constants.EXTERNAL_LOGICAL_VIEW_DDL_REPLACE, - Boolean.toString(replace)); - SessionStateUtil.addResource(context.getConf(), Constants.EXTERNAL_LOGICAL_VIEW_CREATE_IF_NOT_EXISTS, - Boolean.toString(ifNotExists)); - } - private void clearStorageHandlerProp(Table oldview) { Map params = oldview.getParameters(); if (params == null) { From 3b8d48b7581e7235ff65b0f08a624426fe965567 Mon Sep 17 00:00:00 2001 From: Dmitriy Fingerman Date: Mon, 1 Jun 2026 13:31:22 -0400 Subject: [PATCH 5/6] code review comments June 02. --- .../hive/IcebergNativeLogicalViewSupport.java | 63 ++++++++++++------- .../apache/iceberg/hive/MetastoreUtil.java | 63 +++++++++++-------- .../hive/client/HiveRESTCatalogClient.java | 19 +++--- .../TestIcebergNativeLogicalViewSupport.java | 36 ++++++++--- .../client/TestHiveRESTCatalogClient.java | 3 +- .../mr/hive/BaseHiveIcebergMetaHook.java | 2 + .../iceberg/mr/hive/HiveIcebergMetaHook.java | 28 ++++----- .../positive/iceberg_rest_catalog_gravitino.q | 12 +++- .../iceberg_native_logical_view.q.out | 19 +++--- .../llap/iceberg_rest_catalog_gravitino.q.out | 63 ++++++++++++++++++- .../hive/TestHiveRESTCatalogClientITBase.java | 60 ++++++++++++++---- .../ddl/view/create/CreateViewAnalyzer.java | 6 +- 12 files changed, 257 insertions(+), 117 deletions(-) diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergNativeLogicalViewSupport.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergNativeLogicalViewSupport.java index 58c1f20ba565..2a940cf73e50 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergNativeLogicalViewSupport.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergNativeLogicalViewSupport.java @@ -50,19 +50,47 @@ private IcebergNativeLogicalViewSupport() { } /** - * Creates or replaces a view in the Iceberg catalog. - * - * @return {@code false} if skipped because the view already exists and {@code replace} is false + * Loads the native Iceberg logical view definition and applies SQL, schema, and Iceberg params to {@code hmsTable} */ - public static boolean createOrReplaceNativeView( + public static void enrichHmsTableFromIcebergView( + org.apache.hadoop.hive.metastore.api.Table hmsTable, Configuration conf) { + TableIdentifier identifier = TableIdentifier.of(hmsTable.getDbName(), hmsTable.getTableName()); + String catalogName = IcebergCatalogProperties.getCatalogName(conf); + Map catalogProps = IcebergCatalogProperties.getCatalogProperties(conf, catalogName); + Catalog catalog = CatalogUtil.buildIcebergCatalog(catalogName, catalogProps, conf); + + try { + if (catalog instanceof Closeable closeable) { + try (Closeable ignored = closeable) { + loadAndApplyView(hmsTable, conf, catalog, catalogName, identifier); + } + } else { + loadAndApplyView(hmsTable, conf, catalog, catalogName, identifier); + } + } catch (IOException e) { + throw new UncheckedIOException("Failed to close Iceberg catalog", e); + } + } + + private static void loadAndApplyView( + org.apache.hadoop.hive.metastore.api.Table hmsTable, + Configuration conf, + Catalog catalog, + String catalogName, + TableIdentifier identifier) { + ViewCatalog viewCatalog = asViewCatalog(catalog, catalogName); + MetastoreUtil.applyIcebergViewToHmsTable(hmsTable, viewCatalog.loadView(identifier), conf); + } + + /** Creates or replaces a view in the Iceberg catalog. */ + public static void createOrReplaceNativeView( Configuration conf, String databaseName, String viewName, List fieldSchemas, String viewSql, Map tblProperties, - String comment, - boolean replace) { + String comment) { TableIdentifier identifier = TableIdentifier.of(databaseName, viewName); String catalogName = IcebergCatalogProperties.getCatalogName(conf); @@ -71,30 +99,24 @@ public static boolean createOrReplaceNativeView( if (catalog instanceof Closeable closeable) { try (Closeable ignored = closeable) { - return commitNativeView( - catalog, catalogName, identifier, fieldSchemas, viewSql, tblProperties, comment, replace); + commitNativeView(catalog, catalogName, identifier, fieldSchemas, viewSql, tblProperties, comment); } catch (IOException e) { throw new UncheckedIOException("Failed to close Iceberg catalog", e); } + } else { + commitNativeView(catalog, catalogName, identifier, fieldSchemas, viewSql, tblProperties, comment); } - - return commitNativeView( - catalog, catalogName, identifier, fieldSchemas, viewSql, tblProperties, comment, replace); } - private static boolean commitNativeView( + private static void commitNativeView( Catalog catalog, String catalogName, TableIdentifier identifier, List fieldSchemas, String viewSql, Map tblProperties, - String comment, - boolean replace) { + String comment) { ViewCatalog viewCatalog = asViewCatalog(catalog, catalogName); - if (!replace && viewCatalog.viewExists(identifier)) { - return false; - } ViewBuilder builder = viewCatalog @@ -116,12 +138,7 @@ private static boolean commitNativeView( } } - if (replace) { - builder.createOrReplace(); - } else { - builder.create(); - } - return true; + builder.createOrReplace(); } private static ViewCatalog asViewCatalog(Catalog catalog, String catalogName) { diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java index a7f3a96d4cc7..0991ffa2a860 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java @@ -36,8 +36,10 @@ import org.apache.hadoop.hive.metastore.api.SkewedInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.hadoop.hive.serde.serdeConstants; +import org.apache.iceberg.BaseMetastoreTableOperations; import org.apache.iceberg.BaseTable; import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.Schema; @@ -46,7 +48,6 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; -import org.apache.iceberg.util.PropertyUtil; import org.apache.iceberg.view.BaseView; import org.apache.iceberg.view.SQLViewRepresentation; import org.apache.iceberg.view.View; @@ -158,37 +159,44 @@ public static Table toHiveTable(org.apache.iceberg.Table table, Configuration co } /** - * Builds a Hive metastore {@link Table} representation for an Iceberg {@link View}, for clients - * (e.g. {@code HiveRESTCatalogClient}) that bridge Iceberg catalog metadata into the HMS API. + * Builds a minimal HMS {@link Table} shell for a native Iceberg logical view (identity, view type, + * and Iceberg storage-handler markers only). The storage handler {@code postGetTable} hook enriches + * this object via {@link IcebergNativeLogicalViewSupport#enrichHmsTableFromIcebergView} (view SQL, + * schema, and Iceberg parameters). */ - public static Table toHiveView(View view, Configuration conf) { + public static Table buildMinimalHMSView(String catName, String dbName, String tableName, Configuration conf) { Table result = new Table(); - TableName tableName = - TableName.fromString( - view.name(), MetaStoreUtils.getDefaultCatalog(conf), Warehouse.DEFAULT_DATABASE_NAME); - result.setCatName(tableName.getCat()); - result.setDbName(tableName.getDb()); - result.setTableName(tableName.getTable()); + result.setCatName(catName); + result.setDbName(dbName); + result.setTableName(tableName); result.setTableType(TableType.VIRTUAL_VIEW.toString()); - ViewMetadata metadata = ((BaseView) view).operations().current(); - String sqlText = viewSqlText(view, metadata); - result.setViewOriginalText(sqlText); - result.setViewExpandedText(sqlText); - - long nowMillis = System.currentTimeMillis(); - int nowSec = (int) (nowMillis / 1000); - String owner = - PropertyUtil.propertyAsString( - metadata.properties(), HiveCatalog.HMS_TABLE_OWNER, System.getProperty("user.name")); - result.setOwner(owner); + int nowSec = (int) (System.currentTimeMillis() / 1000); + result.setOwner(System.getProperty("user.name")); result.setCreateTime(nowSec); result.setLastAccessTime(nowSec); result.setRetention(Integer.MAX_VALUE); + Map parameters = Maps.newHashMap(); + parameters.put( + BaseMetastoreTableOperations.TABLE_TYPE_PROP, + IcebergNativeLogicalViewSupport.ICEBERG_VIEW_HMS_TABLE_TYPE_VALUE); + parameters.put( + hive_metastoreConstants.META_TABLE_STORAGE, HMSTablePropertyHelper.HIVE_ICEBERG_STORAGE_HANDLER); + result.setParameters(parameters); + return result; + } + + /** + * Applies Iceberg view metadata (SQL, schema, params) onto an existing HMS {@link Table}. + */ + public static void applyIcebergViewToHmsTable(Table hmsTable, View view, Configuration conf) { + ViewMetadata metadata = ((BaseView) view).operations().current(); + String sqlText = viewSqlText(view, metadata); + boolean hiveEngineEnabled = false; - result.setSd(HiveOperationsBase.storageDescriptor(metadata.schema(), metadata.location(), hiveEngineEnabled)); - StorageDescriptor sd = result.getSd(); + hmsTable.setSd(HiveOperationsBase.storageDescriptor(metadata.schema(), metadata.location(), hiveEngineEnabled)); + StorageDescriptor sd = hmsTable.getSd(); if (sd.getBucketCols() == null) { sd.setBucketCols(Lists.newArrayList()); @@ -204,17 +212,20 @@ public static Table toHiveView(View view, Configuration conf) { HiveOperationsBase.HIVE_TABLE_PROPERTY_MAX_SIZE_DEFAULT); HMSTablePropertyHelper.updateHmsTableForIcebergView( metadata.metadataFileLocation(), - result, + hmsTable, metadata, Collections.emptySet(), maxHiveTablePropertySize, null); + // In-memory overlay for compile/describe: authoritative SQL comes from Iceberg metadata. + hmsTable.setViewOriginalText(sqlText); + hmsTable.setViewExpandedText(sqlText); + String catalogType = IcebergCatalogProperties.getCatalogType(conf); if (!StringUtils.isEmpty(catalogType) && !IcebergCatalogProperties.NO_CATALOG_TYPE.equals(catalogType)) { - result.getParameters().put(CatalogUtil.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.getCatalogType(conf)); + hmsTable.getParameters().put(CatalogUtil.ICEBERG_CATALOG_TYPE, IcebergCatalogProperties.getCatalogType(conf)); } - return result; } private static String viewSqlText(View view, ViewMetadata metadata) { diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java index 27c7efd662fa..1798b6811c75 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java @@ -49,7 +49,6 @@ import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.catalog.ViewCatalog; import org.apache.iceberg.exceptions.NoSuchTableException; -import org.apache.iceberg.exceptions.NoSuchViewException; import org.apache.iceberg.hive.HMSTablePropertyHelper; import org.apache.iceberg.hive.HiveSchemaUtil; import org.apache.iceberg.hive.IcebergCatalogProperties; @@ -61,7 +60,6 @@ import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.rest.RESTCatalog; -import org.apache.iceberg.view.View; import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -212,12 +210,11 @@ public Table getTable(GetTableRequest tableRequest) throws TException { return MetastoreUtil.toHiveTable(icebergTable, conf); } catch (NoSuchTableException tableMissing) { if (restCatalog instanceof ViewCatalog viewCatalog) { - try { - View icebergView = viewCatalog.loadView(id); - return MetastoreUtil.toHiveView(icebergView, conf); - } catch (NoSuchViewException viewMissing) { + if (!viewCatalog.viewExists(id)) { throw new NoSuchObjectException(); } + return MetastoreUtil.buildMinimalHMSView( + tableRequest.getCatName(), tableRequest.getDbName(), tableRequest.getTblName(), conf); } throw new NoSuchObjectException(); } @@ -240,7 +237,7 @@ public void alter_table(String catName, String dbName, String tblName, Table new EnvironmentContext envContext, String validWriteIdList) throws TException { validateCurrentCatalog(catName); if (hasIcebergNativeViewTableType(newTable) && restCatalog instanceof ViewCatalog) { - createOrReplaceLogicalView(newTable, dbName, tblName, true); + createOrReplaceLogicalView(newTable, dbName, tblName); } } @@ -248,8 +245,7 @@ public void alter_table(String catName, String dbName, String tblName, Table new public void createTable(CreateTableRequest request) throws TException { Table table = request.getTable(); if (hasIcebergNativeViewTableType(table) && restCatalog instanceof ViewCatalog) { - createOrReplaceLogicalView( - table, table.getDbName(), table.getTableName(), false); + createOrReplaceLogicalView(table, table.getDbName(), table.getTableName()); } else { List cols = Lists.newArrayList(table.getSd().getCols()); if (table.isSetPartitionKeys() && !table.getPartitionKeys().isEmpty()) { @@ -273,8 +269,7 @@ public void createTable(CreateTableRequest request) throws TException { } } - private void createOrReplaceLogicalView( - Table table, String dbName, String tableName, boolean replace) { + private void createOrReplaceLogicalView(Table table, String dbName, String tableName) { List cols = Lists.newArrayList(table.getSd().getCols()); if (table.isSetPartitionKeys() && !table.getPartitionKeys().isEmpty()) { @@ -286,7 +281,7 @@ private void createOrReplaceLogicalView( String comment = tblProps.get("comment"); IcebergNativeLogicalViewSupport.createOrReplaceNativeView( - conf, dbName, tableName, cols, table.getViewExpandedText(), tblProps, comment, replace); + conf, dbName, tableName, cols, table.getViewExpandedText(), tblProps, comment); } @Override diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergNativeLogicalViewSupport.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergNativeLogicalViewSupport.java index 88c88645ff5e..3850f1d760d9 100644 --- a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergNativeLogicalViewSupport.java +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergNativeLogicalViewSupport.java @@ -33,6 +33,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.view.BaseView; import org.apache.iceberg.view.View; +import org.apache.thrift.TException; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -85,10 +86,8 @@ void testCreateCommitsNativeViewWithUserProperties() { String sql = String.format("select id, name from %s.src_tbl", DB); Map props = Collections.singletonMap("k1", "v1"); - boolean created = - IcebergNativeLogicalViewSupport.createOrReplaceNativeView( - conf, DB, VIEW, cols, sql, props, "hello-view", false); - assertThat(created).isTrue(); + IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + conf, DB, VIEW, cols, sql, props, "hello-view"); HiveCatalog cat = loadCatalog(); TableIdentifier id = TableIdentifier.of(DB, VIEW); @@ -103,23 +102,40 @@ void testCreateCommitsNativeViewWithUserProperties() { } @Test - void testCreateOrReplaceNativeViewReplacesExistingWhenReplaceFlagTrue() { + void testCreateOrReplaceNativeViewReplacesExisting() { HiveConf conf = nativeViewConf(); List cols = Collections.singletonList(new FieldSchema("id", "int", null)); TableIdentifier id = TableIdentifier.of(DB, VIEW); IcebergNativeLogicalViewSupport.createOrReplaceNativeView( - conf, DB, VIEW, cols, "select 1 as id", null, null, false); + conf, DB, VIEW, cols, "select 1 as id", null, null); View afterCreate = loadCatalog().loadView(id); assertThat(afterCreate.sqlFor("hive").sql().trim()).isEqualTo("select 1 as id"); - assertThat( - IcebergNativeLogicalViewSupport.createOrReplaceNativeView( - conf, DB, VIEW, cols, "select 2 as id", null, null, true)) - .isTrue(); + IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + conf, DB, VIEW, cols, "select 2 as id", null, null); assertThat(loadCatalog().viewExists(id)).isTrue(); View afterReplace = loadCatalog().loadView(id); assertThat(afterReplace.sqlFor("hive").sql().trim()).isEqualTo("select 2 as id"); } + + @Test + void testEnrichHmsTableFromIcebergViewOverridesStaleHmsSql() throws TException { + HiveConf conf = nativeViewConf(); + List cols = Collections.singletonList(new FieldSchema("id", "int", null)); + String sql = "select 42 as id"; + + IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + conf, DB, VIEW, cols, sql, null, null); + + org.apache.hadoop.hive.metastore.api.Table hmsTable = + HIVE_METASTORE_EXTENSION.metastoreClient().getTable(DB, VIEW); + hmsTable.setViewOriginalText("select 0"); + hmsTable.setViewExpandedText("select 0"); + + IcebergNativeLogicalViewSupport.enrichHmsTableFromIcebergView(hmsTable, conf); + assertThat(hmsTable.getViewExpandedText()).isEqualTo(sql); + assertThat(hmsTable.getViewOriginalText()).isEqualTo(sql); + } } diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/client/TestHiveRESTCatalogClient.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/client/TestHiveRESTCatalogClient.java index 3bcbc77419ee..70e441d6109c 100644 --- a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/client/TestHiveRESTCatalogClient.java +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/client/TestHiveRESTCatalogClient.java @@ -228,8 +228,7 @@ public void testAlterNativeIcebergView() throws TException { any(), eq("select 1"), any(), - eq(null), - eq(true))); + eq(null))); } } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java index 708c48294e23..6e8811f71c50 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java @@ -533,6 +533,7 @@ public void postGetTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) { if (hmsTable != null) { try { if (isNativeIcebergLogicalView(hmsTable)) { + IcebergNativeLogicalViewSupport.enrichHmsTableFromIcebergView(hmsTable, conf); return; } Table tbl = IcebergTableUtil.getTable(conf, hmsTable); @@ -562,4 +563,5 @@ private static boolean isHiveIcebergStorageHandler(String storageHandler) { throw new RuntimeException("Error checking storage handler class", e); } } + } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index 7dcfd4e97471..64010a363907 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -181,20 +181,17 @@ public HiveIcebergMetaHook(Configuration conf) { public void commitCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) { if (isNativeIcebergLogicalView(hmsTable)) { tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); - if (Catalogs.hiveCatalog(conf, tableProperties)) { - Map tblProps = - hmsTable.getParameters() == null ? Maps.newHashMap() : Maps.newHashMap(hmsTable.getParameters()); - String comment = tblProps.get("comment"); - IcebergNativeLogicalViewSupport.createOrReplaceNativeView( - conf, - hmsTable.getDbName(), - hmsTable.getTableName(), - hmsTable.getSd().getCols(), - hmsTable.getViewExpandedText(), - tblProps, - comment, - false); - } + Map tblProps = + hmsTable.getParameters() == null ? Maps.newHashMap() : Maps.newHashMap(hmsTable.getParameters()); + String comment = tblProps.get("comment"); + IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + conf, + hmsTable.getDbName(), + hmsTable.getTableName(), + hmsTable.getSd().getCols(), + hmsTable.getViewExpandedText(), + tblProps, + comment); return; } if (icebergTable == null) { @@ -519,8 +516,7 @@ public void commitAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable hmsTable.getSd().getCols(), hmsTable.getViewExpandedText(), tblProps, - comment, - true); + comment); return; } if (isTableMigration) { diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q index 16f99a010a6f..5cdab1267f20 100644 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_rest_catalog_gravitino.q @@ -80,12 +80,22 @@ select * from ice_orc2; --! Native Iceberg logical view with TBLPROPERTIES ('view-format'='iceberg') on a REST catalog table ----------------------------------------------------------------------------------------------------- -create view if not exists ice_v1 tblproperties ('view-format'='iceberg') +create view ice_v1 tblproperties ('view-format'='iceberg') as select first_name, last_name from ice_orc2 where dept_id in (1, 3); select * from ice_v1; desc formatted ice_v1; +------- if-not-exists view test - view should not change ------------------------- + +create view if not exists ice_v1 tblproperties ('view-format'='iceberg') +as select * from ice_orc2 where dept_id = 10000; + +select * from ice_v1; +desc formatted ice_v1; + +------- replace view test - view should be replaced ------------------------------ + create or replace view ice_v1 tblproperties ('view-format'='iceberg') as select first_name || '-' || dept_id from ice_orc2 where dept_id = 2; diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_native_logical_view.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_native_logical_view.q.out index 82cbb661dc49..ad072ca1c23b 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_native_logical_view.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_native_logical_view.q.out @@ -124,8 +124,9 @@ Retention: 0 Table Type: VIRTUAL_VIEW Table Parameters: bucketing_version 2 - current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"_c0\",\"required\":false,\"type\":\"string\"}]} + current-schema {\"type\":\"struct\",\"schema-id\":1,\"fields\":[{\"id\":1,\"name\":\"_c0\",\"required\":false,\"type\":\"string\"}]} metadata_location hdfs://### HDFS PATH ### + previous_metadata_location hdfs://### HDFS PATH ### storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG-VIEW #### A masked pattern was here #### @@ -140,7 +141,7 @@ Compressed: No Sort Columns: [] # View Information -Original Query: select first_name || '-' || dept_id from src_ice where dept_id = 1 +Original Query: select `src_ice`.`first_name` || '-' || `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` where `src_ice`.`dept_id` = 1 Expanded Query: select `src_ice`.`first_name` || '-' || `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` where `src_ice`.`dept_id` = 1 PREHOOK: query: create view if not exists v_def as select first_name, last_name, dept_id from src_ice where dept_id = 2 @@ -213,19 +214,22 @@ Retention: 0 Table Type: VIRTUAL_VIEW Table Parameters: bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"},{\"id\":3,\"name\":\"dept_id\",\"required\":false,\"type\":\"long\"}]} + metadata_location hdfs://### HDFS PATH ### storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler table_type ICEBERG-VIEW #### A masked pattern was here #### + uuid #Masked# # Storage Information -SerDe Library: null -InputFormat: org.apache.hadoop.mapred.TextInputFormat -OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat Compressed: No Sort Columns: [] # View Information -Original Query: select first_name, last_name, dept_id from src_ice where dept_id = 2 +Original Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` where `src_ice`.`dept_id` = 2 Expanded Query: select `src_ice`.`first_name`, `src_ice`.`last_name`, `src_ice`.`dept_id` from `ice_native_view_db`.`src_ice` where `src_ice`.`dept_id` = 2 PREHOOK: query: drop view v_def PREHOOK: type: DROPVIEW @@ -346,8 +350,9 @@ Retention: 0 Table Type: VIRTUAL_VIEW Table Parameters: bucketing_version 2 - current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"_c0\",\"required\":false,\"type\":\"string\"}]} + current-schema {\"type\":\"struct\",\"schema-id\":1,\"fields\":[{\"id\":1,\"name\":\"_c0\",\"required\":false,\"type\":\"string\"}]} metadata_location hdfs://### HDFS PATH ### + previous_metadata_location hdfs://### HDFS PATH ### table_type ICEBERG-VIEW #### A masked pattern was here #### uuid #Masked# diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out index aadc224194cb..759d028e7b81 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out @@ -178,13 +178,13 @@ POSTHOOK: Input: ice_rest@ice_orc2 fn1 ln1 1 10 100 fn2 ln2 2 20 100 fn3 ln3 3 30 100 -PREHOOK: query: create view if not exists ice_v1 tblproperties ('view-format'='iceberg') +PREHOOK: query: create view ice_v1 tblproperties ('view-format'='iceberg') as select first_name, last_name from ice_orc2 where dept_id in (1, 3) PREHOOK: type: CREATEVIEW PREHOOK: Input: ice_rest@ice_orc2 PREHOOK: Output: database:ice_rest PREHOOK: Output: ice_rest@ice_v1 -POSTHOOK: query: create view if not exists ice_v1 tblproperties ('view-format'='iceberg') +POSTHOOK: query: create view ice_v1 tblproperties ('view-format'='iceberg') as select first_name, last_name from ice_orc2 where dept_id in (1, 3) POSTHOOK: type: CREATEVIEW POSTHOOK: Input: ice_rest@ice_orc2 @@ -236,6 +236,65 @@ OutputFormat: org.apache.hadoop.mapred.FileOutputFormat Compressed: No Sort Columns: [] +# View Information +Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name` from `ice_rest`.`ice_orc2` where `ice_orc2`.`dept_id` in (1, 3) +Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name` from `ice_rest`.`ice_orc2` where `ice_orc2`.`dept_id` in (1, 3) +PREHOOK: query: create view if not exists ice_v1 tblproperties ('view-format'='iceberg') +as select * from ice_orc2 where dept_id = 10000 +PREHOOK: type: CREATEVIEW +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Output: database:ice_rest +PREHOOK: Output: ice_rest@ice_v1 +POSTHOOK: query: create view if not exists ice_v1 tblproperties ('view-format'='iceberg') +as select * from ice_orc2 where dept_id = 10000 +POSTHOOK: type: CREATEVIEW +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Output: database:ice_rest +POSTHOOK: Output: ice_rest@ice_v1 +PREHOOK: query: select * from ice_v1 +PREHOOK: type: QUERY +PREHOOK: Input: ice_rest@ice_orc2 +PREHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +POSTHOOK: query: select * from ice_v1 +POSTHOOK: type: QUERY +POSTHOOK: Input: ice_rest@ice_orc2 +POSTHOOK: Input: ice_rest@ice_v1 +#### A masked pattern was here #### +fn1 ln1 +fn3 ln3 +PREHOOK: query: desc formatted ice_v1 +PREHOOK: type: DESCTABLE +PREHOOK: Input: ice_rest@ice_v1 +POSTHOOK: query: desc formatted ice_v1 +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: ice_rest@ice_v1 +# col_name data_type comment +first_name string +last_name string + +# Detailed Table Information +Database: ice_rest +#### A masked pattern was here #### +Retention: 2147483647 +Table Type: VIRTUAL_VIEW +Table Parameters: + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":0,\"fields\":[{\"id\":1,\"name\":\"first_name\",\"required\":false,\"type\":\"string\"},{\"id\":2,\"name\":\"last_name\",\"required\":false,\"type\":\"string\"}]} +#### A masked pattern was here #### + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG-VIEW + type rest + uuid #Masked# + view-format iceberg + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.FileInputFormat +OutputFormat: org.apache.hadoop.mapred.FileOutputFormat +Compressed: No +Sort Columns: [] + # View Information Original Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name` from `ice_rest`.`ice_orc2` where `ice_orc2`.`dept_id` in (1, 3) Expanded Query: select `ice_orc2`.`first_name`, `ice_orc2`.`last_name` from `ice_rest`.`ice_orc2` where `ice_orc2`.`dept_id` in (1, 3) diff --git a/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java index e4ff29376a0d..32ddeae6b5d7 100644 --- a/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java +++ b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat; import org.apache.hadoop.hive.ql.metadata.Hive; @@ -47,7 +48,6 @@ import org.apache.iceberg.TableProperties; import org.apache.iceberg.hive.HiveSchemaUtil; import org.apache.iceberg.hive.IcebergCatalogProperties; -import org.apache.iceberg.hive.IcebergNativeLogicalViewSupport; import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; @@ -115,6 +115,20 @@ void setup() throws Exception { msClient = new HiveMetaStoreClient(conf, hookLoader); hiveConf = new HiveConf(conf, HiveConf.class); hive = Hive.get(hiveConf); + dropDatabaseIfExists(VIEW_DB_NAME); + } + + private void dropDatabaseIfExists(String dbName) { + try { + msClient.dropTable(CATALOG_NAME, dbName, NATIVE_VIEW_NAME); + } catch (Exception ignored) { + // view may not exist + } + try { + msClient.dropDatabase(dbName); + } catch (Exception ignored) { + // database may not exist + } } @AfterEach @@ -219,28 +233,22 @@ public void testNativeIcebergLogicalView() throws Exception { hive.createDatabase(db, true); List cols = Collections.singletonList(new FieldSchema("x", "int", "")); - IcebergNativeLogicalViewSupport.createOrReplaceNativeView( - hiveConf, - VIEW_DB_NAME, - NATIVE_VIEW_NAME, - cols, - "select 1 as x", - null, - "rest-native-view", - false); + createNativeIcebergLogicalView(VIEW_DB_NAME, NATIVE_VIEW_NAME, cols, "select 1 as x", "rest-native-view"); + Assertions.assertTrue(msClient.tableExists(CATALOG_NAME, VIEW_DB_NAME, NATIVE_VIEW_NAME)); + GetTableRequest getTableRequest = new GetTableRequest(); getTableRequest.setCatName(CATALOG_NAME); getTableRequest.setDbName(VIEW_DB_NAME); getTableRequest.setTblName(NATIVE_VIEW_NAME); Table view = msClient.getTable(getTableRequest); + + Assertions.assertNotNull(view); Assertions.assertEquals(TableType.VIRTUAL_VIEW.name(), view.getTableType()); String tableTypeProp = view.getParameters().get(BaseMetastoreTableOperations.TABLE_TYPE_PROP); Assertions.assertNotNull(tableTypeProp); Assertions.assertEquals(ICEBERG_VIEW_TYPE_VALUE, tableTypeProp.toLowerCase()); - Assertions.assertTrue(msClient.tableExists(CATALOG_NAME, VIEW_DB_NAME, NATIVE_VIEW_NAME)); - List names = msClient.getTables(CATALOG_NAME, VIEW_DB_NAME, "*"); Assertions.assertTrue(names.contains(NATIVE_VIEW_NAME)); @@ -250,7 +258,33 @@ public void testNativeIcebergLogicalView() throws Exception { msClient.dropDatabase(VIEW_DB_NAME); } - private static Table createPartitionedTable(IMetaStoreClient db, String catName, String dbName, String tableName, + private void createNativeIcebergLogicalView( + String dbName, String viewName, List cols, String viewSql, String comment) throws Exception { + Table view = new Table(); + view.setCatName(CATALOG_NAME); + view.setDbName(dbName); + view.setTableName(viewName); + view.setTableType(TableType.VIRTUAL_VIEW.toString()); + view.setViewOriginalText(viewSql); + view.setViewExpandedText(viewSql); + + StorageDescriptor sd = new StorageDescriptor(); + sd.setCols(cols); + sd.setSerdeInfo(new SerDeInfo()); + sd.getSerdeInfo().setParameters(new java.util.HashMap<>()); + view.setSd(sd); + + view.setParameters(new java.util.HashMap<>()); + view.getParameters().put(hive_metastoreConstants.META_TABLE_STORAGE, HIVE_ICEBERG_STORAGE_HANDLER); + view.getParameters().put("view-format", "iceberg"); + if (comment != null) { + view.getParameters().put("comment", comment); + } + + msClient.createTable(view); + } + + private static Table createPartitionedTable(IMetaStoreClient db, String catName, String dbName, String tableName, Map tableParameters) throws Exception { db.dropTable(catName, dbName, tableName); Table table = new Table(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java index 62d7a876f143..73b6da2cb28e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewAnalyzer.java @@ -224,7 +224,6 @@ private String resolveViewStorageHandlerClass(String storageHandlerClassFromTabl return null; } - boolean explicitViewFormat = storageHandlerClassFromTableProps != null; try { HiveStorageHandler storageHandler = HiveUtils.getStorageHandler(conf, storageHandlerClass); @@ -232,10 +231,7 @@ private String resolveViewStorageHandlerClass(String storageHandlerClassFromTabl return storageHandlerClass; } } catch (HiveException e) { - - if (explicitViewFormat) { - throw new SemanticException(ErrorMsg.VIEW_STORAGE_HANDLER_UNSUPPORTED.format(storageHandlerClass), e); - } + throw new SemanticException(e); } return null; From 136c3d91d91b56faeb46b77acc2a72247e4fe7e0 Mon Sep 17 00:00:00 2001 From: Dmitriy Fingerman Date: Wed, 3 Jun 2026 12:38:24 -0400 Subject: [PATCH 6/6] code review comments June 03. --- .../iceberg/hive/HiveOperationsBase.java | 2 +- ...rt.java => IcebergLogicalViewSupport.java} | 24 ++---- .../apache/iceberg/hive/MetastoreUtil.java | 20 ++--- .../hive/client/HiveRESTCatalogClient.java | 76 ++++++++++--------- ...ava => TestIcebergLogicalViewSupport.java} | 18 +++-- .../client/TestHiveRESTCatalogClient.java | 23 +++--- .../mr/hive/BaseHiveIcebergMetaHook.java | 17 +++-- .../iceberg/mr/hive/HiveIcebergMetaHook.java | 14 ++-- .../mr/hive/HiveIcebergStorageHandler.java | 12 +-- .../llap/iceberg_rest_catalog_gravitino.q.out | 8 +- .../llap/iceberg_rest_catalog_hms.q.out | 4 +- .../hive/TestHiveRESTCatalogClientITBase.java | 11 ++- .../hive/ql/parse/TestParseCreateView.java | 19 ++--- .../ddl/view/create/CreateViewOperation.java | 2 - .../CreateExternalLogicalViewRequest.java | 2 +- .../hive/ql/metadata/HiveStorageHandler.java | 1 - 16 files changed, 126 insertions(+), 127 deletions(-) rename iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/{IcebergNativeLogicalViewSupport.java => IcebergLogicalViewSupport.java} (84%) rename iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/{TestIcebergNativeLogicalViewSupport.java => TestIcebergLogicalViewSupport.java} (89%) diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveOperationsBase.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveOperationsBase.java index 937939b48160..a5a0657cc12e 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveOperationsBase.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveOperationsBase.java @@ -44,7 +44,7 @@ import org.slf4j.LoggerFactory; /** All the HMS operations like table,view,materialized_view should implement this. */ -interface HiveOperationsBase { +public interface HiveOperationsBase { Logger LOG = LoggerFactory.getLogger(HiveOperationsBase.class); // The max size is based on HMS backend database. For Hive versions below 2.3, the max table diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergNativeLogicalViewSupport.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergLogicalViewSupport.java similarity index 84% rename from iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergNativeLogicalViewSupport.java rename to iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergLogicalViewSupport.java index 2a940cf73e50..9f11c308ddc4 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergNativeLogicalViewSupport.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/IcebergLogicalViewSupport.java @@ -40,13 +40,9 @@ * Commits a native Iceberg view through the configured default Iceberg catalog (HiveCatalog or REST * catalog, etc.) when {@code Catalog} also implements {@link ViewCatalog}. */ -public final class IcebergNativeLogicalViewSupport { +public final class IcebergLogicalViewSupport { - /** Value for HMS {@code table_type} on native Iceberg logical views (uppercase, HMS convention). */ - public static final String ICEBERG_VIEW_HMS_TABLE_TYPE_VALUE = - HiveOperationsBase.ICEBERG_VIEW_TYPE_VALUE.toUpperCase(java.util.Locale.ENGLISH); - - private IcebergNativeLogicalViewSupport() { + private IcebergLogicalViewSupport() { } /** @@ -83,7 +79,7 @@ private static void loadAndApplyView( } /** Creates or replaces a view in the Iceberg catalog. */ - public static void createOrReplaceNativeView( + public static void createOrReplaceView( Configuration conf, String databaseName, String viewName, @@ -99,16 +95,16 @@ public static void createOrReplaceNativeView( if (catalog instanceof Closeable closeable) { try (Closeable ignored = closeable) { - commitNativeView(catalog, catalogName, identifier, fieldSchemas, viewSql, tblProperties, comment); + commitView(catalog, catalogName, identifier, fieldSchemas, viewSql, tblProperties, comment); } catch (IOException e) { throw new UncheckedIOException("Failed to close Iceberg catalog", e); } } else { - commitNativeView(catalog, catalogName, identifier, fieldSchemas, viewSql, tblProperties, comment); + commitView(catalog, catalogName, identifier, fieldSchemas, viewSql, tblProperties, comment); } } - private static void commitNativeView( + private static void commitView( Catalog catalog, String catalogName, TableIdentifier identifier, @@ -132,11 +128,7 @@ private static void commitNativeView( Map tblProps = tblProperties == null ? Maps.newHashMap() : Maps.newHashMap(tblProperties); - for (Map.Entry e : tblProps.entrySet()) { - if (e.getKey() != null && e.getValue() != null) { - builder = builder.withProperty(e.getKey(), e.getValue()); - } - } + builder.withProperties(tblProps); builder.createOrReplace(); } @@ -149,6 +141,6 @@ private static ViewCatalog asViewCatalog(Catalog catalog, String catalogName) { String.format( "Iceberg catalog '%s' does not implement ViewCatalog.", catalogName) + - " Native views require a catalog that implements ViewCatalog (e.g. HiveCatalog or REST)."); + " Iceberg views require a catalog that implements ViewCatalog (e.g. HiveCatalog or REST)."); } } diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java index 0991ffa2a860..cf57c00d5b4e 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/MetastoreUtil.java @@ -48,6 +48,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; +import org.apache.iceberg.util.PropertyUtil; import org.apache.iceberg.view.BaseView; import org.apache.iceberg.view.SQLViewRepresentation; import org.apache.iceberg.view.View; @@ -161,26 +162,19 @@ public static Table toHiveTable(org.apache.iceberg.Table table, Configuration co /** * Builds a minimal HMS {@link Table} shell for a native Iceberg logical view (identity, view type, * and Iceberg storage-handler markers only). The storage handler {@code postGetTable} hook enriches - * this object via {@link IcebergNativeLogicalViewSupport#enrichHmsTableFromIcebergView} (view SQL, + * this object via {@link IcebergLogicalViewSupport#enrichHmsTableFromIcebergView} (view SQL, * schema, and Iceberg parameters). */ - public static Table buildMinimalHMSView(String catName, String dbName, String tableName, Configuration conf) { + public static Table buildMinimalHMSView(String catName, String dbName, String tableName) { Table result = new Table(); result.setCatName(catName); result.setDbName(dbName); result.setTableName(tableName); result.setTableType(TableType.VIRTUAL_VIEW.toString()); - int nowSec = (int) (System.currentTimeMillis() / 1000); - result.setOwner(System.getProperty("user.name")); - result.setCreateTime(nowSec); - result.setLastAccessTime(nowSec); - result.setRetention(Integer.MAX_VALUE); - Map parameters = Maps.newHashMap(); parameters.put( - BaseMetastoreTableOperations.TABLE_TYPE_PROP, - IcebergNativeLogicalViewSupport.ICEBERG_VIEW_HMS_TABLE_TYPE_VALUE); + BaseMetastoreTableOperations.TABLE_TYPE_PROP, HiveOperationsBase.ICEBERG_VIEW_TYPE_VALUE); parameters.put( hive_metastoreConstants.META_TABLE_STORAGE, HMSTablePropertyHelper.HIVE_ICEBERG_STORAGE_HANDLER); result.setParameters(parameters); @@ -218,6 +212,12 @@ public static void applyIcebergViewToHmsTable(Table hmsTable, View view, Configu maxHiveTablePropertySize, null); + hmsTable.setCreateTime((int) (metadata.version(1).timestampMillis() / 1000)); + hmsTable.setLastAccessTime((int) (metadata.currentVersion().timestampMillis() / 1000)); + hmsTable.setOwner( + PropertyUtil.propertyAsString( + metadata.properties(), HiveCatalog.HMS_TABLE_OWNER, HiveHadoopUtil.currentUser())); + // In-memory overlay for compile/describe: authoritative SQL comes from Iceberg metadata. hmsTable.setViewOriginalText(sqlText); hmsTable.setViewExpandedText(sqlText); diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java index 1798b6811c75..d8684b768e11 100644 --- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java +++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/client/HiveRESTCatalogClient.java @@ -50,9 +50,10 @@ import org.apache.iceberg.catalog.ViewCatalog; import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.hive.HMSTablePropertyHelper; +import org.apache.iceberg.hive.HiveOperationsBase; import org.apache.iceberg.hive.HiveSchemaUtil; import org.apache.iceberg.hive.IcebergCatalogProperties; -import org.apache.iceberg.hive.IcebergNativeLogicalViewSupport; +import org.apache.iceberg.hive.IcebergLogicalViewSupport; import org.apache.iceberg.hive.IcebergTableProperties; import org.apache.iceberg.hive.MetastoreUtil; import org.apache.iceberg.hive.RuntimeMetaException; @@ -214,13 +215,13 @@ public Table getTable(GetTableRequest tableRequest) throws TException { throw new NoSuchObjectException(); } return MetastoreUtil.buildMinimalHMSView( - tableRequest.getCatName(), tableRequest.getDbName(), tableRequest.getTblName(), conf); + tableRequest.getCatName(), tableRequest.getDbName(), tableRequest.getTblName()); } throw new NoSuchObjectException(); } } - private static boolean hasIcebergNativeViewTableType(Table table) { + private static boolean hasIcebergViewTableType(Table table) { if (!TableType.VIRTUAL_VIEW.toString().equals(table.getTableType())) { return false; } @@ -228,60 +229,63 @@ private static boolean hasIcebergNativeViewTableType(Table table) { if (params == null) { return false; } - return IcebergNativeLogicalViewSupport.ICEBERG_VIEW_HMS_TABLE_TYPE_VALUE.equals( + return HiveOperationsBase.ICEBERG_VIEW_TYPE_VALUE.equalsIgnoreCase( params.get(BaseMetastoreTableOperations.TABLE_TYPE_PROP)); } @Override public void alter_table(String catName, String dbName, String tblName, Table newTable, - EnvironmentContext envContext, String validWriteIdList) throws TException { + EnvironmentContext envContext, String validWriteIdList) { validateCurrentCatalog(catName); - if (hasIcebergNativeViewTableType(newTable) && restCatalog instanceof ViewCatalog) { - createOrReplaceLogicalView(newTable, dbName, tblName); + if (hasIcebergViewTableType(newTable) && restCatalog instanceof ViewCatalog) { + createOrReplaceIcebergView(newTable, dbName, tblName); } } @Override public void createTable(CreateTableRequest request) throws TException { Table table = request.getTable(); - if (hasIcebergNativeViewTableType(table) && restCatalog instanceof ViewCatalog) { - createOrReplaceLogicalView(table, table.getDbName(), table.getTableName()); + if (hasIcebergViewTableType(table) && restCatalog instanceof ViewCatalog) { + createOrReplaceIcebergView(table, table.getDbName(), table.getTableName()); } else { - List cols = Lists.newArrayList(table.getSd().getCols()); - if (table.isSetPartitionKeys() && !table.getPartitionKeys().isEmpty()) { - cols.addAll(table.getPartitionKeys()); - } - Properties tableProperties = IcebergTableProperties.getTableProperties(table, conf); - Schema schema = HiveSchemaUtil.convert(cols, Collections.emptyMap(), true); - Map envCtxProps = Optional.ofNullable(request.getEnvContext()) - .map(EnvironmentContext::getProperties) - .orElse(Collections.emptyMap()); - org.apache.iceberg.PartitionSpec partitionSpec = - HMSTablePropertyHelper.getPartitionSpec(envCtxProps, schema); - SortOrder sortOrder = HMSTablePropertyHelper.getSortOrder(tableProperties, schema); - - restCatalog.buildTable(TableIdentifier.of(table.getDbName(), table.getTableName()), schema) - .withPartitionSpec(partitionSpec) - .withLocation(tableProperties.getProperty(IcebergTableProperties.LOCATION)) - .withSortOrder(sortOrder) - .withProperties(Maps.fromProperties(tableProperties)) - .create(); + createIcebergTable(request); } } - private void createOrReplaceLogicalView(Table table, String dbName, String tableName) { + private void createIcebergTable(CreateTableRequest request) { + Table table = request.getTable(); + Properties tableProperties = IcebergTableProperties.getTableProperties(table, conf); + Schema schema = HiveSchemaUtil.convert(hmsTableColumns(table), Collections.emptyMap(), true); + Map envCtxProps = Optional.ofNullable(request.getEnvContext()) + .map(EnvironmentContext::getProperties) + .orElse(Collections.emptyMap()); + org.apache.iceberg.PartitionSpec partitionSpec = + HMSTablePropertyHelper.getPartitionSpec(envCtxProps, schema); + SortOrder sortOrder = HMSTablePropertyHelper.getSortOrder(tableProperties, schema); + + restCatalog + .buildTable(TableIdentifier.of(table.getDbName(), table.getTableName()), schema) + .withPartitionSpec(partitionSpec) + .withLocation(tableProperties.getProperty(IcebergTableProperties.LOCATION)) + .withSortOrder(sortOrder) + .withProperties(Maps.fromProperties(tableProperties)) + .create(); + } + private void createOrReplaceIcebergView(Table table, String dbName, String tableName) { + Map tblProps = + table.getParameters() == null ? Maps.newHashMap() : Maps.newHashMap(table.getParameters()); + String comment = tblProps.get("comment"); + IcebergLogicalViewSupport.createOrReplaceView( + conf, dbName, tableName, hmsTableColumns(table), table.getViewExpandedText(), tblProps, comment); + } + + private static List hmsTableColumns(Table table) { List cols = Lists.newArrayList(table.getSd().getCols()); if (table.isSetPartitionKeys() && !table.getPartitionKeys().isEmpty()) { cols.addAll(table.getPartitionKeys()); } - - Map tblProps = - table.getParameters() == null ? Maps.newHashMap() : Maps.newHashMap(table.getParameters()); - - String comment = tblProps.get("comment"); - IcebergNativeLogicalViewSupport.createOrReplaceNativeView( - conf, dbName, tableName, cols, table.getViewExpandedText(), tblProps, comment); + return cols; } @Override diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergNativeLogicalViewSupport.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergLogicalViewSupport.java similarity index 89% rename from iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergNativeLogicalViewSupport.java rename to iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergLogicalViewSupport.java index 3850f1d760d9..e48a2613f74c 100644 --- a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergNativeLogicalViewSupport.java +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/TestIcebergLogicalViewSupport.java @@ -33,6 +33,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.view.BaseView; import org.apache.iceberg.view.View; +import org.apache.iceberg.view.ViewMetadata; import org.apache.thrift.TException; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; @@ -42,7 +43,7 @@ import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE; import static org.assertj.core.api.Assertions.assertThat; -class TestIcebergNativeLogicalViewSupport { +class TestIcebergLogicalViewSupport { private static final String DB = "native_vw_db"; private static final String VIEW = "native_vw"; @@ -86,7 +87,7 @@ void testCreateCommitsNativeViewWithUserProperties() { String sql = String.format("select id, name from %s.src_tbl", DB); Map props = Collections.singletonMap("k1", "v1"); - IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + IcebergLogicalViewSupport.createOrReplaceView( conf, DB, VIEW, cols, sql, props, "hello-view"); HiveCatalog cat = loadCatalog(); @@ -102,17 +103,17 @@ void testCreateCommitsNativeViewWithUserProperties() { } @Test - void testCreateOrReplaceNativeViewReplacesExisting() { + void testCreateOrReplaceViewReplacesExisting() { HiveConf conf = nativeViewConf(); List cols = Collections.singletonList(new FieldSchema("id", "int", null)); TableIdentifier id = TableIdentifier.of(DB, VIEW); - IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + IcebergLogicalViewSupport.createOrReplaceView( conf, DB, VIEW, cols, "select 1 as id", null, null); View afterCreate = loadCatalog().loadView(id); assertThat(afterCreate.sqlFor("hive").sql().trim()).isEqualTo("select 1 as id"); - IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + IcebergLogicalViewSupport.createOrReplaceView( conf, DB, VIEW, cols, "select 2 as id", null, null); assertThat(loadCatalog().viewExists(id)).isTrue(); @@ -126,7 +127,7 @@ void testEnrichHmsTableFromIcebergViewOverridesStaleHmsSql() throws TException { List cols = Collections.singletonList(new FieldSchema("id", "int", null)); String sql = "select 42 as id"; - IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + IcebergLogicalViewSupport.createOrReplaceView( conf, DB, VIEW, cols, sql, null, null); org.apache.hadoop.hive.metastore.api.Table hmsTable = @@ -134,8 +135,11 @@ void testEnrichHmsTableFromIcebergViewOverridesStaleHmsSql() throws TException { hmsTable.setViewOriginalText("select 0"); hmsTable.setViewExpandedText("select 0"); - IcebergNativeLogicalViewSupport.enrichHmsTableFromIcebergView(hmsTable, conf); + IcebergLogicalViewSupport.enrichHmsTableFromIcebergView(hmsTable, conf); assertThat(hmsTable.getViewExpandedText()).isEqualTo(sql); assertThat(hmsTable.getViewOriginalText()).isEqualTo(sql); + + ViewMetadata metadata = ((BaseView) loadCatalog().loadView(TableIdentifier.of(DB, VIEW))).operations().current(); + assertThat(hmsTable.getCreateTime()).isEqualTo((int) (metadata.version(1).timestampMillis() / 1000)); } } diff --git a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/client/TestHiveRESTCatalogClient.java b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/client/TestHiveRESTCatalogClient.java index 70e441d6109c..fd10b13df02b 100644 --- a/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/client/TestHiveRESTCatalogClient.java +++ b/iceberg/iceberg-catalog/src/test/java/org/apache/iceberg/hive/client/TestHiveRESTCatalogClient.java @@ -46,8 +46,9 @@ import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.catalog.ViewCatalog; +import org.apache.iceberg.hive.HiveOperationsBase; import org.apache.iceberg.hive.HiveSchemaUtil; -import org.apache.iceberg.hive.IcebergNativeLogicalViewSupport; +import org.apache.iceberg.hive.IcebergLogicalViewSupport; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.LocationProvider; import org.apache.iceberg.relocated.com.google.common.collect.Maps; @@ -65,7 +66,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; -public class TestHiveRESTCatalogClient { +class TestHiveRESTCatalogClient { private static HiveRESTCatalogClient spyHiveRESTCatalogClient; private static RESTCatalog mockRestCatalog; @@ -142,13 +143,13 @@ public void after() { } @Test - public void testGetTable() throws TException { + void testGetTable() throws TException { spyHiveRESTCatalogClient.getTable("default", "tableName"); Mockito.verify(mockRestCatalog).loadTable(TableIdentifier.of("default", "tableName")); } @Test - public void testCreateTable() throws TException { + void testCreateTable() throws TException { Table table = new Table(); table.setTableName("tableName"); table.setDbName("default"); @@ -160,7 +161,7 @@ public void testCreateTable() throws TException { } @Test - public void testCreatePartitionedTable() throws TException { + void testCreatePartitionedTable() throws TException { Table table = new Table(); table.setTableName("tableName"); table.setDbName("default"); @@ -197,14 +198,14 @@ public void testCreatePartitionedTable() throws TException { } @Test - public void testGetDatabase() throws TException { + void testGetDatabase() throws TException { Database aDefault = spyHiveRESTCatalogClient.getDatabase("default"); assertThat(aDefault.getName()).isEqualTo("default"); Mockito.verify(mockRestCatalog).listNamespaces(Namespace.empty()); } @Test - public void testAlterNativeIcebergView() throws TException { + void testAlterIcebergLogicalView() { RESTCatalog viewCapableCatalog = Mockito.mock(RESTCatalog.class, Mockito.withSettings().extraInterfaces(ViewCatalog.class)); Mockito.doReturn("hive").when(viewCapableCatalog).name(); @@ -216,12 +217,12 @@ public void testAlterNativeIcebergView() throws TException { configuration.set("iceberg.catalog.ice01.uri", "http://localhost"); HiveRESTCatalogClient client = new HiveRESTCatalogClient(configuration); - try (MockedStatic viewSupport = - Mockito.mockStatic(IcebergNativeLogicalViewSupport.class)) { + try (MockedStatic viewSupport = + Mockito.mockStatic(IcebergLogicalViewSupport.class)) { client.alter_table("hive", "ice_db", "ice_v1", createLogicalView(), null, null); viewSupport.verify( () -> - IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + IcebergLogicalViewSupport.createOrReplaceView( any(), eq("ice_db"), eq("ice_v1"), @@ -244,7 +245,7 @@ private static Table createLogicalView() { Maps.newHashMap( Map.of( BaseMetastoreTableOperations.TABLE_TYPE_PROP, - IcebergNativeLogicalViewSupport.ICEBERG_VIEW_HMS_TABLE_TYPE_VALUE))); + HiveOperationsBase.ICEBERG_VIEW_TYPE_VALUE))); return view; } } diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java index 6e8811f71c50..be02c1ececc2 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/BaseHiveIcebergMetaHook.java @@ -61,9 +61,10 @@ import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.exceptions.NotFoundException; import org.apache.iceberg.hive.HMSTablePropertyHelper; +import org.apache.iceberg.hive.HiveOperationsBase; import org.apache.iceberg.hive.HiveSchemaUtil; import org.apache.iceberg.hive.IcebergCatalogProperties; -import org.apache.iceberg.hive.IcebergNativeLogicalViewSupport; +import org.apache.iceberg.hive.IcebergLogicalViewSupport; import org.apache.iceberg.hive.IcebergTableProperties; import org.apache.iceberg.mr.Catalogs; import org.apache.iceberg.mr.InputFormatConfig; @@ -117,7 +118,7 @@ public BaseHiveIcebergMetaHook(Configuration conf) { this.conf = conf; } - public static boolean isNativeIcebergLogicalView(org.apache.hadoop.hive.metastore.api.Table hmsTable) { + public static boolean isIcebergLogicalView(org.apache.hadoop.hive.metastore.api.Table hmsTable) { if (hmsTable == null || hmsTable.getParameters() == null || !TableType.VIRTUAL_VIEW.toString().equals(hmsTable.getTableType())) { @@ -139,8 +140,8 @@ public void preCreateTable(CreateTableRequest request) { if (hmsTable.isTemporary()) { throw new UnsupportedOperationException("Creation of temporary iceberg tables is not supported."); } - if (isNativeIcebergLogicalView(hmsTable)) { - preCreateNativeIcebergLogicalView(request); + if (isIcebergLogicalView(hmsTable)) { + preCreateIcebergLogicalView(request); return; } this.tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); @@ -220,7 +221,7 @@ public void preCreateTable(CreateTableRequest request) { setSortOrder(hmsTable, schema, tableProperties); } - private void preCreateNativeIcebergLogicalView(CreateTableRequest request) { + private void preCreateIcebergLogicalView(CreateTableRequest request) { org.apache.hadoop.hive.metastore.api.Table hmsTable = request.getTable(); tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); @@ -229,7 +230,7 @@ private void preCreateNativeIcebergLogicalView(CreateTableRequest request) { .getParameters() .put( BaseMetastoreTableOperations.TABLE_TYPE_PROP, - IcebergNativeLogicalViewSupport.ICEBERG_VIEW_HMS_TABLE_TYPE_VALUE); + HiveOperationsBase.ICEBERG_VIEW_TYPE_VALUE); } /** @@ -532,8 +533,8 @@ protected void setWriteModeDefaults(Table icebergTbl, Map newPro public void postGetTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) { if (hmsTable != null) { try { - if (isNativeIcebergLogicalView(hmsTable)) { - IcebergNativeLogicalViewSupport.enrichHmsTableFromIcebergView(hmsTable, conf); + if (isIcebergLogicalView(hmsTable)) { + IcebergLogicalViewSupport.enrichHmsTableFromIcebergView(hmsTable, conf); return; } Table tbl = IcebergTableUtil.getTable(conf, hmsTable); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java index 64010a363907..5e280a388a6f 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java @@ -117,7 +117,7 @@ import org.apache.iceberg.hive.HiveLock; import org.apache.iceberg.hive.HiveSchemaUtil; import org.apache.iceberg.hive.HiveTableOperations; -import org.apache.iceberg.hive.IcebergNativeLogicalViewSupport; +import org.apache.iceberg.hive.IcebergLogicalViewSupport; import org.apache.iceberg.hive.IcebergTableProperties; import org.apache.iceberg.hive.MetastoreLock; import org.apache.iceberg.hive.NoLock; @@ -179,12 +179,12 @@ public HiveIcebergMetaHook(Configuration conf) { @Override public void commitCreateTable(org.apache.hadoop.hive.metastore.api.Table hmsTable) { - if (isNativeIcebergLogicalView(hmsTable)) { + if (isIcebergLogicalView(hmsTable)) { tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); Map tblProps = hmsTable.getParameters() == null ? Maps.newHashMap() : Maps.newHashMap(hmsTable.getParameters()); String comment = tblProps.get("comment"); - IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + IcebergLogicalViewSupport.createOrReplaceView( conf, hmsTable.getDbName(), hmsTable.getTableName(), @@ -266,7 +266,7 @@ public void commitDropTable(org.apache.hadoop.hive.metastore.api.Table hmsTable, @Override public void preAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable, EnvironmentContext context) throws MetaException { - if (BaseHiveIcebergMetaHook.isNativeIcebergLogicalView(hmsTable)) { + if (BaseHiveIcebergMetaHook.isIcebergLogicalView(hmsTable)) { currentAlterTableOp = null; if (commitLock == null) { commitLock = new NoLock(); @@ -503,13 +503,12 @@ public void commitAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable if (commitLock == null) { throw new IllegalStateException("Hive commit lock should already be set"); } - commitLock.unlock(); - if (BaseHiveIcebergMetaHook.isNativeIcebergLogicalView(hmsTable)) { + if (BaseHiveIcebergMetaHook.isIcebergLogicalView(hmsTable)) { tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); Map tblProps = hmsTable.getParameters() == null ? Maps.newHashMap() : Maps.newHashMap(hmsTable.getParameters()); String comment = tblProps.get("comment"); - IcebergNativeLogicalViewSupport.createOrReplaceNativeView( + IcebergLogicalViewSupport.createOrReplaceView( conf, hmsTable.getDbName(), hmsTable.getTableName(), @@ -519,6 +518,7 @@ public void commitAlterTable(org.apache.hadoop.hive.metastore.api.Table hmsTable comment); return; } + commitLock.unlock(); if (isTableMigration) { tableProperties = IcebergTableProperties.getTableProperties(hmsTable, conf); tableProperties.put(InputFormatConfig.TABLE_SCHEMA, SchemaParser.toJson(preAlterTableProperties.schema)); diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java index 4a4c66fe3f10..166e5e548281 100644 --- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java +++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergStorageHandler.java @@ -433,7 +433,7 @@ public boolean canProvidePartitionStatistics(org.apache.hadoop.hive.ql.metadata. if (!getStatsSource().equals(HiveMetaHook.ICEBERG)) { return false; } - if (BaseHiveIcebergMetaHook.isNativeIcebergLogicalView(hmsTable.getTTable())) { + if (BaseHiveIcebergMetaHook.isIcebergLogicalView(hmsTable.getTTable())) { return false; } Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); @@ -899,7 +899,7 @@ public boolean supportsPartitionTransform() { @Override public List getPartitionTransformSpec(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { - if (BaseHiveIcebergMetaHook.isNativeIcebergLogicalView(hmsTable.getTTable())) { + if (BaseHiveIcebergMetaHook.isIcebergLogicalView(hmsTable.getTTable())) { return Collections.emptyList(); } Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); @@ -916,7 +916,7 @@ public List getPartitionTransformSpec(org.apache.hadoop.hive.ql.m @Override public Map> getPartitionTransformSpecs( org.apache.hadoop.hive.ql.metadata.Table hmsTable) { - if (BaseHiveIcebergMetaHook.isNativeIcebergLogicalView(hmsTable.getTTable())) { + if (BaseHiveIcebergMetaHook.isIcebergLogicalView(hmsTable.getTTable())) { return Collections.emptyMap(); } Table table = IcebergTableUtil.getTable(conf, hmsTable.getTTable()); @@ -1584,7 +1584,7 @@ public boolean supportsSortColumns() { @Override public List sortColumns(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { - if (BaseHiveIcebergMetaHook.isNativeIcebergLogicalView(hmsTable.getTTable())) { + if (BaseHiveIcebergMetaHook.isIcebergLogicalView(hmsTable.getTTable())) { return Collections.emptyList(); } TableDesc tableDesc = Utilities.getTableDesc(hmsTable); @@ -2150,7 +2150,7 @@ public List getPartitions(org.apache.hadoop.hive.ql.metadata.Table hm } public boolean isPartitioned(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { - if (BaseHiveIcebergMetaHook.isNativeIcebergLogicalView(hmsTable.getTTable())) { + if (BaseHiveIcebergMetaHook.isIcebergLogicalView(hmsTable.getTTable())) { List partCols = hmsTable.getPartCols(); return partCols != null && !partCols.isEmpty(); } @@ -2299,7 +2299,7 @@ public boolean canPerformMetadataDelete(org.apache.hadoop.hive.ql.metadata.Table @Override public List getPartitionKeys(org.apache.hadoop.hive.ql.metadata.Table hmsTable) { - if (BaseHiveIcebergMetaHook.isNativeIcebergLogicalView(hmsTable.getTTable())) { + if (BaseHiveIcebergMetaHook.isIcebergLogicalView(hmsTable.getTTable())) { List partCols = hmsTable.getPartCols(); return partCols != null ? partCols : Collections.emptyList(); } diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out index 759d028e7b81..05c7e7bcfb4b 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_gravitino.q.out @@ -217,7 +217,7 @@ last_name string # Detailed Table Information Database: ice_rest #### A masked pattern was here #### -Retention: 2147483647 +Retention: 0 Table Type: VIRTUAL_VIEW Table Parameters: bucketing_version 2 @@ -276,7 +276,7 @@ last_name string # Detailed Table Information Database: ice_rest #### A masked pattern was here #### -Retention: 2147483647 +Retention: 0 Table Type: VIRTUAL_VIEW Table Parameters: bucketing_version 2 @@ -333,7 +333,7 @@ _c0 string # Detailed Table Information Database: ice_rest #### A masked pattern was here #### -Retention: 2147483647 +Retention: 0 Table Type: VIRTUAL_VIEW Table Parameters: bucketing_version 2 @@ -402,7 +402,7 @@ _c1 string # Detailed Table Information Database: ice_rest #### A masked pattern was here #### -Retention: 2147483647 +Retention: 0 Table Type: VIRTUAL_VIEW Table Parameters: bucketing_version 2 diff --git a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out index 09dfb18b6e3e..a92beb820d1c 100644 --- a/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out +++ b/iceberg/iceberg-handler/src/test/results/positive/llap/iceberg_rest_catalog_hms.q.out @@ -217,7 +217,7 @@ last_name string # Detailed Table Information Database: ice_rest #### A masked pattern was here #### -Retention: 2147483647 +Retention: 0 Table Type: VIRTUAL_VIEW Table Parameters: bucketing_version 2 @@ -287,7 +287,7 @@ team_id bigint # Detailed Table Information Database: ice_rest #### A masked pattern was here #### -Retention: 2147483647 +Retention: 0 Table Type: VIRTUAL_VIEW Table Parameters: bucketing_version 2 diff --git a/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java index 32ddeae6b5d7..ec5f801c8aaa 100644 --- a/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java +++ b/itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java @@ -46,6 +46,7 @@ import org.apache.iceberg.PartitionSpecParser; import org.apache.iceberg.Schema; import org.apache.iceberg.TableProperties; +import org.apache.iceberg.hive.HiveOperationsBase; import org.apache.iceberg.hive.HiveSchemaUtil; import org.apache.iceberg.hive.IcebergCatalogProperties; import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension; @@ -77,7 +78,6 @@ public abstract class TestHiveRESTCatalogClientITBase { static final String HIVE_ICEBERG_STORAGE_HANDLER = "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler"; static final String REST_CATALOG_PREFIX = String.format("%s%s.", IcebergCatalogProperties.CATALOG_CONFIG_PREFIX, CATALOG_NAME); - static final String ICEBERG_VIEW_TYPE_VALUE = "iceberg-view"; HiveConf hiveConf; Configuration conf; @@ -222,7 +222,7 @@ public void testIceberg() throws Exception { } @Test - public void testNativeIcebergLogicalView() throws Exception { + public void testIcebergLogicalView() throws Exception { Database db = new Database(); db.setCatalogName(CATALOG_NAME); db.setName(VIEW_DB_NAME); @@ -233,7 +233,7 @@ public void testNativeIcebergLogicalView() throws Exception { hive.createDatabase(db, true); List cols = Collections.singletonList(new FieldSchema("x", "int", "")); - createNativeIcebergLogicalView(VIEW_DB_NAME, NATIVE_VIEW_NAME, cols, "select 1 as x", "rest-native-view"); + createIcebergLogicalView(VIEW_DB_NAME, NATIVE_VIEW_NAME, cols, "select 1 as x", "rest-native-view"); Assertions.assertTrue(msClient.tableExists(CATALOG_NAME, VIEW_DB_NAME, NATIVE_VIEW_NAME)); @@ -243,11 +243,10 @@ public void testNativeIcebergLogicalView() throws Exception { getTableRequest.setTblName(NATIVE_VIEW_NAME); Table view = msClient.getTable(getTableRequest); - Assertions.assertNotNull(view); Assertions.assertEquals(TableType.VIRTUAL_VIEW.name(), view.getTableType()); String tableTypeProp = view.getParameters().get(BaseMetastoreTableOperations.TABLE_TYPE_PROP); Assertions.assertNotNull(tableTypeProp); - Assertions.assertEquals(ICEBERG_VIEW_TYPE_VALUE, tableTypeProp.toLowerCase()); + Assertions.assertEquals(HiveOperationsBase.ICEBERG_VIEW_TYPE_VALUE, tableTypeProp.toLowerCase()); List names = msClient.getTables(CATALOG_NAME, VIEW_DB_NAME, "*"); Assertions.assertTrue(names.contains(NATIVE_VIEW_NAME)); @@ -258,7 +257,7 @@ public void testNativeIcebergLogicalView() throws Exception { msClient.dropDatabase(VIEW_DB_NAME); } - private void createNativeIcebergLogicalView( + private void createIcebergLogicalView( String dbName, String viewName, List cols, String viewSql, String comment) throws Exception { Table view = new Table(); view.setCatName(CATALOG_NAME); diff --git a/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseCreateView.java b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseCreateView.java index cab826e5963b..22c31f74d0b3 100644 --- a/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseCreateView.java +++ b/parser/src/test/org/apache/hadoop/hive/ql/parse/TestParseCreateView.java @@ -3,17 +3,18 @@ * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ package org.apache.hadoop.hive.ql.parse; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java index 17638a0fe804..4001b1a70b44 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/view/create/CreateViewOperation.java @@ -22,7 +22,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.TableName; -import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.ddl.DDLOperation; @@ -34,7 +33,6 @@ import org.apache.hadoop.hive.ql.metadata.Table; import org.apache.hadoop.hive.ql.parse.HiveTableName; import org.apache.hadoop.hive.ql.parse.StorageFormat; -import org.apache.hadoop.hive.ql.session.SessionStateUtil; import java.util.Map; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/CreateExternalLogicalViewRequest.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/CreateExternalLogicalViewRequest.java index aad1e2c3f7dd..d6b6f2cb4e35 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/CreateExternalLogicalViewRequest.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/CreateExternalLogicalViewRequest.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema; /** - * Parameters for {@link HiveStorageHandler#createOrReplaceExternalLogicalView} + * Parameters for {@link HiveStorageHandler#createOrReplaceExternalLogicalView}. */ public final class CreateExternalLogicalViewRequest implements Serializable { private static final long serialVersionUID = 1L; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java index af2699ebe5f3..31a14688f5b1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveStorageHandler.java @@ -27,7 +27,6 @@ import java.util.concurrent.ExecutorService; import com.google.common.collect.Maps; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.hive.common.classification.InterfaceAudience;