From 5888f657ae204e616666e5224a3e48a804abcc2e Mon Sep 17 00:00:00 2001 From: tanishq-chugh Date: Sun, 15 Mar 2026 23:43:10 +0530 Subject: [PATCH 1/6] HIVE-29342: Iceberg: [V3] Display initial and write defaults in describe formatted column output --- .../positive/iceberg_defaults_in_desc_cols.q | 54 +++ .../iceberg_defaults_in_desc_cols.q.out | 406 ++++++++++++++++++ .../ql/ddl/table/info/desc/DescTableDesc.java | 7 +- .../formatter/TextDescTableFormatter.java | 73 +++- 4 files changed, 534 insertions(+), 6 deletions(-) create mode 100644 iceberg/iceberg-handler/src/test/queries/positive/iceberg_defaults_in_desc_cols.q create mode 100644 iceberg/iceberg-handler/src/test/results/positive/iceberg_defaults_in_desc_cols.q.out diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_defaults_in_desc_cols.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_defaults_in_desc_cols.q new file mode 100644 index 000000000000..5be220422896 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_defaults_in_desc_cols.q @@ -0,0 +1,54 @@ +-- Mask random uuid +--! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/ +-- Mask a random snapshot id +--! qt:replace:/(\s+current-snapshot-id\s+)\S+(\s*)/$1#Masked#/ +-- Mask added file size +--! qt:replace:/(\S\"added-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/ +-- Mask total file size +--! qt:replace:/(\S\"total-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/ +-- Mask current-snapshot-timestamp-ms +--! qt:replace:/(\s+current-snapshot-timestamp-ms\s+)\S+(\s*)/$1#Masked#$2/ + +CREATE TABLE ice_parq ( + id INT) +STORED BY ICEBERG stored as parquet +TBLPROPERTIES ('format-version'='3'); + +INSERT INTO ice_parq (id) VALUES (1); + +ALTER TABLE ice_parq ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', + name STRING DEFAULT 'unknown', + age INT DEFAULT 25); + +INSERT INTO ice_parq (id) VALUES (2); + +SELECT * FROM ice_parq ORDER BY id; + +DESCRIBE FORMATTED ice_parq; + +DESCRIBE FORMATTED ice_parq id; +DESCRIBE FORMATTED ice_parq point; +DESCRIBE FORMATTED ice_parq name; +DESCRIBE FORMATTED ice_parq age; + +CREATE TABLE ice_orc ( + id INT) +STORED BY ICEBERG stored as orc +TBLPROPERTIES ('format-version'='3'); + +INSERT INTO ice_orc (id) VALUES (1); + +ALTER TABLE ice_orc ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', + name STRING DEFAULT 'unknown', + age INT DEFAULT 25); + +INSERT INTO ice_orc (id) VALUES (2); + +SELECT * FROM ice_orc ORDER BY id; + +DESCRIBE FORMATTED ice_orc; + +DESCRIBE FORMATTED ice_orc id; +DESCRIBE FORMATTED ice_orc point; +DESCRIBE FORMATTED ice_orc name; +DESCRIBE FORMATTED ice_orc age; diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_defaults_in_desc_cols.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_defaults_in_desc_cols.q.out new file mode 100644 index 000000000000..edc3d0398bd3 --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_defaults_in_desc_cols.q.out @@ -0,0 +1,406 @@ +PREHOOK: query: CREATE TABLE ice_parq ( + id INT) +STORED BY ICEBERG stored as parquet +TBLPROPERTIES ('format-version'='3') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_parq +POSTHOOK: query: CREATE TABLE ice_parq ( + id INT) +STORED BY ICEBERG stored as parquet +TBLPROPERTIES ('format-version'='3') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_parq +PREHOOK: query: INSERT INTO ice_parq (id) VALUES (1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_parq +POSTHOOK: query: INSERT INTO ice_parq (id) VALUES (1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_parq +PREHOOK: query: ALTER TABLE ice_parq ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', + name STRING DEFAULT 'unknown', + age INT DEFAULT 25) +PREHOOK: type: ALTERTABLE_ADDCOLS +PREHOOK: Input: default@ice_parq +PREHOOK: Output: default@ice_parq +POSTHOOK: query: ALTER TABLE ice_parq ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', + name STRING DEFAULT 'unknown', + age INT DEFAULT 25) +POSTHOOK: type: ALTERTABLE_ADDCOLS +POSTHOOK: Input: default@ice_parq +POSTHOOK: Output: default@ice_parq +PREHOOK: query: INSERT INTO ice_parq (id) VALUES (2) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_parq +POSTHOOK: query: INSERT INTO ice_parq (id) VALUES (2) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_parq +PREHOOK: query: SELECT * FROM ice_parq ORDER BY id +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_parq +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM ice_parq ORDER BY id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_parq +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 NULL unknown 25 +2 {"x":100,"y":99} unknown 25 +PREHOOK: query: DESCRIBE FORMATTED ice_parq +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_parq +POSTHOOK: query: DESCRIBE FORMATTED ice_parq +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_parq +# col_name data_type comment +id int +point struct +name string +age int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: EXTERNAL_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + EXTERNAL TRUE + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":1,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"point\",\"required\":false,\"type\":{\"type\":\"struct\",\"fields\":[{\"id\":3,\"name\":\"x\",\"required\":false,\"type\":\"int\",\"write-default\":100},{\"id\":4,\"name\":\"y\",\"required\":false,\"type\":\"int\",\"write-default\":99}]}},{\"id\":5,\"name\":\"name\",\"required\":false,\"type\":\"string\",\"initial-default\":\"unknown\",\"write-default\":\"unknown\"},{\"id\":6,\"name\":\"age\",\"required\":false,\"type\":\"int\",\"initial-default\":25,\"write-default\":25}]} + current-snapshot-id #Masked# + current-snapshot-summary {\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"2\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"Apache Iceberg 1.10.1 (commit ccb8bc435062171e64bc8b7e5f56e6aed9c5b934)\"} + current-snapshot-timestamp-ms #Masked# + format-version 3 + iceberg.orc.files.only false + metadata_location hdfs://### HDFS PATH ### + numFiles 2 + numRows 2 + parquet.compression zstd + previous_metadata_location hdfs://### HDFS PATH ### + rawDataSize 0 + serialization.format 1 + snapshot-count 2 + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG + totalSize #Masked# +#### A masked pattern was here #### + uuid #Masked# + write.delete.mode merge-on-read + write.format.default parquet + write.merge.mode merge-on-read + write.metadata.delete-after-commit.enabled true + write.update.mode merge-on-read + +# Storage Information +SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe +InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat +OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat +Compressed: No +Sort Columns: [] + +# Constraints + +# Default Constraints +Table: default.ice_parq +Constraint Name: #### A masked pattern was here #### +Column Name:age Default Value:25 + +Constraint Name: #### A masked pattern was here #### +Column Name:point Default Value:'x:100,y:99' + +Constraint Name: #### A masked pattern was here #### +Column Name:name Default Value:'unknown' + +PREHOOK: query: DESCRIBE FORMATTED ice_parq id +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_parq +POSTHOOK: query: DESCRIBE FORMATTED ice_parq id +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_parq +col_name id +data_type int +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bit_vector +comment +initial_default +write_default +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: DESCRIBE FORMATTED ice_parq point +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_parq +POSTHOOK: query: DESCRIBE FORMATTED ice_parq point +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_parq +col_name point +data_type struct +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bit_vector +comment +initial_default +write_default x:100,y:99 +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: DESCRIBE FORMATTED ice_parq name +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_parq +POSTHOOK: query: DESCRIBE FORMATTED ice_parq name +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_parq +col_name name +data_type string +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bit_vector +comment +initial_default unknown +write_default unknown +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: DESCRIBE FORMATTED ice_parq age +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_parq +POSTHOOK: query: DESCRIBE FORMATTED ice_parq age +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_parq +col_name age +data_type int +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bit_vector +comment +initial_default 25 +write_default 25 +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: CREATE TABLE ice_orc ( + id INT) +STORED BY ICEBERG stored as orc +TBLPROPERTIES ('format-version'='3') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_orc +POSTHOOK: query: CREATE TABLE ice_orc ( + id INT) +STORED BY ICEBERG stored as orc +TBLPROPERTIES ('format-version'='3') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_orc +PREHOOK: query: INSERT INTO ice_orc (id) VALUES (1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: INSERT INTO ice_orc (id) VALUES (1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: ALTER TABLE ice_orc ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', + name STRING DEFAULT 'unknown', + age INT DEFAULT 25) +PREHOOK: type: ALTERTABLE_ADDCOLS +PREHOOK: Input: default@ice_orc +PREHOOK: Output: default@ice_orc +POSTHOOK: query: ALTER TABLE ice_orc ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', + name STRING DEFAULT 'unknown', + age INT DEFAULT 25) +POSTHOOK: type: ALTERTABLE_ADDCOLS +POSTHOOK: Input: default@ice_orc +POSTHOOK: Output: default@ice_orc +PREHOOK: query: INSERT INTO ice_orc (id) VALUES (2) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: INSERT INTO ice_orc (id) VALUES (2) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: SELECT * FROM ice_orc ORDER BY id +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_orc +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM ice_orc ORDER BY id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_orc +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 NULL NULL NULL +2 {"x":100,"y":99} unknown 25 +PREHOOK: query: DESCRIBE FORMATTED ice_orc +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_orc +POSTHOOK: query: DESCRIBE FORMATTED ice_orc +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_orc +# col_name data_type comment +id int +point struct +name string +age int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: EXTERNAL_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + EXTERNAL TRUE + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":1,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"point\",\"required\":false,\"type\":{\"type\":\"struct\",\"fields\":[{\"id\":3,\"name\":\"x\",\"required\":false,\"type\":\"int\",\"write-default\":100},{\"id\":4,\"name\":\"y\",\"required\":false,\"type\":\"int\",\"write-default\":99}]}},{\"id\":5,\"name\":\"name\",\"required\":false,\"type\":\"string\",\"write-default\":\"unknown\"},{\"id\":6,\"name\":\"age\",\"required\":false,\"type\":\"int\",\"write-default\":25}]} + current-snapshot-id #Masked# + current-snapshot-summary {\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"2\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"Apache Iceberg 1.10.1 (commit ccb8bc435062171e64bc8b7e5f56e6aed9c5b934)\"} + current-snapshot-timestamp-ms #Masked# + format-version 3 + iceberg.orc.files.only true + metadata_location hdfs://### HDFS PATH ### + numFiles 2 + numRows 2 + parquet.compression zstd + previous_metadata_location hdfs://### HDFS PATH ### + rawDataSize 0 + serialization.format 1 + snapshot-count 2 + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG + totalSize #Masked# +#### A masked pattern was here #### + uuid #Masked# + write.delete.mode merge-on-read + write.format.default orc + write.merge.mode merge-on-read + write.metadata.delete-after-commit.enabled true + write.update.mode merge-on-read + +# Storage Information +SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe +InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat +OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat +Compressed: No +Sort Columns: [] + +# Constraints + +# Default Constraints +Table: default.ice_orc +Constraint Name: #### A masked pattern was here #### +Column Name:point Default Value:'x:100,y:99' + +Constraint Name: #### A masked pattern was here #### +Column Name:age Default Value:25 + +Constraint Name: #### A masked pattern was here #### +Column Name:name Default Value:'unknown' + +PREHOOK: query: DESCRIBE FORMATTED ice_orc id +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_orc +POSTHOOK: query: DESCRIBE FORMATTED ice_orc id +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_orc +col_name id +data_type int +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bit_vector +comment +initial_default +write_default +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: DESCRIBE FORMATTED ice_orc point +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_orc +POSTHOOK: query: DESCRIBE FORMATTED ice_orc point +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_orc +col_name point +data_type struct +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bit_vector +comment +initial_default +write_default x:100,y:99 +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: DESCRIBE FORMATTED ice_orc name +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_orc +POSTHOOK: query: DESCRIBE FORMATTED ice_orc name +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_orc +col_name name +data_type string +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bit_vector +comment +initial_default +write_default unknown +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} +PREHOOK: query: DESCRIBE FORMATTED ice_orc age +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_orc +POSTHOOK: query: DESCRIBE FORMATTED ice_orc age +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_orc +col_name age +data_type int +min +max +num_nulls +distinct_count +avg_col_len +max_col_len +num_trues +num_falses +bit_vector +comment +initial_default +write_default 25 +COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java index f0d979e80474..112b7db910dd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java @@ -110,7 +110,7 @@ public boolean isFormatted() { return isFormatted; } - public static List getColumnStatisticsHeaders(boolean histogramEnabled) { + public static List getColumnStatisticsHeaders(boolean histogramEnabled, boolean isIcebergTable) { ImmutableList.Builder builder = ImmutableList.builder() .add("col_name") .add("data_type") @@ -121,6 +121,11 @@ public static List getColumnStatisticsHeaders(boolean histogramEnabled) } builder.add("comment"); + + if (isIcebergTable) { + builder.add("initial_default") + .add("write_default"); + } return builder.build(); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java index b1dd9738572a..b9b793595007 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java @@ -18,8 +18,11 @@ package org.apache.hadoop.hive.ql.ddl.table.info.desc.formatter; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringEscapeUtils; import org.apache.hadoop.hive.common.MaterializationSnapshot; import org.apache.hadoop.hive.common.StatsSetupConst; @@ -73,6 +76,7 @@ import java.util.stream.Collectors; import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.TABLE_IS_CTAS; +import static org.apache.hadoop.hive.ql.ddl.DDLUtils.isIcebergTable; import static org.apache.hadoop.hive.ql.ddl.ShowUtils.ALIGNMENT; import static org.apache.hadoop.hive.ql.ddl.ShowUtils.DEFAULT_STRINGBUILDER_SIZE; import static org.apache.hadoop.hive.ql.ddl.ShowUtils.FIELD_DELIM; @@ -83,12 +87,13 @@ * Formats DESC TABLE results to text format. */ class TextDescTableFormatter extends DescTableFormatter { + public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @Override public void describeTable(HiveConf conf, DataOutputStream out, String columnPath, String tableName, Table table, Partition partition, List columns, boolean isFormatted, boolean isExtended, boolean isOutputPadded, List columnStats) throws HiveException { try { - addStatsData(out, conf, columnPath, columns, isFormatted, columnStats, isOutputPadded); + addStatsData(out, conf, columnPath, columns, table, isFormatted, columnStats, isOutputPadded); addPartitionData(out, conf, columnPath, table, isFormatted, isOutputPadded); boolean isIcebergMetaTable = table.getMetaTable() != null; @@ -131,21 +136,30 @@ private void addPartitionTransformData(DataOutputStream out, Table table, boolea } private void addStatsData(DataOutputStream out, HiveConf conf, String columnPath, List columns, - boolean isFormatted, List columnStats, boolean isOutputPadded) throws IOException { + Table table, boolean isFormatted, List columnStats, boolean isOutputPadded) + throws IOException { String statsData = ""; TextMetaDataTable metaDataTable = new TextMetaDataTable(); boolean needColStats = isFormatted && columnPath != null; boolean histogramEnabled = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL); + boolean isIcebergTable = isIcebergTable(table); if (needColStats) { - metaDataTable.addRow(DescTableDesc.getColumnStatisticsHeaders(histogramEnabled).toArray(new String[0])); + metaDataTable.addRow(DescTableDesc.getColumnStatisticsHeaders(histogramEnabled, isIcebergTable) + .toArray(new String[0])); } else if (isFormatted && !SessionState.get().isHiveServerQuery()) { statsData += "# "; metaDataTable.addRow(DescTableDesc.SCHEMA.split("#")[0].split(",")); } + JsonNode FieldsNode = OBJECT_MAPPER.readTree(table.getParameters().get("current-schema")).get("fields"); for (FieldSchema column : columns) { - metaDataTable.addRow(ShowUtils.extractColumnValues(column, needColStats, - getColumnStatisticsObject(column.getName(), column.getType(), columnStats), histogramEnabled)); + List values = new ArrayList<>(List.of(ShowUtils.extractColumnValues(column, needColStats, + getColumnStatisticsObject(column.getName(), column.getType(), columnStats), histogramEnabled))); + if (needColStats && isIcebergTable) { + values.add(getColumnDefaults(FieldsNode, column.getName(), "initial-default")); + values.add(getColumnDefaults(FieldsNode, column.getName(), "write-default")); + } + metaDataTable.addRow(values.toArray(new String[0])); } if (needColStats) { metaDataTable.transpose(); @@ -154,6 +168,55 @@ private void addStatsData(DataOutputStream out, HiveConf conf, String columnPath out.write(statsData.getBytes(StandardCharsets.UTF_8)); } + private String getColumnDefaults(JsonNode node, String colName, String defaultType) { + if (node == null || colName == null) { + return StringUtils.EMPTY; + } + + JsonNode targetNode = node; + if (node.isArray()) { + targetNode = null; + for (JsonNode field : node) { + if (field.has("name") && colName.equalsIgnoreCase(field.get("name").asText())) { + targetNode = field; + break; + } + } + } + + if (targetNode == null) { + return StringUtils.EMPTY; + } else if (targetNode.has(defaultType)) { + return targetNode.get(defaultType).asText(); + } + + // In case of struct, extract defaults recursively from its nested fields + JsonNode typeNode = targetNode.get("type"); + if (typeNode != null && typeNode.isObject() && typeNode.has("type") && + "struct".equalsIgnoreCase(typeNode.get("type").asText())) { + JsonNode structFields = typeNode.get("fields"); + if (structFields != null && structFields.isArray()) { + List fieldDefaults = new ArrayList<>(); + boolean hasDefaults = false; + for (JsonNode childField : structFields) { + String childName = childField.has("name") ? childField.get("name").asText() : ""; + String childDefault = getColumnDefaults(childField, childName, defaultType); + if (childDefault != null && !childDefault.isEmpty()) { + hasDefaults = true; + fieldDefaults.add(childName + ":" + childDefault); + } else { + fieldDefaults.add(childName + ":"); + } + } + + if (hasDefaults) { + return String.join(",", fieldDefaults); + } + } + } + return StringUtils.EMPTY; + } + private ColumnStatisticsObj getColumnStatisticsObject(String columnName, String columnType, List columnStats) { if (CollectionUtils.isNotEmpty(columnStats)) { From 6d53665f59e1ed1baf8de627f3ff1f855173a43d Mon Sep 17 00:00:00 2001 From: tanishq-chugh Date: Mon, 16 Mar 2026 23:41:00 +0530 Subject: [PATCH 2/6] Fix NPE caused for non iceberg DESC formatted table queries --- .../table/info/desc/formatter/TextDescTableFormatter.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java index b9b793595007..f4d3ef25bca3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java @@ -151,13 +151,14 @@ private void addStatsData(DataOutputStream out, HiveConf conf, String columnPath statsData += "# "; metaDataTable.addRow(DescTableDesc.SCHEMA.split("#")[0].split(",")); } - JsonNode FieldsNode = OBJECT_MAPPER.readTree(table.getParameters().get("current-schema")).get("fields"); + JsonNode fieldsNode = needColStats && isIcebergTable ? + OBJECT_MAPPER.readTree(table.getParameters().get("current-schema")).get("fields") : null; for (FieldSchema column : columns) { List values = new ArrayList<>(List.of(ShowUtils.extractColumnValues(column, needColStats, getColumnStatisticsObject(column.getName(), column.getType(), columnStats), histogramEnabled))); if (needColStats && isIcebergTable) { - values.add(getColumnDefaults(FieldsNode, column.getName(), "initial-default")); - values.add(getColumnDefaults(FieldsNode, column.getName(), "write-default")); + values.add(getColumnDefaults(fieldsNode, column.getName(), "initial-default")); + values.add(getColumnDefaults(fieldsNode, column.getName(), "write-default")); } metaDataTable.addRow(values.toArray(new String[0])); } From f28e04d45f7dfdacc20c89fb160b542445c17d6f Mon Sep 17 00:00:00 2001 From: tanishq-chugh Date: Mon, 6 Apr 2026 18:22:22 +0530 Subject: [PATCH 3/6] Revert "Fix NPE caused for non iceberg DESC formatted table queries" This reverts commit 6d53665f59e1ed1baf8de627f3ff1f855173a43d. --- .../table/info/desc/formatter/TextDescTableFormatter.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java index f4d3ef25bca3..b9b793595007 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java @@ -151,14 +151,13 @@ private void addStatsData(DataOutputStream out, HiveConf conf, String columnPath statsData += "# "; metaDataTable.addRow(DescTableDesc.SCHEMA.split("#")[0].split(",")); } - JsonNode fieldsNode = needColStats && isIcebergTable ? - OBJECT_MAPPER.readTree(table.getParameters().get("current-schema")).get("fields") : null; + JsonNode FieldsNode = OBJECT_MAPPER.readTree(table.getParameters().get("current-schema")).get("fields"); for (FieldSchema column : columns) { List values = new ArrayList<>(List.of(ShowUtils.extractColumnValues(column, needColStats, getColumnStatisticsObject(column.getName(), column.getType(), columnStats), histogramEnabled))); if (needColStats && isIcebergTable) { - values.add(getColumnDefaults(fieldsNode, column.getName(), "initial-default")); - values.add(getColumnDefaults(fieldsNode, column.getName(), "write-default")); + values.add(getColumnDefaults(FieldsNode, column.getName(), "initial-default")); + values.add(getColumnDefaults(FieldsNode, column.getName(), "write-default")); } metaDataTable.addRow(values.toArray(new String[0])); } From 9c59751121fca78476498f9adf442c7fd6e7c951 Mon Sep 17 00:00:00 2001 From: tanishq-chugh Date: Mon, 6 Apr 2026 18:22:22 +0530 Subject: [PATCH 4/6] Revert "HIVE-29342: Iceberg: [V3] Display initial and write defaults in describe formatted column output" This reverts commit 5888f657ae204e616666e5224a3e48a804abcc2e. --- .../positive/iceberg_defaults_in_desc_cols.q | 54 --- .../iceberg_defaults_in_desc_cols.q.out | 406 ------------------ .../ql/ddl/table/info/desc/DescTableDesc.java | 7 +- .../formatter/TextDescTableFormatter.java | 73 +--- 4 files changed, 6 insertions(+), 534 deletions(-) delete mode 100644 iceberg/iceberg-handler/src/test/queries/positive/iceberg_defaults_in_desc_cols.q delete mode 100644 iceberg/iceberg-handler/src/test/results/positive/iceberg_defaults_in_desc_cols.q.out diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_defaults_in_desc_cols.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_defaults_in_desc_cols.q deleted file mode 100644 index 5be220422896..000000000000 --- a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_defaults_in_desc_cols.q +++ /dev/null @@ -1,54 +0,0 @@ --- Mask random uuid ---! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/ --- Mask a random snapshot id ---! qt:replace:/(\s+current-snapshot-id\s+)\S+(\s*)/$1#Masked#/ --- Mask added file size ---! qt:replace:/(\S\"added-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/ --- Mask total file size ---! qt:replace:/(\S\"total-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/ --- Mask current-snapshot-timestamp-ms ---! qt:replace:/(\s+current-snapshot-timestamp-ms\s+)\S+(\s*)/$1#Masked#$2/ - -CREATE TABLE ice_parq ( - id INT) -STORED BY ICEBERG stored as parquet -TBLPROPERTIES ('format-version'='3'); - -INSERT INTO ice_parq (id) VALUES (1); - -ALTER TABLE ice_parq ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', - name STRING DEFAULT 'unknown', - age INT DEFAULT 25); - -INSERT INTO ice_parq (id) VALUES (2); - -SELECT * FROM ice_parq ORDER BY id; - -DESCRIBE FORMATTED ice_parq; - -DESCRIBE FORMATTED ice_parq id; -DESCRIBE FORMATTED ice_parq point; -DESCRIBE FORMATTED ice_parq name; -DESCRIBE FORMATTED ice_parq age; - -CREATE TABLE ice_orc ( - id INT) -STORED BY ICEBERG stored as orc -TBLPROPERTIES ('format-version'='3'); - -INSERT INTO ice_orc (id) VALUES (1); - -ALTER TABLE ice_orc ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', - name STRING DEFAULT 'unknown', - age INT DEFAULT 25); - -INSERT INTO ice_orc (id) VALUES (2); - -SELECT * FROM ice_orc ORDER BY id; - -DESCRIBE FORMATTED ice_orc; - -DESCRIBE FORMATTED ice_orc id; -DESCRIBE FORMATTED ice_orc point; -DESCRIBE FORMATTED ice_orc name; -DESCRIBE FORMATTED ice_orc age; diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_defaults_in_desc_cols.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_defaults_in_desc_cols.q.out deleted file mode 100644 index edc3d0398bd3..000000000000 --- a/iceberg/iceberg-handler/src/test/results/positive/iceberg_defaults_in_desc_cols.q.out +++ /dev/null @@ -1,406 +0,0 @@ -PREHOOK: query: CREATE TABLE ice_parq ( - id INT) -STORED BY ICEBERG stored as parquet -TBLPROPERTIES ('format-version'='3') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@ice_parq -POSTHOOK: query: CREATE TABLE ice_parq ( - id INT) -STORED BY ICEBERG stored as parquet -TBLPROPERTIES ('format-version'='3') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@ice_parq -PREHOOK: query: INSERT INTO ice_parq (id) VALUES (1) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@ice_parq -POSTHOOK: query: INSERT INTO ice_parq (id) VALUES (1) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@ice_parq -PREHOOK: query: ALTER TABLE ice_parq ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', - name STRING DEFAULT 'unknown', - age INT DEFAULT 25) -PREHOOK: type: ALTERTABLE_ADDCOLS -PREHOOK: Input: default@ice_parq -PREHOOK: Output: default@ice_parq -POSTHOOK: query: ALTER TABLE ice_parq ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', - name STRING DEFAULT 'unknown', - age INT DEFAULT 25) -POSTHOOK: type: ALTERTABLE_ADDCOLS -POSTHOOK: Input: default@ice_parq -POSTHOOK: Output: default@ice_parq -PREHOOK: query: INSERT INTO ice_parq (id) VALUES (2) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@ice_parq -POSTHOOK: query: INSERT INTO ice_parq (id) VALUES (2) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@ice_parq -PREHOOK: query: SELECT * FROM ice_parq ORDER BY id -PREHOOK: type: QUERY -PREHOOK: Input: default@ice_parq -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT * FROM ice_parq ORDER BY id -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ice_parq -POSTHOOK: Output: hdfs://### HDFS PATH ### -1 NULL unknown 25 -2 {"x":100,"y":99} unknown 25 -PREHOOK: query: DESCRIBE FORMATTED ice_parq -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@ice_parq -POSTHOOK: query: DESCRIBE FORMATTED ice_parq -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@ice_parq -# col_name data_type comment -id int -point struct -name string -age int - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: EXTERNAL_TABLE -Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} - EXTERNAL TRUE - bucketing_version 2 - current-schema {\"type\":\"struct\",\"schema-id\":1,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"point\",\"required\":false,\"type\":{\"type\":\"struct\",\"fields\":[{\"id\":3,\"name\":\"x\",\"required\":false,\"type\":\"int\",\"write-default\":100},{\"id\":4,\"name\":\"y\",\"required\":false,\"type\":\"int\",\"write-default\":99}]}},{\"id\":5,\"name\":\"name\",\"required\":false,\"type\":\"string\",\"initial-default\":\"unknown\",\"write-default\":\"unknown\"},{\"id\":6,\"name\":\"age\",\"required\":false,\"type\":\"int\",\"initial-default\":25,\"write-default\":25}]} - current-snapshot-id #Masked# - current-snapshot-summary {\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"2\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"Apache Iceberg 1.10.1 (commit ccb8bc435062171e64bc8b7e5f56e6aed9c5b934)\"} - current-snapshot-timestamp-ms #Masked# - format-version 3 - iceberg.orc.files.only false - metadata_location hdfs://### HDFS PATH ### - numFiles 2 - numRows 2 - parquet.compression zstd - previous_metadata_location hdfs://### HDFS PATH ### - rawDataSize 0 - serialization.format 1 - snapshot-count 2 - storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler - table_type ICEBERG - totalSize #Masked# -#### A masked pattern was here #### - uuid #Masked# - write.delete.mode merge-on-read - write.format.default parquet - write.merge.mode merge-on-read - write.metadata.delete-after-commit.enabled true - write.update.mode merge-on-read - -# Storage Information -SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe -InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat -OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat -Compressed: No -Sort Columns: [] - -# Constraints - -# Default Constraints -Table: default.ice_parq -Constraint Name: #### A masked pattern was here #### -Column Name:age Default Value:25 - -Constraint Name: #### A masked pattern was here #### -Column Name:point Default Value:'x:100,y:99' - -Constraint Name: #### A masked pattern was here #### -Column Name:name Default Value:'unknown' - -PREHOOK: query: DESCRIBE FORMATTED ice_parq id -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@ice_parq -POSTHOOK: query: DESCRIBE FORMATTED ice_parq id -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@ice_parq -col_name id -data_type int -min -max -num_nulls -distinct_count -avg_col_len -max_col_len -num_trues -num_falses -bit_vector -comment -initial_default -write_default -COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} -PREHOOK: query: DESCRIBE FORMATTED ice_parq point -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@ice_parq -POSTHOOK: query: DESCRIBE FORMATTED ice_parq point -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@ice_parq -col_name point -data_type struct -min -max -num_nulls -distinct_count -avg_col_len -max_col_len -num_trues -num_falses -bit_vector -comment -initial_default -write_default x:100,y:99 -COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} -PREHOOK: query: DESCRIBE FORMATTED ice_parq name -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@ice_parq -POSTHOOK: query: DESCRIBE FORMATTED ice_parq name -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@ice_parq -col_name name -data_type string -min -max -num_nulls -distinct_count -avg_col_len -max_col_len -num_trues -num_falses -bit_vector -comment -initial_default unknown -write_default unknown -COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} -PREHOOK: query: DESCRIBE FORMATTED ice_parq age -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@ice_parq -POSTHOOK: query: DESCRIBE FORMATTED ice_parq age -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@ice_parq -col_name age -data_type int -min -max -num_nulls -distinct_count -avg_col_len -max_col_len -num_trues -num_falses -bit_vector -comment -initial_default 25 -write_default 25 -COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} -PREHOOK: query: CREATE TABLE ice_orc ( - id INT) -STORED BY ICEBERG stored as orc -TBLPROPERTIES ('format-version'='3') -PREHOOK: type: CREATETABLE -PREHOOK: Output: database:default -PREHOOK: Output: default@ice_orc -POSTHOOK: query: CREATE TABLE ice_orc ( - id INT) -STORED BY ICEBERG stored as orc -TBLPROPERTIES ('format-version'='3') -POSTHOOK: type: CREATETABLE -POSTHOOK: Output: database:default -POSTHOOK: Output: default@ice_orc -PREHOOK: query: INSERT INTO ice_orc (id) VALUES (1) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@ice_orc -POSTHOOK: query: INSERT INTO ice_orc (id) VALUES (1) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@ice_orc -PREHOOK: query: ALTER TABLE ice_orc ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', - name STRING DEFAULT 'unknown', - age INT DEFAULT 25) -PREHOOK: type: ALTERTABLE_ADDCOLS -PREHOOK: Input: default@ice_orc -PREHOOK: Output: default@ice_orc -POSTHOOK: query: ALTER TABLE ice_orc ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', - name STRING DEFAULT 'unknown', - age INT DEFAULT 25) -POSTHOOK: type: ALTERTABLE_ADDCOLS -POSTHOOK: Input: default@ice_orc -POSTHOOK: Output: default@ice_orc -PREHOOK: query: INSERT INTO ice_orc (id) VALUES (2) -PREHOOK: type: QUERY -PREHOOK: Input: _dummy_database@_dummy_table -PREHOOK: Output: default@ice_orc -POSTHOOK: query: INSERT INTO ice_orc (id) VALUES (2) -POSTHOOK: type: QUERY -POSTHOOK: Input: _dummy_database@_dummy_table -POSTHOOK: Output: default@ice_orc -PREHOOK: query: SELECT * FROM ice_orc ORDER BY id -PREHOOK: type: QUERY -PREHOOK: Input: default@ice_orc -PREHOOK: Output: hdfs://### HDFS PATH ### -POSTHOOK: query: SELECT * FROM ice_orc ORDER BY id -POSTHOOK: type: QUERY -POSTHOOK: Input: default@ice_orc -POSTHOOK: Output: hdfs://### HDFS PATH ### -1 NULL NULL NULL -2 {"x":100,"y":99} unknown 25 -PREHOOK: query: DESCRIBE FORMATTED ice_orc -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@ice_orc -POSTHOOK: query: DESCRIBE FORMATTED ice_orc -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@ice_orc -# col_name data_type comment -id int -point struct -name string -age int - -# Detailed Table Information -Database: default -#### A masked pattern was here #### -Retention: 0 -#### A masked pattern was here #### -Table Type: EXTERNAL_TABLE -Table Parameters: - COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} - EXTERNAL TRUE - bucketing_version 2 - current-schema {\"type\":\"struct\",\"schema-id\":1,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"point\",\"required\":false,\"type\":{\"type\":\"struct\",\"fields\":[{\"id\":3,\"name\":\"x\",\"required\":false,\"type\":\"int\",\"write-default\":100},{\"id\":4,\"name\":\"y\",\"required\":false,\"type\":\"int\",\"write-default\":99}]}},{\"id\":5,\"name\":\"name\",\"required\":false,\"type\":\"string\",\"write-default\":\"unknown\"},{\"id\":6,\"name\":\"age\",\"required\":false,\"type\":\"int\",\"write-default\":25}]} - current-snapshot-id #Masked# - current-snapshot-summary {\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"2\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"Apache Iceberg 1.10.1 (commit ccb8bc435062171e64bc8b7e5f56e6aed9c5b934)\"} - current-snapshot-timestamp-ms #Masked# - format-version 3 - iceberg.orc.files.only true - metadata_location hdfs://### HDFS PATH ### - numFiles 2 - numRows 2 - parquet.compression zstd - previous_metadata_location hdfs://### HDFS PATH ### - rawDataSize 0 - serialization.format 1 - snapshot-count 2 - storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler - table_type ICEBERG - totalSize #Masked# -#### A masked pattern was here #### - uuid #Masked# - write.delete.mode merge-on-read - write.format.default orc - write.merge.mode merge-on-read - write.metadata.delete-after-commit.enabled true - write.update.mode merge-on-read - -# Storage Information -SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe -InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat -OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat -Compressed: No -Sort Columns: [] - -# Constraints - -# Default Constraints -Table: default.ice_orc -Constraint Name: #### A masked pattern was here #### -Column Name:point Default Value:'x:100,y:99' - -Constraint Name: #### A masked pattern was here #### -Column Name:age Default Value:25 - -Constraint Name: #### A masked pattern was here #### -Column Name:name Default Value:'unknown' - -PREHOOK: query: DESCRIBE FORMATTED ice_orc id -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@ice_orc -POSTHOOK: query: DESCRIBE FORMATTED ice_orc id -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@ice_orc -col_name id -data_type int -min -max -num_nulls -distinct_count -avg_col_len -max_col_len -num_trues -num_falses -bit_vector -comment -initial_default -write_default -COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} -PREHOOK: query: DESCRIBE FORMATTED ice_orc point -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@ice_orc -POSTHOOK: query: DESCRIBE FORMATTED ice_orc point -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@ice_orc -col_name point -data_type struct -min -max -num_nulls -distinct_count -avg_col_len -max_col_len -num_trues -num_falses -bit_vector -comment -initial_default -write_default x:100,y:99 -COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} -PREHOOK: query: DESCRIBE FORMATTED ice_orc name -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@ice_orc -POSTHOOK: query: DESCRIBE FORMATTED ice_orc name -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@ice_orc -col_name name -data_type string -min -max -num_nulls -distinct_count -avg_col_len -max_col_len -num_trues -num_falses -bit_vector -comment -initial_default -write_default unknown -COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} -PREHOOK: query: DESCRIBE FORMATTED ice_orc age -PREHOOK: type: DESCTABLE -PREHOOK: Input: default@ice_orc -POSTHOOK: query: DESCRIBE FORMATTED ice_orc age -POSTHOOK: type: DESCTABLE -POSTHOOK: Input: default@ice_orc -col_name age -data_type int -min -max -num_nulls -distinct_count -avg_col_len -max_col_len -num_trues -num_falses -bit_vector -comment -initial_default -write_default 25 -COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java index 112b7db910dd..f0d979e80474 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/DescTableDesc.java @@ -110,7 +110,7 @@ public boolean isFormatted() { return isFormatted; } - public static List getColumnStatisticsHeaders(boolean histogramEnabled, boolean isIcebergTable) { + public static List getColumnStatisticsHeaders(boolean histogramEnabled) { ImmutableList.Builder builder = ImmutableList.builder() .add("col_name") .add("data_type") @@ -121,11 +121,6 @@ public static List getColumnStatisticsHeaders(boolean histogramEnabled, } builder.add("comment"); - - if (isIcebergTable) { - builder.add("initial_default") - .add("write_default"); - } return builder.build(); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java index b9b793595007..b1dd9738572a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java @@ -18,11 +18,8 @@ package org.apache.hadoop.hive.ql.ddl.table.info.desc.formatter; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringEscapeUtils; import org.apache.hadoop.hive.common.MaterializationSnapshot; import org.apache.hadoop.hive.common.StatsSetupConst; @@ -76,7 +73,6 @@ import java.util.stream.Collectors; import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.TABLE_IS_CTAS; -import static org.apache.hadoop.hive.ql.ddl.DDLUtils.isIcebergTable; import static org.apache.hadoop.hive.ql.ddl.ShowUtils.ALIGNMENT; import static org.apache.hadoop.hive.ql.ddl.ShowUtils.DEFAULT_STRINGBUILDER_SIZE; import static org.apache.hadoop.hive.ql.ddl.ShowUtils.FIELD_DELIM; @@ -87,13 +83,12 @@ * Formats DESC TABLE results to text format. */ class TextDescTableFormatter extends DescTableFormatter { - public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @Override public void describeTable(HiveConf conf, DataOutputStream out, String columnPath, String tableName, Table table, Partition partition, List columns, boolean isFormatted, boolean isExtended, boolean isOutputPadded, List columnStats) throws HiveException { try { - addStatsData(out, conf, columnPath, columns, table, isFormatted, columnStats, isOutputPadded); + addStatsData(out, conf, columnPath, columns, isFormatted, columnStats, isOutputPadded); addPartitionData(out, conf, columnPath, table, isFormatted, isOutputPadded); boolean isIcebergMetaTable = table.getMetaTable() != null; @@ -136,30 +131,21 @@ private void addPartitionTransformData(DataOutputStream out, Table table, boolea } private void addStatsData(DataOutputStream out, HiveConf conf, String columnPath, List columns, - Table table, boolean isFormatted, List columnStats, boolean isOutputPadded) - throws IOException { + boolean isFormatted, List columnStats, boolean isOutputPadded) throws IOException { String statsData = ""; TextMetaDataTable metaDataTable = new TextMetaDataTable(); boolean needColStats = isFormatted && columnPath != null; boolean histogramEnabled = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.STATS_FETCH_KLL); - boolean isIcebergTable = isIcebergTable(table); if (needColStats) { - metaDataTable.addRow(DescTableDesc.getColumnStatisticsHeaders(histogramEnabled, isIcebergTable) - .toArray(new String[0])); + metaDataTable.addRow(DescTableDesc.getColumnStatisticsHeaders(histogramEnabled).toArray(new String[0])); } else if (isFormatted && !SessionState.get().isHiveServerQuery()) { statsData += "# "; metaDataTable.addRow(DescTableDesc.SCHEMA.split("#")[0].split(",")); } - JsonNode FieldsNode = OBJECT_MAPPER.readTree(table.getParameters().get("current-schema")).get("fields"); for (FieldSchema column : columns) { - List values = new ArrayList<>(List.of(ShowUtils.extractColumnValues(column, needColStats, - getColumnStatisticsObject(column.getName(), column.getType(), columnStats), histogramEnabled))); - if (needColStats && isIcebergTable) { - values.add(getColumnDefaults(FieldsNode, column.getName(), "initial-default")); - values.add(getColumnDefaults(FieldsNode, column.getName(), "write-default")); - } - metaDataTable.addRow(values.toArray(new String[0])); + metaDataTable.addRow(ShowUtils.extractColumnValues(column, needColStats, + getColumnStatisticsObject(column.getName(), column.getType(), columnStats), histogramEnabled)); } if (needColStats) { metaDataTable.transpose(); @@ -168,55 +154,6 @@ private void addStatsData(DataOutputStream out, HiveConf conf, String columnPath out.write(statsData.getBytes(StandardCharsets.UTF_8)); } - private String getColumnDefaults(JsonNode node, String colName, String defaultType) { - if (node == null || colName == null) { - return StringUtils.EMPTY; - } - - JsonNode targetNode = node; - if (node.isArray()) { - targetNode = null; - for (JsonNode field : node) { - if (field.has("name") && colName.equalsIgnoreCase(field.get("name").asText())) { - targetNode = field; - break; - } - } - } - - if (targetNode == null) { - return StringUtils.EMPTY; - } else if (targetNode.has(defaultType)) { - return targetNode.get(defaultType).asText(); - } - - // In case of struct, extract defaults recursively from its nested fields - JsonNode typeNode = targetNode.get("type"); - if (typeNode != null && typeNode.isObject() && typeNode.has("type") && - "struct".equalsIgnoreCase(typeNode.get("type").asText())) { - JsonNode structFields = typeNode.get("fields"); - if (structFields != null && structFields.isArray()) { - List fieldDefaults = new ArrayList<>(); - boolean hasDefaults = false; - for (JsonNode childField : structFields) { - String childName = childField.has("name") ? childField.get("name").asText() : ""; - String childDefault = getColumnDefaults(childField, childName, defaultType); - if (childDefault != null && !childDefault.isEmpty()) { - hasDefaults = true; - fieldDefaults.add(childName + ":" + childDefault); - } else { - fieldDefaults.add(childName + ":"); - } - } - - if (hasDefaults) { - return String.join(",", fieldDefaults); - } - } - } - return StringUtils.EMPTY; - } - private ColumnStatisticsObj getColumnStatisticsObject(String columnName, String columnType, List columnStats) { if (CollectionUtils.isNotEmpty(columnStats)) { From 4811a37574a670e9ca2e4a9793514a57c05dd33c Mon Sep 17 00:00:00 2001 From: tanishq-chugh Date: Mon, 6 Apr 2026 18:20:16 +0530 Subject: [PATCH 5/6] Refactor Code to add defaults in Defaults Constraint section of Describe formatted table output --- .../positive/iceberg_defaults_in_desc_cols.q | 44 ++++ .../iceberg_defaults_in_desc_cols.q.out | 238 ++++++++++++++++++ .../formatter/TextDescTableFormatter.java | 97 ++++++- 3 files changed, 375 insertions(+), 4 deletions(-) create mode 100644 iceberg/iceberg-handler/src/test/queries/positive/iceberg_defaults_in_desc_cols.q create mode 100644 iceberg/iceberg-handler/src/test/results/positive/iceberg_defaults_in_desc_cols.q.out diff --git a/iceberg/iceberg-handler/src/test/queries/positive/iceberg_defaults_in_desc_cols.q b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_defaults_in_desc_cols.q new file mode 100644 index 000000000000..291315d8d3ff --- /dev/null +++ b/iceberg/iceberg-handler/src/test/queries/positive/iceberg_defaults_in_desc_cols.q @@ -0,0 +1,44 @@ +-- Mask random uuid +--! qt:replace:/(\s+uuid\s+)\S+(\s*)/$1#Masked#$2/ +-- Mask a random snapshot id +--! qt:replace:/(\s+current-snapshot-id\s+)\S+(\s*)/$1#Masked#/ +-- Mask added file size +--! qt:replace:/(\S\"added-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/ +-- Mask total file size +--! qt:replace:/(\S\"total-files-size\\\":\\\")(\d+)(\\\")/$1#Masked#$3/ +-- Mask current-snapshot-timestamp-ms +--! qt:replace:/(\s+current-snapshot-timestamp-ms\s+)\S+(\s*)/$1#Masked#$2/ + +CREATE TABLE ice_parq ( + id INT) +STORED BY ICEBERG stored as parquet +TBLPROPERTIES ('format-version'='3'); + +INSERT INTO ice_parq (id) VALUES (1); + +ALTER TABLE ice_parq ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', + name STRING DEFAULT 'unknown', + age INT DEFAULT 25); + +INSERT INTO ice_parq (id) VALUES (2); + +SELECT * FROM ice_parq ORDER BY id; + +DESCRIBE FORMATTED ice_parq; + +CREATE TABLE ice_orc ( + id INT) +STORED BY ICEBERG stored as orc +TBLPROPERTIES ('format-version'='3'); + +INSERT INTO ice_orc (id) VALUES (1); + +ALTER TABLE ice_orc ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', + name STRING DEFAULT 'unknown', + age INT DEFAULT 25); + +INSERT INTO ice_orc (id) VALUES (2); + +SELECT * FROM ice_orc ORDER BY id; + +DESCRIBE FORMATTED ice_orc; diff --git a/iceberg/iceberg-handler/src/test/results/positive/iceberg_defaults_in_desc_cols.q.out b/iceberg/iceberg-handler/src/test/results/positive/iceberg_defaults_in_desc_cols.q.out new file mode 100644 index 000000000000..88f46ab0107a --- /dev/null +++ b/iceberg/iceberg-handler/src/test/results/positive/iceberg_defaults_in_desc_cols.q.out @@ -0,0 +1,238 @@ +PREHOOK: query: CREATE TABLE ice_parq ( + id INT) +STORED BY ICEBERG stored as parquet +TBLPROPERTIES ('format-version'='3') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_parq +POSTHOOK: query: CREATE TABLE ice_parq ( + id INT) +STORED BY ICEBERG stored as parquet +TBLPROPERTIES ('format-version'='3') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_parq +PREHOOK: query: INSERT INTO ice_parq (id) VALUES (1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_parq +POSTHOOK: query: INSERT INTO ice_parq (id) VALUES (1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_parq +PREHOOK: query: ALTER TABLE ice_parq ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', + name STRING DEFAULT 'unknown', + age INT DEFAULT 25) +PREHOOK: type: ALTERTABLE_ADDCOLS +PREHOOK: Input: default@ice_parq +PREHOOK: Output: default@ice_parq +POSTHOOK: query: ALTER TABLE ice_parq ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', + name STRING DEFAULT 'unknown', + age INT DEFAULT 25) +POSTHOOK: type: ALTERTABLE_ADDCOLS +POSTHOOK: Input: default@ice_parq +POSTHOOK: Output: default@ice_parq +PREHOOK: query: INSERT INTO ice_parq (id) VALUES (2) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_parq +POSTHOOK: query: INSERT INTO ice_parq (id) VALUES (2) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_parq +PREHOOK: query: SELECT * FROM ice_parq ORDER BY id +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_parq +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM ice_parq ORDER BY id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_parq +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 NULL unknown 25 +2 {"x":100,"y":99} unknown 25 +PREHOOK: query: DESCRIBE FORMATTED ice_parq +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_parq +POSTHOOK: query: DESCRIBE FORMATTED ice_parq +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_parq +# col_name data_type comment +id int +point struct +name string +age int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: EXTERNAL_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + EXTERNAL TRUE + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":1,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"point\",\"required\":false,\"type\":{\"type\":\"struct\",\"fields\":[{\"id\":3,\"name\":\"x\",\"required\":false,\"type\":\"int\",\"write-default\":100},{\"id\":4,\"name\":\"y\",\"required\":false,\"type\":\"int\",\"write-default\":99}]}},{\"id\":5,\"name\":\"name\",\"required\":false,\"type\":\"string\",\"initial-default\":\"unknown\",\"write-default\":\"unknown\"},{\"id\":6,\"name\":\"age\",\"required\":false,\"type\":\"int\",\"initial-default\":25,\"write-default\":25}]} + current-snapshot-id #Masked# + current-snapshot-summary {\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"2\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"Apache Iceberg 1.10.1 (commit ccb8bc435062171e64bc8b7e5f56e6aed9c5b934)\"} + current-snapshot-timestamp-ms #Masked# + format-version 3 + iceberg.orc.files.only false + metadata_location hdfs://### HDFS PATH ### + numFiles 2 + numRows 2 + parquet.compression zstd + previous_metadata_location hdfs://### HDFS PATH ### + rawDataSize 0 + serialization.format 1 + snapshot-count 2 + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG + totalSize #Masked# +#### A masked pattern was here #### + uuid #Masked# + write.delete.mode merge-on-read + write.format.default parquet + write.merge.mode merge-on-read + write.metadata.delete-after-commit.enabled true + write.update.mode merge-on-read + +# Storage Information +SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe +InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat +OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat +Compressed: No +Sort Columns: [] + +# Constraints + +# Default Constraints +Table: default.ice_parq +Constraint Name: #### A masked pattern was here #### +Column Name:age Initial Default Value:25 Write Default Value:25 + +Constraint Name: #### A masked pattern was here #### +Column Name:point Initial Default Value: Write Default Value:'x:100,y:99' + +Constraint Name: #### A masked pattern was here #### +Column Name:name Initial Default Value:'unknown' Write Default Value:'unknown' + +PREHOOK: query: CREATE TABLE ice_orc ( + id INT) +STORED BY ICEBERG stored as orc +TBLPROPERTIES ('format-version'='3') +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@ice_orc +POSTHOOK: query: CREATE TABLE ice_orc ( + id INT) +STORED BY ICEBERG stored as orc +TBLPROPERTIES ('format-version'='3') +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@ice_orc +PREHOOK: query: INSERT INTO ice_orc (id) VALUES (1) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: INSERT INTO ice_orc (id) VALUES (1) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: ALTER TABLE ice_orc ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', + name STRING DEFAULT 'unknown', + age INT DEFAULT 25) +PREHOOK: type: ALTERTABLE_ADDCOLS +PREHOOK: Input: default@ice_orc +PREHOOK: Output: default@ice_orc +POSTHOOK: query: ALTER TABLE ice_orc ADD COLUMNS (point STRUCT DEFAULT 'x:100,y:99', + name STRING DEFAULT 'unknown', + age INT DEFAULT 25) +POSTHOOK: type: ALTERTABLE_ADDCOLS +POSTHOOK: Input: default@ice_orc +POSTHOOK: Output: default@ice_orc +PREHOOK: query: INSERT INTO ice_orc (id) VALUES (2) +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@ice_orc +POSTHOOK: query: INSERT INTO ice_orc (id) VALUES (2) +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@ice_orc +PREHOOK: query: SELECT * FROM ice_orc ORDER BY id +PREHOOK: type: QUERY +PREHOOK: Input: default@ice_orc +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT * FROM ice_orc ORDER BY id +POSTHOOK: type: QUERY +POSTHOOK: Input: default@ice_orc +POSTHOOK: Output: hdfs://### HDFS PATH ### +1 NULL NULL NULL +2 {"x":100,"y":99} unknown 25 +PREHOOK: query: DESCRIBE FORMATTED ice_orc +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@ice_orc +POSTHOOK: query: DESCRIBE FORMATTED ice_orc +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@ice_orc +# col_name data_type comment +id int +point struct +name string +age int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: EXTERNAL_TABLE +Table Parameters: + COLUMN_STATS_ACCURATE {\"BASIC_STATS\":\"true\"} + EXTERNAL TRUE + bucketing_version 2 + current-schema {\"type\":\"struct\",\"schema-id\":1,\"fields\":[{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"},{\"id\":2,\"name\":\"point\",\"required\":false,\"type\":{\"type\":\"struct\",\"fields\":[{\"id\":3,\"name\":\"x\",\"required\":false,\"type\":\"int\",\"write-default\":100},{\"id\":4,\"name\":\"y\",\"required\":false,\"type\":\"int\",\"write-default\":99}]}},{\"id\":5,\"name\":\"name\",\"required\":false,\"type\":\"string\",\"write-default\":\"unknown\"},{\"id\":6,\"name\":\"age\",\"required\":false,\"type\":\"int\",\"write-default\":25}]} + current-snapshot-id #Masked# + current-snapshot-summary {\"added-data-files\":\"1\",\"added-records\":\"1\",\"added-files-size\":\"#Masked#\",\"changed-partition-count\":\"1\",\"total-records\":\"2\",\"total-files-size\":\"#Masked#\",\"total-data-files\":\"2\",\"total-delete-files\":\"0\",\"total-position-deletes\":\"0\",\"total-equality-deletes\":\"0\",\"iceberg-version\":\"Apache Iceberg 1.10.1 (commit ccb8bc435062171e64bc8b7e5f56e6aed9c5b934)\"} + current-snapshot-timestamp-ms #Masked# + format-version 3 + iceberg.orc.files.only true + metadata_location hdfs://### HDFS PATH ### + numFiles 2 + numRows 2 + parquet.compression zstd + previous_metadata_location hdfs://### HDFS PATH ### + rawDataSize 0 + serialization.format 1 + snapshot-count 2 + storage_handler org.apache.iceberg.mr.hive.HiveIcebergStorageHandler + table_type ICEBERG + totalSize #Masked# +#### A masked pattern was here #### + uuid #Masked# + write.delete.mode merge-on-read + write.format.default orc + write.merge.mode merge-on-read + write.metadata.delete-after-commit.enabled true + write.update.mode merge-on-read + +# Storage Information +SerDe Library: org.apache.iceberg.mr.hive.HiveIcebergSerDe +InputFormat: org.apache.iceberg.mr.hive.HiveIcebergInputFormat +OutputFormat: org.apache.iceberg.mr.hive.HiveIcebergOutputFormat +Compressed: No +Sort Columns: [] + +# Constraints + +# Default Constraints +Table: default.ice_orc +Constraint Name: #### A masked pattern was here #### +Column Name:point Initial Default Value: Write Default Value:'x:100,y:99' + +Constraint Name: #### A masked pattern was here #### +Column Name:age Initial Default Value: Write Default Value:25 + +Constraint Name: #### A masked pattern was here #### +Column Name:name Initial Default Value: Write Default Value:'unknown' + diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java index b1dd9738572a..9f4476abd4f4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java @@ -18,8 +18,12 @@ package org.apache.hadoop.hive.ql.ddl.table.info.desc.formatter; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.text.StringEscapeUtils; import org.apache.hadoop.hive.common.MaterializationSnapshot; import org.apache.hadoop.hive.common.StatsSetupConst; @@ -73,6 +77,7 @@ import java.util.stream.Collectors; import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.TABLE_IS_CTAS; +import static org.apache.hadoop.hive.ql.ddl.DDLUtils.isIcebergTable; import static org.apache.hadoop.hive.ql.ddl.ShowUtils.ALIGNMENT; import static org.apache.hadoop.hive.ql.ddl.ShowUtils.DEFAULT_STRINGBUILDER_SIZE; import static org.apache.hadoop.hive.ql.ddl.ShowUtils.FIELD_DELIM; @@ -83,6 +88,7 @@ * Formats DESC TABLE results to text format. */ class TextDescTableFormatter extends DescTableFormatter { + public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @Override public void describeTable(HiveConf conf, DataOutputStream out, String columnPath, String tableName, Table table, Partition partition, List columns, boolean isFormatted, boolean isExtended, boolean isOutputPadded, @@ -454,7 +460,7 @@ private void displayAllParameters(Map params, StringBuilder tabl } } - private String getConstraintsInformation(Table table) { + private String getConstraintsInformation(Table table) throws IOException { StringBuilder constraintsInfo = new StringBuilder(DEFAULT_STRINGBUILDER_SIZE); constraintsInfo.append(LINE_DELIM).append("# Constraints").append(LINE_DELIM); @@ -476,7 +482,7 @@ private String getConstraintsInformation(Table table) { } if (DefaultConstraint.isNotEmpty(table.getDefaultConstraint())) { constraintsInfo.append(LINE_DELIM).append("# Default Constraints").append(LINE_DELIM); - getDefaultConstraintsInformation(constraintsInfo, table.getDefaultConstraint()); + getDefaultConstraintsInformation(constraintsInfo, table); } if (CheckConstraint.isNotEmpty(table.getCheckConstraint())) { constraintsInfo.append(LINE_DELIM).append("# Check Constraints").append(LINE_DELIM); @@ -558,12 +564,23 @@ private void getNotNullConstraintsInformation(StringBuilder constraintsInfo, Not } } - private void getDefaultConstraintsInformation(StringBuilder constraintsInfo, DefaultConstraint constraint) { + private void getDefaultConstraintsInformation(StringBuilder constraintsInfo, Table table) throws IOException { + DefaultConstraint constraint = table.getDefaultConstraint(); formatOutput("Table:", constraint.getDatabaseName() + "." + constraint.getTableName(), constraintsInfo); Map> defaultConstraints = constraint.getDefaultConstraints(); if (MapUtils.isNotEmpty(defaultConstraints)) { + boolean isIceberg = isIcebergTable(table); + JsonNode fieldsNode = null; + if (isIceberg && table.getParameters().get("current-schema") != null) { + fieldsNode = OBJECT_MAPPER.readTree(table.getParameters().get("current-schema")).get("fields"); + } + for (Map.Entry> entry : defaultConstraints.entrySet()) { - getDefaultConstraintRelInformation(constraintsInfo, entry.getKey(), entry.getValue()); + if (isIceberg) { + getIcebergDefaultConstraintRelInformation(constraintsInfo, entry.getKey(), entry.getValue(), fieldsNode); + } else { + getDefaultConstraintRelInformation(constraintsInfo, entry.getKey(), entry.getValue()); + } } } } @@ -582,6 +599,78 @@ private void getDefaultConstraintRelInformation(StringBuilder constraintsInfo, S constraintsInfo.append(LINE_DELIM); } + private void getIcebergDefaultConstraintRelInformation(StringBuilder constraintsInfo, String constraintName, + List columns, JsonNode fieldsNode) { + formatOutput("Constraint Name:", constraintName, constraintsInfo); + if (CollectionUtils.isNotEmpty(columns)) { + for (DefaultConstraintCol column : columns) { + String[] fields = new String[3]; + fields[0] = "Column Name:" + column.colName; + fields[1] = "Initial Default Value:" + getColumnDefaults(fieldsNode, column.colName, "initial-default"); + fields[2] = "Write Default Value:" + getColumnDefaults(fieldsNode, column.colName, "write-default"); + formatOutput(fields, constraintsInfo); + } + } + constraintsInfo.append(LINE_DELIM); + } + + private String getColumnDefaults(JsonNode node, String colName, String defaultType) { + if (node == null || colName == null) { + return StringUtils.EMPTY; + } + + JsonNode targetNode = node; + if (node.isArray()) { + targetNode = null; + for (JsonNode field : node) { + if (field.has("name") && colName.equalsIgnoreCase(field.get("name").asText())) { + targetNode = field; + break; + } + } + } + + if (targetNode == null) { + return StringUtils.EMPTY; + } else if (targetNode.has(defaultType)) { + JsonNode defaultNode = targetNode.get(defaultType); + if (defaultNode.isTextual()) { + return quoteString(defaultNode.asText()); + } + return defaultNode.asText(); + } + + // In case of struct, extract defaults recursively from its nested fields + JsonNode typeNode = targetNode.get("type"); + if (typeNode != null && typeNode.isObject() && typeNode.has("type") && + "struct".equalsIgnoreCase(typeNode.get("type").asText())) { + JsonNode structFields = typeNode.get("fields"); + if (structFields != null && structFields.isArray()) { + List fieldDefaults = new ArrayList<>(); + boolean hasDefaults = false; + for (JsonNode childField : structFields) { + String childName = childField.has("name") ? childField.get("name").asText() : ""; + String childDefault = getColumnDefaults(childField, childName, defaultType); + if (childDefault != null && !childDefault.isEmpty()) { + hasDefaults = true; + fieldDefaults.add(childName + ":" + childDefault); + } else { + fieldDefaults.add(childName + ":"); + } + } + + if (hasDefaults) { + return quoteString(String.join(",", fieldDefaults)); + } + } + } + return StringUtils.EMPTY; + } + + private static String quoteString(String input) { + return "'" + input + "'"; + } + private void getCheckConstraintsInformation(StringBuilder constraintsInfo, CheckConstraint constraint) { formatOutput("Table:", constraint.getDatabaseName() + "." + constraint.getTableName(), constraintsInfo); Map> checkConstraints = constraint.getCheckConstraints(); From d8e05a362c80f02af34d56d8259ec844d35b7ea3 Mon Sep 17 00:00:00 2001 From: tanishq-chugh Date: Mon, 6 Apr 2026 22:38:35 +0530 Subject: [PATCH 6/6] Remove unused import --- .../ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java | 1 - 1 file changed, 1 deletion(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java index 9f4476abd4f4..fe0ce74406e0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/info/desc/formatter/TextDescTableFormatter.java @@ -18,7 +18,6 @@ package org.apache.hadoop.hive.ql.ddl.table.info.desc.formatter; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.collections4.CollectionUtils;