HCK-16464: Tables and view REed from the instance with limited number of columns and weird dummy text

reverse_engineering/api.js

@@ -364,10 +364,11 @@ module.exports = {
 						let viewSample = [];
 
 						try {
-							viewSchema = await fetchRequestHelper.fetchEntitySchema({
+							viewSchema = await fetchRequestHelper.fetchViewSchema({
 								connectionInfo: connectionData,
 								dbName,
 								entityName: name,
+								catalogName: dbData.dbProperties.catalogName,
 								logger,
 							});
 							viewSample = await fetchRequestHelper.fetchSample({

reverse_engineering/grammars/SqlBase.g4

@@ -1030,7 +1030,7 @@ columnConstraint
 columnConstraintType
     : NOT NULL
     | PRIMARY KEY
-    | KW_DEFAULT defaultValue
+    | DEFAULT defaultValue
     | columnGeneratedAs
     ;
 
@@ -1049,7 +1049,7 @@ generatedAsExpression
     ;
 
 generatedAsIdentity
-    : (KW_ALWAYS | BY KW_DEFAULT) AS KW_IDENTITY ('(' identityOptions ')')?
+    : (KW_ALWAYS | BY DEFAULT) AS KW_IDENTITY ('(' identityOptions ')')?
     ;
 
 identityOptions
@@ -1524,7 +1524,6 @@ nonReserved
     | ITEMS
     | KEY
     | KEYS
-    | KW_DEFAULT
     | LANGUAGE
 	| LAST
     | LATERAL
@@ -1976,7 +1975,6 @@ ACTION : A C T I O N;
 
 KW_GENERATED: G E N E R A T E D;
 KW_ALWAYS: A L W A Y S;
-KW_DEFAULT: D E F A U L T;
 KW_IDENTITY: I D E N T I T Y;
 KW_INCREMENT: I N C R E M E N T;
 KW_MASK: M A S K;

reverse_engineering/helpers/fetchRequestHelper.js

@@ -269,6 +269,74 @@ const fetchEntitySchema = async ({ connectionInfo, dbName, entityName, logger })
 	}
 };
 
+const getDescribeColumnRowsFromDesc = viewSchema => {
+	if (!Array.isArray(viewSchema)) {
+		return [];
+	}
+
+	const columnRows = [];
+
+	for (const row of viewSchema) {
+		const columnName = row?.[0];
+		if (typeof columnName !== 'string') {
+			continue;
+		}
+
+		const trimmedName = columnName.trim();
+		if (!trimmedName || trimmedName.startsWith('#')) {
+			break;
+		}
+
+		columnRows.push(row);
+	}
+
+	return columnRows;
+};
+
+const mapSparkSchemaColumnsToDescribeRows = columns => {
+	if (!Array.isArray(columns)) {
+		return [];
+	}
+
+	return columns.map(({ name, colType }) => [name, colType]);
+};
+
+const fetchViewSchema = async ({ connectionInfo, dbName, entityName, catalogName, logger }) => {
+	const fullName = buildSparkTableFullNameForPython({
+		schemaName: prepareName(dbName),
+		tableName: prepareName(entityName),
+		catalogName: prepareName(catalogName || connectionInfo.catalogName),
+	});
+
+	try {
+		const command = getTableSchemaColumnsForDdlFallback({ fullName });
+		const raw = await executeCommand({
+			connectionInfo,
+			command,
+			language: SPARK_LANGUAGE.python,
+			logger,
+		});
+		const columns = JSON.parse(coercePythonNotebookOutput(raw));
+
+		logger.log(
+			'info',
+			{ message: `View schema from spark.table(${fullName})`, dbName, entityName },
+			'Getting view schema',
+		);
+
+		return mapSparkSchemaColumnsToDescribeRows(columns);
+	} catch (pythonError) {
+		logger.log(
+			'info',
+			{ message: pythonError.message, dbName, entityName },
+			'fetchViewSchema Python failed; falling back to DESC',
+		);
+
+		const descRows = await fetchEntitySchema({ connectionInfo, dbName, entityName, logger });
+		return getDescribeColumnRowsFromDesc(descRows);
+	}
+};
+
 const fetchSample = async ({ connectionInfo, dbName, entityName, logger }) => {
 	try {
 		const sqlQuery = `SELECT * FROM \`${dbName}\`.\`${entityName}\` LIMIT 1`;
@@ -1103,6 +1171,7 @@ module.exports = {
 	fetchClusterTablesNames,
 	fetchDatabaseViewsNamesViaPython,
 	fetchEntitySchema,
+	fetchViewSchema,
 	useCatalog,
 	fetchSample,
 	fetchTagsForUnityCatalogs,

reverse_engineering/parser/SQLBase/SqlBase.tokens

@@ -304,45 +304,44 @@ MATCH=303
 ACTION=304
 KW_GENERATED=305
 KW_ALWAYS=306
-KW_DEFAULT=307
-KW_IDENTITY=308
-KW_INCREMENT=309
-KW_MASK=310
-EQ=311
-NSEQ=312
-NEQ=313
-NEQJ=314
-LT=315
-LTE=316
-GT=317
-GTE=318
-FAT_ARROW=319
-PLUS=320
-MINUS=321
-ASTERISK=322
-SLASH=323
-PERCENT=324
-TILDE=325
-AMPERSAND=326
-PIPE=327
-CONCAT_PIPE=328
-HAT=329
-STRING=330
-BIGINT_LITERAL=331
-SMALLINT_LITERAL=332
-TINYINT_LITERAL=333
-INTEGER_VALUE=334
-EXPONENT_VALUE=335
-DECIMAL_VALUE=336
-FLOAT_LITERAL=337
-DOUBLE_LITERAL=338
-BIGDECIMAL_LITERAL=339
-IDENTIFIER=340
-BACKQUOTED_IDENTIFIER=341
-SIMPLE_COMMENT=342
-BRACKETED_COMMENT=343
-WS=344
-UNRECOGNIZED=345
+KW_IDENTITY=307
+KW_INCREMENT=308
+KW_MASK=309
+EQ=310
+NSEQ=311
+NEQ=312
+NEQJ=313
+LT=314
+LTE=315
+GT=316
+GTE=317
+FAT_ARROW=318
+PLUS=319
+MINUS=320
+ASTERISK=321
+SLASH=322
+PERCENT=323
+TILDE=324
+AMPERSAND=325
+PIPE=326
+CONCAT_PIPE=327
+HAT=328
+STRING=329
+BIGINT_LITERAL=330
+SMALLINT_LITERAL=331
+TINYINT_LITERAL=332
+INTEGER_VALUE=333
+EXPONENT_VALUE=334
+DECIMAL_VALUE=335
+FLOAT_LITERAL=336
+DOUBLE_LITERAL=337
+BIGDECIMAL_LITERAL=338
+IDENTIFIER=339
+BACKQUOTED_IDENTIFIER=340
+SIMPLE_COMMENT=341
+BRACKETED_COMMENT=342
+WS=343
+UNRECOGNIZED=344
 ';'=1
 '('=2
 ','=3
@@ -354,19 +353,19 @@ UNRECOGNIZED=345
 '['=9
 ']'=10
 ':'=11
-'<=>'=312
-'<>'=313
-'!='=314
-'<'=315
-'>'=317
-'=>'=319
-'+'=320
-'-'=321
-'*'=322
-'/'=323
-'%'=324
-'~'=325
-'&'=326
-'|'=327
-'||'=328
-'^'=329
+'<=>'=311
+'<>'=312
+'!='=313
+'<'=314
+'>'=316
+'=>'=318
+'+'=319
+'-'=320
+'*'=321
+'/'=322
+'%'=323
+'~'=324
+'&'=325
+'|'=326
+'||'=327
+'^'=328