diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FunctionMetadataDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FunctionMetadataDatasource.java new file mode 100644 index 00000000000..f9b68515e3d --- /dev/null +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FunctionMetadataDatasource.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.app.function; + +import java.util.Objects; + +import org.apache.asterix.metadata.api.IDatasourceFunction; +import org.apache.asterix.metadata.declared.DataSourceId; +import org.apache.asterix.metadata.declared.FunctionDataSource; +import org.apache.asterix.metadata.declared.MetadataProvider; +import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain; + +public class FunctionMetadataDatasource extends FunctionDataSource { + + private static final DataSourceId FUNCTION_METADATA_DATASOURCE_ID = + createDataSourceId(FunctionMetadataRewriter.FUNCTION_METADATA); + + public FunctionMetadataDatasource(INodeDomain domain) throws AlgebricksException { + super(FUNCTION_METADATA_DATASOURCE_ID, FunctionMetadataRewriter.FUNCTION_METADATA, domain); + } + + @Override + protected IDatasourceFunction createFunction(MetadataProvider metadataProvider, + AlgebricksAbsolutePartitionConstraint locations) { + // The builtin function registry is identical on every node, so run on a single + // location to avoid emitting duplicate rows. + return new FunctionMetadataFunction( + AlgebricksAbsolutePartitionConstraint.randomLocation(locations.getLocations())); + } + + @Override + protected boolean sameFunctionDatasource(FunctionDataSource other) { + return Objects.equals(this.functionId, other.getFunctionId()); + } +} diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FunctionMetadataFunction.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FunctionMetadataFunction.java new file mode 100644 index 00000000000..d8c9ba22217 --- /dev/null +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FunctionMetadataFunction.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.app.function; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.asterix.external.api.IRecordReader; +import org.apache.asterix.metadata.declared.AbstractDatasourceFunction; +import org.apache.asterix.om.functions.BuiltinFunctionInfo; +import org.apache.asterix.om.functions.BuiltinFunctions; +import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; +import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; +import org.apache.hyracks.api.context.IHyracksTaskContext; +import org.apache.hyracks.api.exceptions.HyracksDataException; + +public class FunctionMetadataFunction extends AbstractDatasourceFunction { + + private static final long serialVersionUID = 1L; + + public FunctionMetadataFunction(AlgebricksAbsolutePartitionConstraint locations) { + super(locations); + } + + @Override + public IRecordReader createRecordReader(IHyracksTaskContext ctx, int partition) + throws HyracksDataException { + List records = new ArrayList<>(); + for (FunctionIdentifier fi : BuiltinFunctions.getBuiltinFunctionIdentifiers()) { + records.add(toRecord(fi)); + } + return new FunctionMetadataReader(records.toArray(new String[0])); + } + + private static String toRecord(FunctionIdentifier fi) { + BuiltinFunctionInfo info = BuiltinFunctions.getBuiltinFunctionInfo(fi); + boolean isPrivate = info != null && info.isPrivate(); + return "{\"name\":\"" + escape(fi.getName()) + "\",\"arity\":" + fi.getArity() + ",\"category\":\"" + + category(fi) + "\",\"private\":" + isPrivate + "}"; + } + + private static String category(FunctionIdentifier fi) { + if (BuiltinFunctions.isWindowFunction(fi) || BuiltinFunctions.getWindowFunction(fi) != null) { + // isWindowFunction matches the internal *-impl ids; getWindowFunction matches the + // user-facing names (row_number, rank, ...) that map to those impls. + return "window"; + } + if (BuiltinFunctions.isBuiltinAggregateFunction(fi)) { + return "aggregate"; + } + if (BuiltinFunctions.isBuiltinScalarAggregateFunction(fi)) { + return "aggregate-scalar"; + } + if (BuiltinFunctions.isBuiltinUnnestingFunction(fi)) { + return "unnest"; + } + if (BuiltinFunctions.getDatasourceTransformer(fi) != null) { + return "datasource"; + } + return "scalar"; + } + + private static String escape(String s) { + StringBuilder sb = new StringBuilder(s.length()); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c == '"' || c == '\\') { + sb.append('\\').append(c); + } else if (c < 0x20) { + sb.append(String.format("\\u%04x", (int) c)); + } else { + sb.append(c); + } + } + return sb.toString(); + } +} diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FunctionMetadataReader.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FunctionMetadataReader.java new file mode 100644 index 00000000000..27f6531c5dd --- /dev/null +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FunctionMetadataReader.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.app.function; + +import java.io.IOException; + +import org.apache.asterix.external.api.IRawRecord; +import org.apache.asterix.external.input.record.CharArrayRecord; + +public class FunctionMetadataReader extends FunctionReader { + + private final String[] functions; + private int pos = 0; + + public FunctionMetadataReader(String[] functions) { + this.functions = functions; + } + + @Override + public boolean hasNext() throws Exception { + return pos < functions.length; + } + + @Override + public IRawRecord next() throws IOException, InterruptedException { + CharArrayRecord record = new CharArrayRecord(); + record.set(functions[pos++]); + record.endRecord(); + return record; + } +} diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FunctionMetadataRewriter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FunctionMetadataRewriter.java new file mode 100644 index 00000000000..ae6027f7225 --- /dev/null +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/FunctionMetadataRewriter.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.app.function; + +import org.apache.asterix.common.functions.FunctionConstants; +import org.apache.asterix.metadata.declared.FunctionDataSource; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext; +import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; +import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; + +public class FunctionMetadataRewriter extends FunctionRewriter { + + public static final FunctionIdentifier FUNCTION_METADATA = FunctionConstants.newAsterix("function-metadata", 0); + public static final FunctionMetadataRewriter INSTANCE = new FunctionMetadataRewriter(FUNCTION_METADATA); + + private FunctionMetadataRewriter(FunctionIdentifier functionId) { + super(functionId); + } + + @Override + protected FunctionDataSource toDatasource(IOptimizationContext context, AbstractFunctionCallExpression f) + throws AlgebricksException { + return new FunctionMetadataDatasource(context.getComputationNodeDomain()); + } +} diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java index e86eb8a0adb..9866ee0561a 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/util/MetadataBuiltinFunctions.java @@ -24,6 +24,7 @@ import org.apache.asterix.app.function.DatasetRewriter; import org.apache.asterix.app.function.DumpIndexRewriter; import org.apache.asterix.app.function.FeedRewriter; +import org.apache.asterix.app.function.FunctionMetadataRewriter; import org.apache.asterix.app.function.JobSummariesRewriter; import org.apache.asterix.app.function.PingRewriter; import org.apache.asterix.app.function.QueryIndexRewriter; @@ -119,6 +120,12 @@ public class MetadataBuiltinFunctions { BuiltinFunctions.addUnnestFun(CollectionEstimateColumnCountRewriter.ESTIMATED_COLLECTION_COLUMN_COUNT, true); BuiltinFunctions.addDatasourceFunction(CollectionEstimateColumnCountRewriter.ESTIMATED_COLLECTION_COLUMN_COUNT, CollectionEstimateColumnCountRewriter.INSTANCE); + // Function metadata function + BuiltinFunctions.addFunction(FunctionMetadataRewriter.FUNCTION_METADATA, + (expression, env, mp) -> RecordUtil.FULLY_OPEN_RECORD_TYPE, true); + BuiltinFunctions.addUnnestFun(FunctionMetadataRewriter.FUNCTION_METADATA, true); + BuiltinFunctions.addDatasourceFunction(FunctionMetadataRewriter.FUNCTION_METADATA, + FunctionMetadataRewriter.INSTANCE, BuiltinFunctions.DataSourceFunctionProperty.MIN_MEMORY_BUDGET); } private MetadataBuiltinFunctions() { diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/function_metadata/function_metadata.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/function_metadata/function_metadata.1.query.sqlpp new file mode 100644 index 00000000000..59c2cc48538 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/function_metadata/function_metadata.1.query.sqlpp @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +select value f +from function_metadata() f +where f.name in ["rank", "row_number"] +order by f.name; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/function_metadata/function_metadata.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/function_metadata/function_metadata.1.adm new file mode 100644 index 00000000000..8f5580025d2 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/function_metadata/function_metadata.1.adm @@ -0,0 +1,2 @@ +{ "name": "rank", "arity": 0, "category": "window", "private": false } +{ "name": "row_number", "arity": 0, "category": "window", "private": false } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml index b4672e17cdb..9dd96d13048 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml @@ -7548,6 +7548,11 @@ jobs + + + function_metadata + + completed_requests diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java index 63543dd551a..e6fb45a570d 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java @@ -3079,6 +3079,10 @@ public static BuiltinFunctionInfo getBuiltinFunctionInfo(FunctionIdentifier fi) return registeredFunctions.get(fi); } + public static Set getBuiltinFunctionIdentifiers() { + return Collections.unmodifiableSet(registeredFunctions.keySet()); + } + public static BuiltinFunctionInfo resolveBuiltinFunction(String name, int arity) { //TODO:optimize BuiltinFunctionInfo finfo;