Implement function signature table

plugins/elasticsearch/pom.xml

@@ -104,6 +104,10 @@
           <groupId>log4j</groupId>
           <artifactId>*</artifactId>
         </exclusion>
+        <exclusion>
+          <groupId>org.yaml</groupId>
+          <artifactId>snakeyaml</artifactId>
+        </exclusion>
       </exclusions>
     </dependency>
     <dependency>

sabot/kernel/src/main/java/com/dremio/exec/expr/fn/FunctionImplementationRegistry.java

@@ -17,8 +17,11 @@
 
 import java.util.List;
 import java.util.Set;
+import java.util.*;
 import java.util.concurrent.TimeUnit;
 
+import com.dremio.exec.store.sys.functions.FunctionParameterInfo;
+import com.dremio.exec.store.sys.functions.SysTableFunctionsInfo;
 import org.apache.arrow.vector.types.pojo.ArrowType;
 
 import com.dremio.common.config.SabotConfig;
@@ -39,6 +42,10 @@
 import com.google.inject.Guice;
 import com.google.inject.Injector;
 import com.google.inject.ProvisionException;
+import org.apache.calcite.sql.*;
+import org.apache.calcite.sql.type.SqlOperandTypeInference;
+import org.apache.calcite.sql.type.SqlReturnTypeInference;
+import org.apache.commons.lang3.StringUtils;
 
 /**
  * This class offers the registry for functions. Notably, in addition to Dremio its functions
@@ -55,15 +62,16 @@ public class FunctionImplementationRegistry implements FunctionLookupContext {
   private static final Set<String> aggrFunctionNames = Sets.newHashSet("sum", "$sum0", "min",
     "max", "hll");
   protected boolean isDecimalV2Enabled;
+  private SabotConfig sabotConfig;
 
-  public FunctionImplementationRegistry(SabotConfig config, ScanResult classpathScan){
+  public FunctionImplementationRegistry(SabotConfig config, ScanResult classpathScan) {
     Stopwatch w = Stopwatch.createStarted();
-
+    this.sabotConfig = config;
     logger.debug("Generating function registry.");
     functionRegistry = new FunctionRegistry(classpathScan);
     initializePrimaryRegistries();
     Set<Class<? extends PluggableFunctionRegistry>> registryClasses =
-        classpathScan.getImplementations(PluggableFunctionRegistry.class);
+      classpathScan.getImplementations(PluggableFunctionRegistry.class);
 
     // Create a small Guice module
     final Injector injector = Guice.createInjector(new AbstractModule() {
@@ -110,6 +118,7 @@ public ArrayListMultimap<String, AbstractFunctionHolder> getRegisteredFunctions(
 
   /**
    * Register functions in given operator table.
+   *
    * @param operatorTable
    */
   public void register(OperatorTable operatorTable) {
@@ -118,7 +127,7 @@ public void register(OperatorTable operatorTable) {
       registry.register(operatorTable, isDecimalV2Enabled);
     }
 
-    for(PluggableFunctionRegistry registry : pluggableFuncRegistries) {
+    for (PluggableFunctionRegistry registry : pluggableFuncRegistries) {
       registry.register(operatorTable, isDecimalV2Enabled);
     }
   }
@@ -179,7 +188,7 @@ private FunctionCall getDecimalV2NamesIfEnabled(FunctionCall functionCall) {
   /**
    * Find function implementation for given <code>functionCall</code> in non-Dremio function registries such as Hive UDF
    * registry.
-   *
+   * <p>
    * Note: Order of searching is same as order of {@link com.dremio.exec.expr.fn.PluggableFunctionRegistry}
    * implementations found on classpath.
    *
@@ -188,7 +197,7 @@ private FunctionCall getDecimalV2NamesIfEnabled(FunctionCall functionCall) {
    */
   @Override
   public AbstractFunctionHolder findNonFunction(FunctionCall functionCall) {
-    for(PluggableFunctionRegistry registry : pluggableFuncRegistries) {
+    for (PluggableFunctionRegistry registry : pluggableFuncRegistries) {
       AbstractFunctionHolder h = registry.getFunction(functionCall);
       if (h != null) {
         return h;
@@ -207,7 +216,7 @@ public OptionManager getOptionManager() {
   public boolean isFunctionComplexOutput(String name) {
     List<AbstractFunctionHolder> methods = functionRegistry.getMethods(name);
     for (AbstractFunctionHolder holder : methods) {
-      if (((BaseFunctionHolder)holder).getReturnValue().isComplexWriter()) {
+      if (((BaseFunctionHolder) holder).getReturnValue().isComplexWriter()) {
         return true;
       }
     }
@@ -222,4 +231,114 @@ public FunctionRegistry getFunctionRegistry() {
   public boolean isDecimalV2Enabled() {
     return isDecimalV2Enabled;
   }
+
+  public Map<String, Map<String, Object>> generateMapWithRegisteredFunctions() {
+    // Retrieve the registered functions on the function registry
+    ArrayListMultimap<String, AbstractFunctionHolder> functions = this.getRegisteredFunctions();
+
+    Map<String, Map<String, Object>> functionsToSave = new HashMap<>();
+    Map<String, Object> stringObjectMap;
+    // Iterate over each registered function to extract the available information
+    for (Map.Entry<String, AbstractFunctionHolder> holders : functions.entries()) {
+      String functionName = holders.getKey().toLowerCase();
+      BaseFunctionHolder value = (BaseFunctionHolder) holders.getValue();
+
+      stringObjectMap = functionsToSave.get(functionName);
+      if (stringObjectMap == null) {
+        stringObjectMap = new HashMap<>();
+        stringObjectMap.put("functionName", functionName.toUpperCase());
+        stringObjectMap.put("dremioVersions", null);
+        stringObjectMap.put("description", null);
+        stringObjectMap.put("extendedDescription", null);
+        stringObjectMap.put("useCaseExamples", null);
+      }
+      List<Map<String, Object>> signaturesList = (List<Map<String, Object>>) stringObjectMap.get("signatures");
+      if (signaturesList == null) {
+        signaturesList = new ArrayList<>();
+      }
+
+      // Define signatures values
+      Map<String, Object> signaturesValues = new HashMap<>();
+      List<Map<String, Object>> parametersList = new ArrayList<>();
+      int count = 1;
+      for (BaseFunctionHolder.ValueReference parameter : value.getParameters()) {
+        int finalCount = count;
+        parametersList.add(
+          new HashMap<String, Object>() {
+            {
+              put("ordinalPosition", finalCount);
+              put("parameterName", parameter.getName());
+              put("parameterType", parameter.getType().toString());
+              try {
+                put("parameterFormat", (parameter.getType().toMinorType()));
+              } catch (Exception ignored) {
+              }
+            }
+          }
+        );
+        count++;
+      }
+      signaturesValues.put("parameterList", parametersList);
+      signaturesValues.put("returnType", value.getReturnValue().getType().toString());
+      signaturesList.add(signaturesValues);
+      stringObjectMap.put("signatures", signaturesList);
+
+      functionsToSave.put(functionName, stringObjectMap);
+    }
+    return functionsToSave;
+  }
+
+  public List<SysTableFunctionsInfo> generateListWithCalciteFunctions() {
+    OperatorTable operatorTable = new OperatorTable(this);
+    List<SysTableFunctionsInfo> functionsInfoList = new ArrayList<>();
+
+    for (SqlOperator operator : operatorTable.getOperatorList()) {
+      String opName = operator.getName();
+      // Only SQL's functions are uppercased
+      opName = opName.toLowerCase(Locale.ROOT);
+      logger.info("opName: {}", opName);
+      //  SqlReturnTypeInference returnType = operator.getReturnTypeInference().inferReturnType();
+      String returnType = "";
+      List<FunctionParameterInfo> parameterInfoList = new ArrayList<>();
+      SqlSyntax syntax = operator.getSyntax();
+      if (syntax.toString().equals("FUNCTION")) {
+        String signaturesString = "";
+        try {
+          // Get the function's signature, it will come like FUNCTION_NAME(<PARAM1_TYPE> <PARAM2_TYPE>)
+          signaturesString = operator.getAllowedSignatures();
+        } catch (Exception e) {
+          logger.warn("Failed to read Calcite {} function's allowed signatures, with exception {}. ", opName, e);
+        }
+        // Get a list of allowed signatures that come in a single string
+        String[] signatures = signaturesString.split("\\r?\\n");
+        for (String signature : signatures) {
+          // Get only the param's type for each signature
+          String[] ps = StringUtils.substringsBetween(signature, "<", ">");
+          if (ps != null) {
+            List<String> params = Arrays.asList(ps);
+            for (int i = 0; i < params.size() ; i++) {
+              // Based on the doc, operators have similar, straightforward strategies,
+              // such as to take the type of the first operand to be its return type
+              if  (i == 0) {
+                returnType = params.get(i).toLowerCase(Locale.ROOT);
+              }
+              parameterInfoList.add(new FunctionParameterInfo("param-" + i,
+                params.get(i).toLowerCase(Locale.ROOT),
+                false));
+            }
+          }
+          if (returnType.isEmpty()) {
+            returnType = "inferred at runtime";
+          }
+          SysTableFunctionsInfo toAdd = new SysTableFunctionsInfo(opName, returnType, parameterInfoList.toString());
+
+          // Avoid possible duplicates
+          if (!functionsInfoList.contains(toAdd)) {
+            functionsInfoList.add(toAdd);
+          }
+        }
+      }
+    }
+    return functionsInfoList;
+  }
 }

sabot/kernel/src/main/java/com/dremio/exec/store/pojo/PojoDataType.java

@@ -18,6 +18,7 @@
 import java.lang.reflect.Field;
 import java.lang.reflect.Modifier;
 import java.sql.Timestamp;
+import java.util.Collection;
 import java.util.List;
 
 import org.apache.calcite.sql.type.SqlTypeName;
@@ -61,6 +62,8 @@ public PojoDataType(Class<?> pojoClass) {
         types.add(SqlTypeName.VARCHAR);
       } else if (type == Timestamp.class || type == DateTime.class) {
         types.add(SqlTypeName.TIMESTAMP);
+      } else if (type == List.class || type == Collection.class) {
+        types.add(SqlTypeName.ARRAY);
       } else {
         throw new RuntimeException(String.format("PojoDataType doesn't yet support conversions from type [%s].", type));
       }

sabot/kernel/src/main/java/com/dremio/exec/store/sys/SystemTable.java

@@ -18,6 +18,8 @@
 import java.util.Iterator;
 import java.util.stream.StreamSupport;
 
+import com.dremio.exec.store.sys.functions.FunctionsInfoIterator;
+import com.dremio.exec.store.sys.functions.SysTableFunctionsInfo;
 import org.apache.calcite.rel.type.RelDataType;
 
 import com.dremio.connector.metadata.DatasetHandle;
@@ -260,6 +262,14 @@ public Iterator<?> getIterator(final SabotContext sabotContext, final OperatorCo
     public Iterator<?> getIterator(final SabotContext sContext, final OperatorContext context) {
       return sContext.getStatisticsListManagerProvider().get().getStatisticsInfos().iterator();
     }
+  },
+
+  FUNCTIONS(false, SysTableFunctionsInfo.class, "functions") {
+    @Override
+    public Iterator<?> getIterator(final SabotContext sabotContext, final OperatorContext context) {
+      FunctionsInfoIterator iterator = new FunctionsInfoIterator(sabotContext);
+      return iterator;
+    }
   };
 
   private static final long RECORD_COUNT = 100L;

sabot/kernel/src/main/java/com/dremio/exec/store/sys/functions/FunctionParameterInfo.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright (C) 2017-2019 Dremio Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.dremio.exec.store.sys.functions;
+
+import java.util.Objects;
+
+/**
+ * Struct for functions parameters.
+ */
+public class FunctionParameterInfo {
+  public final String name;
+  public final String data_type;
+  public final Boolean is_optional;
+
+  public FunctionParameterInfo(String name, String data_type, Boolean is_optional) {
+    this.name = name;
+    this.data_type = data_type;
+    this.is_optional = is_optional;
+  }
+
+  public String getName() {
+    return name;
+  }
+
+  public String getData_type() {
+    return data_type;
+  }
+
+  public Boolean getIs_optional() {
+    return is_optional;
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(this.name, this.data_type, this.is_optional);
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) {
+      return true;
+    }
+    if (o == null || getClass() != o.getClass()) {
+      return false;
+    }
+
+    FunctionParameterInfo other = (FunctionParameterInfo) o;
+    return Objects.equals(this.name, other.getName()) &&
+      Objects.equals(this.data_type, other.getData_type()) &&
+      Objects.equals(this.is_optional, other.getIs_optional());
+  }
+
+  @Override
+  public String toString() {
+    return new StringBuilder()
+      .append("{")
+      .append("\"name\":\"" + name + "\",")
+      .append("\"data_type\":\"" + data_type + "\",")
+      .append("\"is_optional\":\"" + is_optional + "\",")
+      .append("}")
+      .toString();
+  }
+}