Adds serializability checks for UDFs

-- adds check to catch any issues with serializability at UDF
initialization time (i.e when the udf object is created)
-- moves internal UDFProcessing/FatalErrors to udf package
-- wraps error on loading udfs to propagate to a UDFFatalError
-- adds associated tests/test classes

DAFFODIL-2235

daffodil-cli/src/it/scala/org/apache/daffodil/CLI/Util.scala

@@ -148,6 +148,14 @@ object Util {
     }
   }
 
+  def makeMultipleCmds(cmds: Array[String]): String = {
+    if (isWindows) {
+      cmds.mkString(" & ")
+    } else {
+      cmds.mkString("; ")
+    }
+  }
+
   def md5sum(blob_path: String): String = {
     if (isWindows) {
       String.format("certutil -hashfile %s MD5", blob_path)

daffodil-cli/src/it/scala/org/apache/daffodil/udf/TestCLIUdfs.scala

@@ -66,7 +66,7 @@ class TestCLIUdfs {
       shell.expect(
         allOf(
           contains("<fn_func>"),
-          contains("data>strng</data>"),
+          contains("<data>strng</data>"),
           contains("<value>Hello,strng</value>"),
           contains("</fn_func>")))
 
@@ -406,7 +406,7 @@ class TestCLIUdfs {
         1,
         allOf(
           contains("[error] User Defined Function 'ssudf:reverse' Error: UDF Error!"),
-          contains("org.apache.daffodil.dpath.UserDefinedFunctionFatalErrorException: "),
+          contains("org.apache.daffodil.udf.UserDefinedFunctionFatalErrorException: "),
           contains("at org.sbadudfs.udfexceptions.evaluating.StringFunctions.Reverse.evaluate")))
 
       shell.send("exit\n")
@@ -471,7 +471,9 @@ class TestCLIUdfs {
           contains("[error] Error initializing User Defined Function:"),
           contains("http://example.com/scala/udf:rev-words."),
           contains("Error thrown: org.sbadudfs.udfexceptions2.StringFunctions.ReverseWords$CustomException: UDF Error!"),
-          contains("[error] Schema Definition Error: Unsupported function: ssudf:rev-words")))
+          contains("[error] User Defined Function 'http://example.com/scala/udf:rev-words' Error: UDF Error!"),
+          contains("org.apache.daffodil.udf.UserDefinedFunctionFatalErrorException:"),
+          contains("at org.sbadudfs.udfexceptions2.StringFunctions.ReverseWords")))
 
       shell.send("exit\n")
       shell.expect(eof)
@@ -588,4 +590,113 @@ class TestCLIUdfs {
       shell.close()
     }
   }
+
+  /**
+   * Tests the case when a provider class:
+   *   incorrectly implements createUserDefinedFunction that results in an exception
+   */
+  @Test def test_UDFPClass_incorrectUDFCreateImplementation() {
+    val schemaFile = Util.daffodilPath("daffodil-udf/src/test/resources/org/apache/daffodil/udf/genericUdfSchema.xsd")
+    val (testSchemaFile) = if (Util.isWindows) (Util.cmdConvert(schemaFile)) else (schemaFile)
+
+    val metaInfForSomeUdfA = "daffodil-udf/src/test/scala/org/sbadudfs/functionclasses/StringFunctions/"
+
+    val dafClassPath =
+      (testUdfsPaths :+ Util.daffodilPath(metaInfForSomeUdfA))
+        .mkString(java.io.File.pathSeparator)
+
+    val shell = Util.startIncludeErrors("", envp = Map("DAFFODIL_CLASSPATH" -> dafClassPath))
+
+    try {
+      val cmd = String.format(Util.echoN("strng") + "| %s -v parse -s %s -r user_func2", Util.binPath, testSchemaFile)
+      shell.sendLine(cmd)
+      shell.expectIn(
+        1,
+        allOf(
+          contains("[error] Error initializing User Defined Function:"),
+          contains("http://example.com/scala/udf:reverse."),
+          contains("Error thrown: scala.MatchError:"),
+          contains("org.apache.daffodil.udf.UserDefinedFunctionFatalErrorException:"),
+          contains("at org.sbadudfs.functionclasses.StringFunctions.StringFunctionsProvider.createUserDefinedFunction")))
+
+      shell.send("exit\n")
+      shell.expect(eof)
+      shell.close()
+    } finally {
+      shell.close()
+    }
+  }
+
+  /**
+   * Tests the case when a UDF class:
+   *    contains a non serializable member
+   */
+  @Test def test_UDFClass_serializability() {
+    val schemaFile = Util.daffodilPath("daffodil-udf/src/test/resources/org/apache/daffodil/udf/genericUdfSchema.xsd")
+    val (testSchemaFile) = if (Util.isWindows) (Util.cmdConvert(schemaFile)) else (schemaFile)
+
+    val metaInfForSomeUdfA = "daffodil-udf/src/test/scala/org/sbadudfs/functionclasses2/StringFunctions/"
+
+    val dafClassPath =
+      (testUdfsPaths :+ Util.daffodilPath(metaInfForSomeUdfA))
+        .mkString(java.io.File.pathSeparator)
+
+    val shell = Util.startIncludeErrors("", envp = Map("DAFFODIL_CLASSPATH" -> dafClassPath))
+
+    try {
+      val cmd = String.format("%s -v save-parser -s %s -r user_func4", Util.binPath, testSchemaFile)
+      shell.sendLine(cmd)
+      shell.expectIn(
+        1,
+        allOf(
+          contains("[error] Error serializing initialized User Defined Function: org.sbadudfs.functionclasses2.StringFunctions.GetNonSerializableState"),
+          contains("Could not serialize member of class: org.sbadudfs.functionclasses2.StringFunctions.SomeNonSerializableClass"),
+          contains("[error] Schema Definition Error: Unsupported function: ssudf:get-nonserializable-state")))
+      shell.send("exit\n")
+      shell.expect(eof)
+      shell.close()
+    } finally {
+      shell.close()
+    }
+  }
+
+  /**
+   * Tests the case when a UDF class:
+   *    contains serializable member
+   */
+  @Test def test_UDFClass_serializability2() {
+    val schemaFile = Util.daffodilPath("daffodil-udf/src/test/resources/org/apache/daffodil/udf/genericUdfSchema.xsd")
+    val (testSchemaFile) = if (Util.isWindows) (Util.cmdConvert(schemaFile)) else (schemaFile)
+
+    val savedParserFile = java.io.File.createTempFile("testParser_", ".tmp")
+    savedParserFile.deleteOnExit
+    val metaInfForSomeUdfA = "daffodil-udf/src/test/scala/org/sbadudfs/functionclasses2/StringFunctions/"
+
+    val dafClassPath =
+      (testUdfsPaths :+ Util.daffodilPath(metaInfForSomeUdfA))
+        .mkString(java.io.File.pathSeparator)
+
+    val shell = Util.startIncludeErrors("", envp = Map("DAFFODIL_CLASSPATH" -> dafClassPath))
+
+    try {
+      val cmds = Array(
+        String.format("%s -v save-parser -s %s -r user_func5 %s", Util.binPath, testSchemaFile, savedParserFile.getAbsolutePath),
+        String.format(Util.echoN("strng") + "| %s -v parse -P %s", Util.binPath, savedParserFile.getAbsolutePath))
+      val cmd = Util.makeMultipleCmds(cmds)
+      shell.sendLine(cmd)
+      shell.expectIn(
+        0,
+        allOf(
+          contains("<user_func5>"),
+          contains("<data>strng</data>"),
+          contains("<value>Serializable State</value>"),
+          contains("</user_func5>")))
+      shell.send("exit\n")
+      shell.expect(eof)
+      shell.close()
+    } finally {
+      shell.close()
+      savedParserFile.delete
+    }
+  }
 }

daffodil-cli/src/main/scala/org/apache/daffodil/Main.scala

@@ -100,7 +100,7 @@ import org.apache.daffodil.io.DataDumper
 import java.nio.ByteBuffer
 import org.apache.daffodil.io.FormatInfo
 import org.apache.daffodil.schema.annotation.props.gen.BitOrder
-import org.apache.daffodil.dpath.UserDefinedFunctionFatalErrorException
+import org.apache.daffodil.udf.UserDefinedFunctionFatalErrorException
 
 class NullOutputStream extends OutputStream {
   override def close() {}

daffodil-runtime1/src/main/scala/org/apache/daffodil/dpath/DPath.scala

@@ -34,6 +34,7 @@ import org.apache.daffodil.api.Diagnostic
 import org.apache.daffodil.util.Numbers._
 import org.apache.daffodil.processors.parsers.DoSDEMixin
 import org.apache.daffodil.processors.parsers.PState
+import org.apache.daffodil.udf.UserDefinedFunctionProcessingErrorException
 
 class ExpressionEvaluationException(e: Throwable, s: ParseOrUnparseState)
   extends ProcessingError(

daffodil-runtime1/src/main/scala/org/apache/daffodil/dpath/UserDefinedFunctionBase.scala

@@ -18,51 +18,14 @@
 package org.apache.daffodil.dpath
 
 import org.apache.daffodil.udf.UserDefinedFunction
+import org.apache.daffodil.udf.UserDefinedFunctionProcessingErrorException
+import org.apache.daffodil.udf.exceptions.UserDefinedFunctionProcessingError
+import org.apache.daffodil.udf.UserDefinedFunctionFatalErrorException
+import org.apache.daffodil.udf.exceptions.UserDefinedFunctionFatalException
 import org.apache.daffodil.udf.UserDefinedFunctionService.UserDefinedFunctionMethod
 import java.lang.reflect.Method
 import java.lang.reflect.InvocationTargetException
-import org.apache.daffodil.udf.exceptions.UserDefinedFunctionFatalException
-import org.apache.daffodil.udf.exceptions.UserDefinedFunctionProcessingError
-import org.apache.daffodil.processors.ProcessingError
 import org.apache.daffodil.util.Maybe
-import org.apache.daffodil.exceptions.SchemaFileLocation
-import org.apache.daffodil.api.DataLocation
-import org.apache.daffodil.exceptions.Abort
-
-/**
- * User Defined Function Exception class to wrap processing errors from the UDF
- * into Daffodil Processing Errors
- */
-case class UserDefinedFunctionProcessingErrorException(
-  errorInfo: String,
-  schemaContext: Maybe[SchemaFileLocation],
-  dataContext: Maybe[DataLocation],
-  errorCause: Maybe[Throwable],
-  errorStr: Maybe[String])
-  extends ProcessingError(errorInfo, schemaContext, dataContext, errorCause, errorStr)
-
-/**
- * User Defined Function Exception class to wrap fatal errors from the UDF
- * into Daffodil Aborts
- */
-case class UserDefinedFunctionFatalErrorException(description: String = "", cause: Throwable, classOfInterest: String = "")
-  extends Abort(description + ": " + cause.getMessage) {
-
-  /*
-   * This will replace the stacktrace of the fatal error with just the UDF relevant
-   * portions; removing the Daffodil (incl our evaluate invocation) and the Method
-   * reflection portions.
-   *
-   */
-  if (classOfInterest.nonEmpty) {
-    val curStackTrace = cause.getStackTrace
-    val indexLastUdfEntry = curStackTrace.lastIndexWhere(_.getClassName == classOfInterest)
-    val finalStackTrace =
-      if (indexLastUdfEntry >= 0) curStackTrace.slice(0, indexLastUdfEntry + 1)
-      else curStackTrace
-    this.setStackTrace(finalStackTrace)
-  }
-}
 
 /**
  * Both the serializable evaluate method and the User Defined Function instance are passed in,

daffodil-runtime1/src/main/scala/org/apache/daffodil/udf/UserDefinedFunctionErrors.scala

@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.daffodil.udf
+
+import org.apache.daffodil.exceptions.Abort
+import org.apache.daffodil.processors.ProcessingError
+import org.apache.daffodil.util.Maybe
+import org.apache.daffodil.exceptions.SchemaFileLocation
+import org.apache.daffodil.api.DataLocation
+import org.apache.daffodil.util.Misc
+
+/**
+ * User Defined Function Exception class to wrap processing errors from the UDF
+ * into Daffodil Processing Errors
+ */
+case class UserDefinedFunctionProcessingErrorException(
+  errorInfo: String,
+  schemaContext: Maybe[SchemaFileLocation],
+  dataContext: Maybe[DataLocation],
+  errorCause: Maybe[Throwable],
+  errorStr: Maybe[String])
+  extends ProcessingError(errorInfo, schemaContext, dataContext, errorCause, errorStr)
+
+/**
+ * User Defined Function Exception class to wrap fatal errors from the UDF
+ * into Daffodil Aborts
+ */
+case class UserDefinedFunctionFatalErrorException(description: String = "", cause: Throwable, udfOfInterest: String = "", providerOfInterest: String = "")
+  extends Abort(description + ": " + cause.getMessage) {
+
+  /*
+   * This will replace the stacktrace of the fatal error with just the UDF relevant
+   * portions; removing the Daffodil (incl our evaluate invocation) and the Method
+   * reflection portions.
+   *
+   */
+  val classesOfInterest = List(udfOfInterest, providerOfInterest).filterNot(Misc.isNullOrBlank)
+  if (classesOfInterest.nonEmpty) {
+    val curStackTrace = cause.getStackTrace
+    val indexLastUdfEntry = curStackTrace.lastIndexWhere(ste => classesOfInterest.exists(_ == ste.getClassName))
+    val finalStackTrace =
+      if (indexLastUdfEntry >= 0) curStackTrace.slice(0, indexLastUdfEntry + 1)
+      else curStackTrace
+    this.setStackTrace(finalStackTrace)
+  }
+}

daffodil-runtime1/src/main/scala/org/apache/daffodil/udf/UserDefinedFunctionService.scala

@@ -29,6 +29,9 @@ import org.apache.daffodil.dpath.NodeInfo
 import java.io.Serializable
 import java.io.ObjectInputStream
 import scala.language.existentials
+import java.io.ByteArrayOutputStream
+import java.io.ObjectOutputStream
+import java.io.NotSerializableException
 
 /**
  * Loads, validates and caches (for use at schema compile time) all User Defined Functions
@@ -107,7 +110,6 @@ object UserDefinedFunctionService extends Logging {
           udfc =>
             val nonAnn = !udfc.isAnnotationPresent(classUserDefinedFunctionIdentification)
             val nonUdf = !classUserDefinedFunction.isAssignableFrom(udfc)
-
             if (nonAnn) {
               log(LogLevel.Warning, "User Defined Function ignored: %s. Missing %s annotation",
                 udfc.getName, classUserDefinedFunctionIdentification.getName)
@@ -226,12 +228,35 @@ object UserDefinedFunctionService extends Logging {
           val maybeUdf =
             try {
               val udf = udfInfo.provider.createUserDefinedFunction(namespaceURI, fname)
-              Option(udf)
+              /*
+               * This is to check for any errors thrown when if we try to serialize the
+               * UDF, such as when using save-parser
+               */
+              try {
+                new ObjectOutputStream(new ByteArrayOutputStream()).writeObject(udf)
+                Option(udf)
+              } catch {
+                case e: NotSerializableException =>
+                  log(LogLevel.Error, "Error serializing initialized User Defined Function: %s. Could not serialize member of class: %s",
+                    udf.getClass.getName, e.getMessage)
+                  None
+              }
             } catch {
-              case e: ReflectiveOperationException => {
+              /*
+               * This is to protect against any errors thrown when we are trying to
+               * initialize the UDFs. It will catch any exceptions and emit them as the reason the UDF
+               * is unsupported
+               */
+              case e: Exception => {
+                val actualCause = e match {
+                  case _: ReflectiveOperationException => e.getCause
+                  case x => x
+                }
                 log(LogLevel.Error, "Error initializing User Defined Function: %s. Error thrown: %s",
-                  udfid, e.getCause)
-                None
+                  udfid, actualCause)
+                throw new UserDefinedFunctionFatalErrorException(
+                  s"User Defined Function '$udfid' Error",
+                  actualCause, udfInfo.udfClass.getName, udfInfo.provider.getClass.getName)
               }
             }
 

daffodil-udf/src/test/resources/org/apache/daffodil/udf/genericUdfSchema.xsd

@@ -74,6 +74,28 @@
     </xs:complexType>
   </xs:element>
 
+  <xs:element name="user_func4">
+    <xs:complexType>
+      <xs:sequence>
+        <xs:element name="data" type="xs:string"
+          dfdl:lengthKind="pattern" dfdl:lengthPattern=".*\s*" />
+        <xs:element name="value" type="xs:string"
+          dfdl:inputValueCalc="{ ssudf:get-nonserializable-state() }" />
+      </xs:sequence>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:element name="user_func5">
+    <xs:complexType>
+      <xs:sequence>
+        <xs:element name="data" type="xs:string"
+          dfdl:lengthKind="pattern" dfdl:lengthPattern=".*\s*" />
+        <xs:element name="value" type="xs:string"
+          dfdl:inputValueCalc="{ ssudf:get-serializable-state() }" />
+      </xs:sequence>
+    </xs:complexType>
+  </xs:element>
+
   <xs:element name="fn_func">
     <xs:complexType>
       <xs:sequence>

daffodil-udf/src/test/scala/org/sbadudfs/functionclasses/StringFunctions/StringFunctionsProvider.scala

@@ -32,8 +32,10 @@ class StringFunctionsProvider extends UserDefinedFunctionProvider {
 
   override def createUserDefinedFunction(namespaceURI: String, fName: String) = {
     val udf = s"$namespaceURI:$fName" match {
+      // incorrect object
       case "http://example.com/scala/udf:rev-words" => new Reverse
-      case "http://example.com/scala/udf:reverse" => new ReverseWords
+      // incorrect udfid
+      case "http://example.com/scala/udf:reversee" => new Reverse
     }
     udf
   }