apache spark - Registering UDF's dynamically using scala reflect not working

Question

I am registering my UDF's dynamically using scala reflect as shown below and this code works fine. However when I try to list spark functions using spark.catalog then I don't see it. Can you please help me understanding what I am missing here:

   spark.catalog.listFunctions().foreach{
      fun =>
        if (fun.name == "ModelIdToModelYear") {
          println(fun.name)
        }
    }

def registerUDF(spark: SparkSession) : Unit = {
    val runtimeMirror = scala.reflect.runtime.universe.runtimeMirror(getClass.getClassLoader)
    val moduleSymbol = runtimeMirror.moduleSymbol(Class.forName("com.local.practice.udf.UdfModelIdToModelYear"))
    val targetMethod = moduleSymbol.typeSignature.members.filter{
      x => x.isMethod && x.name.toString == "ModelIdToModelYear"
    }.head.asMethod
    val function = runtimeMirror.reflect(runtimeMirror.reflectModule(moduleSymbol).instance).reflectMethod(targetMethod)
    function(spark.udf)
  }

Below is my UDF definition

package com.local.practice.udf

import org.apache.spark.sql.expressions.UserDefinedFunction
import org.apache.spark.sql.functions.udf

//noinspection ScalaStyle
object UdfModelIdToModelYear {

  val ModelIdToModelYear: UserDefinedFunction = udf((model_id : String) => {
    val numPattern = "(\d{2})_.+".r
     numPattern.findFirstIn(model_id).getOrElse("0").toInt
  })

}

question from:https://stackoverflow.com/questions/66058571/registering-udfs-dynamically-using-scala-reflect-not-working

Answer 1

Waitting for answers