Exception while creating a udf #429
Description
I get an exception as soon as any code containing udf is encountered, eg:
@udf("int")
def night(hour, weekday):
if ((16 <= hour <= 20) and (weekday < 5)):
return int(1)
else:
return int(0)
Using Ray 2.41, pyarrow 19.0.0.
File "/home/ankur/dev/apps/ML/learn/ray/ray_spark/taxi_data_preprocess.py", line 35, in add_time_features
@udf("int")
^^^^^^^^^^
File "/home/ankur/miniconda3/envs/py3_12/lib/python3.12/site-packages/pyspark/sql/udf.py", line 127, in _create_py_udf
else session.conf.get("spark.sql.execution.pythonUDF.arrow.enabled") == "true"
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/ankur/miniconda3/envs/py3_12/lib/python3.12/site-packages/pyspark/sql/conf.py", line 54, in get
return self._jconf.get(key)
^^^^^^^^^^^^^^^^^^^^
File "/home/ankur/miniconda3/envs/py3_12/lib/python3.12/site-packages/py4j/java_gateway.py", line 1322, in call
return_value = get_return_value(
^^^^^^^^^^^^^^^^^
File "/home/ankur/miniconda3/envs/py3_12/lib/python3.12/site-packages/pyspark/errors/exceptions/captured.py", line 179, in deco
return f(*a, **kw)
^^^^^^^^^^^
File "/home/ankur/miniconda3/envs/py3_12/lib/python3.12/site-packages/py4j/protocol.py", line 326, in get_return_value
raise Py4JJavaError(
py4j.protocol.Py4JJavaError: An error occurred while calling o53.get.
: java.util.NoSuchElementException: spark.sql.execution.pythonUDF.arrow.enabled
at org.apache.spark.sql.errors.QueryExecutionErrors$.noSuchElementExceptionError(QueryExecutionErrors.scala:1678)
at org.apache.spark.sql.internal.SQLConf.$anonfun$getConfString$3(SQLConf.scala:4568)
at scala.Option.getOrElse(Option.scala:189)
at org.apache.spark.sql.internal.SQLConf.getConfString(SQLConf.scala:4568)
at org.apache.spark.sql.RuntimeConfig.get(RuntimeConfig.scala:72)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77)
at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.base/java.lang.reflect.Method.invoke(Method.java:569)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
at py4j.ClientServerConnection.run(ClientServerConnection.java:106)
at java.base/java.lang.Thread.run(Thread.java:840)