Skip to content
The Chat Archive
gpt-3.5-turbo-0301· OpenAI WildChat-4.8M (AI2)

Py4JJavaErrorTraceback (most recent call last) Cell In[49], line 4 2 evaluator =…

  1. User

    Py4JJavaErrorTraceback (most recent call last) Cell In[49], line 4 2 evaluator = MulticlassClassificationEvaluator(metricName="accuracy") 3 evaluator = evaluator.setLabelCol("PE") ----> 4 print("Test set accuracy = " + str(evaluator.evaluate(predictionAndLabels)))

    File /usr/local/spark/python/pyspark/ml/evaluation.py:111, in Evaluator.evaluate(self, dataset, params) 109 return self.copy(params)._evaluate(dataset) 110 else: --> 111 return self._evaluate(dataset) 112 else: 113 raise TypeError("Params must be a param map but got %s." % type(params))

    File /usr/local/spark/python/pyspark/ml/evaluation.py:148, in JavaEvaluator._evaluate(self, dataset) 146 self._transfer_params_to_java() 147 assert self._java_obj is not None --> 148 return self._java_obj.evaluate(dataset._jdf)

    File /usr/local/spark/python/lib/py4j-0.10.9.7-src.zip/py4j/java_gateway.py:1322, in JavaMember.call(self, *args) 1316 command = proto.CALL_COMMAND_NAME +
    1317 self.command_header +
    1318 args_command +
    1319 proto.END_COMMAND_PART 1321 answer = self.gateway_client.send_command(command) -> 1322 return_value = get_return_value( 1323 answer, self.gateway_client, self.target_id, self.name) 1325 for temp_arg in temp_args: 1326 if hasattr(temp_arg, "_detach"):

    File /usr/local/spark/python/pyspark/errors/exceptions/captured.py:169, in capture_sql_exception..deco(*a, **kw) 167 def deco(*a: Any, **kw: Any) -> Any: 168 try: --> 169 return f(*a, **kw) 170 except Py4JJavaError as e: 171 converted = convert_exception(e.java_exception)

    File /usr/local/spark/python/lib/py4j-0.10.9.7-src.zip/py4j/protocol.py:326, in get_return_value(answer, gateway_client, target_id, name) 324 value = OUTPUT_CONVERTER[type](answer[2:], gateway_client) 325 if answer[1] == REFERENCE_TYPE: --> 326 raise Py4JJavaError( 327 "An error occurred while calling {0}{1}{2}.\n". 328 format(target_id, ".", name), value) 329 else: 330 raise Py4JError( 331 "An error occurred while calling {0}{1}{2}. Trace:\n{3}\n". 332 format(target_id, ".", name, value))

    Py4JJavaError: An error occurred while calling o1415.evaluate. : org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 103.0 failed 1 times, most recent failure: Lost task 0.0 in stage 103.0 (TID 84) (839eac4168dc executor driver): org.apache.spark.SparkException: [FAILED_EXECUTE_UDF] Failed to execute user defined function (ProbabilisticClassificationModelLambda$4156/0x0000000801e23800: (struct<type:tinyint,size:int,indices:array<int>,values:array<double>>) => struct<type:tinyint,size:int,indices:array<int>,values:array<double>>). at org.apache.spark.sql.errors.QueryExecutionErrors$.failedExecuteUserDefinedFunctionError(QueryExecutionErrors.scala:217) at org.apache.spark.sql.errors.QueryExecutionErrors.failedExecuteUserDefinedFunctionError(QueryExecutionErrors.scala) at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) at org.apache.spark.sql.execution.WholeStageCodegenExecanon1.hasNext(WholeStageCodegenExec.scala:760) at scala.collection.Iterator$$anon10.hasNext(Iterator.scala:460) at scala.collection.Iteratoranon$10.hasNext(Iterator.scala:460) at scala.collection.Iteratoranon10.hasNext(Iterator.scala:460) at scala.collection.Iterator$$anon10.hasNext(Iterator.scala:460) at org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:197) at org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63) at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:101) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53) at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:161) at org.apache.spark.scheduler.Task.run(Task.scala:139) at org.apache.spark.executor.ExecutorTaskRunner.TaskRunner.anonfunrunrun3(Executor.scala:554) at org.apache.spark.util.Utils.tryWithSafeFinally(Utils.scala:1529)atorg.apache.spark.executor.Executor.tryWithSafeFinally(Utils.scala:1529) at org.apache.spark.executor.ExecutorTaskRunner.run(Executor.scala:557) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutorWorker.run(ThreadPoolExecutor.java:635) at java.base/java.lang.Thread.run(Thread.java:833) Caused by: java.lang.IllegalArgumentException: requirement failed: A & B Dimension mismatch! at scala.Predef.require(Predef.scala:281) at org.apache.spark.ml.ann.BreezeUtil.dgemm(BreezeUtil.scala:42)atorg.apache.spark.ml.ann.AffineLayerModel.eval(Layer.scala:164)atorg.apache.spark.ml.ann.FeedForwardModel.forward(Layer.scala:508)atorg.apache.spark.ml.ann.FeedForwardModel.predictRaw(Layer.scala:561)atorg.apache.spark.ml.classification.MultilayerPerceptronClassificationModel.predictRaw(MultilayerPerceptronClassifier.scala:337)atorg.apache.spark.ml.classification.MultilayerPerceptronClassificationModel.predictRaw(MultilayerPerceptronClassifier.scala:279)atorg.apache.spark.ml.classification.ProbabilisticClassificationModel..dgemm(BreezeUtil.scala:42) at org.apache.spark.ml.ann.AffineLayerModel.eval(Layer.scala:164) at org.apache.spark.ml.ann.FeedForwardModel.forward(Layer.scala:508) at org.apache.spark.ml.ann.FeedForwardModel.predictRaw(Layer.scala:561) at org.apache.spark.ml.classification.MultilayerPerceptronClassificationModel.predictRaw(MultilayerPerceptronClassifier.scala:337) at org.apache.spark.ml.classification.MultilayerPerceptronClassificationModel.predictRaw(MultilayerPerceptronClassifier.scala:279) at org.apache.spark.ml.classification.ProbabilisticClassificationModel.anonfuntransformtransform2(ProbabilisticClassifier.scala:121) ... 20 more

    Driver stacktrace: at org.apache.spark.scheduler.DAGScheduler.failJobAndIndependentStages(DAGScheduler.scala:2785) at org.apache.spark.scheduler.DAGScheduler.anonfunanonfunabortStage2(DAGScheduler.scala:2721)atorg.apache.spark.scheduler.DAGScheduler.2(DAGScheduler.scala:2721) at org.apache.spark.scheduler.DAGScheduler.anonfunabortStageabortStage2adapted(DAGScheduler.scala:2720)atscala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62)atscala.collection.mutable.ResizableArray.foreachadapted(DAGScheduler.scala:2720) at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:62) at scala.collection.mutable.ResizableArray.foreach(ResizableArray.scala:55) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:49) at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:2720) at org.apache.spark.scheduler.DAGScheduler.anonfunanonfunhandleTaskSetFailed1(DAGScheduler.scala:1206)atorg.apache.spark.scheduler.DAGScheduler.1(DAGScheduler.scala:1206) at org.apache.spark.scheduler.DAGScheduler.anonfunhandleTaskSetFailedhandleTaskSetFailed1adapted(DAGScheduler.scala:1206) at scala.Option.foreach(Option.scala:407) at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:1206) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2984) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2923) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2912) at org.apache.spark.util.EventLoop$$anon1.run(EventLoop.scala:49) at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:971) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2263) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2284) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2303) at org.apache.spark.SparkContext.runJob(SparkContext.scala:2328) at org.apache.spark.rdd.RDD.anonfunanonfuncollect1(RDD.scala:1019)atorg.apache.spark.rdd.RDDOperationScope1(RDD.scala:1019) at org.apache.spark.rdd.RDDOperationScope.withScope(RDDOperationScope.scala:151) at org.apache.spark.rdd.RDDOperationScope.withScope(RDDOperationScope.scala:112)atorg.apache.spark.rdd.RDD.withScope(RDD.scala:405)atorg.apache.spark.rdd.RDD.collect(RDD.scala:1018)atorg.apache.spark.rdd.PairRDDFunctions..withScope(RDDOperationScope.scala:112) at org.apache.spark.rdd.RDD.withScope(RDD.scala:405) at org.apache.spark.rdd.RDD.collect(RDD.scala:1018) at org.apache.spark.rdd.PairRDDFunctions.anonfuncollectAsMapcollectAsMap1(PairRDDFunctions.scala:738) at org.apache.spark.rdd.RDDOperationScope.withScope(RDDOperationScope.scala:151)atorg.apache.spark.rdd.RDDOperationScope.withScope(RDDOperationScope.scala:151) at org.apache.spark.rdd.RDDOperationScope.withScope(RDDOperationScope.scala:112) at org.apache.spark.rdd.RDD.withScope(RDD.scala:405) at org.apache.spark.rdd.PairRDDFunctions.collectAsMap(PairRDDFunctions.scala:737) at org.apache.spark.mllib.evaluation.MulticlassMetrics.confusionslzycompute(MulticlassMetrics.scala:61)atorg.apache.spark.mllib.evaluation.MulticlassMetrics.confusions(MulticlassMetrics.scala:52)atorg.apache.spark.mllib.evaluation.MulticlassMetrics.tpByClasslzycompute(MulticlassMetrics.scala:61) at org.apache.spark.mllib.evaluation.MulticlassMetrics.confusions(MulticlassMetrics.scala:52) at org.apache.spark.mllib.evaluation.MulticlassMetrics.tpByClasslzycompute(MulticlassMetrics.scala:78) at org.apache.spark.mllib.evaluation.MulticlassMetrics.tpByClass(MulticlassMetrics.scala:76) at org.apache.spark.mllib.evaluation.MulticlassMetrics.accuracylzycompute(MulticlassMetrics.scala:188) at org.apache.spark.mllib.evaluation.MulticlassMetrics.accuracy(MulticlassMetrics.scala:188) at org.apache.spark.ml.evaluation.MulticlassClassificationEvaluator.evaluate(MulticlassClassificationEvaluator.scala:153) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) at py4j.Gateway.invoke(Gateway.java:282) at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) at py4j.commands.CallCommand.execute(CallCommand.java:79) at py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) at py4j.ClientServerConnection.run(ClientServerConnection.java:106) at java.base/java.lang.Thread.run(Thread.java:833) Caused by: org.apache.spark.SparkException: [FAILED_EXECUTE_UDF] Failed to execute user defined function (ProbabilisticClassificationModel$$Lambda4156/0x0000000801e23800: (struct<type:tinyint,size:int,indices:array,values:array>) => struct<type:tinyint,size:int,indices:array,values:array>). at org.apache.spark.sql.errors.QueryExecutionErrors.failedExecuteUserDefinedFunctionError(QueryExecutionErrors.scala:217)atorg.apache.spark.sql.errors.QueryExecutionErrors.failedExecuteUserDefinedFunctionError(QueryExecutionErrors.scala)atorg.apache.spark.sql.catalyst.expressions.GeneratedClass.failedExecuteUserDefinedFunctionError(QueryExecutionErrors.scala:217) at org.apache.spark.sql.errors.QueryExecutionErrors.failedExecuteUserDefinedFunctionError(QueryExecutionErrors.scala) at org.apache.spark.sql.catalyst.expressions.GeneratedClassGeneratedIteratorForCodegenStage1.processNext(Unknown Source) at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) at org.apache.spark.sql.execution.WholeStageCodegenExecanon$1.hasNext(WholeStageCodegenExec.scala:760) at scala.collection.Iteratoranon10.hasNext(Iterator.scala:460) at scala.collection.Iterator$$anon10.hasNext(Iterator.scala:460) at scala.collection.Iteratoranon$10.hasNext(Iterator.scala:460) at scala.collection.Iteratoranon10.hasNext(Iterator.scala:460)atorg.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:197)atorg.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63)atorg.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59)atorg.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:101)atorg.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)atorg.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:161)atorg.apache.spark.scheduler.Task.run(Task.scala:139)atorg.apache.spark.executor.Executor10.hasNext(Iterator.scala:460) at org.apache.spark.util.collection.ExternalSorter.insertAll(ExternalSorter.scala:197) at org.apache.spark.shuffle.sort.SortShuffleWriter.write(SortShuffleWriter.scala:63) at org.apache.spark.shuffle.ShuffleWriteProcessor.write(ShuffleWriteProcessor.scala:59) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:101) at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53) at org.apache.spark.TaskContext.runTaskWithListeners(TaskContext.scala:161) at org.apache.spark.scheduler.Task.run(Task.scala:139) at org.apache.spark.executor.ExecutorTaskRunner.anonfunanonfunrun3(Executor.scala:554)atorg.apache.spark.util.Utils3(Executor.scala:554) at org.apache.spark.util.Utils.tryWithSafeFinally(Utils.scala:1529) at org.apache.spark.executor.ExecutorTaskRunner.run(Executor.scala:557)atjava.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)atjava.base/java.util.concurrent.ThreadPoolExecutorTaskRunner.run(Executor.scala:557) at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base/java.util.concurrent.ThreadPoolExecutorWorker.run(ThreadPoolExecutor.java:635) ... 1 more Caused by: java.lang.IllegalArgumentException: requirement failed: A & B Dimension mismatch! at scala.Predef.require(Predef.scala:281)atorg.apache.spark.ml.ann.BreezeUtil.require(Predef.scala:281) at org.apache.spark.ml.ann.BreezeUtil.dgemm(BreezeUtil.scala:42) at org.apache.spark.ml.ann.AffineLayerModel.eval(Layer.scala:164) at org.apache.spark.ml.ann.FeedForwardModel.forward(Layer.scala:508) at org.apache.spark.ml.ann.FeedForwardModel.predictRaw(Layer.scala:561) at org.apache.spark.ml.classification.MultilayerPerceptronClassificationModel.predictRaw(MultilayerPerceptronClassifier.scala:337) at org.apache.spark.ml.classification.MultilayerPerceptronClassificationModel.predictRaw(MultilayerPerceptronClassifier.scala:279) at org.apache.spark.ml.classification.ProbabilisticClassificationModel.anonfunanonfuntransform$2(ProbabilisticClassifier.scala:121) ... 20 more

  2. Assistant

    The error message suggests that there is a dimension mismatch in the input data to the multilayer perceptron classifier model, which causes a failure in executing a user-defined function. This could be due to incompatible input dimensions between the features and label columns, or due to incompatible layer configurations in the model. More information on the data and model configuration is needed to diagnose the specific cause of the error.