spark unit testing with SharedSparkSession (org.apache.spark.sql.test.SharedSparkSession.eventually)

Question

I tried to write a simple unit for Spark join from the tutorial - Apache Spark Unit Testing Part 2 — Spark SQL. Added all dependencies from reference and

  "org.apache.spark" %% "spark-core" % "2.4.7" % Test,
  "org.apache.spark" %% "spark-core" % "2.4.7" % Test classifier "tests",
  "org.scalatest" %% "scalatest" % "3.2.3" % Test,

but can't add extended QueryTest because it involves AnyFunSuite in private class. Now the code looks like

class TestScalaCheck extends AnyFunSuite with SharedSparkSession {
  import testImplicits._

  test("join - join using") {
    val df = Seq(1, 2, 3).map(i => (i, i.toString)).toDF("int", "str")
    val df2 = Seq(1, 2, 3).map(i => (i, (i + 1).toString)).toDF("int", "str")

    checkAnswer(
      df.join(df2, "int"),
      Row(1, "1", "2") :: Row(2, "2", "3") :: Row(3, "3", "5") :: Nil)
  }

And it generates the exception message:

An exception or error caused a run to abort: org.apache.spark.sql.test.SharedSparkSession.eventually(Lorg/scalatest/concurrent/PatienceConfiguration$Timeout;Lorg/scalatest/concurrent/PatienceConfiguration$Interval;Lscala/Function0;Lorg/scalactic/source/Position;)Ljava/lang/Object; 
java.lang.NoSuchMethodError: org.apache.spark.sql.test.SharedSparkSession.eventually(Lorg/scalatest/concurrent/PatienceConfiguration$Timeout;Lorg/scalatest/concurrent/PatienceConfiguration$Interval;Lscala/Function0;Lorg/scalactic/source/Position;)Ljava/lang/Object;
    at org.apache.spark.sql.test.SharedSparkSession.afterEach(SharedSparkSession.scala:135)
    at org.apache.spark.sql.test.SharedSparkSession.afterEach$(SharedSparkSession.scala:129)
    at lesson2.TestScalaCheck.afterEach(TestScalaCheck.scala:14)
    at org.scalatest.BeforeAndAfterEach.$anonfun$runTest$1(BeforeAndAfterEach.scala:247)
    at org.scalatest.Status.$anonfun$withAfterEffect$1(Status.scala:377)
    at org.scalatest.Status.$anonfun$withAfterEffect$1$adapted(Status.scala:373)
    at org.scalatest.SucceededStatus$.whenCompleted(Status.scala:462)
    at org.scalatest.Status.withAfterEffect(Status.scala:373)
    at org.scalatest.Status.withAfterEffect$(Status.scala:371)
    at org.scalatest.SucceededStatus$.withAfterEffect(Status.scala:434)
    at org.scalatest.BeforeAndAfterEach.runTest(BeforeAndAfterEach.scala:246)
    at org.scalatest.BeforeAndAfterEach.runTest$(BeforeAndAfterEach.scala:227)
    at lesson2.TestScalaCheck.runTest(TestScalaCheck.scala:14)
    at org.scalatest.funsuite.AnyFunSuiteLike.$anonfun$runTests$1(AnyFunSuiteLike.scala:233)
    at org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:413)
    at scala.collection.immutable.List.foreach(List.scala:392)
    at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:401)
    at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:396)
    at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:475)
    at org.scalatest.funsuite.AnyFunSuiteLike.runTests(AnyFunSuiteLike.scala:233)
    at org.scalatest.funsuite.AnyFunSuiteLike.runTests$(AnyFunSuiteLike.scala:232)
    at org.scalatest.funsuite.AnyFunSuite.runTests(AnyFunSuite.scala:1563)
    at org.scalatest.Suite.run(Suite.scala:1112)
    at org.scalatest.Suite.run$(Suite.scala:1094)
    at org.scalatest.funsuite.AnyFunSuite.org$scalatest$funsuite$AnyFunSuiteLike$$super$run(AnyFunSuite.scala:1563)
    at org.scalatest.funsuite.AnyFunSuiteLike.$anonfun$run$1(AnyFunSuiteLike.scala:237)
    at org.scalatest.SuperEngine.runImpl(Engine.scala:535)
    at org.scalatest.funsuite.AnyFunSuiteLike.run(AnyFunSuiteLike.scala:237)
    at org.scalatest.funsuite.AnyFunSuiteLike.run$(AnyFunSuiteLike.scala:236)
    at lesson2.TestScalaCheck.org$scalatest$BeforeAndAfterAll$$super$run(TestScalaCheck.scala:14)
    at org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213)
    at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210)
    at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
    at lesson2.TestScalaCheck.run(TestScalaCheck.scala:14)
    at org.scalatest.tools.SuiteRunner.run(SuiteRunner.scala:45)
    at org.scalatest.tools.Runner$.$anonfun$doRunRunRunDaDoRunRun$13(Runner.scala:1320)
    at org.scalatest.tools.Runner$.$anonfun$doRunRunRunDaDoRunRun$13$adapted(Runner.scala:1314)
    at scala.collection.immutable.List.foreach(List.scala:392)
    at org.scalatest.tools.Runner$.doRunRunRunDaDoRunRun(Runner.scala:1314)
    at org.scalatest.tools.Runner$.$anonfun$runOptionallyWithPassFailReporter$24(Runner.scala:993)
    at org.scalatest.tools.Runner$.$anonfun$runOptionallyWithPassFailReporter$24$adapted(Runner.scala:971)
    at org.scalatest.tools.Runner$.withClassLoaderAndDispatchReporter(Runner.scala:1480)
    at org.scalatest.tools.Runner$.runOptionallyWithPassFailReporter(Runner.scala:971)
    at org.scalatest.tools.Runner$.run(Runner.scala:798)
    at org.scalatest.tools.Runner.run(Runner.scala)
    at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.runScalaTest2or3(ScalaTestRunner.java:38)
    at org.jetbrains.plugins.scala.testingSupport.scalaTest.ScalaTestRunner.main(ScalaTestRunner.java:25)
 

Could you help me to resolve it?

The original code is - DataFrameJoinSuite.scala

question from:https://stackoverflow.com/questions/65915911/spark-unit-testing-with-sharedsparksession-org-apache-spark-sql-test-sharedspar

Answer 1

Waitting for answers