Use Spark 3.5.7 for the Maven project (dev only).

yruslan · yruslan · commit 917de9a3661a · 2025-11-11T15:45:27.000+01:00
diff --git a/pramen/core/pom.xml b/pramen/core/pom.xml
@@ -77,13 +77,12 @@
         <dependency>
             <groupId>org.apache.httpcomponents</groupId>
             <artifactId>httpclient</artifactId>
-            <version>4.5.13</version>
         </dependency>
 
         <!-- Delta format -->
         <dependency>
             <groupId>io.delta</groupId>
-            <artifactId>delta-core_${scala.compat.version}</artifactId>
+            <artifactId>delta-spark_${scala.compat.version}</artifactId>
         </dependency>
 
         <!-- MongoDB driver -->
@@ -97,34 +96,26 @@
             <groupId>org.postgresql</groupId>
             <artifactId>postgresql</artifactId>
         </dependency>
+
         <dependency>
             <groupId>net.sourceforge.jtds</groupId>
             <artifactId>jtds</artifactId>
-            <version>1.3.1</version>
         </dependency>
         <dependency>
             <groupId>com.microsoft.sqlserver</groupId>
             <artifactId>mssql-jdbc</artifactId>
-            <!--            <version>12.8.1.jre8</version>-->
         </dependency>
 
         <dependency>
             <groupId>com.mysql</groupId>
             <artifactId>mysql-connector-j</artifactId>
-            <version>8.3.0</version>
         </dependency>
 
         <dependency>
             <groupId>com.typesafe.slick</groupId>
             <artifactId>slick_${scala.compat.version}</artifactId>
         </dependency>
 
-        <!--        <dependency>-->
-        <!--            <groupId>com.luckycatlabs</groupId>-->
-        <!--            <artifactId>SunriseSunsetCalculator</artifactId>-->
-        <!--            <version>1.2</version>-->
-        <!--        </dependency>-->
-
         <dependency>
             <groupId>com.typesafe.slick</groupId>
             <artifactId>slick-hikaricp_${scala.compat.version}</artifactId>
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/integration/IncrementalPipelineJdbcLongSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/integration/IncrementalPipelineJdbcLongSuite.scala
@@ -42,7 +42,7 @@ class IncrementalPipelineJdbcLongSuite extends AnyWordSpec
   with TextComparisonFixture {
 
   val jdbcConfig: JdbcConfig = JdbcConfig(driver, Some(url), Nil, None, Some(user), Some(password))
-  val pramenDb: PramenDb = PramenDb(jdbcConfig)
+  lazy val pramenDb: PramenDb = PramenDb(jdbcConfig)
 
   private val infoDate = LocalDate.of(2021, 2, 18)
 
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/integration/IncrementalPipelineLongFixture.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/integration/IncrementalPipelineLongFixture.scala
@@ -46,7 +46,7 @@ class IncrementalPipelineLongFixture extends AnyWordSpec
   with TextComparisonFixture {
 
   val jdbcConfig: JdbcConfig = JdbcConfig(driver, Some(url), Nil, None, Some(user), Some(password))
-  val pramenDb: PramenDb = PramenDb(jdbcConfig)
+  lazy val pramenDb: PramenDb = PramenDb(jdbcConfig)
 
   before {
     pramenDb.rdb.executeDDL("DROP SCHEMA PUBLIC CASCADE;")
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/metadata/MetadataManagerJdbcSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/metadata/MetadataManagerJdbcSuite.scala
@@ -27,7 +27,7 @@ import java.time.{LocalDate, ZoneOffset}
 
 class MetadataManagerJdbcSuite extends AnyWordSpec with RelationalDbFixture with BeforeAndAfter with BeforeAndAfterAll {
   val jdbcConfig: JdbcConfig = JdbcConfig(driver, Some(url), Nil, None, Some(user), Some(password))
-  val pramenDb: PramenDb = PramenDb(jdbcConfig)
+  lazy val pramenDb: PramenDb = PramenDb(jdbcConfig)
   private val infoDate = LocalDate.of(2021, 2, 18)
   private val exampleInstant = infoDate.atStartOfDay().toInstant(ZoneOffset.UTC)
 
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/bookkeeper/BookkeeperJdbcSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/bookkeeper/BookkeeperJdbcSuite.scala
@@ -25,7 +25,7 @@ import za.co.absa.pramen.core.reader.model.JdbcConfig
 class BookkeeperJdbcSuite extends BookkeeperCommonSuite with RelationalDbFixture with BeforeAndAfter with BeforeAndAfterAll {
 
   val jdbcConfig: JdbcConfig = JdbcConfig(driver, Some(url), Nil, None, Some(user), Some(password))
-  val pramenDb: PramenDb = PramenDb(jdbcConfig)
+  lazy val pramenDb: PramenDb = PramenDb(jdbcConfig)
 
   before {
     pramenDb.rdb.executeDDL("DROP SCHEMA PUBLIC CASCADE;")
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/bookkeeper/BookkeeperSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/bookkeeper/BookkeeperSuite.scala
@@ -43,7 +43,7 @@ class BookkeeperSuite extends AnyWordSpec
   import za.co.absa.pramen.core.bookkeeper.BookkeeperMongoDb._
 
   val jdbcConfig: JdbcConfig = JdbcConfig(driver, Some(url), Nil, None, Option(user), Option(password))
-  val pramenDb: PramenDb = PramenDb(jdbcConfig)
+  lazy val pramenDb: PramenDb = PramenDb(jdbcConfig)
 
   before {
     pramenDb.rdb.executeDDL("DROP SCHEMA PUBLIC CASCADE;")
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/bookkeeper/OffsetManagerJdbcSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/bookkeeper/OffsetManagerJdbcSuite.scala
@@ -29,7 +29,7 @@ import java.time.{Instant, LocalDate}
 
 class OffsetManagerJdbcSuite extends AnyWordSpec with RelationalDbFixture with BeforeAndAfter with BeforeAndAfterAll {
   val jdbcConfig: JdbcConfig = JdbcConfig(driver, Some(url), Nil, None, Some(user), Some(password))
-  val pramenDb: PramenDb = PramenDb(jdbcConfig)
+  lazy val pramenDb: PramenDb = PramenDb(jdbcConfig)
 
   private val infoDate = LocalDate.of(2023, 8, 25)
 
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/journal/JournalJdbcSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/journal/JournalJdbcSuite.scala
@@ -28,7 +28,7 @@ class JournalJdbcSuite extends AnyWordSpec with SparkTestBase with BeforeAndAfte
   import TestCases._
 
   val jdbcConfig: JdbcConfig = JdbcConfig(driver, Some(url), Nil, None, Option(user), Option(password))
-  val pramenDb: PramenDb = PramenDb(jdbcConfig)
+  lazy val pramenDb: PramenDb = PramenDb(jdbcConfig)
 
   before {
     pramenDb.rdb.executeDDL("DROP SCHEMA PUBLIC CASCADE;")
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/lock/TokenLockJdbcSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/lock/TokenLockJdbcSuite.scala
@@ -16,18 +16,19 @@
 
 package za.co.absa.pramen.core.tests.lock
 
+import org.scalatest.concurrent.Eventually._
 import org.scalatest.wordspec.AnyWordSpec
 import org.scalatest.{BeforeAndAfter, BeforeAndAfterAll}
 import za.co.absa.pramen.core.fixtures.RelationalDbFixture
 import za.co.absa.pramen.core.lock.{TokenLock, TokenLockJdbc}
 import za.co.absa.pramen.core.rdb.PramenDb
 import za.co.absa.pramen.core.reader.model.JdbcConfig
+
 import scala.concurrent.duration._
-import org.scalatest.concurrent.Eventually._
 
 class TokenLockJdbcSuite extends AnyWordSpec with RelationalDbFixture with BeforeAndAfter with BeforeAndAfterAll {
   val jdbcConfig: JdbcConfig = JdbcConfig(driver, Some(url), Nil, None, Option(user), Option(password))
-  val pramenDb: PramenDb = PramenDb(jdbcConfig)
+  lazy val pramenDb: PramenDb = PramenDb(jdbcConfig)
 
   before {
     pramenDb.rdb.executeDDL("DROP SCHEMA PUBLIC CASCADE;")
diff --git a/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorSasSuite.scala b/pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorSasSuite.scala
@@ -42,7 +42,7 @@ class SqlGeneratorSasSuite extends AnyWordSpec with RelationalDbFixture {
 
   val driverSas = "com.sas.rio.MVADriver"
 
-  val connection: Connection = getConnection
+  lazy val connection: Connection = getConnection
 
   val gen: SqlGenerator = getSqlGenerator(driverSas, sqlConfigDate)
   val genStr: SqlGenerator = getSqlGenerator(driverSas, sqlConfigString)
diff --git a/pramen/pom.xml b/pramen/pom.xml
@@ -110,9 +110,9 @@
 
         <scala.version>2.12.20</scala.version>
         <scala.compat.version>2.12</scala.compat.version>
-        <spark.version>3.2.4</spark.version>
-        <hadoop.version>3.2.1</hadoop.version>
-        <delta.version>2.0.0</delta.version>
+        <spark.version>3.5.7</spark.version>
+        <hadoop.version>3.3.6</hadoop.version>
+        <delta.version>3.3.2</delta.version>
         <abris.version>6.4.1</abris.version>
 
         <!-- Scala 2.13 and Spark 3.2.2 -->
@@ -136,7 +136,6 @@
         <maven.surefire.version>3.5.3</maven.surefire.version>
 
         <!-- Frameworks and libraries -->
-        <jackson.version>2.10.3</jackson.version>
         <scala_logging.version>3.7.2</scala_logging.version>
         <scodec_bits.version>1.1.4</scodec_bits.version>
         <scodec_core.version>1.10.3</scodec_core.version>
@@ -193,6 +192,12 @@
                 <version>${hadoop.version}</version>
             </dependency>
 
+            <dependency>
+                <groupId>org.apache.httpcomponents</groupId>
+                <artifactId>httpclient</artifactId>
+                <version>4.5.13</version>
+            </dependency>
+
             <!-- ABRIS -->
             <dependency>
                 <groupId>za.co.absa</groupId>
@@ -221,12 +226,27 @@
                 <version>42.7.7</version>
             </dependency>
 
+            <!-- My SQL JDBC driver -->
+            <dependency>
+                <groupId>com.mysql</groupId>
+                <artifactId>mysql-connector-j</artifactId>
+                <version>8.3.0</version>
+            </dependency>
+
+            <!-- MS SQL JDBC driver -->
             <dependency>
                 <groupId>com.microsoft.sqlserver</groupId>
                 <artifactId>mssql-jdbc</artifactId>
                 <version>12.8.1.jre8</version>
             </dependency>
 
+            <!-- MS SQL OSS JDBC driver -->
+            <dependency>
+                <groupId>net.sourceforge.jtds</groupId>
+                <artifactId>jtds</artifactId>
+                <version>1.3.1</version>
+            </dependency>
+
             <!-- Oracle JDBC driver -->
             <dependency>
                 <groupId>com.oracle</groupId>
@@ -270,10 +290,10 @@
                 <version>3.3.3</version>
             </dependency>
 
-            <!-- https://mvnrepository.com/artifact/io.delta/delta-core -->
+            <!-- https://mvnrepository.com/artifact/io.delta/delta-spark -->
             <dependency>
                 <groupId>io.delta</groupId>
-                <artifactId>delta-core_${scala.compat.version}</artifactId>
+                <artifactId>delta-spark_${scala.compat.version}</artifactId>
                 <version>${delta.version}</version>
             </dependency>
 
