#660 Make spark-avro dependency provided because it is tightly coupled with Spark version.

Author: yruslan

pramen/api/src/main/scala/za/co/absa/pramen/api/Source.scala

@@ -112,7 +112,7 @@ trait Source extends ExternalChannel {
   def getDataIncremental(query: Query, onlyForInfoDate: Option[LocalDate], offsetFromOpt: Option[OffsetValue], offsetToOpt: Option[OffsetValue], columns: Seq[String]): SourceResult
 
   /**
-    * This method is called after the ingestion is finished. You can query the output table form the output information
+    * This method is called after the ingestion is finished. You can query the output table from the output information
     * data and the data should be there.
     *
     * @param query           The query used to read the data from the source.

pramen/core/src/main/scala/za/co/absa/pramen/core/runner/task/TaskRunnerBase.scala

@@ -365,7 +365,7 @@ abstract class TaskRunnerBase(conf: Config,
           case None => runResult.data
         }
 
-        val dfWithInfoDate = if (dfWithTimestamp.schema.exists(f => f.name.equals(task.job.outputTable.infoDateColumn)) || task.job.outputTable.infoDateColumn.isEmpty) {
+        val dfWithInfoDate = if (task.job.outputTable.infoDateColumn.isEmpty) {
           dfWithTimestamp
         } else {
           dfWithTimestamp.withColumn(task.job.outputTable.infoDateColumn, lit(Date.valueOf(task.infoDate)))

pramen/project/Dependencies.scala

@@ -46,9 +46,9 @@ object Dependencies {
 
   def ExtrasJobsDependencies(scalaVersion: String): Seq[ModuleID] = Seq(
     "org.apache.spark"          %% "spark-sql"                  % sparkVersion(scalaVersion) % Provided,
+    "org.apache.spark"          %% "spark-avro"                 % sparkVersion(scalaVersion) % Provided,
     "net.sourceforge.jtds"      %  "jtds"                       % msSqlDriverVersion,
     "org.apache.httpcomponents" %  "httpclient"                 % httpClientVersion,
-    "org.apache.spark"          %% "spark-avro"                 % sparkVersion(scalaVersion),
     "org.scalatest"             %% "scalatest"                  % scalatestVersion           % Test,
     "org.mockito"               %  "mockito-core"               % mockitoVersion             % Test,
     "org.hsqldb"                %  "hsqldb"                     % hsqlDbVersion              % Test classifier "jdk8"