Skip to content

Commit b4551c1

Browse files
yruslanyruslan
committed
#667 Fix Denodo filter pushdown for timestamp-based queries.
1 parent 0ffaeba commit b4551c1

File tree

2 files changed

+31
-36
lines changed

2 files changed

+31
-36
lines changed

‎pramen/core/src/main/scala/za/co/absa/pramen/core/sql/SqlGeneratorDenodo.scala‎

Lines changed: 14 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import za.co.absa.pramen.api.sql.{SqlColumnType, SqlConfig, SqlGeneratorBase}
2323
import za.co.absa.pramen.core.sql.dialects.DenodoDialect
2424

2525
import java.time.format.DateTimeFormatter
26-
import java.time.{LocalDate, LocalDateTime}
26+
import java.time.{LocalDate, LocalTime, ZonedDateTime}
2727

2828
object SqlGeneratorDenodo {
2929
private val log = LoggerFactory.getLogger(this.getClass)
@@ -40,6 +40,7 @@ object SqlGeneratorDenodo {
4040

4141
class SqlGeneratorDenodo(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConfig) {
4242
private val dateFormatterApp = DateTimeFormatter.ofPattern(sqlConfig.dateFormatApp)
43+
private val timestampDbFormatter: DateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSXXX")
4344

4445
SqlGeneratorDenodo.registerDialect
4546

@@ -76,22 +77,15 @@ class SqlGeneratorDenodo(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConfi
7677
}
7778

7879
override def getWhere(dateBegin: LocalDate, dateEnd: LocalDate): String = {
79-
val dateBeginLit = getDateLiteral(dateBegin)
80-
val dateEndLit = getDateLiteral(dateEnd)
80+
if (sqlConfig.infoDateType == SqlColumnType.DATE && dateBegin == dateEnd) {
81+
val dateBeginLit = getDateLiteral(dateBegin)
8182

82-
val dateTypes: Array[SqlColumnType] = Array(SqlColumnType.DATETIME)
83-
84-
val infoDateColumnAdjusted =
85-
if (dateTypes.contains(sqlConfig.infoDateType)) {
86-
s"CAST($infoDateColumn AS DATE)"
87-
} else {
88-
infoDateColumn
89-
}
90-
91-
if (dateBeginLit == dateEndLit) {
92-
s"$infoDateColumnAdjusted = $dateBeginLit"
83+
s"$infoDateColumn = $dateBeginLit"
9384
} else {
94-
s"$infoDateColumnAdjusted >= $dateBeginLit AND $infoDateColumnAdjusted <= $dateEndLit"
85+
val dateBeginLit = getDateLiteral(dateBegin)
86+
val dateEndLit = getDateLiteral(dateEnd.plusDays(1))
87+
88+
s"$infoDateColumn >= $dateBeginLit AND $infoDateColumn < $dateEndLit"
9589
}
9690
}
9791

@@ -101,8 +95,9 @@ class SqlGeneratorDenodo(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConfi
10195
val dateStr = DateTimeFormatter.ISO_LOCAL_DATE.format(date)
10296
s"date'$dateStr'"
10397
case SqlColumnType.DATETIME =>
104-
val dateStr = DateTimeFormatter.ISO_LOCAL_DATE.format(date)
105-
s"date'$dateStr'"
98+
val zdt = ZonedDateTime.of(date, LocalTime.MIDNIGHT, sqlConfig.serverTimeZone)
99+
val tsLiteral = timestampDbFormatter.format(zdt)
100+
s"TIMESTAMP '$tsLiteral'"
106101
case SqlColumnType.STRING =>
107102
val dateStr = dateFormatterApp.format(date)
108103
s"'$dateStr'"
@@ -115,8 +110,8 @@ class SqlGeneratorDenodo(sqlConfig: SqlConfig) extends SqlGeneratorBase(sqlConfi
115110
override def getOffsetWhereCondition(column: String, condition: String, offset: OffsetValue): String = {
116111
offset match {
117112
case OffsetValue.DateTimeValue(ts) =>
118-
val ldt = LocalDateTime.ofInstant(ts, sqlConfig.serverTimeZone)
119-
val tsLiteral = timestampGenericDbFormatter.format(ldt)
113+
val zdt = ZonedDateTime.ofInstant(ts, sqlConfig.serverTimeZone)
114+
val tsLiteral = timestampDbFormatter.format(zdt)
120115
s"$column $condition TIMESTAMP '$tsLiteral'"
121116
case OffsetValue.IntegralValue(value) =>
122117
s"$column $condition $value"

‎pramen/core/src/test/scala/za/co/absa/pramen/core/tests/sql/SqlGeneratorDenodoSuite.scala‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -71,35 +71,35 @@ class SqlGeneratorDenodoSuite extends AnyWordSpec {
7171
assert(gen.getCountQuery("A", date1, date1) ==
7272
"SELECT COUNT(*) FROM A WHERE D = date'2020-08-17'")
7373
assert(gen.getCountQuery("A", date1, date2) ==
74-
"SELECT COUNT(*) FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
74+
"SELECT COUNT(*) FROM A WHERE D >= date'2020-08-17' AND D < date'2020-08-31'")
7575
}
7676

7777
"date is in DATETIME format" in {
7878
assert(genDateTime.getCountQuery("A", date1, date1) ==
79-
"SELECT COUNT(*) FROM A WHERE CAST(D AS DATE) = date'2020-08-17'")
79+
"SELECT COUNT(*) FROM A WHERE D >= TIMESTAMP '2020-08-17 00:00:00.000+02:00' AND D < TIMESTAMP '2020-08-18 00:00:00.000+02:00'")
8080
assert(genDateTime.getCountQuery("A", date1, date2) ==
81-
"SELECT COUNT(*) FROM A WHERE CAST(D AS DATE) >= date'2020-08-17' AND CAST(D AS DATE) <= date'2020-08-30'")
81+
"SELECT COUNT(*) FROM A WHERE D >= TIMESTAMP '2020-08-17 00:00:00.000+02:00' AND D < TIMESTAMP '2020-08-31 00:00:00.000+02:00'")
8282
}
8383

8484
"date is in STRING format" in {
8585
assert(genStr.getCountQuery("A", date1, date1) ==
86-
"SELECT COUNT(*) FROM A WHERE D = '2020-08-17'")
86+
"SELECT COUNT(*) FROM A WHERE D >= '2020-08-17' AND D < '2020-08-18'")
8787
assert(genStr.getCountQuery("A", date1, date2) ==
88-
"SELECT COUNT(*) FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
88+
"SELECT COUNT(*) FROM A WHERE D >= '2020-08-17' AND D < '2020-08-31'")
8989
}
9090

9191
"date is in NUMBER format" in {
9292
assert(genNum.getCountQuery("A", date1, date1) ==
93-
"SELECT COUNT(*) FROM A WHERE D = 20200817")
93+
"SELECT COUNT(*) FROM A WHERE D >= 20200817 AND D < 20200818")
9494
assert(genNum.getCountQuery("A", date1, date2) ==
95-
"SELECT COUNT(*) FROM A WHERE D >= 20200817 AND D <= 20200830")
95+
"SELECT COUNT(*) FROM A WHERE D >= 20200817 AND D < 20200831")
9696
}
9797

9898
"the table name and column name need to be escaped" in {
9999
assert(genEscaped.getCountQuery("Input Table", date1, date1) ==
100100
"SELECT COUNT(*) FROM \"Input Table\" WHERE \"Info date\" = date'2020-08-17'")
101101
assert(genEscapedAuto.getCountQuery("Input Table", date1, date2) ==
102-
"SELECT COUNT(*) FROM \"Input Table\" WHERE \"Info date\" >= date'2020-08-17' AND \"Info date\" <= date'2020-08-30'")
102+
"SELECT COUNT(*) FROM \"Input Table\" WHERE \"Info date\" >= date'2020-08-17' AND \"Info date\" < date'2020-08-31'")
103103
}
104104
}
105105

@@ -108,35 +108,35 @@ class SqlGeneratorDenodoSuite extends AnyWordSpec {
108108
assert(gen.getDataQuery("A", date1, date1, Nil, None) ==
109109
"SELECT * FROM A WHERE D = date'2020-08-17'")
110110
assert(gen.getDataQuery("A", date1, date2, Nil, None) ==
111-
"SELECT * FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
111+
"SELECT * FROM A WHERE D >= date'2020-08-17' AND D < date'2020-08-31'")
112112
}
113113

114114
"date is in DATETIME format" in {
115115
assert(genDateTime.getDataQuery("A", date1, date1, Nil, None) ==
116-
"SELECT * FROM A WHERE CAST(D AS DATE) = date'2020-08-17'")
116+
"SELECT * FROM A WHERE D >= TIMESTAMP '2020-08-17 00:00:00.000+02:00' AND D < TIMESTAMP '2020-08-18 00:00:00.000+02:00'")
117117
assert(genDateTime.getDataQuery("A", date1, date2, Nil, None) ==
118-
"SELECT * FROM A WHERE CAST(D AS DATE) >= date'2020-08-17' AND CAST(D AS DATE) <= date'2020-08-30'")
118+
"SELECT * FROM A WHERE D >= TIMESTAMP '2020-08-17 00:00:00.000+02:00' AND D < TIMESTAMP '2020-08-31 00:00:00.000+02:00'")
119119
}
120120

121121
"date is in STRING format" in {
122122
assert(genStr.getDataQuery("A", date1, date1, Nil, None) ==
123-
"SELECT * FROM A WHERE D = '2020-08-17'")
123+
"SELECT * FROM A WHERE D >= '2020-08-17' AND D < '2020-08-18'")
124124
assert(genStr.getDataQuery("A", date1, date2, Nil, None) ==
125-
"SELECT * FROM A WHERE D >= '2020-08-17' AND D <= '2020-08-30'")
125+
"SELECT * FROM A WHERE D >= '2020-08-17' AND D < '2020-08-31'")
126126
}
127127

128128
"date is in NUMBER format" in {
129129
assert(genNum.getDataQuery("A", date1, date1, Nil, None) ==
130-
"SELECT * FROM A WHERE D = 20200817")
130+
"SELECT * FROM A WHERE D >= 20200817 AND D < 20200818")
131131
assert(genNum.getDataQuery("A", date1, date2, Nil, None) ==
132-
"SELECT * FROM A WHERE D >= 20200817 AND D <= 20200830")
132+
"SELECT * FROM A WHERE D >= 20200817 AND D < 20200831")
133133
}
134134

135135
"with limit records" in {
136136
assert(gen.getDataQuery("A", date1, date1, Nil, Some(100)) ==
137137
"SELECT * FROM A WHERE D = date'2020-08-17'")
138138
assert(gen.getDataQuery("A", date1, date2, Nil, Some(100)) ==
139-
"SELECT * FROM A WHERE D >= date'2020-08-17' AND D <= date'2020-08-30'")
139+
"SELECT * FROM A WHERE D >= date'2020-08-17' AND D < date'2020-08-31'")
140140
}
141141
}
142142

@@ -184,7 +184,7 @@ class SqlGeneratorDenodoSuite extends AnyWordSpec {
184184
val actual = gen.asInstanceOf[SqlGeneratorBase]
185185
.getOffsetWhereCondition("offset", ">", OffsetValue.DateTimeValue(Instant.ofEpochMilli(1727761000)))
186186

187-
assert(actual == "offset > TIMESTAMP '1970-01-21 01:56:01.000'")
187+
assert(actual == "offset > TIMESTAMP '1970-01-21 01:56:01.000+02:00'")
188188
}
189189

190190
"return the correct condition for string offsets" in {

0 commit comments

Comments
 (0)