support setting default format via env vars for Instant and DateTime types

Author: cwensel

tessellate-main/build.gradle.kts

@@ -125,7 +125,7 @@ dependencies {
     testImplementation("io.hosuaby:inject-resources-junit-jupiter:$injectResources")
 
     // https://github.com/webcompere/system-stubs
-    val systemStubs = "2.1.6"
+    val systemStubs = "2.1.8"
     testImplementation("uk.org.webcompere:system-stubs-core:$systemStubs")
     testImplementation("uk.org.webcompere:system-stubs-jupiter:$systemStubs")
     testImplementation("org.mockito:mockito-inline:5.2.0")

tessellate-main/src/main/antora/modules/reference/pages/types.adoc

@@ -31,10 +31,16 @@ Some types are canonically a custom or special Java type, but also have metadata
 These types take the form `type[|metadata]` where the optional metadata can be a format string, in the case of the temporal types.
 
 `Instant|format`:: Canonically a `java.time.Instant`.
-Supports nanos precision, format defaults to ISO-8601
-instant format, e.g. `2011-12-03T10:15:30Z`
-`DateTime|format`:: Canonically a `Long`, format defaults to `yyyy-MM-dd HH:mm:ss.SSSSSS z`.
-`json`:: Canonically a `com.fasterxml.jackson.databind.JsonNode` values, nested objects, and arrays.
+- Supports nanos precision, format defaults to ISO-8601 instant format, e.g. `2011-12-03T10:15:30Z`
+- May also be a Clusterless time interval: `Day`, `Hours`, `Fourths`, `Sixth`, or `Twelfths`
+- The default format may be changed via the `INSTANT_TYPE_FORMAT` environment variable
+
+`DateTime|format`:: Canonically a `Long`.
+- Format defaults to `yyyy-MM-dd HH:mm:ss.SSSSSS z`.
+- The default format may be changed via the `DATE_TYPE_FORMAT` environment variable
+
+`json`:: Canonically `com.fasterxml.jackson.databind.JsonNode`.
+- Maybe either a value, nested object, or array.
 
 When reading source data, fields can be parsed and coerced into specific types, as declared, or when using formats like Parquet, type information will be inherited.
 

tessellate-main/src/main/java/io/clusterless/tessellate/parser/FieldParser.java

@@ -110,6 +110,15 @@ public class FieldParser {
     public static final Parser<List<Rel>> RELATION_LIST =
             RELATION.sepBy1(Parsers.sequence(Scanners.many(IS_WHITESPACE)));
 
+    public static Optional<FieldTypeParam> parseFieldTypeParam(String param) {
+        if (param == null || param.isEmpty()) {
+            return Optional.empty();
+        }
+
+        // parser expects the |, hack so we don't mess with the parser
+        return BaseParser.parse(typeParam, "|" + param);
+    }
+
     public static Field parseField(String field) {
         return BaseParser.parse(fullFieldDeclaration, field);
     }

tessellate-main/src/main/java/io/clusterless/tessellate/parser/FieldsParser.java

@@ -29,6 +29,7 @@
 import java.time.temporal.TemporalUnit;
 import java.util.List;
 import java.util.Locale;
+import java.util.Optional;
 import java.util.TimeZone;
 
 /**
@@ -49,6 +50,8 @@
  */
 public class FieldsParser {
     public static final FieldsParser INSTANCE = new FieldsParser();
+    public static final String INSTANT_TYPE_FORMAT = "INSTANT_TYPE_FORMAT";
+    public static final String DATE_TYPE_FORMAT = "DATE_TYPE_FORMAT";
 
     private DateType defaultDateTimeType = new DateType("yyyy-MM-dd HH:mm:ss.SSSSSS z", TimeZone.getTimeZone("UTC"));
     private InstantType defaultInstantType = InstantType.ISO_MICROS;
@@ -128,8 +131,18 @@ protected Type resolveType(FieldType fieldType) {
 
         String typeName = fieldType.name().name();
 
-        String first = fieldType.param().map(FieldTypeParam::param1).orElse(null);
-        String second = fieldType.param().map(FieldTypeParam::param2).orElse(null);
+        Optional<FieldTypeParam> param = fieldType.param();
+
+        if (param.isEmpty()) {
+            if (typeName.equalsIgnoreCase("DateTime")) {
+                param = FieldParser.parseFieldTypeParam(System.getenv(DATE_TYPE_FORMAT));
+            } else if (typeName.equalsIgnoreCase("Instant")) {
+                param = FieldParser.parseFieldTypeParam(System.getenv(INSTANT_TYPE_FORMAT));
+            }
+        }
+
+        String first = param.map(FieldTypeParam::param1).orElse(null);
+        String second = param.map(FieldTypeParam::param2).orElse(null);
 
         if (typeName.equalsIgnoreCase("string")) {
             type = String.class;
@@ -194,8 +207,8 @@ public String resolveTypeName(Type type) {
         return typeNames[0];
     }
 
-    private static DateTimeFormatter createPattern(String splitType) {
-        return DateTimeFormatter.ofPattern(splitType)
+    public static DateTimeFormatter createPattern(String format) {
+        return DateTimeFormatter.ofPattern(format)
                 .withZone(ZoneId.of("UTC"));
     }
 

tessellate-main/src/test/java/io/clusterless/tessellate/parser/FieldParserTest.java

@@ -8,12 +8,23 @@
 
 package io.clusterless.tessellate.parser;
 
+import cascading.tuple.type.DateType;
+import cascading.tuple.type.InstantType;
+import clusterless.commons.temporal.IntervalUnits;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import uk.org.webcompere.systemstubs.environment.EnvironmentVariables;
+import uk.org.webcompere.systemstubs.jupiter.SystemStub;
+import uk.org.webcompere.systemstubs.jupiter.SystemStubsExtension;
 
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.*;
 
+@ExtendWith(SystemStubsExtension.class)
 public class FieldParserTest {
+
+    @SystemStub
+    private EnvironmentVariables variables = new EnvironmentVariables();
+
     @Test
     void parseFields() {
         assertNotNull(FieldParser.parseField("@field"));
@@ -56,4 +67,52 @@ void parseFieldsList() {
         assertEquals(4, FieldParser.parseFieldList("@field1 + @field2+@field3 +@field4").size());
         assertEquals(5, FieldParser.parseFieldList("@field1+ @field|DateTime +@field|DateTime|yyyyMMdd+1|DateTime|yyyyMMdd + @field|Instant|twelfths|yyyyMMdd").size());
     }
+
+    @Test
+    void parseFieldsWithTypeDefaults() {
+        assertEquals(
+                "yyyyMMdd",
+                assertInstanceOf(
+                        DateType.class,
+                        FieldsParser.INSTANCE.asFields(FieldParser.parseField("@field|DateTime|yyyyMMdd")).getType(0)
+                ).getDateFormat().toLocalizedPattern()
+        );
+
+        variables.set(FieldsParser.DATE_TYPE_FORMAT, "yyyyMMdd");
+        assertEquals(
+                "yyyyMMdd",
+                assertInstanceOf(
+                        DateType.class,
+                        FieldsParser.INSTANCE.asFields(FieldParser.parseField("@field|DateTime")).getType(0)
+                ).getDateFormat().toLocalizedPattern()
+        );
+
+        assertEquals(
+                FieldsParser.createPattern("yyyyMMdd").toString(),
+                assertInstanceOf(
+                        InstantType.class,
+                        FieldsParser.INSTANCE.asFields(FieldParser.parseField("@field|Instant|twelfths|yyyyMMdd")).getType(0)
+                ).getDateTimeFormatter().toString()
+        );
+
+        variables.set(FieldsParser.INSTANT_TYPE_FORMAT, "twelfths|yyyyMMdd");
+
+        assertEquals(
+                FieldsParser.createPattern("yyyyMMdd").toString(),
+                assertInstanceOf(
+                        InstantType.class,
+                        FieldsParser.INSTANCE.asFields(FieldParser.parseField("@field|Instant")).getType(0)
+                ).getDateTimeFormatter().toString()
+        );
+
+        variables.set(FieldsParser.INSTANT_TYPE_FORMAT, "twelfths");
+
+        assertEquals(
+                IntervalUnits.formatter(IntervalUnits.find("twelfths")).toString(),
+                assertInstanceOf(
+                        InstantType.class,
+                        FieldsParser.INSTANCE.asFields(FieldParser.parseField("@field|Instant")).getType(0)
+                ).getDateTimeFormatter().toString()
+        );
+    }
 }