allow the assignment operator to insert null

Author: cwensel

tessellate-main/src/main/antora/modules/reference/pages/transforms.adoc

@@ -34,7 +34,9 @@ Apply function:: Apply intrinsic functions against one or more fields.
 There are three transform operators:
 
 `pass:[=>]`:: Assign a literal value to a new field.
-Format::: `literal pass:[=>] new_field|type`
+Format:::
+- `literal pass:[=>] new_field|type`
+- `pass:[=>] new_field|type` # insert `null` into the field
 `+>`:: Retain the input field, and assign the result value to a new field.
 Format::: `field +> new_field|type`
 `pass:[->]`:: Discard the input fields, and assign the result value to a new field.

tessellate-main/src/main/java/io/clusterless/tessellate/parser/StatementParser.java

@@ -149,11 +149,18 @@ private static Parser<Void> op(String op) {
             );
 
     public static Parser<Assignment> LITERAL_ASSIGNMENT =
-            Parsers.sequence(
-                    LITERAL_VALUE,
-                    ASSIGNMENT.followedBy(Scanners.many(IS_WHITESPACE)),
-                    FieldParser.fullFieldDeclaration.followedBy(EOF),
-                    Assignment::new
+            Parsers.or(
+                    Parsers.sequence(
+                            LITERAL_VALUE,
+                            ASSIGNMENT.followedBy(Scanners.many(IS_WHITESPACE)),
+                            FieldParser.fullFieldDeclaration.followedBy(EOF),
+                            Assignment::new
+                    ),
+                    Parsers.sequence(
+                            ASSIGNMENT.followedBy(Scanners.many(IS_WHITESPACE)),
+                            FieldParser.fullFieldDeclaration.followedBy(EOF),
+                            Assignment::new
+                    )
             );
 
     private static final Parser<JoinType> JOIN_NAME = Parsers.sequence(

tessellate-main/src/main/java/io/clusterless/tessellate/parser/ast/Assignment.java

@@ -22,6 +22,12 @@ public Assignment(String literal, Op op, Field result) {
         this.result = result;
     }
 
+    public Assignment(Op op, Field result) {
+        this.literal = null;
+        this.op = op;
+        this.result = result;
+    }
+
     public String literal() {
         return literal;
     }

tessellate-main/src/test/java/io/clusterless/tessellate/parser/StatementParserTest.java

@@ -51,6 +51,7 @@ void parse() {
         assertInstanceOf(Intrinsic.class, assertInstanceOf(Operation.class, StatementParser.parse("^toJson{} -> json")).exp());
 
         assertNotNull(StatementParser.parse("five => intoField|type"));
+        assertNotNull(StatementParser.parse("=> emptyValue"));
     }
 
     @Test

tessellate-main/src/test/java/io/clusterless/tessellate/pipeline/PipelineParseTest.java

@@ -19,6 +19,7 @@
 import io.clusterless.tessellate.model.PipelineDef;
 import io.clusterless.tessellate.model.SourcePartition;
 import io.clusterless.tessellate.model.Transform;
+import io.clusterless.tessellate.parser.ast.Assignment;
 import io.clusterless.tessellate.parser.ast.Op;
 import io.clusterless.tessellate.type.WrappedCoercibleType;
 import io.clusterless.tessellate.util.json.JSONUtil;
@@ -28,6 +29,7 @@
 import java.util.TimeZone;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
 
 @TestWithResources
 public class PipelineParseTest {
@@ -51,12 +53,14 @@ void name(@GivenTextResource("config/pipeline.json") String pipelineJson) throws
 
         Transform transform = pipeline.transform();
 
-        assertEquals(6, transform.statements().size());
+        assertEquals(7, transform.statements().size());
         assertEquals(new Op(), transform.statements().get(0).op());
         assertEquals(new Op("->"), transform.statements().get(1).op());
         assertEquals(new Op("+>"), transform.statements().get(2).op());
         assertEquals(new Op("->"), transform.statements().get(3).op());
         assertEquals(new Op("=>"), transform.statements().get(4).op());
         assertEquals(new Op("=>"), transform.statements().get(5).op());
+        assertEquals(new Op("=>"), transform.statements().get(6).op());
+        assertNull(((Assignment) transform.statements().get(6)).literal());
     }
 }

tessellate-main/src/test/java/io/clusterless/tessellate/pipeline/PipelineTest.java

@@ -319,7 +319,8 @@ void awsS3AccessLogWithTransforms(@PathForResource("/data/aws-s3-access-log.txt"
                         "httpStatus->httpStatusString|String",// rename: "httpStatus|Integer"
                         "httpStatusString|Integer",// coerce: "httpStatusString|String"
                         "requestID->",// discard: "httpStatusString|String"
-                        "200=>code|Integer" // insert
+                        "200=>code|Integer", // insert
+                        "=> _empty|Integer" // insert
                 ))
                 .withSink(Sink.builder()
                         .withOutput(output)
@@ -346,7 +347,7 @@ void awsS3AccessLogWithTransforms(@PathForResource("/data/aws-s3-access-log.txt"
         CascadingTesting.validateEntries(
                 pipeline.flow().openSink(),
                 l -> assertEquals(4, l, "wrong length"), // headers are declared so aren't counted
-                l -> assertEquals(merged.source().schema().declared().size() + 1 + 1, l, "wrong size"),
+                l -> assertEquals(merged.source().schema().declared().size() + 1 + 1 + 1, l, "wrong size"),
                 l -> {
                 }
         );

tessellate-main/src/test/resources/config/pipeline.json

@@ -24,6 +24,7 @@
     "three+>@three|DateTime|yyyyMMdd",
     "four->",
     "five=>_five",
-    "1689820455=>six|DateTime|yyyyMMdd"
+    "1689820455=>six|DateTime|yyyyMMdd",
+    "=> _null"
   ]
 }