Support enclosing character #1

Author: kazuki43zoo

README.md

@@ -11,17 +11,19 @@ Support following features.
 * Updating columns by specified expression(fixed value or dynamic value)
 * Ordering columns by specified order
 
+## Related libraries document
+
+* [SpEL provided by Spring Framework](https://docs.spring.io/spring-framework/docs/current/reference/html/core.html#expressions)
+* [FlatFileItemReader/FlatFileItemWriter provided by Spring Batch](https://docs.spring.io/spring-batch/docs/current/reference/html/readersAndWriters.html#flatFiles)
+
 ## Support CSV file specifications
 
 * The first line is header
 * The Record separator is LF or CRLF (Writing separator is OS dependent character)
-* No support an enclosing character (no support include a comma and record separator in column value)
+* Support an enclosing character(`"`) (support include a record separator(`,`) in column value)
+* Support custom delimiter character(e.g. `\t`)
 * The default encoding is UTF-8 (can change an any encoding using command line argument)
 
-> **NOTE:**
-> 
-> Will have a plan to supporting various csv format at future.
-
 ## How to specify target files
 
 Search files that matches conditions specified by `--dir` and `--files`.
@@ -95,6 +97,10 @@ Search files that matches conditions specified by `--dir` and `--files`.
        bar:
          "10": "2"
          "20": "1"
+  --delimiter
+       delimiter character (default: ",")
+  --ignore-escaped-enclosure
+       whether escape a enclosing character on writing (default: false)
   --h (--help)
        print help
 
@@ -166,3 +172,9 @@ $ ./mvnw clean verify -DskipTests
 ```
 $ java -jar target/csv-bulk-commands.jar
 ```
+
+### How to specify Tab character on delimiter with bash
+
+```
+$ java -jar target/csv-bulk-commands.jar ... --delimiter=$'\t'
+```

pom.xml

@@ -21,6 +21,10 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.springframework.batch</groupId>
+            <artifactId>spring-batch-infrastructure</artifactId>
+        </dependency>
 
         <dependency>
             <groupId>org.springframework.boot</groupId>

src/main/java/com/example/tools/AbstractColumnProcessor.java

@@ -0,0 +1,144 @@
+package com.example.tools;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.batch.item.ExecutionContext;
+import org.springframework.batch.item.ItemStreamReader;
+import org.springframework.batch.item.ItemStreamWriter;
+import org.springframework.batch.item.file.builder.FlatFileItemReaderBuilder;
+import org.springframework.batch.item.file.builder.FlatFileItemWriterBuilder;
+import org.springframework.batch.item.file.separator.DefaultRecordSeparatorPolicy;
+import org.springframework.batch.item.file.transform.DefaultFieldSet;
+import org.springframework.batch.item.file.transform.ExtractorLineAggregator;
+import org.springframework.batch.item.file.transform.FieldSet;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.core.io.FileSystemResource;
+import org.springframework.util.StringUtils;
+
+import java.nio.charset.Charset;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+public abstract class AbstractColumnProcessor {
+
+  protected final Logger logger = LoggerFactory.getLogger(getClass());
+
+  protected List<String> readHeaderColumns(Path file, Charset encoding, String delimiter) throws Exception {
+    List<String> headerColumns = new ArrayList<>();
+    ItemStreamReader<String[]> headerReader = new FlatFileItemReaderBuilder<String[]>()
+        .recordSeparatorPolicy(new DefaultRecordSeparatorPolicy())
+        .lineTokenizer(x -> new DefaultFieldSet(StringUtils.delimitedListToStringArray(x, Optional.ofNullable(delimiter).orElse(","))))
+        .fieldSetMapper(FieldSet::getValues)
+        .encoding(encoding.name())
+        .resource(new FileSystemResource(file.toFile()))
+        .name("default")
+        .linesToSkip(0)
+        .maxItemCount(1)
+        .build();
+    try {
+      headerReader.open(new ExecutionContext());
+      String[] line = headerReader.read();
+      if (line == null) {
+        return null;
+      }
+      headerColumns.addAll(Arrays.asList(line));
+    } finally {
+      headerReader.close();
+    }
+    return headerColumns;
+  }
+
+  protected List<String[]> readDataColumns(String[] columnNames, Path file, Charset encoding, String delimiter) throws Exception {
+    ItemStreamReader<String[]> reader = new FlatFileItemReaderBuilder<String[]>()
+        .delimited().delimiter(Optional.ofNullable(delimiter).orElse(","))
+        .names(columnNames)
+        .fieldSetMapper(FieldSet::getValues)
+        .recordSeparatorPolicy(new DefaultRecordSeparatorPolicy())
+        .encoding(encoding.name())
+        .resource(new FileSystemResource(file.toFile()))
+        .name("default")
+        .linesToSkip(1).build();
+    List<String[]> lines = new ArrayList<>();
+    reader.open(new ExecutionContext());
+    try {
+      String[] items;
+      while ((items = reader.read()) != null) {
+        lines.add(items);
+      }
+    } finally {
+      reader.close();
+    }
+    return lines;
+  }
+
+  protected void writeLines(List<String[]> lines, Path file, Charset encoding, String delimiter, Boolean ignoreEscapedEnclosure) throws Exception {
+    EnclosableDelimitedLineAggregator<String[]> aggregator = new EnclosableDelimitedLineAggregator<>();
+    aggregator.setIgnoreEscapedEnclosure(Optional.ofNullable(ignoreEscapedEnclosure).orElse(false));
+    aggregator.setDelimiter(Optional.ofNullable(delimiter).orElse(",").toCharArray()[0]);
+    aggregator.afterPropertiesSet();
+    ItemStreamWriter<String[]> itemWriter = new FlatFileItemWriterBuilder<String[]>()
+        .lineAggregator(aggregator)
+        .encoding(encoding.name())
+        .resource(new FileSystemResource(file.toFile()))
+        .name("default")
+        .shouldDeleteIfEmpty(false)
+        .transactional(false)
+        .build();
+    itemWriter.open(new ExecutionContext());
+    try {
+      itemWriter.write(lines);
+    } finally {
+      itemWriter.close();
+    }
+  }
+
+  private static class EnclosableDelimitedLineAggregator<T> extends ExtractorLineAggregator<T> implements InitializingBean {
+    protected boolean ignoreEscapedEnclosure = false;
+    private final String enclosure = "\"";
+    private final String escapedEnclosure;
+    private String delimiter;
+
+    public EnclosableDelimitedLineAggregator() {
+      this.escapedEnclosure = this.enclosure + this.enclosure;
+      this.delimiter = ",";
+    }
+
+    public void setIgnoreEscapedEnclosure(boolean ignoreEscapedEnclosure) {
+      this.ignoreEscapedEnclosure = ignoreEscapedEnclosure;
+    }
+
+    public void setDelimiter(char delimiter) {
+      this.delimiter = String.valueOf(delimiter);
+    }
+
+    public void afterPropertiesSet() {
+      if (this.enclosure.equals(this.delimiter)) {
+        throw new IllegalStateException("the delimiter and enclosure must be different. [value:" + this.enclosure + "]");
+      }
+    }
+
+    protected String doAggregate(Object[] fields) {
+      return Arrays.stream(fields)
+          .map(Object::toString)
+          .map((field) -> this.hasTargetChar(field) ? this.encloseAndEscape(field) : field)
+          .collect(Collectors.joining(this.delimiter));
+    }
+
+    private boolean hasTargetChar(String field) {
+      return field.contains(this.delimiter) || field.contains(this.enclosure) || this.containsCrlf(field);
+    }
+
+    private boolean containsCrlf(String field) {
+      return field.contains("\r") || field.contains("\n");
+    }
+
+    private String encloseAndEscape(String field) {
+      return this.enclosure + (ignoreEscapedEnclosure ? field : field.replace(this.enclosure, this.escapedEnclosure)) + this.enclosure;
+    }
+  }
+
+}

src/main/java/com/example/tools/AddingColumnProcessor.java

@@ -1,17 +1,13 @@
 package com.example.tools;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.springframework.expression.Expression;
 import org.springframework.expression.ExpressionParser;
 import org.springframework.expression.spel.standard.SpelExpressionParser;
 import org.springframework.expression.spel.support.StandardEvaluationContext;
-import org.springframework.util.StringUtils;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.nio.charset.Charset;
-import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -20,24 +16,23 @@
 import java.util.Map;
 import java.util.Objects;
 
-public class AddingColumnProcessor {
+public class AddingColumnProcessor extends AbstractColumnProcessor {
 
   static final AddingColumnProcessor INSTANCE = new AddingColumnProcessor();
-  private static final Logger LOGGER = LoggerFactory.getLogger(AddingColumnProcessor.class);
   private static final ExpressionParser EXPRESSION_PARSER = new SpelExpressionParser();
 
   private AddingColumnProcessor() {
     // NOP
   }
 
-  void execute(List<String> columnNames, List<String> columnValues, Path file, Charset encoding, Map<String, Object> valueMappings) {
+  void execute(List<String> columnNames, List<String> columnValues, Path file, Charset encoding, Map<String, Object> valueMappings, String delimiter, Boolean ignoreEscapedEnclosure) {
     try {
-      List<String> lines = Files.readAllLines(file, encoding);
-      if (lines.isEmpty()) {
-        LOGGER.warn("Skip adding because file is empty. file:{}", file);
+      final List<String> originalHeaderColumns = readHeaderColumns(file, encoding, delimiter);
+      if (originalHeaderColumns == null) {
+        logger.warn("Skip adding because file is empty. file:{}", file);
         return;
       }
-      List<String> headerColumns = new ArrayList<>(Arrays.asList(StringUtils.commaDelimitedListToStringArray(lines.remove(0))));
+      final List<String> headerColumns = new ArrayList<>(originalHeaderColumns);
       Map<String, Integer> headerIndexMap = new LinkedHashMap<>();
       for (String column : headerColumns) {
         headerIndexMap.put(column, headerIndexMap.size());
@@ -56,23 +51,26 @@ void execute(List<String> columnNames, List<String> columnValues, Path file, Cha
           validColumnValues.add(columnValues.get(entry.getKey()));
         }
       }
-      List<String> saveLines = new ArrayList<>();
-      saveLines.add(StringUtils.collectionToCommaDelimitedString(headerColumns));
-      for (String line : lines) {
-        List<String> valueColumns = new ArrayList<>(Arrays.asList(StringUtils.commaDelimitedListToStringArray(line)));
+      List<String[]> lines = readDataColumns(originalHeaderColumns.toArray(new String[0]), file, encoding, delimiter);
+      List<String[]> saveLines = new ArrayList<>();
+      saveLines.add(headerColumns.toArray(new String[0]));
+      for (String[] items : lines) {
+        List<String> valueColumns = new ArrayList<>(Arrays.asList(items));
         StandardEvaluationContext context = new StandardEvaluationContext();
         context.setVariable("_valueMappings", valueMappings);
         headerIndexMap.forEach((name, index) -> context.setVariable(name, valueColumns.get(index)));
         for (String value : validColumnValues) {
           Expression expression = EXPRESSION_PARSER.parseExpression(value);
           valueColumns.add(Objects.toString(expression.getValue(context)));
         }
-        saveLines.add(StringUtils.collectionToCommaDelimitedString(valueColumns));
+        saveLines.add(valueColumns.toArray(new String[0]));
       }
-
-      Files.write(file, saveLines, encoding);
+      writeLines(saveLines, file, encoding, delimiter, ignoreEscapedEnclosure);
     } catch (IOException e) {
       throw new UncheckedIOException(e);
+    } catch (Exception e) {
+      throw new IllegalStateException(e);
     }
   }
+
 }

src/main/java/com/example/tools/CsvBulkCommandsApplicationRunner.java

@@ -54,6 +54,10 @@ public void run(ApplicationArguments args) throws IOException {
       System.out.println("       bar:");
       System.out.println("         \"10\": \"2\"");
       System.out.println("         \"20\": \"1\"");
+      System.out.println("  --delimiter");
+      System.out.println("       delimiter character (default: \",\")");
+      System.out.println("  --ignore-escaped-enclosure");
+      System.out.println("       whether escape a enclosing character on writing (default: false)");
       System.out.println("  --h (--help)");
       System.out.println("       print help");
       System.out.println();
@@ -155,30 +159,38 @@ public void run(ApplicationArguments args) throws IOException {
       valueMappings = Collections.emptyMap();
     }
 
-    LOGGER.info("Start. command:{} dir:{} files:{} column-names:{} column-values:{} encoding:{} value-mappings:{}", command, dir, files, columnNames, columnValues, encoding, valueMappings);
+    String delimiter = args.containsOption("delimiter") ?
+        args.getOptionValues("delimiter").stream().findFirst().orElse(",") :
+        ",";
+
+    Boolean ignoreEscapedEnclosure = args.containsOption("ignore-escaped-enclosure") &&
+        Boolean.parseBoolean(args.getOptionValues("ignore-escaped-enclosure").stream().findFirst().orElse(null));
+
+    LOGGER.info("Start. command:{} dir:{} files:{} column-names:{} column-values:{} encoding:{} value-mappings:{} delimiter:{} ignore-escaped-enclosure:{}",
+        command, dir, files, columnNames, columnValues, encoding, valueMappings, delimiter, ignoreEscapedEnclosure);
 
     Files.walk(Paths.get(dir))
         .filter(Files::isRegularFile)
         .filter(file -> files.stream().anyMatch(x -> file.toString().replace('\\', '/').endsWith(x)))
-        .sorted().forEach(file -> execute(command, columnNames, columnValues, file, encoding, valueMappings));
+        .sorted().forEach(file -> execute(command, columnNames, columnValues, file, encoding, valueMappings, delimiter, ignoreEscapedEnclosure));
 
     LOGGER.info("End.");
   }
 
-  private void execute(String command, List<String> columnNames, List<String> columnValues, Path file, Charset encoding, Map<String, Object> valueMappings) {
+  private void execute(String command, List<String> columnNames, List<String> columnValues, Path file, Charset encoding, Map<String, Object> valueMappings, String delimiter, Boolean ignoreEscapedEnclosure) {
     LOGGER.info("processing file:{}", file);
     switch (command) {
       case "adding-columns":
-        AddingColumnProcessor.INSTANCE.execute(columnNames, columnValues, file, encoding, valueMappings);
+        AddingColumnProcessor.INSTANCE.execute(columnNames, columnValues, file, encoding, valueMappings, delimiter, ignoreEscapedEnclosure);
         break;
       case "deleting-columns":
-        DeletingColumnProcessor.INSTANCE.execute(columnNames, file, encoding);
+        DeletingColumnProcessor.INSTANCE.execute(columnNames, file, encoding, delimiter, ignoreEscapedEnclosure);
         break;
       case "updating-columns":
-        UpdatingColumnProcessor.INSTANCE.execute(columnNames, columnValues, file, encoding, valueMappings);
+        UpdatingColumnProcessor.INSTANCE.execute(columnNames, columnValues, file, encoding, valueMappings, delimiter, ignoreEscapedEnclosure);
         break;
       case "ordering-columns":
-        OrderingColumnProcessor.INSTANCE.execute(columnNames, file, encoding);
+        OrderingColumnProcessor.INSTANCE.execute(columnNames, file, encoding, delimiter, ignoreEscapedEnclosure);
         break;
       default:
         throw new UnsupportedOperationException(String.format("'%s' command not support. valid-commands:%s", command, "[adding-columns, deleting-columns, updating-columns, ordering-columns]"));