Keep an escaped value that equals the null string (#626).

PR plus fixes to the PR to pass the build.

Author: garydgregory

src/changes/changes.xml

@@ -76,6 +76,7 @@
       <action type="fix" dev="ggregory" due-to="Gary Gregory, Naveed Khan">Fix quadratic CSVFormat.printWithEscapes(Reader) delimiter look-ahead (#631).</action>
       <action type="fix" dev="ggregory" due-to="Gary Gregory">General Javadoc improvements.</action>
       <action type="fix" dev="ggregory" due-to="Gary Gregory, Naveed Khan">Validate CSVFormat invariants when deserializing (#632).</action>
+      <action type="fix" dev="ggregory" due-to="Gary Gregory, saleem malik">Keep an escaped value that equals the null string (#626).</action>
       <!-- ADD -->
       <action type="add" dev="ggregory" due-to="Gary Gregory, Indy, Sylvia van Os" issue="CSV-307">Add an "Android Compatibility" section to the web site.</action>
       <action type="add" dev="ggregory" due-to="Ruiqi Dong, Gary Gregory" issue="CSV-325">Add CSVParser.Builder.setByteOffset(long) (#604).</action>

src/main/java/org/apache/commons/csv/CSVParser.java

@@ -819,8 +819,11 @@ private String handleNull(final String input) {
         final String nullString = format.getNullString();
         final boolean strictQuoteMode = isStrictQuoteMode();
         if (input.equals(nullString)) {
+            // A token that needed an escape translation cannot be the null marker: printing null emits the null
+            // string without escaping it (it may be quoted, but never escaped), so an escaped "\N" for nullString
+            // "\N" can only have come from a field whose value really is "\N".
             // nullString = NULL(String), distinguish between "NULL" and NULL in ALL_NON_NULL or NON_NUMERIC quote mode
-            return strictQuoteMode && isQuoted ? input : null;
+            return reusableToken.isEscaped || strictQuoteMode && isQuoted ? input : null;
         }
         // don't set nullString, distinguish between "" and ,, (absent values) in All_NON_NULL or NON_NUMERIC quote mode
         return strictQuoteMode && nullString == null && input.isEmpty() && !isQuoted ? null : input;

src/main/java/org/apache/commons/csv/Lexer.java

@@ -76,12 +76,15 @@ final class Lexer implements Closeable {
     private void appendNextEscapedCharacterToToken(final Token token) throws IOException {
         if (isEscapeDelimiter()) {
             token.content.append(delimiter);
+            token.isEscaped = true;
         } else {
             final int unescaped = readEscape();
             if (unescaped == EOF) { // unexpected char after escape
+                // The escape character is kept verbatim, so nothing was translated.
                 token.content.append((char) escape).append((char) reader.getLastChar());
             } else {
                 token.content.append((char) unescaped);
+                token.isEscaped = true;
             }
         }
     }

src/main/java/org/apache/commons/csv/Token.java

@@ -61,11 +61,15 @@ enum Type {
 
     boolean isQuoted;
 
+    /** True when an escape sequence in the input was translated while building {@link #content}. */
+    boolean isEscaped;
+
     void reset() {
         content.setLength(0);
         type = INVALID;
         isReady = false;
         isQuoted = false;
+        isEscaped = false;
     }
 
     /**

src/test/java/org/apache/commons/csv/CSVParserTest.java

@@ -662,6 +662,24 @@ void testEndOfFileBehaviorExcel() throws Exception {
         }
     }
 
+    @ParameterizedTest
+    @EnumSource(value = CSVFormat.Predefined.class, names = { "MySQL", "PostgreSQLCsv", "PostgreSQLText", "Oracle" })
+    void testEscapedNullStringIsAValue(final CSVFormat.Predefined predefined) throws Exception {
+        // "\N" is the null string for MySQL, PostgreSQL Text and Oracle; PostgreSQL CSV uses an empty null
+        // string. In every case a field whose value equals "\N" must round trip as that value, not as null.
+        final String valueEqualToNullString = "\\N";
+        final CSVFormat format = predefined.getFormat();
+        final StringWriter writer = new StringWriter();
+        try (CSVPrinter printer = new CSVPrinter(writer, format)) {
+            printer.printRecord(valueEqualToNullString, null);
+        }
+        try (CSVParser parser = CSVParser.parse(writer.toString(), format)) {
+            final CSVRecord record = parser.nextRecord();
+            assertEquals(valueEqualToNullString, record.get(0));
+            assertNull(record.get(1));
+        }
+    }
+
     @Test
     void testExcelFormat1() throws IOException {
         final String code = "value1,value2,value3,value4\r\na,b,c,d\r\n  x,,,\r\n\r\n\"\"\"hello\"\"\",\"  \"\"world\"\"\",\"abc\ndef\",\r\n";

src/test/java/org/apache/commons/csv/CSVPrinterTest.java

@@ -158,13 +158,18 @@ private void doRandom(final CSVFormat format, final int iter) throws Exception {
     }
 
     /**
-     * Converts an input CSV array into expected output values, including NULLs. NULL strings are converted to null values because the parser will convert
-     * these strings to null.
+     * Converts an input CSV array into expected output values, including NULLs. A value equal to the null string is expected back as null only when the
+     * printer writes it verbatim (neither escaped nor quoted); once it is escaped or quoted the parser can tell it apart from a real null and reads it back
+     * as the value it is.
      */
-    private <T> T[] expectNulls(final T[] original, final CSVFormat csvFormat) {
+    private <T> T[] expectNulls(final T[] original, final CSVFormat csvFormat) throws IOException {
         final T[] fixed = original.clone();
+        final String nullString = csvFormat.getNullString();
+        if (nullString == null || !printsVerbatim(csvFormat, nullString)) {
+            return fixed;
+        }
         for (int i = 0; i < fixed.length; i++) {
-            if (Objects.equals(csvFormat.getNullString(), fixed[i])) {
+            if (Objects.equals(nullString, fixed[i])) {
                 fixed[i] = null;
             }
         }
@@ -196,6 +201,13 @@ private String printNullRecord(final CSVFormat format) throws IOException {
         return sw.toString();
     }
 
+    /** Tests whether the format prints the given value unchanged, in which case the parser cannot tell it from the null string. */
+    private boolean printsVerbatim(final CSVFormat csvFormat, final String value) throws IOException {
+        final StringBuilder sb = new StringBuilder();
+        csvFormat.print(value, sb, true);
+        return value.contentEquals(sb);
+    }
+
     private CSVPrinter printWithHeaderComments(final StringWriter sw, final Date now, final CSVFormat baseFormat) throws IOException {
         // Use withHeaderComments first to test CSV-145
         // @formatter:off