Add context options to handle BOM

malarzm · malarzm · commit 618b90d50fad · 2019-10-08T00:10:31.000+02:00
diff --git a/src/Symfony/Component/Serializer/ByteOrderMark.php b/src/Symfony/Component/Serializer/ByteOrderMark.php
@@ -0,0 +1,31 @@
+<?php
+
+namespace Symfony\Component\Serializer;
+
+final class ByteOrderMark
+{
+    /**
+     *  UTF-8 BOM sequence.
+     */
+    const BOM_UTF8 = "\xEF\xBB\xBF";
+
+    /**
+     * UTF-16 BE BOM sequence.
+     */
+    const BOM_UTF16_BE = "\xFE\xFF";
+
+    /**
+     * UTF-16 LE BOM sequence.
+     */
+    const BOM_UTF16_LE = "\xFF\xFE";
+
+    /**
+     * UTF-32 BE BOM sequence.
+     */
+    const BOM_UTF32_BE = "\x00\x00\xFE\xFF";
+
+    /**
+     * UTF-32 LE BOM sequence.
+     */
+    const BOM_UTF32_LE = "\xFF\xFE\x00\x00";
+}
diff --git a/src/Symfony/Component/Serializer/CHANGELOG.md b/src/Symfony/Component/Serializer/CHANGELOG.md
@@ -5,6 +5,9 @@ CHANGELOG
 -----
 
  * deprecated the `XmlEncoder::TYPE_CASE_ATTRIBUTES` constant, use `XmlEncoder::TYPE_CAST_ATTRIBUTES` instead
+ * added common BOMs in `ByteOrderMark`
+ * added option to output BOM in CSV encoder via `CsvEncoder::OUTPUT_BOM` context option
+ * added option to skip BOM in CSV being decoded via `CsvEncoder::SKIP_INPUT_BOM` (defaults to `true`)
 
 4.3.0
 -----
diff --git a/src/Symfony/Component/Serializer/Encoder/CsvEncoder.php b/src/Symfony/Component/Serializer/Encoder/CsvEncoder.php
@@ -11,6 +11,7 @@
 
 namespace Symfony\Component\Serializer\Encoder;
 
+use Symfony\Component\Serializer\ByteOrderMark;
 use Symfony\Component\Serializer\Exception\InvalidArgumentException;
 
 /**
@@ -30,6 +31,8 @@ class CsvEncoder implements EncoderInterface, DecoderInterface
     const ESCAPE_FORMULAS_KEY = 'csv_escape_formulas';
     const AS_COLLECTION_KEY = 'as_collection';
     const NO_HEADERS_KEY = 'no_headers';
+    const OUTPUT_BOM = 'output_bom';
+    const SKIP_INPUT_BOM = 'skip_input_bom';
 
     private $formulasStartCharacters = ['=', '-', '+', '@'];
     private $defaultContext = [
@@ -40,6 +43,8 @@ class CsvEncoder implements EncoderInterface, DecoderInterface
         self::HEADERS_KEY => [],
         self::KEY_SEPARATOR_KEY => '.',
         self::NO_HEADERS_KEY => false,
+        self::OUTPUT_BOM => '',
+        self::SKIP_INPUT_BOM => true,
     ];
 
     /**
@@ -90,7 +95,7 @@ public function encode($data, $format, array $context = [])
             }
         }
 
-        list($delimiter, $enclosure, $escapeChar, $keySeparator, $headers, $escapeFormulas) = $this->getCsvOptions($context);
+        list($delimiter, $enclosure, $escapeChar, $keySeparator, $headers, $escapeFormulas, $outputBOM) = $this->getCsvOptions($context);
 
         foreach ($data as &$value) {
             $flattened = [];
@@ -101,6 +106,8 @@ public function encode($data, $format, array $context = [])
 
         $headers = array_merge(array_values($headers), array_diff($this->extractHeaders($data), $headers));
 
+        fwrite($handle, $outputBOM);
+
         if (!($context[self::NO_HEADERS_KEY] ?? false)) {
             fputcsv($handle, $headers, $delimiter, $enclosure, $escapeChar);
         }
@@ -134,6 +141,11 @@ public function decode($data, $format, array $context = [])
         fwrite($handle, $data);
         rewind($handle);
 
+        if (($context[self::SKIP_INPUT_BOM] ?? true) === false) {
+            $inputBom = $this->determineBom(substr($data, 0, 4));
+            fseek($handle, strlen($inputBom));
+        }
+
         $headers = null;
         $nbHeaders = 0;
         $headerCount = [];
@@ -238,12 +250,13 @@ private function getCsvOptions(array $context): array
         $keySeparator = $context[self::KEY_SEPARATOR_KEY] ?? $this->defaultContext[self::KEY_SEPARATOR_KEY];
         $headers = $context[self::HEADERS_KEY] ?? $this->defaultContext[self::HEADERS_KEY];
         $escapeFormulas = $context[self::ESCAPE_FORMULAS_KEY] ?? $this->defaultContext[self::ESCAPE_FORMULAS_KEY];
+        $outputBOM = $context[self::OUTPUT_BOM] ?? $this->defaultContext[self::OUTPUT_BOM];
 
         if (!\is_array($headers)) {
             throw new InvalidArgumentException(sprintf('The "%s" context variable must be an array or null, given "%s".', self::HEADERS_KEY, \gettype($headers)));
         }
 
-        return [$delimiter, $enclosure, $escapeChar, $keySeparator, $headers, $escapeFormulas];
+        return [$delimiter, $enclosure, $escapeChar, $keySeparator, $headers, $escapeFormulas, $outputBOM];
     }
 
     /**
@@ -281,4 +294,18 @@ private function extractHeaders(iterable $data): array
 
         return $headers;
     }
+
+    private function determineBom(string $text): string
+    {
+        static $list;
+
+        $list = $list ?? (new \ReflectionClass(ByteOrderMark::class))->getConstants();
+        foreach ($list as $sequence) {
+            if (0 === strpos($text, $sequence)) {
+                return $sequence;
+            }
+        }
+
+        return '';
+    }
 }
diff --git a/src/Symfony/Component/Serializer/Tests/Encoder/CsvEncoderTest.php b/src/Symfony/Component/Serializer/Tests/Encoder/CsvEncoderTest.php
@@ -12,6 +12,7 @@
 namespace Symfony\Component\Serializer\Tests\Encoder;
 
 use PHPUnit\Framework\TestCase;
+use Symfony\Component\Serializer\ByteOrderMark;
 use Symfony\Component\Serializer\Encoder\CsvEncoder;
 
 /**
@@ -595,4 +596,29 @@ public function testDecodeWithoutHeader()
             CsvEncoder::NO_HEADERS_KEY => true,
         ]));
     }
+
+    public function testBOMIsAddedOnDemand()
+    {
+        $value = ['foo' => 'hello', 'bar' => 'hey ho'];
+
+        $this->assertEquals(ByteOrderMark::BOM_UTF8 . <<<'CSV'
+foo,bar
+hello,"hey ho"
+
+CSV
+            , $this->encoder->encode($value, 'csv', [CsvEncoder::OUTPUT_BOM => ByteOrderMark::BOM_UTF8]));
+    }
+
+    public function testBOMIsStripped()
+    {
+        $csv = ByteOrderMark::BOM_UTF8 . <<<'CSV'
+foo,bar
+hello,"hey ho"
+
+CSV;
+        $this->assertEquals(
+            ['foo' => 'hello', 'bar' => 'hey ho'],
+            $this->encoder->decode($csv, 'csv', [CsvEncoder::SKIP_INPUT_BOM => false])
+        );
+    }
 }
