Add invalid: :replace for CSV.open (#130)

koic · web-flow · commit 5bf687341c03 · 2020-06-04T12:33:47.000+09:00
This PR adds `invalid: :replace` for `CSV.open`. It is a PR similar to #129.
diff --git a/lib/csv.rb b/lib/csv.rb
@@ -962,8 +962,10 @@ def generate_line(row, **options)
     #
     # possible options elements:
     #   hash form:
-    #     :undef => :replace # replace undefined conversion
-    #     :replace => string # replacement string ("?" or "\uFFFD" if not specified)
+    #     :invalid => nil      # raise error on invalid byte sequence (default)
+    #     :invalid => :replace # replace invalid byte sequence
+    #     :undef => :replace   # replace undefined conversion
+    #     :replace => string   # replacement string ("?" or "\uFFFD" if not specified)
     #
     # This method opens an IO object, and wraps that with CSV. This is intended
     # as the primary interface for writing a CSV file.
@@ -1026,6 +1028,7 @@ def open(filename, mode="r", **options)
       # wrap a File opened with the remaining +args+ with no newline
       # decorator
       file_opts = {universal_newline: false}.merge(options)
+      options.delete(:invalid)
       options.delete(:undef)
       options.delete(:replace)
 
diff --git a/test/csv/interface/test_read.rb b/test/csv/interface/test_read.rb
@@ -135,6 +135,36 @@ def test_open_invalid_byte_sequence_in_utf_8
     end
   end
 
+  def test_open_with_invalid_nil
+    CSV.open(@input.path, "w", encoding: Encoding::CP932, invalid: nil) do |rows|
+      error = assert_raise(Encoding::InvalidByteSequenceError) do
+        rows << ["\x82\xa0"]
+      end
+      assert_equal('"\x82" on UTF-8',
+                   error.message)
+    end
+  end
+
+  def test_open_with_invalid_replace
+    CSV.open(@input.path, "w", encoding: Encoding::CP932, invalid: :replace) do |rows|
+      rows << ["\x82\xa0".force_encoding(Encoding::UTF_8)]
+    end
+    CSV.open(@input.path, encoding: Encoding::CP932) do |csv|
+      assert_equal([["??"]],
+                   csv.to_a)
+    end
+  end
+
+  def test_open_with_invalid_replace_and_replace_string
+    CSV.open(@input.path, "w", encoding: Encoding::CP932, invalid: :replace, replace: "X") do |rows|
+      rows << ["\x82\xa0".force_encoding(Encoding::UTF_8)]
+    end
+    CSV.open(@input.path, encoding: Encoding::CP932) do |csv|
+      assert_equal([["XX"]],
+                   csv.to_a)
+    end
+  end
+
   def test_open_with_undef_replace
     # U+00B7 Middle Dot
     CSV.open(@input.path, "w", encoding: Encoding::CP932, undef: :replace) do |rows|
