Assume strings are utf8 by default and fallback on encoding if necessary

kamaradclimber · kamaradclimber · commit 5856b704007d · 2016-12-07T17:21:18.000+01:00
Before this patch, we always assumed the string was ascii-8bit which does not work correctly with real utf8 strings (french accented chars or chineese chars). This patch tries to use the string as utf8 and fallback if necessary to the the ascii-8bit assumption if necessary. This should help with #295 and might replace the need for #301. Change-Id: Idac63fa10e5aefafa1eb99a6be4138cac5f90ea0
diff --git a/lib/git/lib.rb b/lib/git/lib.rb
@@ -859,13 +859,28 @@ def meets_required_version?
 
     def command_lines(cmd, opts = [], chdir = true, redirect = '')
       cmd_op = command(cmd, opts, chdir)
-      op = cmd_op.encode("UTF-8", "binary", {
-	  	:invalid => :replace,
-		:undef => :replace
-	  })
-      op.split("\n")
+      split_utf8(cmd_op)
     end
 
+    def split_utf8(s)
+      # ruby can think the string is utf8 but any action on the strings chars
+      # will trigger errors. Here we try to split utf8 strings.
+      # It it isn't utf8 or if it fails, we fallback to assume the input is
+      # "binary" encoding
+      result = begin
+                 s.encoding == Encoding::UTF_8 && s.split("\n")
+               rescue Encoding::InvalidByteSequenceError, Encoding::UndefinedConversionError
+                 nil
+               end
+
+      result ||= s.encode("UTF-8", "binary", {
+        :invalid => :replace,
+        :undef => :replace,
+      }).split("\n")
+      result
+    end
+
+
     # Takes the current git's system ENV variables and store them.
     def store_git_system_env_variables
       @git_system_env_variables = {}
