Implementation and tests required to ensure that command output encod…

…ing does not raise an exception (#405) Signed-off-by: James Couball <jcouball@yahoo.com>Signed-off-by: Agora Security <github@agora-security.com>
ruby-git · AgoraSecurity · Aug 22, 2018 · Aug 24, 2018 · Oct 2, 2018 · Feb 17, 2019
commit 84a271f28c4bd5c58b0afb46da7c207740fd2d31
diff --git a/git.gemspec b/git.gemspec
@@ -15,6 +15,8 @@ Gem::Specification.new do |s|
   s.required_rubygems_version = Gem::Requirement.new('>= 0') if s.respond_to?(:required_rubygems_version=)
   s.requirements = ['git 1.6.0.0, or greater']
 
+  s.add_runtime_dependency 'rchardet', '~> 1.8'
+
   s.add_development_dependency 'rake'
   s.add_development_dependency 'rdoc'
   s.add_development_dependency 'test-unit', '>=2', '< 4'

diff --git a/lib/git/diff.rb b/lib/git/diff.rb
@@ -127,12 +127,7 @@ def process_full_diff
         }
         final = {}
         current_file = nil
-        if @full_diff.encoding.name != "UTF-8"
-          full_diff_utf8_encoded = @full_diff.encode("UTF-8", "binary", { :invalid => :replace, :undef => :replace })
-        else
-          full_diff_utf8_encoded = @full_diff
-        end
-        full_diff_utf8_encoded.split("\n").each do |line|
+        @full_diff.split("\n").each do |line|
           if m = /^diff --git a\/(.*?) b\/(.*?)/.match(line)
             current_file = m[1]
             final[current_file] = defaults.merge({:patch => line, :path => current_file})

diff --git a/lib/git/lib.rb b/lib/git/lib.rb
@@ -1,3 +1,4 @@
+require 'rchardet'
 require 'tempfile'
 
 module Git
@@ -901,16 +902,7 @@ def meets_required_version?
     ENV_VARIABLE_NAMES = ['GIT_DIR', 'GIT_WORK_TREE', 'GIT_INDEX_FILE', 'GIT_SSH']
 
     def command_lines(cmd, opts = [], chdir = true, redirect = '')
-      cmd_op = command(cmd, opts, chdir)
-      if cmd_op.encoding.name != "UTF-8"
-        op = cmd_op.encode("UTF-8", "binary", {
-          :invalid => :replace,
-          :undef => :replace
-        })
-      else
-        op = cmd_op
-      end
-      op.split("\n")
+      command(cmd, opts, chdir).lines.map(&:chomp)
     end
 
     # Takes the current git's system ENV variables and store them.
@@ -1040,10 +1032,35 @@ def log_path_options(opts)
       arr_opts
     end
 
+    def default_encoding
+      __ENCODING__.name
+    end
+
+    def best_guess_encoding
+      # Encoding::ASCII_8BIT.name
+      Encoding::UTF_8.name
+    end
+
+    def detected_encoding(str)
+      CharDet.detect(str)['encoding'] || best_guess_encoding
+    end
+
+    def encoding_options
+      { invalid: :replace, undef: :replace }
+    end
+
+    def normalize_encoding(str)
+      return str if str.valid_encoding? && str.encoding == default_encoding
+
+      return str.encode(default_encoding, str.encoding, encoding_options) if str.valid_encoding?
+
+      str.encode(default_encoding, detected_encoding(str), encoding_options)
+    end
+
     def run_command(git_cmd, &block)
       return IO.popen(git_cmd, &block) if block_given?
 
-      `#{git_cmd}`.chomp
+      `#{git_cmd}`.chomp.lines.map { |l| normalize_encoding(l) }.join
     end
 
     def escape(s)

diff --git a/tests/files/encoding/dot_git/COMMIT_EDITMSG b/tests/files/encoding/dot_git/COMMIT_EDITMSG
@@ -0,0 +1 @@
+A file with Japanese text
diff --git a/tests/files/encoding/dot_git/HEAD b/tests/files/encoding/dot_git/HEAD
@@ -0,0 +1 @@
+ref: refs/heads/master
diff --git a/tests/files/encoding/dot_git/config b/tests/files/encoding/dot_git/config
@@ -0,0 +1,7 @@
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = false
+	logallrefupdates = true
+	ignorecase = true
+	precomposeunicode = true
diff --git a/tests/files/encoding/dot_git/description b/tests/files/encoding/dot_git/description
@@ -0,0 +1 @@
+Unnamed repository; edit this file 'description' to name the repository.
diff --git a/tests/files/encoding/dot_git/hooks/applypatch-msg.sample b/tests/files/encoding/dot_git/hooks/applypatch-msg.sample
@@ -0,0 +1,15 @@
+#!/bin/sh
+#
+# An example hook script to check the commit log message taken by
+# applypatch from an e-mail message.
+#
+# The hook should exit with non-zero status after issuing an
+# appropriate message if it wants to stop the commit.  The hook is
+# allowed to edit the commit message file.
+#
+# To enable this hook, rename this file to "applypatch-msg".
+
+. git-sh-setup
+commitmsg="$(git rev-parse --git-path hooks/commit-msg)"
+test -x "$commitmsg" && exec "$commitmsg" ${1+"$@"}
+:
diff --git a/tests/files/encoding/dot_git/hooks/commit-msg.sample b/tests/files/encoding/dot_git/hooks/commit-msg.sample
@@ -0,0 +1,24 @@
+#!/bin/sh
+#
+# An example hook script to check the commit log message.
+# Called by "git commit" with one argument, the name of the file
+# that has the commit message.  The hook should exit with non-zero
+# status after issuing an appropriate message if it wants to stop the
+# commit.  The hook is allowed to edit the commit message file.
+#
+# To enable this hook, rename this file to "commit-msg".
+
+# Uncomment the below to add a Signed-off-by line to the message.
+# Doing this in a hook is a bad idea in general, but the prepare-commit-msg
+# hook is more suited to it.
+#
+# SOB=$(git var GIT_AUTHOR_IDENT | sed -n 's/^\(.*>\).*$/Signed-off-by: \1/p')
+# grep -qs "^$SOB" "$1" || echo "$SOB" >> "$1"
+
+# This example catches duplicate Signed-off-by lines.
+
+test "" = "$(grep '^Signed-off-by: ' "$1" |
+	 sort | uniq -c | sed -e '/^[ 	]*1[ 	]/d')" || {
+	echo >&2 Duplicate Signed-off-by lines.
+	exit 1
+}
diff --git a/tests/files/encoding/dot_git/hooks/fsmonitor-watchman.sample b/tests/files/encoding/dot_git/hooks/fsmonitor-watchman.sample
@@ -0,0 +1,114 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use IPC::Open2;
+
+# An example hook script to integrate Watchman
+# (https://facebook.github.io/watchman/) with git to speed up detecting
+# new and modified files.
+#
+# The hook is passed a version (currently 1) and a time in nanoseconds
+# formatted as a string and outputs to stdout all files that have been
+# modified since the given time. Paths must be relative to the root of
+# the working tree and separated by a single NUL.
+#
+# To enable this hook, rename this file to "query-watchman" and set
+# 'git config core.fsmonitor .git/hooks/query-watchman'
+#
+my ($version, $time) = @ARGV;
+
+# Check the hook interface version
+
+if ($version == 1) {
+	# convert nanoseconds to seconds
+	$time = int $time / 1000000000;
+} else {
+	die "Unsupported query-fsmonitor hook version '$version'.\n" .
+	    "Falling back to scanning...\n";
+}
+
+my $git_work_tree;
+if ($^O =~ 'msys' || $^O =~ 'cygwin') {
+	$git_work_tree = Win32::GetCwd();
+	$git_work_tree =~ tr/\\/\//;
+} else {
+	require Cwd;
+	$git_work_tree = Cwd::cwd();
+}
+
+my $retry = 1;
+
+launch_watchman();
+
+sub launch_watchman {
+
+	my $pid = open2(\*CHLD_OUT, \*CHLD_IN, 'watchman -j --no-pretty')
+	    or die "open2() failed: $!\n" .
+	    "Falling back to scanning...\n";
+
+	# In the query expression below we're asking for names of files that
+	# changed since $time but were not transient (ie created after
+	# $time but no longer exist).
+	#
+	# To accomplish this, we're using the "since" generator to use the
+	# recency index to select candidate nodes and "fields" to limit the
+	# output to file names only. Then we're using the "expression" term to
+	# further constrain the results.
+	#
+	# The category of transient files that we want to ignore will have a
+	# creation clock (cclock) newer than $time_t value and will also not
+	# currently exist.
+
+	my $query = <<"	END";
+		["query", "$git_work_tree", {
+			"since": $time,
+			"fields": ["name"],
+			"expression": ["not", ["allof", ["since", $time, "cclock"], ["not", "exists"]]]
+		}]
+	END
+
+	print CHLD_IN $query;
+	close CHLD_IN;
+	my $response = do {local $/; <CHLD_OUT>};
+
+	die "Watchman: command returned no output.\n" .
+	    "Falling back to scanning...\n" if $response eq "";
+	die "Watchman: command returned invalid output: $response\n" .
+	    "Falling back to scanning...\n" unless $response =~ /^\{/;
+
+	my $json_pkg;
+	eval {
+		require JSON::XS;
+		$json_pkg = "JSON::XS";
+		1;
+	} or do {
+		require JSON::PP;
+		$json_pkg = "JSON::PP";
+	};
+
+	my $o = $json_pkg->new->utf8->decode($response);
+
+	if ($retry > 0 and $o->{error} and $o->{error} =~ m/unable to resolve root .* directory (.*) is not watched/) {
+		print STDERR "Adding '$git_work_tree' to watchman's watch list.\n";
+		$retry--;
+		qx/watchman watch "$git_work_tree"/;
+		die "Failed to make watchman watch '$git_work_tree'.\n" .
+		    "Falling back to scanning...\n" if $? != 0;
+
+		# Watchman will always return all files on the first query so
+		# return the fast "everything is dirty" flag to git and do the
+		# Watchman query just to get it over with now so we won't pay
+		# the cost in git to look up each individual file.
+		print "/\0";
+		eval { launch_watchman() };
+		exit 0;
+	}
+
+	die "Watchman: $o->{error}.\n" .
+	    "Falling back to scanning...\n" if $o->{error};
+
+	binmode STDOUT, ":utf8";
+	local $, = "\0";
+	print @{$o->{files}};
+}
diff --git a/tests/files/encoding/dot_git/hooks/post-update.sample b/tests/files/encoding/dot_git/hooks/post-update.sample
@@ -0,0 +1,8 @@
+#!/bin/sh
+#
+# An example hook script to prepare a packed repository for use over
+# dumb transports.
+#
+# To enable this hook, rename this file to "post-update".
+
+exec git update-server-info
diff --git a/tests/files/encoding/dot_git/hooks/pre-applypatch.sample b/tests/files/encoding/dot_git/hooks/pre-applypatch.sample
@@ -0,0 +1,14 @@
+#!/bin/sh
+#
+# An example hook script to verify what is about to be committed
+# by applypatch from an e-mail message.
+#
+# The hook should exit with non-zero status after issuing an
+# appropriate message if it wants to stop the commit.
+#
+# To enable this hook, rename this file to "pre-applypatch".
+
+. git-sh-setup
+precommit="$(git rev-parse --git-path hooks/pre-commit)"
+test -x "$precommit" && exec "$precommit" ${1+"$@"}
+:
diff --git a/tests/files/encoding/dot_git/hooks/pre-commit.sample b/tests/files/encoding/dot_git/hooks/pre-commit.sample
@@ -0,0 +1,49 @@
+#!/bin/sh
+#
+# An example hook script to verify what is about to be committed.
+# Called by "git commit" with no arguments.  The hook should
+# exit with non-zero status after issuing an appropriate message if
+# it wants to stop the commit.
+#
+# To enable this hook, rename this file to "pre-commit".
+
+if git rev-parse --verify HEAD >/dev/null 2>&1
+then
+	against=HEAD
+else
+	# Initial commit: diff against an empty tree object
+	against=$(git hash-object -t tree /dev/null)
+fi
+
+# If you want to allow non-ASCII filenames set this variable to true.
+allownonascii=$(git config --bool hooks.allownonascii)
+
+# Redirect output to stderr.
+exec 1>&2
+
+# Cross platform projects tend to avoid non-ASCII filenames; prevent
+# them from being added to the repository. We exploit the fact that the
+# printable range starts at the space character and ends with tilde.
+if [ "$allownonascii" != "true" ] &&
+	# Note that the use of brackets around a tr range is ok here, (it's
+	# even required, for portability to Solaris 10's /usr/bin/tr), since
+	# the square bracket bytes happen to fall in the designated range.
+	test $(git diff --cached --name-only --diff-filter=A -z $against |
+	  LC_ALL=C tr -d '[ -~]\0' | wc -c) != 0
+then
+	cat <<\EOF
+Error: Attempt to add a non-ASCII file name.
+
+This can cause problems if you want to work with people on other platforms.
+
+To be portable it is advisable to rename the file.
+
+If you know what you are doing you can disable this check using:
+
+  git config hooks.allownonascii true
+EOF
+	exit 1
+fi
+
+# If there are whitespace errors, print the offending file names and fail.
+exec git diff-index --check --cached $against --
diff --git a/tests/files/encoding/dot_git/hooks/pre-push.sample b/tests/files/encoding/dot_git/hooks/pre-push.sample
@@ -0,0 +1,53 @@
+#!/bin/sh
+
+# An example hook script to verify what is about to be pushed.  Called by "git
+# push" after it has checked the remote status, but before anything has been
+# pushed.  If this script exits with a non-zero status nothing will be pushed.
+#
+# This hook is called with the following parameters:
+#
+# $1 -- Name of the remote to which the push is being done
+# $2 -- URL to which the push is being done
+#
+# If pushing without using a named remote those arguments will be equal.
+#
+# Information about the commits which are being pushed is supplied as lines to
+# the standard input in the form:
+#
+#   <local ref> <local sha1> <remote ref> <remote sha1>
+#
+# This sample shows how to prevent push of commits where the log message starts
+# with "WIP" (work in progress).
+
+remote="$1"
+url="$2"
+
+z40=0000000000000000000000000000000000000000
+
+while read local_ref local_sha remote_ref remote_sha
+do
+	if [ "$local_sha" = $z40 ]
+	then
+		# Handle delete
+		:
+	else
+		if [ "$remote_sha" = $z40 ]
+		then
+			# New branch, examine all commits
+			range="$local_sha"
+		else
+			# Update to existing branch, examine new commits
+			range="$remote_sha..$local_sha"
+		fi
+
+		# Check for WIP commit
+		commit=`git rev-list -n 1 --grep '^WIP' "$range"`
+		if [ -n "$commit" ]
+		then
+			echo >&2 "Found WIP commit in $local_ref, not pushing"
+			exit 1
+		fi
+	fi
+done
+
+exit 0
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Unnamed repository; edit this file 'description' to name the repository.