From 7cd263e6a0d53f793ba41b249f4a78f84c4dc016 Mon Sep 17 00:00:00 2001 From: Robert Haines Date: Tue, 3 Apr 2018 23:45:04 +0100 Subject: [PATCH 01/36] Clean up use of file_name in Zip::File::initialize. --- lib/zip/file.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/zip/file.rb b/lib/zip/file.rb index 6952ba99..6ff3463d 100644 --- a/lib/zip/file.rb +++ b/lib/zip/file.rb @@ -69,18 +69,18 @@ def initialize(file_name, create = false, buffer = false, options = {}) @name = file_name @comment = '' @create = create ? true : false # allow any truthy value to mean true - if !buffer && ::File.size?(file_name) + if !buffer && ::File.size?(@name) @create = false - @file_permissions = ::File.stat(file_name).mode - ::File.open(name, 'rb') do |f| + @file_permissions = ::File.stat(@name).mode + ::File.open(@name, 'rb') do |f| read_from_stream(f) end elsif @create @entry_set = EntrySet.new - elsif ::File.zero?(file_name) - raise Error, "File #{file_name} has zero size. Did you mean to pass the create flag?" + elsif ::File.zero?(@name) + raise Error, "File #{@name} has zero size. Did you mean to pass the create flag?" else - raise Error, "File #{file_name} not found" + raise Error, "File #{@name} not found" end @stored_entries = @entry_set.dup @stored_comment = @comment From cfa9441914d56bb866dd70c29fd5ec33bd2a8fc6 Mon Sep 17 00:00:00 2001 From: Robert Haines Date: Tue, 3 Apr 2018 23:48:54 +0100 Subject: [PATCH 02/36] Handle passing an IO to Zip::File.new better. This now actually extracts the path from the IO if one is passed in. --- lib/zip/file.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/zip/file.rb b/lib/zip/file.rb index 6ff3463d..8cb5f03a 100644 --- a/lib/zip/file.rb +++ b/lib/zip/file.rb @@ -64,9 +64,9 @@ class File < CentralDirectory # Opens a zip archive. Pass true as the second parameter to create # a new archive if it doesn't exist already. - def initialize(file_name, create = false, buffer = false, options = {}) + def initialize(path_or_io, create = false, buffer = false, options = {}) super() - @name = file_name + @name = path_or_io.respond_to?(:path) ? path_or_io.path : path_or_io @comment = '' @create = create ? true : false # allow any truthy value to mean true if !buffer && ::File.size?(@name) From 03633933ebb714cdff1e3f5604434f35bad0e0cf Mon Sep 17 00:00:00 2001 From: Robert Haines Date: Wed, 4 Apr 2018 14:31:34 +0100 Subject: [PATCH 03/36] No need to require stringio in Zip::File.open_buffer. It's already required in zip.rb. --- lib/zip/file.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/zip/file.rb b/lib/zip/file.rb index 8cb5f03a..dc86bb72 100644 --- a/lib/zip/file.rb +++ b/lib/zip/file.rb @@ -120,7 +120,6 @@ def open_buffer(io, options = {}) raise "Zip::File.open_buffer expects a String or IO-like argument (responds to #{IO_METHODS.join(', ')}). Found: #{io.class}" end if io.is_a?(::String) - require 'stringio' io = ::StringIO.new(io) elsif io.respond_to?(:binmode) # https://github.com/rubyzip/rubyzip/issues/119 From 15ccc25da1755200f6d2cb29fde49c303a236073 Mon Sep 17 00:00:00 2001 From: Robert Haines Date: Wed, 4 Apr 2018 15:12:22 +0100 Subject: [PATCH 04/36] Fix File.open_buffer when no changes are made. Things are now more carefully set up, and if a buffer is passed in which represents a file that already exists then this is taken into account. All initialization is now done in File.new, rather than being split between there and File.open_buffer. This has also needed a bit of a re-write of Zip::File.initialize. I've tried to bring some logic to it as a result, and have added comments to explain what is now happening. --- lib/zip/file.rb | 24 ++++++++++++++++++++---- test/file_test.rb | 5 +++-- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/lib/zip/file.rb b/lib/zip/file.rb index dc86bb72..c0a44207 100644 --- a/lib/zip/file.rb +++ b/lib/zip/file.rb @@ -69,19 +69,33 @@ def initialize(path_or_io, create = false, buffer = false, options = {}) @name = path_or_io.respond_to?(:path) ? path_or_io.path : path_or_io @comment = '' @create = create ? true : false # allow any truthy value to mean true - if !buffer && ::File.size?(@name) + + if ::File.size?(@name.to_s) + # There is a file, which exists, that is associated with this zip. @create = false @file_permissions = ::File.stat(@name).mode - ::File.open(@name, 'rb') do |f| - read_from_stream(f) + + if buffer + read_from_stream(path_or_io) + else + ::File.open(@name, 'rb') do |f| + read_from_stream(f) + end end + elsif buffer && path_or_io.size > 0 + # This zip is probably a non-empty StringIO. + read_from_stream(path_or_io) elsif @create + # This zip is completely new/empty and is to be created. @entry_set = EntrySet.new elsif ::File.zero?(@name) + # A file exists, but it is empty. raise Error, "File #{@name} has zero size. Did you mean to pass the create flag?" else + # Everything is wrong. raise Error, "File #{@name} not found" end + @stored_entries = @entry_set.dup @stored_comment = @comment @restore_ownership = options[:restore_ownership] || false @@ -119,16 +133,18 @@ def open_buffer(io, options = {}) unless IO_METHODS.map { |method| io.respond_to?(method) }.all? || io.is_a?(String) raise "Zip::File.open_buffer expects a String or IO-like argument (responds to #{IO_METHODS.join(', ')}). Found: #{io.class}" end + if io.is_a?(::String) io = ::StringIO.new(io) elsif io.respond_to?(:binmode) # https://github.com/rubyzip/rubyzip/issues/119 io.binmode end + zf = ::Zip::File.new(io, true, true, options) - zf.read_from_stream(io) return zf unless block_given? yield zf + begin zf.write_buffer(io) rescue IOError => e diff --git a/test/file_test.rb b/test/file_test.rb index 32e21e33..53124bea 100644 --- a/test/file_test.rb +++ b/test/file_test.rb @@ -107,13 +107,14 @@ def test_open_buffer_with_stringio def test_close_buffer_with_stringio string_io = StringIO.new File.read('test/data/rubycode.zip') zf = ::Zip::File.open_buffer string_io - assert(zf.close || true) # Poor man's refute_raises + assert_nil zf.close end def test_close_buffer_with_io f = File.open('test/data/rubycode.zip') zf = ::Zip::File.open_buffer f - assert zf.close + refute zf.commit_required? + assert_nil zf.close f.close end From 84c208982f717f10f7133cdfc1f016607398858d Mon Sep 17 00:00:00 2001 From: Robert Haines Date: Wed, 4 Apr 2018 15:40:38 +0100 Subject: [PATCH 05/36] Switch newly created StringIOs to binmode. StringIO objects created within File.open_buffer were not being switched into binmode, but those passed in were. Fix this inconsistency and add a test. --- lib/zip/file.rb | 10 ++++------ test/file_test.rb | 7 +++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/zip/file.rb b/lib/zip/file.rb index c0a44207..b5b85eea 100644 --- a/lib/zip/file.rb +++ b/lib/zip/file.rb @@ -134,12 +134,10 @@ def open_buffer(io, options = {}) raise "Zip::File.open_buffer expects a String or IO-like argument (responds to #{IO_METHODS.join(', ')}). Found: #{io.class}" end - if io.is_a?(::String) - io = ::StringIO.new(io) - elsif io.respond_to?(:binmode) - # https://github.com/rubyzip/rubyzip/issues/119 - io.binmode - end + io = ::StringIO.new(io) if io.is_a?(::String) + + # https://github.com/rubyzip/rubyzip/issues/119 + io.binmode if io.respond_to?(:binmode) zf = ::Zip::File.new(io, true, true, options) return zf unless block_given? diff --git a/test/file_test.rb b/test/file_test.rb index 53124bea..8bbf7cf8 100644 --- a/test/file_test.rb +++ b/test/file_test.rb @@ -97,6 +97,13 @@ def test_get_output_stream end end + def test_open_buffer_with_string + string = File.read('test/data/rubycode.zip') + ::Zip::File.open_buffer string do |zf| + assert zf.entries.map { |e| e.name }.include?('zippedruby1.rb') + end + end + def test_open_buffer_with_stringio string_io = StringIO.new File.read('test/data/rubycode.zip') ::Zip::File.open_buffer string_io do |zf| From eda8862c59aeb9f55b6e14b614ebe77ce0217332 Mon Sep 17 00:00:00 2001 From: Vipul A M Date: Sun, 26 Aug 2018 11:36:08 +0900 Subject: [PATCH 06/36] Move jruby to allow failures matrix till crc uint 32 issues are resolved --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index ad59d61e..b197c86b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,6 +24,7 @@ matrix: - rvm: ruby-head - rvm: rbx-3 - rvm: jruby-head + - rvm: jruby before_install: - gem update --system - gem install bundler From afb1b79efd34f8d144104bbe4665037eac7c974a Mon Sep 17 00:00:00 2001 From: Mihyaeru Date: Tue, 4 Dec 2018 00:14:32 +0900 Subject: [PATCH 07/36] remove some strange commas --- lib/zip/entry.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/zip/entry.rb b/lib/zip/entry.rb index fddab51e..6e91c213 100644 --- a/lib/zip/entry.rb +++ b/lib/zip/entry.rb @@ -275,10 +275,10 @@ def pack_local_entry zip64 = @extra['Zip64'] [::Zip::LOCAL_ENTRY_SIGNATURE, @version_needed_to_extract, # version needed to extract - @gp_flags, # @gp_flags , + @gp_flags, # @gp_flags @compression_method, - @time.to_binary_dos_time, # @last_mod_time , - @time.to_binary_dos_date, # @last_mod_date , + @time.to_binary_dos_time, # @last_mod_time + @time.to_binary_dos_date, # @last_mod_date @crc, zip64 && zip64.compressed_size ? 0xFFFFFFFF : @compressed_size, zip64 && zip64.original_size ? 0xFFFFFFFF : @size, @@ -432,11 +432,11 @@ def pack_c_dir_entry @header_signature, @version, # version of encoding software @fstype, # filesystem type - @version_needed_to_extract, # @versionNeededToExtract , - @gp_flags, # @gp_flags , + @version_needed_to_extract, # @versionNeededToExtract + @gp_flags, # @gp_flags @compression_method, - @time.to_binary_dos_time, # @last_mod_time , - @time.to_binary_dos_date, # @last_mod_date , + @time.to_binary_dos_time, # @last_mod_time + @time.to_binary_dos_date, # @last_mod_date @crc, zip64 && zip64.compressed_size ? 0xFFFFFFFF : @compressed_size, zip64 && zip64.original_size ? 0xFFFFFFFF : @size, From 9eac0d66e8cf069e8528daa89b7c998a4898d260 Mon Sep 17 00:00:00 2001 From: Adam Spiers Date: Wed, 23 Jan 2019 11:02:17 +0000 Subject: [PATCH 08/36] Add Changelog for 1.2.2 (#378) 1.2.2 was already released in #376, so unfortunately this is too late for inclusion in that, but at least future releases will have it. This is just a list of the titles of all non-merge commits since 1.2.1, so it won't be as concise or readable a summary as for previous releases, but it's better than nothing, and anyone is welcome to volunteer to condense it further. Closes #378. --- Changelog.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/Changelog.md b/Changelog.md index 7318fd10..7ed352dc 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,40 @@ +1.2.2 +===== + +* Expand from root rather than current working directory +* Disable symlinks and check for path traversal +* Consolidate path traversal tests +* Add jwilk's path traversal tests +* Trigger CI again +* Move jruby to allow failures matrix till crc uint 32 issues are resolved +* Fix CVE-2018-1000544 symlink path traversal +* Fix CVE-2018-1000544 absolute path traversal +* Fix jruby version +* When globbing in ZipFSDir, take CWD into account. +* Pass glob through from ZipFileNameMapper. +* Turn off all terminal output in all tests. +* Handle stored files with general purpose bit 3 set +* Fix regression caused by Rubocop cleanup +* Added fix for calling 'close' on a StringIO-backed zip file, and specs +* Bump Ruby versions on Travis CI +* Travis: Typo +* Travis: Workaround a rbx-3 autoload issue +* CI against Ruby 2.2.8, 2.3.5, and 2.4.2 +* Travis: typo +* Travis: Try using rbx-3 +* Travis: update RubyGems +* Travis: drop oraclejdk-7 +* Travis: use JRUBY_OPTS="--debug" +* Travis: use pre-installed Travis rubies +* README: Use a blockquote to make text readable +* add option to force entry names encoding +* Make naming on README more consistent +* Apply automatic correction by rubocop +* Disable Style/MutableConstant because existent code relies on it +* Add rubocop dependency and correct settings +* Save temporary files to a temporary directory +* File.join() is our friend for joining paths + 1.2.1 ===== From a420323c84e32df1ac2b95cd878826c9f41c06b9 Mon Sep 17 00:00:00 2001 From: David Ryskalczyk Date: Sun, 10 Feb 2019 11:51:29 -0500 Subject: [PATCH 09/36] require pathname where it is used --- lib/zip/entry.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/zip/entry.rb b/lib/zip/entry.rb index fddab51e..f50ea31a 100644 --- a/lib/zip/entry.rb +++ b/lib/zip/entry.rb @@ -1,3 +1,4 @@ +require 'pathname' module Zip class Entry STORED = 0 From 74f0d4eabbadb005979aa7595507e41cb67a1950 Mon Sep 17 00:00:00 2001 From: taichi Date: Thu, 28 Feb 2019 01:23:29 +0900 Subject: [PATCH 10/36] fixed errors caused by frozen-string-literal --- lib/zip/entry.rb | 2 +- lib/zip/extra_field.rb | 2 +- lib/zip/inflater.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/zip/entry.rb b/lib/zip/entry.rb index fddab51e..357f74d1 100644 --- a/lib/zip/entry.rb +++ b/lib/zip/entry.rb @@ -602,7 +602,7 @@ def create_file(dest_path, _continue_on_exists_proc = proc { Zip.continue_on_exi get_input_stream do |is| set_extra_attributes_on_path(dest_path) - buf = '' + buf = ''.dup while (buf = is.sysread(::Zip::Decompressor::CHUNK_SIZE, buf)) os << buf end diff --git a/lib/zip/extra_field.rb b/lib/zip/extra_field.rb index cbc2fa8d..72c36764 100644 --- a/lib/zip/extra_field.rb +++ b/lib/zip/extra_field.rb @@ -26,7 +26,7 @@ def extra_field_type_unknown(binstr, len, i) end def create_unknown_item - s = '' + s = ''.dup class << s alias_method :to_c_dir_bin, :to_s alias_method :to_local_bin, :to_s diff --git a/lib/zip/inflater.rb b/lib/zip/inflater.rb index ef952f07..f1b26d45 100644 --- a/lib/zip/inflater.rb +++ b/lib/zip/inflater.rb @@ -3,7 +3,7 @@ class Inflater < Decompressor #:nodoc:all def initialize(input_stream, decrypter = NullDecrypter.new) super(input_stream) @zlib_inflater = ::Zlib::Inflate.new(-Zlib::MAX_WBITS) - @output_buffer = '' + @output_buffer = ''.dup @has_returned_empty_string = false @decrypter = decrypter end From 0e6e626d45bcf85e520de83f5c1cf69cfec93b03 Mon Sep 17 00:00:00 2001 From: taichi Date: Thu, 28 Feb 2019 17:40:12 +0900 Subject: [PATCH 11/36] fixed CI error --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b197c86b..5c70ff27 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,8 +26,10 @@ matrix: - rvm: jruby-head - rvm: jruby before_install: - - gem update --system - - gem install bundler + - "if $(ruby -e 'exit(RUBY_VERSION >= \"2.3.0\")'); then gem update --system; fi" + - "if $(ruby -e 'exit(RUBY_VERSION < \"2.3.0\")'); then gem update --system 2.7.8; fi" + - "if $(ruby -e 'exit(RUBY_VERSION >= \"2.3.0\")'); then gem install bundler; fi" + - "if $(ruby -e 'exit(RUBY_VERSION < \"2.3.0\")'); then gem install bundler --version 1.17.3; fi" - gem --version before_script: - echo `whereis zip` From fa4f7fb1c2e23ab9dc13e680821355e438804a1d Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Fri, 22 Mar 2019 11:05:52 +0200 Subject: [PATCH 12/36] Stop allowing jruby failures --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5c70ff27..1dfd67b8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,11 +24,10 @@ matrix: - rvm: ruby-head - rvm: rbx-3 - rvm: jruby-head - - rvm: jruby before_install: - "if $(ruby -e 'exit(RUBY_VERSION >= \"2.3.0\")'); then gem update --system; fi" - "if $(ruby -e 'exit(RUBY_VERSION < \"2.3.0\")'); then gem update --system 2.7.8; fi" - - "if $(ruby -e 'exit(RUBY_VERSION >= \"2.3.0\")'); then gem install bundler; fi" + - "if $(ruby -e 'exit(RUBY_VERSION >= \"2.3.0\")'); then gem install bundler; fi" - "if $(ruby -e 'exit(RUBY_VERSION < \"2.3.0\")'); then gem install bundler --version 1.17.3; fi" - gem --version before_script: From d2f0f021e67a58b5002c8eb81d207c72bd7d1209 Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Thu, 21 Mar 2019 23:39:17 +0200 Subject: [PATCH 13/36] Enable parallel build support for coveralls --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1dfd67b8..5e13b199 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,3 +36,6 @@ before_script: env: global: - JRUBY_OPTS="--debug" + - COVERALLS_PARALLEL=true +notifications: + webhooks: https://coveralls.io/webhook From 0f36838981669a6242fc579a3579294b274ff6ed Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Fri, 22 Mar 2019 11:31:21 +0200 Subject: [PATCH 14/36] Update ruby dependencies --- .travis.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5e13b199..aad67df0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,10 +4,11 @@ cache: bundler rvm: - 2.0.0 - 2.1.10 - - 2.2.9 - - 2.3.6 - - 2.4.3 - - 2.5.0 + - 2.2.10 + - 2.3.8 + - 2.4.5 + - 2.5.3 + - 2.6.0 - ruby-head matrix: include: @@ -25,10 +26,6 @@ matrix: - rvm: rbx-3 - rvm: jruby-head before_install: - - "if $(ruby -e 'exit(RUBY_VERSION >= \"2.3.0\")'); then gem update --system; fi" - - "if $(ruby -e 'exit(RUBY_VERSION < \"2.3.0\")'); then gem update --system 2.7.8; fi" - - "if $(ruby -e 'exit(RUBY_VERSION >= \"2.3.0\")'); then gem install bundler; fi" - - "if $(ruby -e 'exit(RUBY_VERSION < \"2.3.0\")'); then gem install bundler --version 1.17.3; fi" - gem --version before_script: - echo `whereis zip` From ad15c3c49464097390248220fd93ce4caa8f43e3 Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Sun, 3 Mar 2019 14:46:49 +0000 Subject: [PATCH 15/36] Allow tilde in zip entry names Use absolute_path rather than expand_path to allow tilde to pass through unchanged. Otherwise, we try to expand it to a home directory. --- lib/zip/entry.rb | 2 +- test/data/path_traversal/tilde.zip | Bin 0 -> 577 bytes test/path_traversal_test.rb | 7 +++++++ 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 test/data/path_traversal/tilde.zip diff --git a/lib/zip/entry.rb b/lib/zip/entry.rb index a98c0772..80160b57 100644 --- a/lib/zip/entry.rb +++ b/lib/zip/entry.rb @@ -118,7 +118,7 @@ def name_safe? return false unless cleanpath.relative? root = ::File::SEPARATOR naive_expanded_path = ::File.join(root, cleanpath.to_s) - cleanpath.expand_path(root).to_s == naive_expanded_path + ::File.absolute_path(cleanpath.to_s, root) == naive_expanded_path end def local_entry_offset #:nodoc:all diff --git a/test/data/path_traversal/tilde.zip b/test/data/path_traversal/tilde.zip new file mode 100644 index 0000000000000000000000000000000000000000..0442ab93701831639904d7f1c1538cc476c18dbe GIT binary patch literal 577 zcmWIWW@Zs#-~d9~%0xc~B*4xfz))9`nUj)Q7aGCCu;fm4jGK35%vVMc2JMq)JUyQ> zF$8$Cb9`Q=RLlX?#sIxMlp;NXjpu_ucNbnaD+a{xRpTTKp12kNE>!H z2qA0Ji^t)d=^n|2@557KOANa#M0vebmNHD7lNHcy^QegYI-avp=#@VT9fzO0K zNr#-~Wi5h=db8%tUp+k{V8Y~8;SsYUrpySRG;{i?fED=(35glGvvTH5nUs^0^>$9+ zl=-u>auc7VJ$&`-*|O{xZ=S8le)KGD#p{%em#eb#o;=G)c#x9!Amha$knhoUs4^ literal 0 HcmV?d00001 diff --git a/test/path_traversal_test.rb b/test/path_traversal_test.rb index 9a361a59..e5bdd722 100644 --- a/test/path_traversal_test.rb +++ b/test/path_traversal_test.rb @@ -131,4 +131,11 @@ def test_entry_name_with_relative_symlink refute File.exist?('/tmp/file.txt') end end + + def test_entry_name_with_tilde + in_tmpdir do + extract_path_traversal_zip 'tilde.zip' + assert File.exist?('~tilde~') + end + end end From fb1c230cac322d776bb010748e5e1ac87f15100a Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Fri, 22 Mar 2019 17:51:57 +0200 Subject: [PATCH 16/36] Bump version to 1.2.3 --- Changelog.md | 72 ++++++++++++++++++++++++---------------------- lib/zip/version.rb | 2 +- 2 files changed, 38 insertions(+), 36 deletions(-) diff --git a/Changelog.md b/Changelog.md index 7ed352dc..5cf9622a 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,39 +1,41 @@ +X.X.X (Next) +===== + + + +1.2.3 +===== + +* Allow tilde in zip entry names [#391](https://github.com/rubyzip/rubyzip/pull/391) (fixes regression in 1.2.2 from [#376](https://github.com/rubyzip/rubyzip/pull/376)) +* Support frozen string literals in more files [#390](https://github.com/rubyzip/rubyzip/pull/390) +* Require `pathname` explicitly [#388](https://github.com/rubyzip/rubyzip/pull/388) (fixes regression in 1.2.2 from [#376](https://github.com/rubyzip/rubyzip/pull/376)) + +Tooling / Documentation: + +* CI updates [#392](https://github.com/rubyzip/rubyzip/pull/392) + * Bump supported ruby versions and add 2.6.0 + * JRuby failures are no longer ignored (reverts [#375](https://github.com/rubyzip/rubyzip/pull/375) / part of [#371](https://github.com/rubyzip/rubyzip/pull/371)) +* Add changelog entry that was missing for last release [#387](https://github.com/rubyzip/rubyzip/pull/387) +* Comment cleanup [#385](https://github.com/rubyzip/rubyzip/pull/385) + 1.2.2 ===== -* Expand from root rather than current working directory -* Disable symlinks and check for path traversal -* Consolidate path traversal tests -* Add jwilk's path traversal tests -* Trigger CI again -* Move jruby to allow failures matrix till crc uint 32 issues are resolved -* Fix CVE-2018-1000544 symlink path traversal -* Fix CVE-2018-1000544 absolute path traversal -* Fix jruby version -* When globbing in ZipFSDir, take CWD into account. -* Pass glob through from ZipFileNameMapper. -* Turn off all terminal output in all tests. -* Handle stored files with general purpose bit 3 set -* Fix regression caused by Rubocop cleanup -* Added fix for calling 'close' on a StringIO-backed zip file, and specs -* Bump Ruby versions on Travis CI -* Travis: Typo -* Travis: Workaround a rbx-3 autoload issue -* CI against Ruby 2.2.8, 2.3.5, and 2.4.2 -* Travis: typo -* Travis: Try using rbx-3 -* Travis: update RubyGems -* Travis: drop oraclejdk-7 -* Travis: use JRUBY_OPTS="--debug" -* Travis: use pre-installed Travis rubies -* README: Use a blockquote to make text readable -* add option to force entry names encoding -* Make naming on README more consistent -* Apply automatic correction by rubocop -* Disable Style/MutableConstant because existent code relies on it -* Add rubocop dependency and correct settings -* Save temporary files to a temporary directory -* File.join() is our friend for joining paths +NB: This release drops support for extracting symlinks, because there was no clear way to support this securely. See https://github.com/rubyzip/rubyzip/pull/376#issue-210954555 for details. + +* Fix CVE-2018-1000544 [#376](https://github.com/rubyzip/rubyzip/pull/376) / [#371](https://github.com/rubyzip/rubyzip/pull/371) +* Fix NoMethodError: undefined method `glob' [#363](https://github.com/rubyzip/rubyzip/pull/363) +* Fix handling of stored files (i.e. files not using compression) with general purpose bit 3 set [#358](https://github.com/rubyzip/rubyzip/pull/358) +* Fix `close` on StringIO-backed zip file [#353](https://github.com/rubyzip/rubyzip/pull/353) +* Add `Zip.force_entry_names_encoding` option [#340](https://github.com/rubyzip/rubyzip/pull/340) +* Update rubocop, apply auto-fixes, and fix regressions caused by said auto-fixes [#332](https://github.com/rubyzip/rubyzip/pull/332), [#355](https://github.com/rubyzip/rubyzip/pull/355) +* Save temporary files to temporary directory (rather than current directory) [#325](https://github.com/rubyzip/rubyzip/pull/325) + +Tooling / Documentation: + +* Turn off all terminal output in all tests [#361](https://github.com/rubyzip/rubyzip/pull/361) +* Several CI updates [#346](https://github.com/rubyzip/rubyzip/pull/346), [#347](https://github.com/rubyzip/rubyzip/pull/347), [#350](https://github.com/rubyzip/rubyzip/pull/350), [#352](https://github.com/rubyzip/rubyzip/pull/352) +* Several README improvements [#345](https://github.com/rubyzip/rubyzip/pull/345), [#326](https://github.com/rubyzip/rubyzip/pull/326), [#321](https://github.com/rubyzip/rubyzip/pull/321) 1.2.1 ===== @@ -100,7 +102,7 @@ * Fix compatibility of ::OutputStream::write_buffer (@orien) * Clean up tempfiles from output stream (@iangreenleaf) -1.1.2 +1.1.2 ===== * Fix compatibility of ::Zip::File.write_buffer @@ -113,7 +115,7 @@ * Fix Zip64 writting support (@mrjamesriley) * Fix StringIO support (@simonoff) * Posibility to change default compression level -* Make Zip64 write support optional via configuration +* Make Zip64 write support optional via configuration 1.1.0 ===== diff --git a/lib/zip/version.rb b/lib/zip/version.rb index 14a9f99e..4d6ab8b3 100644 --- a/lib/zip/version.rb +++ b/lib/zip/version.rb @@ -1,3 +1,3 @@ module Zip - VERSION = '1.2.2' + VERSION = '1.2.3' end From a8609e1e2ba306dbfc5c17e2837315577f376d15 Mon Sep 17 00:00:00 2001 From: Olle Jonsson Date: Fri, 22 Mar 2019 17:54:30 +0100 Subject: [PATCH 17/36] CI: update to latest MRI, drop a setting - drop unused Travis configuration: sudo: false - see https://blog.travis-ci.com/2018-11-19-required-linux-infrastructure-migration for historical detail about when it was removed. --- .travis.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index aad67df0..00f3b2d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,4 @@ language: ruby -sudo: false cache: bundler rvm: - 2.0.0 @@ -7,8 +6,8 @@ rvm: - 2.2.10 - 2.3.8 - 2.4.5 - - 2.5.3 - - 2.6.0 + - 2.5.5 + - 2.6.2 - ruby-head matrix: include: From ada408d60a7d3aa708c8560bbab5f6d32694a45a Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Fri, 22 Mar 2019 21:18:40 +0200 Subject: [PATCH 18/36] Add #394 to changelog --- Changelog.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Changelog.md b/Changelog.md index 5cf9622a..20e61c35 100644 --- a/Changelog.md +++ b/Changelog.md @@ -12,8 +12,8 @@ X.X.X (Next) Tooling / Documentation: -* CI updates [#392](https://github.com/rubyzip/rubyzip/pull/392) - * Bump supported ruby versions and add 2.6.0 +* CI updates [#392](https://github.com/rubyzip/rubyzip/pull/392), [#394]((https://github.com/rubyzip/rubyzip/pull/394) + * Bump supported ruby versions and add 2.6 * JRuby failures are no longer ignored (reverts [#375](https://github.com/rubyzip/rubyzip/pull/375) / part of [#371](https://github.com/rubyzip/rubyzip/pull/371)) * Add changelog entry that was missing for last release [#387](https://github.com/rubyzip/rubyzip/pull/387) * Comment cleanup [#385](https://github.com/rubyzip/rubyzip/pull/385) From 9d891f7353e66052283562d3e252fe380bb4b199 Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Thu, 23 May 2019 18:35:24 +0100 Subject: [PATCH 19/36] Fix link typo in changelog --- Changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 20e61c35..591b7ca0 100644 --- a/Changelog.md +++ b/Changelog.md @@ -12,7 +12,7 @@ X.X.X (Next) Tooling / Documentation: -* CI updates [#392](https://github.com/rubyzip/rubyzip/pull/392), [#394]((https://github.com/rubyzip/rubyzip/pull/394) +* CI updates [#392](https://github.com/rubyzip/rubyzip/pull/392), [#394](https://github.com/rubyzip/rubyzip/pull/394) * Bump supported ruby versions and add 2.6 * JRuby failures are no longer ignored (reverts [#375](https://github.com/rubyzip/rubyzip/pull/375) / part of [#371](https://github.com/rubyzip/rubyzip/pull/371)) * Add changelog entry that was missing for last release [#387](https://github.com/rubyzip/rubyzip/pull/387) From 1e21121f6cdb105ee8d6ab7551950b72120a261f Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Fri, 24 May 2019 17:07:35 +0100 Subject: [PATCH 20/36] Update example_recursive in README The sample has been updated several times since the last update to the README. Also ran through prettier for formatting consistency. --- README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index d5dbe76b..8255cd90 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # rubyzip + [![Gem Version](https://badge.fury.io/rb/rubyzip.svg)](http://badge.fury.io/rb/rubyzip) [![Build Status](https://secure.travis-ci.org/rubyzip/rubyzip.svg)](http://travis-ci.org/rubyzip/rubyzip) [![Code Climate](https://codeclimate.com/github/rubyzip/rubyzip.svg)](https://codeclimate.com/github/rubyzip/rubyzip) @@ -19,9 +20,10 @@ gem 'zip-zip' # will load compatibility for old rubyzip API. ## Requirements -* Ruby 1.9.2 or greater +- Ruby 1.9.2 or greater ## Installation + Rubyzip is available on RubyGems: ``` @@ -59,7 +61,8 @@ end ``` ### Zipping a directory recursively -Copy from [here](https://github.com/rubyzip/rubyzip/blob/05916bf89181e1955118fd3ea059f18acac28cc8/samples/example_recursive.rb ) + +Copy from [here](https://github.com/rubyzip/rubyzip/blob/9d891f7353e66052283562d3e252fe380bb4b199/samples/example_recursive.rb) ```ruby require 'zip' @@ -83,7 +86,7 @@ class ZipFileGenerator # Zip the input directory. def write - entries = Dir.entries(@input_dir) - %w(. ..) + entries = Dir.entries(@input_dir) - %w[. ..] ::Zip::File.open(@output_file, ::Zip::File::CREATE) do |zipfile| write_entries entries, '', zipfile @@ -97,7 +100,6 @@ class ZipFileGenerator entries.each do |e| zipfile_path = path == '' ? e : File.join(path, e) disk_file_path = File.join(@input_dir, zipfile_path) - puts "Deflating #{disk_file_path}" if File.directory? disk_file_path recursively_deflate_directory(disk_file_path, zipfile, zipfile_path) @@ -109,14 +111,12 @@ class ZipFileGenerator def recursively_deflate_directory(disk_file_path, zipfile, zipfile_path) zipfile.mkdir zipfile_path - subdir = Dir.entries(disk_file_path) - %w(. ..) + subdir = Dir.entries(disk_file_path) - %w[. ..] write_entries subdir, zipfile_path, zipfile end def put_into_archive(disk_file_path, zipfile, zipfile_path) - zipfile.get_output_stream(zipfile_path) do |f| - f.write(File.open(disk_file_path, 'rb').read) - end + zipfile.add(zipfile_path, disk_file_path) end end ``` @@ -177,7 +177,6 @@ But there is one exception when it is not working - General Purpose Flag Bit 3. > If bit 3 (0x08) of the general-purpose flags field is set, then the CRC-32 and file sizes are not known when the header is written. The fields in the local header are filled with zero, and the CRC-32 and size are appended in a 12-byte structure (optionally preceded by a 4-byte signature) immediately after the compressed data - If `::Zip::InputStream` finds such entry in the zip archive it will raise an exception. ### Password Protection (Experimental) @@ -220,7 +219,7 @@ File.open(new_path, "wb") {|f| f.write(buffer.string) } ## Configuration -By default, rubyzip will not overwrite files if they already exist inside of the extracted path. To change this behavior, you may specify a configuration option like so: +By default, rubyzip will not overwrite files if they already exist inside of the extracted path. To change this behavior, you may specify a configuration option like so: ```ruby Zip.on_exists_proc = true @@ -251,6 +250,7 @@ You can set the default compression level like so: ```ruby Zip.default_compression = Zlib::DEFAULT_COMPRESSION ``` + It defaults to `Zlib::DEFAULT_COMPRESSION`. Possible values are `Zlib::BEST_COMPRESSION`, `Zlib::DEFAULT_COMPRESSION` and `Zlib::NO_COMPRESSION` Sometimes file names inside zip contain non-ASCII characters. If you can assume which encoding was used for such names and want to be able to find such entries using `find_entry` then you can force assumed encoding like so: From 952950e474a07ef8fe2f5cf894bad189c6247ac1 Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Fri, 24 May 2019 17:25:17 +0100 Subject: [PATCH 21/36] Update changelog for #397 Also run changelog through prettier for consistency with README.md. --- Changelog.md | 399 ++++++++++++++++++++++----------------------------- 1 file changed, 170 insertions(+), 229 deletions(-) diff --git a/Changelog.md b/Changelog.md index 591b7ca0..57fccdf4 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,320 +1,261 @@ -X.X.X (Next) -===== +# X.X.X (Next) +- +Tooling / Documentation -1.2.3 -===== +- Update `example_recursive.rb` in README [#397](https://github.com/rubyzip/rubyzip/pull/397) -* Allow tilde in zip entry names [#391](https://github.com/rubyzip/rubyzip/pull/391) (fixes regression in 1.2.2 from [#376](https://github.com/rubyzip/rubyzip/pull/376)) -* Support frozen string literals in more files [#390](https://github.com/rubyzip/rubyzip/pull/390) -* Require `pathname` explicitly [#388](https://github.com/rubyzip/rubyzip/pull/388) (fixes regression in 1.2.2 from [#376](https://github.com/rubyzip/rubyzip/pull/376)) +# 1.2.3 + +- Allow tilde in zip entry names [#391](https://github.com/rubyzip/rubyzip/pull/391) (fixes regression in 1.2.2 from [#376](https://github.com/rubyzip/rubyzip/pull/376)) +- Support frozen string literals in more files [#390](https://github.com/rubyzip/rubyzip/pull/390) +- Require `pathname` explicitly [#388](https://github.com/rubyzip/rubyzip/pull/388) (fixes regression in 1.2.2 from [#376](https://github.com/rubyzip/rubyzip/pull/376)) Tooling / Documentation: -* CI updates [#392](https://github.com/rubyzip/rubyzip/pull/392), [#394](https://github.com/rubyzip/rubyzip/pull/394) - * Bump supported ruby versions and add 2.6 - * JRuby failures are no longer ignored (reverts [#375](https://github.com/rubyzip/rubyzip/pull/375) / part of [#371](https://github.com/rubyzip/rubyzip/pull/371)) -* Add changelog entry that was missing for last release [#387](https://github.com/rubyzip/rubyzip/pull/387) -* Comment cleanup [#385](https://github.com/rubyzip/rubyzip/pull/385) +- CI updates [#392](https://github.com/rubyzip/rubyzip/pull/392), [#394](https://github.com/rubyzip/rubyzip/pull/394) + - Bump supported ruby versions and add 2.6 + - JRuby failures are no longer ignored (reverts [#375](https://github.com/rubyzip/rubyzip/pull/375) / part of [#371](https://github.com/rubyzip/rubyzip/pull/371)) +- Add changelog entry that was missing for last release [#387](https://github.com/rubyzip/rubyzip/pull/387) +- Comment cleanup [#385](https://github.com/rubyzip/rubyzip/pull/385) -1.2.2 -===== +# 1.2.2 NB: This release drops support for extracting symlinks, because there was no clear way to support this securely. See https://github.com/rubyzip/rubyzip/pull/376#issue-210954555 for details. -* Fix CVE-2018-1000544 [#376](https://github.com/rubyzip/rubyzip/pull/376) / [#371](https://github.com/rubyzip/rubyzip/pull/371) -* Fix NoMethodError: undefined method `glob' [#363](https://github.com/rubyzip/rubyzip/pull/363) -* Fix handling of stored files (i.e. files not using compression) with general purpose bit 3 set [#358](https://github.com/rubyzip/rubyzip/pull/358) -* Fix `close` on StringIO-backed zip file [#353](https://github.com/rubyzip/rubyzip/pull/353) -* Add `Zip.force_entry_names_encoding` option [#340](https://github.com/rubyzip/rubyzip/pull/340) -* Update rubocop, apply auto-fixes, and fix regressions caused by said auto-fixes [#332](https://github.com/rubyzip/rubyzip/pull/332), [#355](https://github.com/rubyzip/rubyzip/pull/355) -* Save temporary files to temporary directory (rather than current directory) [#325](https://github.com/rubyzip/rubyzip/pull/325) +- Fix CVE-2018-1000544 [#376](https://github.com/rubyzip/rubyzip/pull/376) / [#371](https://github.com/rubyzip/rubyzip/pull/371) +- Fix NoMethodError: undefined method `glob' [#363](https://github.com/rubyzip/rubyzip/pull/363) +- Fix handling of stored files (i.e. files not using compression) with general purpose bit 3 set [#358](https://github.com/rubyzip/rubyzip/pull/358) +- Fix `close` on StringIO-backed zip file [#353](https://github.com/rubyzip/rubyzip/pull/353) +- Add `Zip.force_entry_names_encoding` option [#340](https://github.com/rubyzip/rubyzip/pull/340) +- Update rubocop, apply auto-fixes, and fix regressions caused by said auto-fixes [#332](https://github.com/rubyzip/rubyzip/pull/332), [#355](https://github.com/rubyzip/rubyzip/pull/355) +- Save temporary files to temporary directory (rather than current directory) [#325](https://github.com/rubyzip/rubyzip/pull/325) Tooling / Documentation: -* Turn off all terminal output in all tests [#361](https://github.com/rubyzip/rubyzip/pull/361) -* Several CI updates [#346](https://github.com/rubyzip/rubyzip/pull/346), [#347](https://github.com/rubyzip/rubyzip/pull/347), [#350](https://github.com/rubyzip/rubyzip/pull/350), [#352](https://github.com/rubyzip/rubyzip/pull/352) -* Several README improvements [#345](https://github.com/rubyzip/rubyzip/pull/345), [#326](https://github.com/rubyzip/rubyzip/pull/326), [#321](https://github.com/rubyzip/rubyzip/pull/321) - -1.2.1 -===== - -* Add accessor to @internal_file_attributes #304 -* Extended globbing #303 -* README updates #283, #289 -* Cleanup after tests #298, #306 -* Fix permissions on new zip files #294, #300 -* Fix examples #297 -* Support cp932 encoding #308 -* Fix Directory traversal vulnerability #315 -* Allow open_buffer to work without a given block #314 +- Turn off all terminal output in all tests [#361](https://github.com/rubyzip/rubyzip/pull/361) +- Several CI updates [#346](https://github.com/rubyzip/rubyzip/pull/346), [#347](https://github.com/rubyzip/rubyzip/pull/347), [#350](https://github.com/rubyzip/rubyzip/pull/350), [#352](https://github.com/rubyzip/rubyzip/pull/352) +- Several README improvements [#345](https://github.com/rubyzip/rubyzip/pull/345), [#326](https://github.com/rubyzip/rubyzip/pull/326), [#321](https://github.com/rubyzip/rubyzip/pull/321) -1.2.0 -===== +# 1.2.1 -* Don't enable JRuby objectspace #252 -* Fixes an exception thrown when decoding some weird .zip files #248 -* Use duck typing with IO methods #244 -* Added error for empty (zero bit) zip file #242 -* Accept StringIO in Zip.open_buffer #238 -* Do something more expected with new file permissions #237 -* Case insensitivity option for #find_entry #222 -* Fixes in documentation and examples +- Add accessor to @internal_file_attributes #304 +- Extended globbing #303 +- README updates #283, #289 +- Cleanup after tests #298, #306 +- Fix permissions on new zip files #294, #300 +- Fix examples #297 +- Support cp932 encoding #308 +- Fix Directory traversal vulnerability #315 +- Allow open_buffer to work without a given block #314 -1.1.7 -===== +# 1.2.0 -* Fix UTF-8 support for comments -* `Zip.sort_entries` working for zip output -* Prevent tempfile path from being unlinked by garbage collection -* NTFS Extra Field (0x000a) support -* Use String#tr instead of String#gsub -* Ability to not show warning about incorrect date -* Be smarter about handling buffer file modes. -* Support for Traditional Encryption (ZipCrypto) +- Don't enable JRuby objectspace #252 +- Fixes an exception thrown when decoding some weird .zip files #248 +- Use duck typing with IO methods #244 +- Added error for empty (zero bit) zip file #242 +- Accept StringIO in Zip.open_buffer #238 +- Do something more expected with new file permissions #237 +- Case insensitivity option for #find_entry #222 +- Fixes in documentation and examples -1.1.6 -===== +# 1.1.7 -* Revert "Return created zip file from Zip::File.open when supplied a block" +- Fix UTF-8 support for comments +- `Zip.sort_entries` working for zip output +- Prevent tempfile path from being unlinked by garbage collection +- NTFS Extra Field (0x000a) support +- Use String#tr instead of String#gsub +- Ability to not show warning about incorrect date +- Be smarter about handling buffer file modes. +- Support for Traditional Encryption (ZipCrypto) -1.1.5 -===== +# 1.1.6 -* Treat empty file as non-exists (@layerssss) -* Revert regression commit -* Return created zip file from Zip::File.open when supplied a block (@tpickett66) -* Zip::Entry::DEFLATED is forced on every file (@mehmetc) -* Add InputStream#ungetc (@zacstewart) -* Alias for legacy error names (@orien) +- Revert "Return created zip file from Zip::File.open when supplied a block" -1.1.4 -===== +# 1.1.5 -* Don't send empty string to stream (@mrloop) -* Zip::Entry::DEFLATED was forced on every file (@mehmetc) -* Alias for legacy error names (@orien) +- Treat empty file as non-exists (@layerssss) +- Revert regression commit +- Return created zip file from Zip::File.open when supplied a block (@tpickett66) +- Zip::Entry::DEFLATED is forced on every file (@mehmetc) +- Add InputStream#ungetc (@zacstewart) +- Alias for legacy error names (@orien) -1.1.3 -===== +# 1.1.4 -* Fix compatibility of ::OutputStream::write_buffer (@orien) -* Clean up tempfiles from output stream (@iangreenleaf) +- Don't send empty string to stream (@mrloop) +- Zip::Entry::DEFLATED was forced on every file (@mehmetc) +- Alias for legacy error names (@orien) -1.1.2 -===== +# 1.1.3 -* Fix compatibility of ::Zip::File.write_buffer +- Fix compatibility of ::OutputStream::write_buffer (@orien) +- Clean up tempfiles from output stream (@iangreenleaf) -1.1.1 -===== +# 1.1.2 -* Speedup deflater (@loadhigh) -* Less Arrays and Strings allocations (@srawlins) -* Fix Zip64 writting support (@mrjamesriley) -* Fix StringIO support (@simonoff) -* Posibility to change default compression level -* Make Zip64 write support optional via configuration +- Fix compatibility of ::Zip::File.write_buffer -1.1.0 -===== +# 1.1.1 -* StringIO Support -* Zip64 Support -* Better jRuby Support -* Order of files in the archive can be sorted -* Other small fixes +- Speedup deflater (@loadhigh) +- Less Arrays and Strings allocations (@srawlins) +- Fix Zip64 writing support (@mrjamesriley) +- Fix StringIO support (@simonoff) +- Possibility to change default compression level +- Make Zip64 write support optional via configuration -1.0.0 -===== +# 1.1.0 -* Removed support for Ruby 1.8 -* Changed the API for gem. Now it can be used without require param in Gemfile. -* Added read-only support for Zip64 files. -* Added support for setting Unicode file names. +- StringIO Support +- Zip64 Support +- Better jRuby Support +- Order of files in the archive can be sorted +- Other small fixes -0.9.9 -===== +# 1.0.0 -* Added support for backslashes in zip files (generated by the default Windows zip packer for example) and comment sections with the comment length set to zero even though there is actually a comment. +- Removed support for Ruby 1.8 +- Changed the API for gem. Now it can be used without require param in Gemfile. +- Added read-only support for Zip64 files. +- Added support for setting Unicode file names. -0.9.8 -===== +# 0.9.9 -* Fixed: "Unitialized constant NullInputStream" error +- Added support for backslashes in zip files (generated by the default Windows zip packer for example) and comment sections with the comment length set to zero even though there is actually a comment. -0.9.5 -===== +# 0.9.8 -* Removed support for loading ruby in zip files (ziprequire.rb). +- Fixed: "Unitialized constant NullInputStream" error -0.9.4 -===== +# 0.9.5 -* Changed ZipOutputStream.put_next_entry signature (API CHANGE!). Now allows comment, extra field and compression method to be specified. +- Removed support for loading ruby in zip files (ziprequire.rb). -0.9.3 -===== +# 0.9.4 -* Fixed: Added ZipEntry::name_encoding which retrieves the character -encoding of the name and comment of the entry. -* Added convenience methods ZipEntry::name_in(enc) and ZipEntry::comment_in(enc) for -getting zip entry names and comments in a specified character -encoding. +- Changed ZipOutputStream.put_next_entry signature (API CHANGE!). Now allows comment, extra field and compression method to be specified. -0.9.2 -===== +# 0.9.3 -* Fixed: Renaming an entry failed if the entry's new name was a different length than its old name. (Diego Barros) +- Fixed: Added ZipEntry::name_encoding which retrieves the character encoding of the name and comment of the entry. +- Added convenience methods ZipEntry::name_in(enc) and ZipEntry::comment_in(enc) for getting zip entry names and comments in a specified character encoding. -0.9.1 -===== +# 0.9.2 -* Added symlink support and support for unix file permissions. Reduced memory usage during decompression. -* New methods ZipFile::[follow_symlinks, restore_times, restore_permissions, restore_ownership]. -* New methods ZipEntry::unix_perms, ZipInputStream::eof?. -* Added documentation and test for new ZipFile::extract. -* Added some of the API suggestions from sf.net #1281314. -* Applied patch for sf.net bug #1446926. -* Applied patch for sf.net bug #1459902. -* Rework ZipEntry and delegate classes. +- Fixed: Renaming an entry failed if the entry's new name was a different length than its old name. (Diego Barros) -0.5.12 -====== +# 0.9.1 -* Fixed problem with writing binary content to a ZipFile in MS Windows. +- Added symlink support and support for unix file permissions. Reduced memory usage during decompression. +- New methods ZipFile::[follow_symlinks, restore_times, restore_permissions, restore_ownership]. +- New methods ZipEntry::unix_perms, ZipInputStream::eof?. +- Added documentation and test for new ZipFile::extract. +- Added some of the API suggestions from sf.net #1281314. +- Applied patch for sf.net bug #1446926. +- Applied patch for sf.net bug #1459902. +- Rework ZipEntry and delegate classes. -0.5.11 -====== +# 0.5.12 -* Fixed name clash file method copy_stream from fileutils.rb. Fixed problem with references to constant CHUNK_SIZE. -* ZipInputStream/AbstractInputStream read is now buffered like ruby IO's read method, which means that read and gets etc can be mixed. The - unbuffered read method has been renamed to sysread. +- Fixed problem with writing binary content to a ZipFile in MS Windows. -0.5.10 -====== +# 0.5.11 -* Fixed method name resolution problem with FileUtils::copy_stream and IOExtras::copy_stream. +- Fixed name clash file method copy_stream from fileutils.rb. Fixed problem with references to constant CHUNK_SIZE. +- ZipInputStream/AbstractInputStream read is now buffered like ruby IO's read method, which means that read and gets etc can be mixed. The unbuffered read method has been renamed to sysread. -0.5.9 -===== +# 0.5.10 -* Fixed serious memory consumption issue +- Fixed method name resolution problem with FileUtils::copy_stream and IOExtras::copy_stream. -0.5.8 -===== +# 0.5.9 -* Fixed install script. +- Fixed serious memory consumption issue -0.5.7 -===== -* install.rb no longer assumes it is being run from the toplevel source -dir. Directory structure changed to reflect common ruby library -project structure. Migrated from RubyUnit to Test::Unit format. Now -uses Rake to build source packages and gems and run unit tests. +# 0.5.8 -0.5.6 -===== -* Fix for FreeBSD 4.9 which returns Errno::EFBIG instead of -Errno::EINVAL for some invalid seeks. Fixed 'version needed to -extract'-field incorrect in local headers. +- Fixed install script. -0.5.5 -===== +# 0.5.7 -* Fix for a problem with writing zip files that concerns only ruby 1.8.1. +- install.rb no longer assumes it is being run from the toplevel source dir. Directory structure changed to reflect common ruby library project structure. Migrated from RubyUnit to Test::Unit format. Now uses Rake to build source packages and gems and run unit tests. -0.5.4 -===== +# 0.5.6 -* Significantly reduced memory footprint when modifying zip files. +- Fix for FreeBSD 4.9 which returns Errno::EFBIG instead of Errno::EINVAL for some invalid seeks. Fixed 'version needed to extract'-field incorrect in local headers. -0.5.3 -===== -* Added optimization to avoid decompressing and recompressing individual -entries when modifying a zip archive. +# 0.5.5 -0.5.2 -===== -* Fixed ZipFile corruption bug in ZipFile class. Added basic unix -extra-field support. +- Fix for a problem with writing zip files that concerns only ruby 1.8.1. -0.5.1 -===== +# 0.5.4 -* Fixed ZipFile.get_output_stream bug. +- Significantly reduced memory footprint when modifying zip files. -0.5.0 -===== +# 0.5.3 -* Ruby 1.8.0 and ruby-zlib 0.6.0 compatibility -* Changed method names from camelCase to rubys underscore style. -* Installs to zip/ subdir instead of directly to site_ruby -* Added ZipFile.directory and ZipFile.file - each method return an -object that can be used like Dir and File only for the contents of the -zip file. -* Added sample application zipfind which works like Find.find, only -Zip::ZipFind.find traverses into zip archives too. -* FIX: AbstractInputStream.each_line with non-default separator +- Added optimization to avoid decompressing and recompressing individual entries when modifying a zip archive. +# 0.5.2 -0.5.0a -====== -Source reorganized. Added ziprequire, which can be used to load ruby -modules from a zip file, in a fashion similar to jar files in -Java. Added gtk_ruby_zip, another sample application. Implemented -ZipInputStream.lineno and ZipInputStream.rewind +- Fixed ZipFile corruption bug in ZipFile class. Added basic unix extra-field support. -Bug fixes: +# 0.5.1 + +- Fixed ZipFile.get_output_stream bug. -* Read and write date and time information correctly for zip entries. -* Fixed read() using separate buffer, causing mix of gets/readline/read to -cause problems. +# 0.5.0 -0.4.2 -===== +- Ruby 1.8.0 and ruby-zlib 0.6.0 compatibility +- Changed method names from camelCase to rubys underscore style. +- Installs to zip/ subdir instead of directly to site_ruby +- Added ZipFile.directory and ZipFile.file - each method return an + object that can be used like Dir and File only for the contents of the + zip file. +- Added sample application zipfind which works like Find.find, only + Zip::ZipFind.find traverses into zip archives too. +- FIX: AbstractInputStream.each_line with non-default separator -* Performance optimizations. Test suite runs in half the time. +# 0.5.0a + +Source reorganized. Added ziprequire, which can be used to load ruby modules from a zip file, in a fashion similar to jar files in Java. Added gtk_ruby_zip, another sample application. Implemented ZipInputStream.lineno and ZipInputStream.rewind + +Bug fixes: -0.4.1 -===== +- Read and write date and time information correctly for zip entries. +- Fixed read() using separate buffer, causing mix of gets/readline/read to cause problems. -* Windows compatibility fixes. +# 0.4.2 -0.4.0 -===== +- Performance optimizations. Test suite runs in half the time. -* Zip::ZipFile is now mutable and provides a more convenient way of -modifying zip archives than Zip::ZipOutputStream. Operations for -adding, extracting, renaming, replacing and removing entries to zip -archives are now available. +# 0.4.1 -* Runs without warnings with -w switch. +- Windows compatibility fixes. -* Install script install.rb added. +# 0.4.0 -0.3.1 -===== +- Zip::ZipFile is now mutable and provides a more convenient way of modifying zip archives than Zip::ZipOutputStream. Operations for adding, extracting, renaming, replacing and removing entries to zip archives are now available. +- Runs without warnings with -w switch. +- Install script install.rb added. -* Rudimentary support for writing zip archives. +# 0.3.1 -0.2.2 -===== +- Rudimentary support for writing zip archives. -* Fixed and extended unit test suite. Updated to work with ruby/zlib -0.5. It doesn't work with earlier versions of ruby/zlib. +# 0.2.2 -0.2.0 -===== +- Fixed and extended unit test suite. Updated to work with ruby/zlib 0.5. It doesn't work with earlier versions of ruby/zlib. -* Class ZipFile added. Where ZipInputStream is used to read the -individual entries in a zip file, ZipFile reads the central directory -in the zip archive, so you can get to any entry in the zip archive -without having to skipping through all the preceeding entries. +# 0.2.0 +- Class ZipFile added. Where ZipInputStream is used to read the individual entries in a zip file, ZipFile reads the central directory in the zip archive, so you can get to any entry in the zip archive without having to skipping through all the preceeding entries. -0.1.0 -===== +# 0.1.0 -* First working version of ZipInputStream. +- First working version of ZipInputStream. From 5152f6f7a0f5515d0fe1717d0c3dcb40c26ab2c9 Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Sun, 7 Jul 2019 17:59:35 +0100 Subject: [PATCH 22/36] Put CI back to trusty Xenial is now the default. Trusty is now out of support but still not end of life. Also omit the ruby patch versions so we don't have to keep updating them. --- .travis.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 00f3b2d6..6b7d6e05 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,14 @@ language: ruby +dist: trusty cache: bundler rvm: - - 2.0.0 - - 2.1.10 - - 2.2.10 - - 2.3.8 - - 2.4.5 - - 2.5.5 - - 2.6.2 + - 2.0 + - 2.1 + - 2.2 + - 2.3 + - 2.4 + - 2.5 + - 2.6 - ruby-head matrix: include: From b2573f6069ef1eecb440d23c93015dfa011d283a Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Sun, 7 Jul 2019 18:18:49 +0100 Subject: [PATCH 23/36] Use rbx-4 in CI --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6b7d6e05..d98d0e27 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,12 +18,10 @@ matrix: jdk: openjdk7 - rvm: jruby-head jdk: oraclejdk8 - - rvm: rbx-3 - env: - - RUBYOPT="-rbundler/deprecate" + - rvm: rbx-4 allow_failures: - rvm: ruby-head - - rvm: rbx-3 + - rvm: rbx-4 - rvm: jruby-head before_install: - gem --version From fc23db2efc8ba7b39e5ef94ddbd0bf23a4d5ba5e Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Sat, 20 Jul 2019 15:06:17 +0100 Subject: [PATCH 24/36] Update changelog for #399 --- Changelog.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Changelog.md b/Changelog.md index 57fccdf4..1240b3de 100644 --- a/Changelog.md +++ b/Changelog.md @@ -5,6 +5,7 @@ Tooling / Documentation - Update `example_recursive.rb` in README [#397](https://github.com/rubyzip/rubyzip/pull/397) +- Fix CI on `trusty` for now, and automatically pick the latest ruby patch version [#399](https://github.com/rubyzip/rubyzip/pull/399) # 1.2.3 From 8dfc95dc79c93c0a4c10cf9407784bc736600564 Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Sat, 20 Jul 2019 15:15:30 +0100 Subject: [PATCH 25/36] Hold jruby at 9.1 on JDK 7 --- .travis.yml | 2 +- Changelog.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d98d0e27..358e2a8a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ matrix: include: - rvm: jruby jdk: oraclejdk8 - - rvm: jruby + - rvm: jruby-9.1 jdk: openjdk7 - rvm: jruby-head jdk: oraclejdk8 diff --git a/Changelog.md b/Changelog.md index 1240b3de..8bb4f6dd 100644 --- a/Changelog.md +++ b/Changelog.md @@ -5,7 +5,7 @@ Tooling / Documentation - Update `example_recursive.rb` in README [#397](https://github.com/rubyzip/rubyzip/pull/397) -- Fix CI on `trusty` for now, and automatically pick the latest ruby patch version [#399](https://github.com/rubyzip/rubyzip/pull/399) +- Hold CI at `trusty` for now, automatically pick the latest ruby patch version, use rbx-4 and hold jruby at 9.1 [#399](https://github.com/rubyzip/rubyzip/pull/399) # 1.2.3 From eeef5073d58253e2044dbf81d1b205efd590b59a Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Thu, 5 Sep 2019 19:00:34 +0100 Subject: [PATCH 26/36] Add test case based on #146 --- test/file_test.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/file_test.rb b/test/file_test.rb index 3c52c778..f2d248e3 100644 --- a/test/file_test.rb +++ b/test/file_test.rb @@ -131,6 +131,15 @@ def test_close_buffer_with_io f.close end + def test_open_buffer_with_io_and_block + File.open('test/data/rubycode.zip') do |io| + io.set_encoding(Encoding::BINARY) # not strictly required but can be set + Zip::File.open_buffer(io) do |zip_io| + # left empty on purpose + end + end + end + def test_open_buffer_without_block string_io = StringIO.new File.read('test/data/rubycode.zip') zf = ::Zip::File.open_buffer string_io From 9a41ce65c432bf90e30824d7a6b60f9a75ccfe0d Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Fri, 6 Sep 2019 17:58:38 +0100 Subject: [PATCH 27/36] Add more explicit test for #280 --- test/file_test.rb | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/test/file_test.rb b/test/file_test.rb index f2d248e3..abe4e4a6 100644 --- a/test/file_test.rb +++ b/test/file_test.rb @@ -123,12 +123,40 @@ def test_close_buffer_with_stringio assert_nil zf.close end - def test_close_buffer_with_io - f = File.open('test/data/rubycode.zip') - zf = ::Zip::File.open_buffer f - refute zf.commit_required? - assert_nil zf.close - f.close + def test_open_buffer_no_op_does_not_change_file + Dir.mktmpdir do |tmp| + test_zip = File.join(tmp, 'test.zip') + FileUtils.cp 'test/data/rubycode.zip', test_zip + + # Note: this may change the file if it is opened with r+b instead of rb. + # The 'extra fields' in this particular zip file get reordered. + File.open(test_zip, 'rb') do |file| + Zip::File.open_buffer(file) do |zf| + nil # do nothing + end + end + + assert_equal \ + File.binread('test/data/rubycode.zip'), + File.binread(test_zip) + end + end + + def test_open_buffer_close_does_not_change_file + Dir.mktmpdir do |tmp| + test_zip = File.join(tmp, 'test.zip') + FileUtils.cp 'test/data/rubycode.zip', test_zip + + File.open(test_zip, 'rb') do |file| + zf = Zip::File.open_buffer(file) + refute zf.commit_required? + assert_nil zf.close + end + + assert_equal \ + File.binread('test/data/rubycode.zip'), + File.binread(test_zip) + end end def test_open_buffer_with_io_and_block From 0d85cb6a49cce7ef51186e64c8f3f147d0fd2b72 Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Fri, 6 Sep 2019 18:01:30 +0100 Subject: [PATCH 28/36] Bump to 1.2.4 --- Changelog.md | 4 ++++ lib/zip/version.rb | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 8bb4f6dd..50fc6e5b 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,6 +2,10 @@ - +# 1.2.4 (2019-09-06) + +- Do not rewrite zip files opened with `open_buffer` that have not changed [#360](https://github.com/rubyzip/rubyzip/pull/360) + Tooling / Documentation - Update `example_recursive.rb` in README [#397](https://github.com/rubyzip/rubyzip/pull/397) diff --git a/lib/zip/version.rb b/lib/zip/version.rb index 4d6ab8b3..afbccee8 100644 --- a/lib/zip/version.rb +++ b/lib/zip/version.rb @@ -1,3 +1,3 @@ module Zip - VERSION = '1.2.3' + VERSION = '1.2.4' end From 72e7ca0d04de580e31717555db20d340c69e68de Mon Sep 17 00:00:00 2001 From: Orien Madgwick <_@orien.io> Date: Thu, 12 Sep 2019 12:56:00 +1000 Subject: [PATCH 29/36] Add project metadata to the gemspec As per https://guides.rubygems.org/specification-reference/#metadata, add metadata to the gemspec file. This'll allow people to more easily access the source code, raise issues and read the changelog. These `bug_tracker_uri`, `changelog_uri`, `documentation_uri`, `wiki_uri` and `source_code_uri` links will appear on the rubygems page at https://rubygems.org/gems/rubyzip and be available via the rubygems API after the next release. --- rubyzip.gemspec | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rubyzip.gemspec b/rubyzip.gemspec index 4ca36c2d..6b873752 100644 --- a/rubyzip.gemspec +++ b/rubyzip.gemspec @@ -16,6 +16,13 @@ Gem::Specification.new do |s| s.test_files = Dir.glob('test/**/*') s.require_paths = ['lib'] s.license = 'BSD 2-Clause' + s.metadata = { + 'bug_tracker_uri' => 'https://github.com/rubyzip/rubyzip/issues', + 'changelog_uri' => "https://github.com/rubyzip/rubyzip/blob/v#{s.version}/Changelog.md", + 'documentation_uri' => "https://www.rubydoc.info/gems/rubyzip/#{s.version}", + 'source_code_uri' => "https://github.com/rubyzip/rubyzip/tree/v#{s.version}", + 'wiki_uri' => 'https://github.com/rubyzip/rubyzip/wiki' + } s.required_ruby_version = '>= 1.9.2' s.add_development_dependency 'rake', '~> 10.3' s.add_development_dependency 'pry', '~> 0.10' From ecb277621852589ecc1557f228665a5338ac0809 Mon Sep 17 00:00:00 2001 From: Robert Haines Date: Sun, 20 May 2018 15:34:55 +0100 Subject: [PATCH 30/36] Zip::File.add_stored() to add uncompressed files. Adding uncompressed files to a zip archive can be overly complex, so this convenience method makes it easier. --- lib/zip/file.rb | 7 +++++++ test/file_test.rb | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/lib/zip/file.rb b/lib/zip/file.rb index b5b85eea..9c7f3cbd 100644 --- a/lib/zip/file.rb +++ b/lib/zip/file.rb @@ -287,6 +287,13 @@ def add(entry, src_path, &continue_on_exists_proc) @entry_set << new_entry end + # Convenience method for adding the contents of a file to the archive + # in Stored format (uncompressed) + def add_stored(entry, src_path, &continue_on_exists_proc) + entry = ::Zip::Entry.new(@name, entry.to_s, nil, nil, nil, nil, ::Zip::Entry::STORED) + add(entry, src_path, &continue_on_exists_proc) + end + # Removes the specified entry. def remove(entry) @entry_set.delete(get_entry(entry)) diff --git a/test/file_test.rb b/test/file_test.rb index abe4e4a6..3b53c2b9 100644 --- a/test/file_test.rb +++ b/test/file_test.rb @@ -204,6 +204,25 @@ def test_add zfRead.get_input_stream(entryName) { |zis| zis.read }) end + def test_add_stored + srcFile = 'test/data/file2.txt' + entryName = 'newEntryName.rb' + assert(::File.exist?(srcFile)) + zf = ::Zip::File.new(EMPTY_FILENAME, ::Zip::File::CREATE) + zf.add_stored(entryName, srcFile) + zf.close + + zfRead = ::Zip::File.new(EMPTY_FILENAME) + entry = zfRead.entries.first + assert_equal('', zfRead.comment) + assert_equal(1, zfRead.entries.length) + assert_equal(entryName, entry.name) + assert_equal(entry.size, entry.compressed_size) + assert_equal(::Zip::Entry::STORED, entry.compression_method) + AssertEntry.assert_contents(srcFile, + zfRead.get_input_stream(entryName) { |zis| zis.read }) + end + def test_recover_permissions_after_add_files_to_archive srcZip = TEST_ZIP.zip_name ::File.chmod(0o664, srcZip) From 93505ca16f0444bdb04f88f4b8f820ae5d628353 Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Sun, 15 Sep 2019 14:58:13 +0100 Subject: [PATCH 31/36] Check expected entry size in add_stored test --- test/file_test.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/test/file_test.rb b/test/file_test.rb index 3b53c2b9..94ff769c 100644 --- a/test/file_test.rb +++ b/test/file_test.rb @@ -217,6 +217,7 @@ def test_add_stored assert_equal('', zfRead.comment) assert_equal(1, zfRead.entries.length) assert_equal(entryName, entry.name) + assert_equal(File.size(srcFile), entry.size) assert_equal(entry.size, entry.compressed_size) assert_equal(::Zip::Entry::STORED, entry.compression_method) AssertEntry.assert_contents(srcFile, From 94b7fa276992933592d69eb6bb17fc09105f8395 Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Sun, 15 Sep 2019 15:03:19 +0100 Subject: [PATCH 32/36] [ci skip] Update changelog --- Changelog.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Changelog.md b/Changelog.md index 50fc6e5b..e8a7e16b 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,6 +1,10 @@ # X.X.X (Next) -- +- Add `add_stored` method to simplify adding entries without compression [#366](https://github.com/rubyzip/rubyzip/pull/366) + +Tooling / Documentation + +- Add more gem metadata links [#402](https://github.com/rubyzip/rubyzip/pull/402) # 1.2.4 (2019-09-06) From 4167f0ce67e42b082605bca75c7bdfd01eb23804 Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Thu, 12 Sep 2019 22:01:38 +0100 Subject: [PATCH 33/36] Validate entry sizes when extracting --- README.md | 67 +++++++++++++++++++++++++++++++-------- lib/zip.rb | 4 ++- lib/zip/entry.rb | 7 ++++ lib/zip/errors.rb | 1 + test/file_extract_test.rb | 62 ++++++++++++++++++++++++++++++++++++ 5 files changed, 127 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 8255cd90..2ff41ed9 100644 --- a/README.md +++ b/README.md @@ -152,12 +152,15 @@ When modifying a zip archive the file permissions of the archive are preserved. ### Reading a Zip file ```ruby +MAX_SIZE = 1024**2 # 1MiB (but of course you can increase this) Zip::File.open('foo.zip') do |zip_file| # Handle entries one by one zip_file.each do |entry| - # Extract to file/directory/symlink puts "Extracting #{entry.name}" - entry.extract(dest_file) + raise 'File too large when extracted' if entry.size > MAX_SIZE + + # Extract to file or directory based on name in the archive + entry.extract # Read into memory content = entry.get_input_stream.read @@ -165,6 +168,7 @@ Zip::File.open('foo.zip') do |zip_file| # Find specific entry entry = zip_file.glob('*.csv').first + raise 'File too large when extracted' if entry.size > MAX_SIZE puts entry.get_input_stream.read end ``` @@ -219,6 +223,8 @@ File.open(new_path, "wb") {|f| f.write(buffer.string) } ## Configuration +### Existing Files + By default, rubyzip will not overwrite files if they already exist inside of the extracted path. To change this behavior, you may specify a configuration option like so: ```ruby @@ -233,18 +239,57 @@ Additionally, if you want to configure rubyzip to overwrite existing files while Zip.continue_on_exists_proc = true ``` +### Non-ASCII Names + If you want to store non-english names and want to open them on Windows(pre 7) you need to set this option: ```ruby Zip.unicode_names = true ``` +Sometimes file names inside zip contain non-ASCII characters. If you can assume which encoding was used for such names and want to be able to find such entries using `find_entry` then you can force assumed encoding like so: + +```ruby +Zip.force_entry_names_encoding = 'UTF-8' +``` + +Allowed encoding names are the same as accepted by `String#force_encoding` + +### Date Validation + Some zip files might have an invalid date format, which will raise a warning. You can hide this warning with the following setting: ```ruby Zip.warn_invalid_date = false ``` +### Size Validation + +By default, `rubyzip`'s `extract` method checks that an entry's reported uncompressed size is not (significantly) smaller than its actual size. This is to help you protect your application against [zip bombs](https://en.wikipedia.org/wiki/Zip_bomb). Before `extract`ing an entry, you should check that its size is in the range you expect. For example, if your application supports processing up to 100 files at once, each up to 10MiB, your zip extraction code might look like: + +```ruby +MAX_FILE_SIZE = 10 * 1024**2 # 10MiB +MAX_FILES = 100 +Zip::File.open('foo.zip') do |zip_file| + num_files = 0 + zip_file.each do |entry| + num_files += 1 if entry.file? + raise 'Too many extracted files' if num_files > MAX_FILES + raise 'File too large when extracted' if entry.size > MAX_FILE_SIZE + entry.extract + end +end +``` + +If you need to extract zip files that report incorrect uncompressed sizes and you really trust them not too be too large, you can disable this setting with +```ruby +Zip.validate_entry_sizes = false +``` + +Note that if you use the lower level `Zip::InputStream` interface, `rubyzip` does *not* check the entry `size`s. In this case, the caller is responsible for making sure it does not read more data than expected from the input stream. + +### Default Compression + You can set the default compression level like so: ```ruby @@ -253,13 +298,17 @@ Zip.default_compression = Zlib::DEFAULT_COMPRESSION It defaults to `Zlib::DEFAULT_COMPRESSION`. Possible values are `Zlib::BEST_COMPRESSION`, `Zlib::DEFAULT_COMPRESSION` and `Zlib::NO_COMPRESSION` -Sometimes file names inside zip contain non-ASCII characters. If you can assume which encoding was used for such names and want to be able to find such entries using `find_entry` then you can force assumed encoding like so: +### Zip64 Support + +By default, Zip64 support is disabled for writing. To enable it do this: ```ruby -Zip.force_entry_names_encoding = 'UTF-8' +Zip.write_zip64_support = true ``` -Allowed encoding names are the same as accepted by `String#force_encoding` +_NOTE_: If you will enable Zip64 writing then you will need zip extractor with Zip64 support to extract archive. + +### Block Form You can set multiple settings at the same time by using a block: @@ -272,14 +321,6 @@ You can set multiple settings at the same time by using a block: end ``` -By default, Zip64 support is disabled for writing. To enable it do this: - -```ruby -Zip.write_zip64_support = true -``` - -_NOTE_: If you will enable Zip64 writing then you will need zip extractor with Zip64 support to extract archive. - ## Developing To run the test you need to do this: diff --git a/lib/zip.rb b/lib/zip.rb index 9145207b..c3a6ed5e 100644 --- a/lib/zip.rb +++ b/lib/zip.rb @@ -42,7 +42,8 @@ module Zip :write_zip64_support, :warn_invalid_date, :case_insensitive_match, - :force_entry_names_encoding + :force_entry_names_encoding, + :validate_entry_sizes def reset! @_ran_once = false @@ -54,6 +55,7 @@ def reset! @write_zip64_support = false @warn_invalid_date = true @case_insensitive_match = false + @validate_entry_sizes = true end def setup diff --git a/lib/zip/entry.rb b/lib/zip/entry.rb index 80160b57..bd3e4f34 100644 --- a/lib/zip/entry.rb +++ b/lib/zip/entry.rb @@ -603,9 +603,16 @@ def create_file(dest_path, _continue_on_exists_proc = proc { Zip.continue_on_exi get_input_stream do |is| set_extra_attributes_on_path(dest_path) + bytes_written = 0 buf = ''.dup while (buf = is.sysread(::Zip::Decompressor::CHUNK_SIZE, buf)) os << buf + + next unless ::Zip.validate_entry_sizes + bytes_written += buf.bytesize + if bytes_written > size + raise ::Zip::EntrySizeError, "Entry #{name} should be #{size}B but is larger when inflated" + end end end end diff --git a/lib/zip/errors.rb b/lib/zip/errors.rb index b2bcccd2..364c6eee 100644 --- a/lib/zip/errors.rb +++ b/lib/zip/errors.rb @@ -4,6 +4,7 @@ class EntryExistsError < Error; end class DestinationFileExistsError < Error; end class CompressionMethodError < Error; end class EntryNameError < Error; end + class EntrySizeError < Error; end class InternalError < Error; end class GPFBit3Error < Error; end diff --git a/test/file_extract_test.rb b/test/file_extract_test.rb index 57833fcb..6103aeae 100644 --- a/test/file_extract_test.rb +++ b/test/file_extract_test.rb @@ -10,6 +10,10 @@ def setup ::File.delete(EXTRACTED_FILENAME) if ::File.exist?(EXTRACTED_FILENAME) end + def teardown + ::Zip.reset! + end + def test_extract ::Zip::File.open(TEST_ZIP.zip_name) do |zf| zf.extract(ENTRY_TO_EXTRACT, EXTRACTED_FILENAME) @@ -80,4 +84,62 @@ def test_extract_non_entry_2 end assert(!File.exist?(outFile)) end + + def test_extract_incorrect_size + # The uncompressed size fields in the zip file cannot be trusted. This makes + # it harder for callers to validate the sizes of the files they are + # extracting, which can lead to denial of service. See also + # https://en.wikipedia.org/wiki/Zip_bomb + Dir.mktmpdir do |tmp| + real_zip = File.join(tmp, 'real.zip') + fake_zip = File.join(tmp, 'fake.zip') + file_name = 'a' + true_size = 500_000 + fake_size = 1 + + ::Zip::File.open(real_zip, ::Zip::File::CREATE) do |zf| + zf.get_output_stream(file_name) do |os| + os.write 'a' * true_size + end + end + + compressed_size = nil + ::Zip::File.open(real_zip) do |zf| + a_entry = zf.find_entry(file_name) + compressed_size = a_entry.compressed_size + assert_equal true_size, a_entry.size + end + + true_size_bytes = [compressed_size, true_size, file_name.size].pack('LLS') + fake_size_bytes = [compressed_size, fake_size, file_name.size].pack('LLS') + + data = File.binread(real_zip) + assert data.include?(true_size_bytes) + data.gsub! true_size_bytes, fake_size_bytes + + File.open(fake_zip, 'wb') do |file| + file.write data + end + + Dir.chdir tmp do + ::Zip::File.open(fake_zip) do |zf| + a_entry = zf.find_entry(file_name) + assert_equal fake_size, a_entry.size + + ::Zip.validate_entry_sizes = false + a_entry.extract + assert_equal true_size, File.size(file_name) + FileUtils.rm file_name + + ::Zip.validate_entry_sizes = true + error = assert_raises ::Zip::EntrySizeError do + a_entry.extract + end + assert_equal \ + 'Entry a should be 1B but is larger when inflated', + error.message + end + end + end + end end From 7849f7362ab0cd23d5730ef8b6f2c39252da2285 Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Sun, 15 Sep 2019 15:23:35 +0100 Subject: [PATCH 34/36] Default validate_entry_sizes to false for 1.3 release --- Changelog.md | 11 +++++++++++ README.md | 8 +++++++- lib/zip.rb | 2 +- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/Changelog.md b/Changelog.md index e8a7e16b..45f14333 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,16 @@ # X.X.X (Next) +- + +# 1.3.0 (Next) + +Security + +- Add `validate_entry_sizes` option so that callers can trust an entry's reported size when using `extract` [#403](https://github.com/rubyzip/rubyzip/pull/403) + - This option defaults to `false` for backward compatibility in this release, but you are strongly encouraged to set it to `true`. It will default to `true` in rubyzip 2.0. + +New Feature + - Add `add_stored` method to simplify adding entries without compression [#366](https://github.com/rubyzip/rubyzip/pull/366) Tooling / Documentation diff --git a/README.md b/README.md index 2ff41ed9..51b275b9 100644 --- a/README.md +++ b/README.md @@ -265,7 +265,13 @@ Zip.warn_invalid_date = false ### Size Validation -By default, `rubyzip`'s `extract` method checks that an entry's reported uncompressed size is not (significantly) smaller than its actual size. This is to help you protect your application against [zip bombs](https://en.wikipedia.org/wiki/Zip_bomb). Before `extract`ing an entry, you should check that its size is in the range you expect. For example, if your application supports processing up to 100 files at once, each up to 10MiB, your zip extraction code might look like: +**This setting defaults to `false` in rubyzip 1.3 for backward compatibility, but it will default to `true` in rubyzip 2.0.** + +If you set +``` +Zip.validate_entry_sizes = true +``` +then `rubyzip`'s `extract` method checks that an entry's reported uncompressed size is not (significantly) smaller than its actual size. This is to help you protect your application against [zip bombs](https://en.wikipedia.org/wiki/Zip_bomb). Before `extract`ing an entry, you should check that its size is in the range you expect. For example, if your application supports processing up to 100 files at once, each up to 10MiB, your zip extraction code might look like: ```ruby MAX_FILE_SIZE = 10 * 1024**2 # 10MiB diff --git a/lib/zip.rb b/lib/zip.rb index c3a6ed5e..eeac96a0 100644 --- a/lib/zip.rb +++ b/lib/zip.rb @@ -55,7 +55,7 @@ def reset! @write_zip64_support = false @warn_invalid_date = true @case_insensitive_match = false - @validate_entry_sizes = true + @validate_entry_sizes = false end def setup From 97cb6aefe6d12bd2429d7a2e119ccb26f259d71d Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Wed, 18 Sep 2019 18:34:23 +0100 Subject: [PATCH 35/36] Warn when an entry size is invalid --- lib/zip/entry.rb | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/zip/entry.rb b/lib/zip/entry.rb index bd3e4f34..677e49ef 100644 --- a/lib/zip/entry.rb +++ b/lib/zip/entry.rb @@ -604,14 +604,19 @@ def create_file(dest_path, _continue_on_exists_proc = proc { Zip.continue_on_exi set_extra_attributes_on_path(dest_path) bytes_written = 0 + warned = false buf = ''.dup while (buf = is.sysread(::Zip::Decompressor::CHUNK_SIZE, buf)) os << buf - - next unless ::Zip.validate_entry_sizes bytes_written += buf.bytesize - if bytes_written > size - raise ::Zip::EntrySizeError, "Entry #{name} should be #{size}B but is larger when inflated" + if bytes_written > size && !warned + message = "Entry #{name} should be #{size}B but is larger when inflated" + if ::Zip.validate_entry_sizes + raise ::Zip::EntrySizeError, message + else + puts "WARNING: #{message}" + warned = true + end end end end From 7c65e1e3595031392f1050b81fb2b95b0f2ee764 Mon Sep 17 00:00:00 2001 From: John Lees-Miller Date: Wed, 25 Sep 2019 19:58:16 +0100 Subject: [PATCH 36/36] Bump version to 1.3.0 --- Changelog.md | 2 +- lib/zip/version.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Changelog.md b/Changelog.md index 45f14333..36ae1009 100644 --- a/Changelog.md +++ b/Changelog.md @@ -2,7 +2,7 @@ - -# 1.3.0 (Next) +# 1.3.0 (2019-09-25) Security diff --git a/lib/zip/version.rb b/lib/zip/version.rb index afbccee8..37fba090 100644 --- a/lib/zip/version.rb +++ b/lib/zip/version.rb @@ -1,3 +1,3 @@ module Zip - VERSION = '1.2.4' + VERSION = '1.3.0' end