diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..ca998f87 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,16 @@ +version: 2 +updates: +- package-ecosystem: bundler + directory: "/" + schedule: + interval: daily + time: "13:00" + open-pull-requests-limit: 10 + ignore: + - dependency-name: rubocop + versions: + - 1.10.0 + - 1.12.0 + - 1.12.1 + - 1.9.0 + - 1.9.1 diff --git a/.github/workflows/rspec.yml b/.github/workflows/rspec.yml new file mode 100644 index 00000000..73185e63 --- /dev/null +++ b/.github/workflows/rspec.yml @@ -0,0 +1,23 @@ +name: rspec + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + rspec: + runs-on: ubuntu-latest + strategy: + matrix: + ruby-version: ["3.0", "3.1", "3.2", "3.3"] + steps: + - uses: actions/checkout@v2 + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true + - name: Run rspec tests + run: bundle exec rspec spec diff --git a/.github/workflows/rubocop.yml b/.github/workflows/rubocop.yml new file mode 100644 index 00000000..752d2862 --- /dev/null +++ b/.github/workflows/rubocop.yml @@ -0,0 +1,23 @@ +name: rubocop + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + rubocop: + runs-on: ubuntu-latest + strategy: + matrix: + ruby-version: ["3.0", "3.1", "3.2", "3.3"] + steps: + - uses: actions/checkout@v2 + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby-version }} + bundler-cache: true + - name: Run rubocop style check + run: bundle exec rubocop diff --git a/.gitignore b/.gitignore index c529daea..7d50bc35 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,7 @@ *.page_info ._* tags +vendor +.bundle +.idea +.byebug_history diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 00000000..f0fa685f --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,50 @@ +# TODO: Enable rubocop-rspec only along with all the fixes it demands. +# require: rubocop-rspec + +AllCops: + TargetRubyVersion: 2.6 + NewCops: enable +Gemspec/DevelopmentDependencies: + Enabled: false +Gemspec/RequireMFA: + Enabled: false +Lint/AmbiguousOperator: + Enabled: false +Lint/FormatParameterMismatch: + Enabled: false +Layout/LineLength: + Max: 120 +Metrics/BlockLength: + Max: 400 +Naming/VariableNumber: + Enabled: false +Style/AccessorGrouping: + EnforcedStyle: separated +Style/StringLiterals: + EnforcedStyle: double_quotes +Style/TrailingCommaInArrayLiteral: + EnforcedStyleForMultiline: consistent_comma +Style/TrailingCommaInHashLiteral: + EnforcedStyleForMultiline: consistent_comma +Style/FormatString: + Enabled: false +Style/FormatStringToken: + Enabled: false +Style/Documentation: + Enabled: false +Metrics/ClassLength: + Enabled: false +Metrics/MethodLength: + Enabled: false +Metrics/AbcSize: + Enabled: false +Metrics/CyclomaticComplexity: + Enabled: false +Metrics/ParameterLists: + Max: 10 +Metrics/PerceivedComplexity: + Enabled: false +Style/SymbolArray: + MinSize: 1 +Style/EmptyCaseCondition: + Enabled: false \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 5edbe961..00000000 --- a/.travis.yml +++ /dev/null @@ -1,2 +0,0 @@ -language: ruby -script: bundle exec rspec spec diff --git a/Gemfile b/Gemfile index d7fab679..be173b20 100644 --- a/Gemfile +++ b/Gemfile @@ -1,5 +1,5 @@ -source 'https://rubygems.org' +# frozen_string_literal: true -gem 'rspec' -gem 'bindata', '>= 1.4.5' -gem 'digest-crc', '>= 0.4.1' +source "https://rubygems.org" + +gemspec diff --git a/Gemfile.lock b/Gemfile.lock index 10dbda10..d7a7040d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,30 +1,85 @@ +PATH + remote: . + specs: + innodb_ruby (0.14.0) + bigdecimal (~> 3.1.8) + bindata (>= 1.4.5, < 3.0) + csv (~> 3.3) + digest-crc (~> 0.4, >= 0.4.1) + getoptlong (~> 0.2.1) + histogram (~> 0.2) + GEM remote: https://rubygems.org/ specs: - bindata (2.3.3) - diff-lcs (1.2.5) - digest-crc (0.4.1) - rspec (3.5.0) - rspec-core (~> 3.5.0) - rspec-expectations (~> 3.5.0) - rspec-mocks (~> 3.5.0) - rspec-core (3.5.3) - rspec-support (~> 3.5.0) - rspec-expectations (3.5.0) + ast (2.4.2) + bigdecimal (3.1.8) + bindata (2.5.0) + csv (3.3.0) + diff-lcs (1.5.1) + digest-crc (0.6.5) + rake (>= 12.0.0, < 14.0.0) + getoptlong (0.2.1) + gnuplot (2.6.2) + histogram (0.2.4.1) + json (2.7.5) + language_server-protocol (3.17.0.3) + parallel (1.26.3) + parser (3.3.5.0) + ast (~> 2.4.1) + racc + racc (1.8.1) + rainbow (3.1.1) + rake (13.2.1) + regexp_parser (2.9.2) + rspec (3.11.0) + rspec-core (~> 3.11.0) + rspec-expectations (~> 3.11.0) + rspec-mocks (~> 3.11.0) + rspec-core (3.11.0) + rspec-support (~> 3.11.0) + rspec-expectations (3.11.1) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.5.0) - rspec-mocks (3.5.0) + rspec-support (~> 3.11.0) + rspec-mocks (3.11.2) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.5.0) - rspec-support (3.5.0) + rspec-support (~> 3.11.0) + rspec-support (3.11.1) + rubocop (1.67.0) + json (~> 2.3) + language_server-protocol (>= 3.17.0) + parallel (~> 1.10) + parser (>= 3.3.0.2) + rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 2.4, < 3.0) + rubocop-ast (>= 1.32.2, < 2.0) + ruby-progressbar (~> 1.7) + unicode-display_width (>= 2.4.0, < 3.0) + rubocop-ast (1.33.0) + parser (>= 3.3.1.0) + rubocop-capybara (2.21.0) + rubocop (~> 1.41) + rubocop-factory_bot (2.26.1) + rubocop (~> 1.61) + rubocop-rspec (2.31.0) + rubocop (~> 1.40) + rubocop-capybara (~> 2.17) + rubocop-factory_bot (~> 2.22) + rubocop-rspec_rails (~> 2.28) + rubocop-rspec_rails (2.29.1) + rubocop (~> 1.61) + ruby-progressbar (1.13.0) + unicode-display_width (2.6.0) PLATFORMS ruby DEPENDENCIES - bindata (>= 1.4.5) - digest-crc (>= 0.4.1) - rspec + gnuplot (~> 2.6.0) + innodb_ruby! + rspec (~> 3.11.0) + rubocop (~> 1.18) + rubocop-rspec (~> 2.4) BUNDLED WITH - 1.13.1 + 2.5.22 diff --git a/README.md b/README.md index 94f60ec7..ee008abc 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,7 @@ -# A parser for InnoDB file formats, in Ruby # +# A parser for InnoDB file formats, in Ruby + +[![rspec test status](https://github.com/jeremycole/innodb_ruby/actions/workflows/rspec.yml/badge.svg)](https://github.com/jeremycole/innodb_ruby/actions/workflows/rspec.yml) +[![rubocop style check status](https://github.com/jeremycole/innodb_ruby/actions/workflows/rubocop.yml/badge.svg)](https://github.com/jeremycole/innodb_ruby/actions/workflows/rubocop.yml) The purpose for this library and tools is to expose some otherwise hidden internals of InnoDB. This code is not intended for critical production usage. It is definitely buggy, and it may be dangerous. Neither its internal APIs or its output are considered stable and are subject to change at any time. @@ -11,11 +14,7 @@ It is intended as for a few purposes: Various parts of this library and the tools included may have wildly differing maturity levels, as it is worked on primarily based on immediate needs of the authors. -# Resources # +# Resources * The [innodb_ruby wiki](https://github.com/jeremycole/innodb_ruby/wiki) contains some additional references and documentation to help you get started. -* Visit the [innodb_ruby mailing list on Google Groups](https://groups.google.com/d/forum/innodb_ruby) or email [innodb_ruby@googlegroups.com](mailto:innodb_ruby@googlegroups.com) — If you have questions about `innodb_ruby` or its usage. * See the [RubyGems page for innodb_ruby](http://rubygems.org/gems/innodb_ruby) — Gem packaged releases are published regularly to RubyGems.org, which also provides online documentation. - - -[![Build Status](https://travis-ci.org/jeremycole/innodb_ruby.svg?branch=master)](https://travis-ci.org/jeremycole/innodb_ruby) diff --git a/bin/innodb_log b/bin/innodb_log index a4c376cb..9954d0a6 100755 --- a/bin/innodb_log +++ b/bin/innodb_log @@ -1,21 +1,14 @@ #!/usr/bin/env ruby -# -*- encoding : utf-8 -*- +# frozen_string_literal: true require "getoptlong" -require "ostruct" require "set" require "innodb" def log_summary(log_group) - puts "%-20s%-15s%-10s%-12s%-10s" % [ - "lsn", - "block", - "length", - "first_rec", - "checkpoint", - ] + puts "%-20s%-15s%-10s%-12s%-10s" % %w[lsn block length first_rec checkpoint] lsn = log_group.start_lsn.first - log_group.each_block do |block_index, block| + log_group.each_block do |_block_index, block| header = block.header puts "%-20i%-15i%-10i%-12i%-10i" % [ lsn, @@ -30,14 +23,7 @@ def log_summary(log_group) end def log_reader_record_summary(reader, follow) - puts "%-10s%-10s%-20s%-10s%-10s%-10s" % [ - "time", - "lsn", - "type", - "size", - "space", - "page" - ] + puts "%-10s%-10s%-20s%-10s%-10s%-10s" % %w[time lsn type size space page] reader.each_record(follow) do |rec| preamble = rec.preamble.dup @@ -72,7 +58,8 @@ end def usage(exit_code, message = nil) print "Error: #{message}\n" unless message.nil? - print <<'END_OF_USAGE' + # rubocop:disable Layout/HeredocIndentation + print <] -f @@ -103,21 +90,31 @@ The following modes are supported: Dump the contents of a log record, using the Ruby pp ("pretty-print") module. END_OF_USAGE + # rubocop:enable Layout/HeredocIndentation exit exit_code end -@options = OpenStruct.new +InnodbLogOptions = Struct.new( + :log_files, + :dump, + :lsn, + keyword_init: true +) + +@options = InnodbLogOptions.new @options.log_files = [] @options.dump = false @options.lsn = nil +# rubocop:disable Layout/SpaceInsideArrayLiteralBrackets getopt_options = [ [ "--help", "-?", GetoptLong::NO_ARGUMENT ], [ "--log-file", "-f", GetoptLong::REQUIRED_ARGUMENT ], [ "--dump-blocks", "-d", GetoptLong::NO_ARGUMENT ], [ "--lsn", "-l", GetoptLong::REQUIRED_ARGUMENT ], ] +# rubocop:enable Layout/SpaceInsideArrayLiteralBrackets getopt = GetoptLong.new(*getopt_options) @@ -136,6 +133,8 @@ end mode = ARGV.shift +# rubocop:disable Style/IfUnlessModifier + unless mode usage 1, "At least one mode must be provided" end @@ -144,10 +143,12 @@ if @options.log_files.empty? usage 1, "At least one log file (-f) must be specified" end -if /^(log-)?record-/.match(mode) and !@options.lsn +if /^(log-)?record-/.match(mode) && !@options.lsn usage 1, "LSN must be specified using -l/--lsn" end +# rubocop:enable Style/IfUnlessModifier + log_group = Innodb::LogGroup.new(@options.log_files.sort) case mode diff --git a/bin/innodb_ruby_generate_mysql_collations b/bin/innodb_ruby_generate_mysql_collations new file mode 100755 index 00000000..5bb91dda --- /dev/null +++ b/bin/innodb_ruby_generate_mysql_collations @@ -0,0 +1,96 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +# To update lib/innodb/mysql_collations.rb, run this with a path to the MySQL source directory +# containing CHARSET_INFO collation definitions, e.g.: + +# bundle exec bin/innodb_ruby_generate_mysql_collations ~/git/mysql-server > lib/innodb/mysql_collations.rb + +MysqlCharsetInfo = Struct.new( + :number, + :primary_number, + :binary_number, + :state, + :csname, + :m_coll_name, + :comment, + :tailoring, + :coll_param, + :ctype, + :to_lower, + :to_upper, + :sort_order, + :uca, + :tab_to_uni, + :tab_from_uni, + :caseinfo, + :state_maps, + :ident_map, + :strxfrm_multiply, + :caseup_multiply, + :casedn_multiply, + :mbminlen, + :mbmaxlen, + :mbmaxlenlen, + :min_sort_char, + :max_sort_char, + :pad_char, + :escape_with_backslash_is_dangerous, + :levels_for_compare, + :cset, + :coll, + :pad_attribute +) + +charset_infos = [] + +raise "First argument must be the path to a modern MySQL source tree" unless (ARGV.size == 1) && Dir.exist?(ARGV[0]) + +Dir.glob(File.join(ARGV[0], "strings/ctype-**.cc")).each do |filename| + content = File.read(filename) + warn "Parsing #{filename}..." + + # Global individual constants e.g. CHARSET_INFO my_charset_utf8mb4_general_ci = { ... } + charset_info_strings = content.scan(/^CHARSET_INFO \w+ = ({.*?})/m).flatten + + # Global array of constants e.g. CHARSET_INFO compiled_charsets[] = { { ... }, { ... } }; + content.match(/CHARSET_INFO \w+\[\] = {\s*(?:{.*?}\s*,\s*)+/m) + &.match(0) + &.gsub(/CHARSET_INFO \w+\[\] = {/, "") + &.scan(/{.*?}/m) + &.each do |s| + charset_info_strings.push(s) + end + + charset_info_strings = charset_info_strings.map do |x| + x.gsub(%r{/\*.*?\*/}, "").gsub(%r{//.*?$}, "").gsub(/\s+/, " ").gsub(/["']/, "") + end + + charset_infos += charset_info_strings.map do |charset_info_string| + matches = charset_info_string.match(/{(?.*?)}/) + + MysqlCharsetInfo.new(*matches[:definition].split(",").map(&:strip).map { |x| x =~ /^[0-9]+$/ ? x.to_i : x }) + end +end + +if charset_infos.empty? + warn "No MySQL collations found... bad path provided?" + exit 1 +end + +warn "Found #{charset_infos.size} collations, generating output." + +puts "# frozen_string_literal: true" +puts +puts "# Generated at #{Time.now.utc} using innodb_ruby_generate_mysql_collations. Do not edit!" +puts + +puts "# rubocop:disable all" +charset_infos.sort_by(&:number).each do |charset_info| + puts format("Innodb::MysqlCollation.add(id: %d, name: %s, character_set_name: %s, mbminlen: %i, mbmaxlen: %i)", + charset_info.number, + charset_info.m_coll_name.inspect, + charset_info.csname.inspect, + charset_info.mbminlen, + charset_info.mbmaxlen) +end diff --git a/bin/innodb_space b/bin/innodb_space index af4a529d..edcd107e 100755 --- a/bin/innodb_space +++ b/bin/innodb_space @@ -1,27 +1,35 @@ #!/usr/bin/env ruby -# -*- encoding : utf-8 -*- +# frozen_string_literal: true require "getoptlong" -require "ostruct" +require "histogram/array" require "innodb" +class String + def squish! + gsub!(/[[:space:]]+/, " ") + strip! + self + end + + def squish + dup.squish! + end +end + # Convert a floating point RGB array into an ANSI color number approximating it. def rgb_to_ansi(rgb) rgb_n = rgb.map { |c| (c * 5.0).round } 16 + (rgb_n[0] * 36) + (rgb_n[1] * 6) + rgb_n[2] end -def rgb_to_rgbhex(rgb) - rgb.map { |c| "%02x" % [(c * 255.0).round] }.join -end - # Interpolate intermediate float-arrays between two float-arrays. Do not # include the points a and b in the result. -def interpolate(a, b, count) - deltas = a.each_index.map { |i| b[i] - a[i] } - steps = a.each_index.map { |i| deltas[i].to_f / (count.to_f + 1) } +def interpolate(ary_a, ary_b, count) + deltas = ary_a.each_index.map { |i| ary_b[i] - ary_a[i] } + steps = ary_a.each_index.map { |i| deltas[i].to_f / (count.to_f + 1) } - count.times.to_a.map { |i| a.each_index.map { |j| a[j] + ((i+1).to_f * steps[j]) } } + count.times.to_a.map { |i| ary_a.each_index.map { |j| ary_a[j] + ((i + 1).to_f * steps[j]) } } end # Interpolate intermediate float-arrays between each step in a sequence of @@ -29,7 +37,7 @@ end def interpolate_sequence(sequence, count) result = [] result << sequence.first - (sequence.size-1).times.map { |n| [sequence[n], sequence[n+1]] }.each do |from, to| + (sequence.size - 1).times.map { |n| [sequence[n], sequence[n + 1]] }.each do |from, to| interpolate(from, to, count).each do |step| result << step end @@ -48,15 +56,11 @@ HEATMAP_PROGRESSION = [ [1.0, 1.0, 0.0], # Yellow [1.0, 0.0, 0.0], # Red [1.0, 0.0, 1.0], # Purple -] +].freeze # Typical heatmap color progression. ANSI_COLORS_HEATMAP = interpolate_sequence(HEATMAP_PROGRESSION, 6).map { |rgb| rgb_to_ansi(rgb) } -RGBHEX_COLORS_HEATMAP = interpolate_sequence(HEATMAP_PROGRESSION, 41).map { |rgb| rgb_to_rgbhex(rgb) } - -RGBHEX_COLORS_RANDOM = 100.times.inject([]) { |a, x| a << rgb_to_rgbhex([rand * 0.7 + 0.25, rand * 0.7 + 0.25, rand * 0.7 + 0.25]) } - # The 24-step grayscale progression. ANSI_COLORS_GRAYSCALE = (0xe8..0xff).to_a @@ -66,79 +70,57 @@ def ansi_color(color, text) end # Zero and 1/8 through 8/8 illustrations. -BLOCK_CHARS_V = ["░", "▁", "▂", "▃", "▄", "▅", "▆", "▇", "█"] -BLOCK_CHARS_H = ["░", "▏", "▎", "▍", "▌", "▋", "▊", "▉", "█"] +BLOCK_CHARS_V = "░▁▂▃▄▅▆▇█".chars.freeze +BLOCK_CHARS_H = "░▏▎▍▌▋▊▉█".chars.freeze # A reasonably large prime number to multiply identifiers by in order to # space out the colors used for similar identifiers. -COLOR_SPACING_PRIME = 999983 +COLOR_SPACING_PRIME = 999_983 # Return a string with a possibly colored filled block for use in printing # to an ANSI-capable Unicode-enabled terminal. -def filled_block(fraction, identifier=nil, block_chars=BLOCK_CHARS_V) - if fraction == 0.0 +def filled_block(fraction, identifier = nil, block_chars = BLOCK_CHARS_V) + if fraction.zero? block = block_chars[0] else - parts = (fraction.to_f * (block_chars.size.to_f-1)).floor - block = block_chars[[block_chars.size-1, parts+1].min] + parts = (fraction.to_f * (block_chars.size.to_f - 1)).floor + block = block_chars[[block_chars.size - 1, parts + 1].min] end if identifier # ANSI 256-color mode, color palette starts at 10 and contains 216 colors. - color = 16 + ((identifier * COLOR_SPACING_PRIME) % 216) + color = 16 + ((identifier * COLOR_SPACING_PRIME) % 216) ansi_color(color, block) else block end end -def svg_join_args(args) - args.map { |arg| "%s=\"%s\"" % [ arg[0], arg[1], ] }.join(" ") -end - -def svg(elem, args, content=nil) - if content - "<" + elem.to_s + " " + svg_join_args(args) + ">" + - content.to_s + - "" - else - "<" + elem.to_s + " " + svg_join_args(args) + " />" - end -end +def center(text, width) + return text if text.size >= width -def svg_path_rounded_rect(x, y, w, h, r) - [ - "M%i,%i" % [ x + r, y - r ], - "L%i,%i" % [ x + w - r, y - r ], - "S%i,%i %i,%i" % [ x + w + r, y - r, x + w + r, y + r ], - "L%i,%i" % [ x + w + r, y + h - r ], - "S%i,%i %i,%i" % [ x + w + r, y + h + r, x + w - r, y + h + r ], - "L%i,%i" % [ x + r, y + h + r ], - "S%i,%i %i,%i" % [ x - r, y + h + r, x - r, y + h - r ], - "L%i,%i" % [ x - r, y + r ], - "S%i,%i %i,%i" % [ x - r, y - r, x + r, y - r ], - "Z", - ].join(" ") + spaces = (width - text.size) / 2 + (" " * spaces) + text + (" " * spaces) end # Print metadata about each list in an array of InnoDB::List objects. def print_lists(lists) - puts "%-20s%-12s%-12s%-12s%-12s%-12s" % [ - "name", - "length", - "f_page", - "f_offset", - "l_page", - "l_offset", + puts "%-20s%-12s%-12s%-12s%-12s%-12s" % %w[ + name + length + f_page + f_offset + l_page + l_offset ] lists.each do |name, list| puts "%-20s%-12i%-12i%-12i%-12i%-12i" % [ name, list.base[:length], - list.base[:first] && list.base[:first][:page] || 0, - list.base[:first] && list.base[:first][:offset] || 0, - list.base[:last] && list.base[:last][:page] || 0, - list.base[:last] && list.base[:last][:offset] || 0, + (list.base[:first] && list.base[:first][:page]) || 0, + (list.base[:first] && list.base[:first][:offset]) || 0, + (list.base[:last] && list.base[:last][:page]) || 0, + (list.base[:last] && list.base[:last][:offset]) || 0, ] end end @@ -146,38 +128,36 @@ end # Print a page usage bitmap for each extent descriptor in an array of # Innodb::XdesEntry objects. def print_xdes_list(space, list) - puts "%-12s%-64s" % [ - "start_page", - "page_used_bitmap" + puts "%-12s%-64s" % %w[ + start_page + page_used_bitmap ] list.each do |entry| - puts "%-12i%-64s" % [ - entry.xdes[:start_page], - entry.each_page_status.inject("") { |bitmap, (page_number, page_status)| - if page_number < space.pages - bitmap += page_status[:free] ? "." : "#" - end - bitmap - }, - ] + display = entry.each_page_status.inject("") do |bitmap, (page_number, page_status)| + if page_number < space.pages + bitmap += page_status[:free] ? "." : "#" + end + bitmap + end + puts "%-12i%-64s" % [entry.xdes[:start_page], display] end end # Print a summary of page usage for all pages in an index. def print_index_page_summary(pages) - puts "%-12s%-8s%-8s%-8s%-8s%-8s" % [ - "page", - "index", - "level", - "data", - "free", - "records", + puts "%-12s%-8s%-8s%-8s%-8s%-8s" % %w[ + page + index + level + data + free + records ] pages.each do |page_number, page| - case page.type - when :INDEX + case + when page.is_a?(Innodb::Page::Index) puts "%-12i%-8i%-8i%-8i%-8i%-8i" % [ page_number, page.page_header[:index_id], @@ -186,142 +166,89 @@ def print_index_page_summary(pages) page.free_space, page.records, ] - when :ALLOCATED - puts "%-12i%-8i%-8i%-8i%-8i%-8i" % [ page_number, 0, 0, 0, page.size, 0 ] + when page.type == :ALLOCATED + puts "%-12i%-8i%-8i%-8i%-8i%-8i" % [page_number, 0, 0, 0, page.size, 0] end end end # Print a summary of all spaces in the InnoDB system. def system_spaces(innodb_system) - puts "%-32s%-12s%-12s" % [ - "name", - "pages", - "indexes", + puts "%-64s%-12s" % %w[ + name + pages ] - print_space_information = lambda do |name, space| - puts "%-32s%-12i%-12i" % [ - name, + innodb_system&.each_space do |space| + puts "%-64s%-12i" % [ + space.name.gsub("#{innodb_system.data_directory}/", ""), space.pages, - space.each_index.to_a.size, ] end - print_space_information.call("(system)", innodb_system.system_space) - - innodb_system.each_table_name do |table_name| - space = innodb_system.space_by_table_name(table_name) - next unless space - print_space_information.call(table_name, space) - end - innodb_system.each_orphan do |table_name| - puts "%-43s (orphan/tmp)" % table_name + puts "%-64s%-12s" % [table_name, "(orphan/tmp)"] end end -# Print the contents of the SYS_TABLES data dictionary table. +# Print the contents of the table list from the data dictionary. def data_dictionary_tables(innodb_system) - puts "%-32s%-12s%-12s%-12s%-12s%-12s%-15s%-12s" % [ - "name", - "id", - "n_cols", - "type", - "mix_id", - "mix_len", - "cluster_name", - "space", + puts "%-64s%-12s%-12s" % %w[ + name + columns + indexes ] - innodb_system.data_dictionary.each_table do |record| - puts "%-32s%-12i%-12i%-12i%-12i%-12i%-15s%-12i" % [ - record["NAME"], - record["ID"], - record["N_COLS"], - record["TYPE"], - record["MIX_ID"], - record["MIX_LEN"], - record["CLUSTER_NAME"], - record["SPACE"], + innodb_system.data_dictionary.tables.each do |table| + puts "%-64s%-12i%-12i" % [ + table.name, + table.columns.count, + table.indexes.count, ] end end -# Print the contents of the SYS_COLUMNS data dictionary table. +# Print the contents of the column list from the data dictionary. def data_dictionary_columns(innodb_system) - puts "%-12s%-6s%-32s%-12s%-12s%-6s%-6s" % [ - "table_id", - "pos", - "name", - "mtype", - "prtype", - "len", - "prec", + puts "%-64s%-32s%-32s" % %w[ + table + name + description ] - innodb_system.data_dictionary.each_column do |record| - puts "%-12i%-6i%-32s%-12i%-12i%-6i%-6i" % [ - record["TABLE_ID"], - record["POS"], - record["NAME"], - record["MTYPE"], - record["PRTYPE"], - record["LEN"], - record["PREC"], + innodb_system.data_dictionary.columns.each do |column| + puts "%-64s%-32s%-32s" % [ + column.table.name, + column.name, + column.description, ] end end -# Print the contents of the SYS_INDEXES data dictionary table. +# Print the contents of the index list from the data dictionary. def data_dictionary_indexes(innodb_system) - puts "%-12s%-12s%-32s%-10s%-6s%-12s%-12s" % [ - "table_id", - "id", - "name", - "n_fields", - "type", - "space", - "page_no", - ] - - innodb_system.data_dictionary.each_index do |record| - puts "%-12i%-12i%-32s%-10i%-6i%-12i%-12i" % [ - record["TABLE_ID"], - record["ID"], - record["NAME"], - record["N_FIELDS"], - record["TYPE"], - record["SPACE"], - record["PAGE_NO"], - ] - end -end - -# Print the contents of the SYS_FIELDS data dictionary table. -def data_dictionary_fields(innodb_system) - puts "%-12s%-12s%-32s" % [ - "index_id", - "pos", - "col_name", + puts "%-64s%-32s%-32s" % %w[ + table + name + columns ] - innodb_system.data_dictionary.each_field do |record| - puts "%-12i%-12i%-32s" % [ - record["INDEX_ID"], - record["POS"], - record["COL_NAME"], + innodb_system.data_dictionary.indexes.each do |index| + puts "%-64s%-32s%-32s" % [ + index.table.name, + index.name, + index.column_references.each.map(&:name), ] end end def space_summary(space, start_page) - puts "%-12s%-20s%-12s%-12s%-20s" % [ - "page", - "type", - "prev", - "next", - "lsn", + puts "%-12s%-20s%-12s%-12s%-20s" % %w[ + page + type + prev + next + lsn ] space.each_page(start_page) do |page_number, page| @@ -344,11 +271,11 @@ def space_index_fseg_pages_summary(space, fseg_id) end def space_page_type_regions(space, start_page) - puts "%-12s%-12s%-12s%-20s" % [ - "start", - "end", - "count", - "type", + puts "%-12s%-12s%-12s%-20s" % %w[ + start + end + count + type ] space.each_page_type_region(start_page) do |region| @@ -367,16 +294,16 @@ def space_page_type_summary(space, start_page) page_count = 0 # A Hash of page type => count. page_type = Hash.new(0) - space.each_page(start_page) do |page_number, page| + space.each_page(start_page) do |_page_number, page| page_count += 1 page_type[page.type] += 1 end - puts "%-20s%-12s%-12s%-20s" % [ - "type", - "count", - "percent", - "description", + puts "%-20s%-12s%-12s%-20s" % %w[ + type + count + percent + description ] # Sort the page type Hash by count, descending. @@ -384,7 +311,7 @@ def space_page_type_summary(space, start_page) puts "%-20s%-12i%-12.2f%-20s" % [ type, type_count, - 100.0 * (type_count.to_f / page_count.to_f), + 100.0 * (type_count.to_f / page_count), Innodb::Page::PAGE_TYPE[type][:description], ] end @@ -397,16 +324,14 @@ end def space_list_iterate(space, list_name) fsp = space.page(0).fsp_header - unless fsp[list_name] && fsp[list_name].is_a?(Innodb::List) - raise "List '#{list_name}' doesn't exist" - end + raise "List '#{list_name}' doesn't exist" unless fsp[list_name].is_a?(Innodb::List) case fsp[list_name] when Innodb::List::Xdes print_xdes_list(space, fsp[list_name]) when Innodb::List::Inode - puts "%-12s" % [ - "page", + puts "%-12s" % %w[ + page ] fsp[list_name].each do |page| puts "%-12i" % [ @@ -417,22 +342,22 @@ def space_list_iterate(space, list_name) end def space_indexes(innodb_system, space) - puts "%-12s%-32s%-12s%-12s%-12s%-12s%-12s%-12s" % [ - "id", - "name", - "root", - "fseg", - "fseg_id", - "used", - "allocated", - "fill_factor", + puts "%-12s%-32s%-12s%-12s%-12s%-12s%-12s%-12s" % %w[ + id + name + root + fseg + fseg_id + used + allocated + fill_factor ] space.each_index do |index| index.each_fseg do |fseg_name, fseg| puts "%-12i%-32s%-12i%-12s%-12i%-12i%-12i%-12s" % [ index.id, - innodb_system ? innodb_system.index_name_by_id(index.id) : "", + innodb_system.data_dictionary.indexes.find(innodb_index_id: index.id)&.name, index.root.offset, fseg_name, fseg.fseg_id, @@ -444,34 +369,36 @@ def space_indexes(innodb_system, space) end end -def space_index_pages_free_plot(space, image, start_page) - unless require "gnuplot" - raise "Couldn't load gnuplot. Is it installed?" - end +def space_index_pages_free_plot(space, start_page) + raise "Could not load gnuplot. Is it installed?" unless require "gnuplot" - index_data = {0 => {:x => [], :y => []}} + index_data = { 0 => { x: [], y: [] } } space.each_page(start_page) do |page_number, page| - case page.type - when :INDEX - data = (index_data[page.page_header[:index_id]] ||= {:x => [], :y => []}) + case + when page.is_a?(Innodb::Page::Index) + data = (index_data[page.page_header[:index_id]] ||= { x: [], y: [] }) data[:x] << page_number data[:y] << page.free_space - when :ALLOCATED + when page.type == :ALLOCATED index_data[0][:x] << page_number index_data[0][:y] << page.size end end - image_file = image + "_free.png" + image_name = space.name + image_name = image_name.sub(%r{^#{space.innodb_system.data_directory}/}, "") if space.innodb_system + image_name = image_name.sub(".ibd", "").gsub(/[^a-zA-Z0-9_]/, "_").sub(/\A_+/, "") + image_file = "#{image_name}_free.png" + # Aim for one horizontal pixel per extent, but min 1k and max 10k width. - image_width = [10000, [1000, space.pages / space.pages_per_extent].max].min + image_width = (space.pages / space.pages_per_extent).clamp(1_000, 10_000) Gnuplot.open do |gp| Gnuplot::Plot.new(gp) do |plot| plot.terminal "png size #{image_width}, 800" plot.output image_file - plot.title image + plot.title image_name.gsub("_", " ") plot.key "reverse left top box horizontal Left textcolor variable" plot.ylabel "free space per page" plot.xlabel "page number" @@ -481,7 +408,7 @@ def space_index_pages_free_plot(space, image, start_page) index_data.sort.each do |id, data| plot.data << Gnuplot::DataSet.new([data[:x], data[:y]]) do |ds| ds.with = "dots" - ds.title = id == 0 ? "Unallocated" : "Index #{id}" + ds.title = id.zero? ? "Unallocated" : "Index #{id}" end end @@ -494,279 +421,99 @@ def space_extents(space) print_xdes_list(space, space.each_xdes) end +# rubocop:disable Metrics/BlockNesting +def space_extents_illustrate_page_status(space, entry, count_by_identifier, identifiers) + entry.each_page_status.with_object("".dup) do |(page_number, page_status), bitmap| + if page_number >= space.pages + bitmap << " " + next + end + + used_fraction = 1.0 + identifier = nil + if page_status[:free] + used_fraction = 0.0 + else + page = space.page(page_number) + used_fraction = page.used_space.to_f / page.size if page.respond_to?(:used_space) + if page.respond_to?(:index_id) + identifier = page.index_id + unless identifiers[identifier] + identifiers[identifier] = page.ibuf_index? ? "Insert Buffer Index" : "Index #{page.index_id}" + if space.innodb_system + dd_index = space.innodb_system.data_dictionary.indexes.find(innodb_index_id: page.index_id) + identifiers[identifier] += " (%s.%s)" % [dd_index.table.name, dd_index.name] if dd_index + end + end + end + end + + bitmap << filled_block(used_fraction, identifier) + + if used_fraction.zero? + count_by_identifier[:free] += 1 + else + count_by_identifier[identifier] += 1 + end + end +end +# rubocop:enable Metrics/BlockNesting + # Illustrate the space by printing each extent and for each page, printing a # filled block colored based on the index the page is part of. Print a legend # for the colors used afterwards. def space_extents_illustrate(space) - line_width = space.pages_per_extent + width = space.pages_per_extent puts - puts "%12s ╭%-#{line_width}s╮" % [ "Start Page", "─" * line_width ] + puts "%12s %-#{width}s " % ["", center(space.name, width)] + puts "%12s ╭%-#{width}s╮" % ["Start Page", "─" * width] identifiers = {} count_by_identifier = Hash.new(0) space.each_xdes do |entry| - puts "%12i │%-#{line_width}s│" % [ + puts "%12i │%-#{width}s│" % [ entry.xdes[:start_page], - entry.each_page_status.inject("") { |bitmap, (page_number, page_status)| - if page_number < space.pages - used_fraction = 1.0 - identifier = nil - if page_status[:free] - used_fraction = 0.0 - else - page = space.page(page_number) - if page.respond_to?(:used_space) - used_fraction = page.used_space.to_f / page.size.to_f - end - if page.respond_to?(:index_id) - identifier = page.index_id - unless identifiers[identifier] - identifiers[identifier] = (page.index_id == Innodb::IbufIndex::INDEX_ID) ? - "Insert Buffer Index" : - "Index #{page.index_id}" - if space.innodb_system - table, index = space.innodb_system.table_and_index_name_by_id(page.index_id) - if table && index - identifiers[identifier] += " (%s.%s)" % [table, index] - end - end - end - end - end - bitmap += filled_block(used_fraction, identifier) - if used_fraction != 0.0 - count_by_identifier[identifier] += 1 - else - count_by_identifier[:free] += 1 - end - else - bitmap += " " - end - bitmap - }, + space_extents_illustrate_page_status(space, entry, count_by_identifier, identifiers), ] end total_pages = count_by_identifier.values.reduce(:+) - puts "%12s ╰%-#{line_width}s╯" % [ "", "─" * line_width ] + puts "%12s ╰%-#{width}s╯" % ["", "─" * width] puts puts "Legend (%s = 1 page):" % [filled_block(1.0, nil)] puts " %-62s %8s %8s" % [ - "Page Type", "Pages", "Ratio" + "Page Type", + "Pages", + "Ratio", ] puts " %s %-60s %8i %7.2f%%" % [ filled_block(1.0, nil), "System", count_by_identifier[nil], - 100.0 * (count_by_identifier[nil].to_f / total_pages.to_f), + 100.0 * (count_by_identifier[nil].to_f / total_pages), ] identifiers.sort.each do |identifier, description| puts " %s %-60s %8i %7.2f%%" % [ filled_block(1.0, identifier), description, count_by_identifier[identifier], - 100.0 * (count_by_identifier[identifier].to_f / total_pages.to_f), + 100.0 * (count_by_identifier[identifier].to_f / total_pages), ] end puts " %s %-60s %8i %7.2f%%" % [ filled_block(0.0, nil), "Free space", count_by_identifier[:free], - 100.0 * (count_by_identifier[:free].to_f / total_pages.to_f), + 100.0 * (count_by_identifier[:free].to_f / total_pages), ] puts end -def svg_extent_legend(x, y, block_size, color=nil, description=nil, pages=nil, ratio=nil) - [ - svg("rect", { - "y" => y, - "x" => x, - "width" => block_size, - "height" => block_size, - "fill" => color ? color : "white", - "stroke" => description ? "black" : "none", - }), - svg("text", { - "y" => y + block_size - 4, - "x" => x + (description ? block_size + 5 : 0), - "font-family" => "monospace", - "font-size" => block_size, - "font-weight" => description ? "normal" : "bold", - "text-anchor" => "start", - }, description ? description : "Page Type"), - svg("text", { - "y" => y + block_size - 4, - "x" => x + block_size + 5 + (40 * block_size), - "font-family" => "monospace", - "font-size" => block_size, - "font-weight" => description ? "normal" : "bold", - "text-anchor" => "end", - }, pages ? pages : "Pages"), - svg("text", { - "y" => y + block_size - 4, - "x" => x + block_size + 5 + (40 * block_size) + (10 * block_size), - "font-family" => "monospace", - "font-size" => block_size, - "font-weight" => description ? "normal" : "bold", - "text-anchor" => "end", - }, ratio ? ("%7.2f%%" % [ratio]) : "Ratio"), - ].join("\n") -end - -# Illustrate the space by printing each extent and for each page, printing a -# filled block colored based on the index the page is part of. Print a legend -# for the colors used afterwards. -def space_extents_illustrate_svg(space) - line_width = space.pages_per_extent - block_size = @options.illustration_block_size - - puts "" - puts "" - - identifiers = {} - count_by_identifier = Hash.new(0) - - graphic_x = 48 - graphic_y = 16 - - puts svg("text", { - "y" => graphic_y - 3, - "x" => graphic_x - 7, - "font-family" => "monospace", - "font-size" => block_size, - "font-weight" => "bold", - "text-anchor" => "end", - }, "Page") - - block_x = 0 - block_y = 0 - space.each_xdes do |entry| - block_x = 0 - - puts svg("text", { - "y" => graphic_y + block_y + block_size, - "x" => graphic_x - 7, - "font-family" => "monospace", - "font-size" => block_size, - "text-anchor" => "end", - }, entry.xdes[:start_page]) - - entry.each_page_status do |page_number, page_status| - if page_number < space.pages - used_fraction = 1.0 - identifier = nil - if page_status[:free] - used_fraction = 0.0 - else - page = space.page(page_number) - if page.respond_to?(:used_space) - used_fraction = page.used_space.to_f / page.size.to_f - end - if page.respond_to?(:index_id) - identifier = page.index_id - unless identifiers[identifier] - identifiers[identifier] = (page.index_id == Innodb::IbufIndex::INDEX_ID) ? - "Insert Buffer Index" : - "Index #{page.index_id}" - if space.innodb_system - table, index = space.innodb_system.table_and_index_name_by_id(page.index_id) - if table && index - identifiers[identifier] += " (%s.%s)" % [table, index] - end - end - end - end - end - if used_fraction != 0.0 - count_by_identifier[identifier] += 1 - else - count_by_identifier[:free] += 1 - end - - block_height = block_size * used_fraction - color = "black" - if identifier - color = "#" + RGBHEX_COLORS_RANDOM[(identifier * COLOR_SPACING_PRIME) % RGBHEX_COLORS_RANDOM.size] - end - puts svg("rect", { - "x" => graphic_x + block_x, - "y" => graphic_y + block_y + (block_size - block_height), - "width" => block_size, - "height" => block_height, - "fill" => color, - }) - end - block_x += block_size - end - block_y += block_size - end - - puts svg("path", { - "stroke" => "black", - "stroke-width" => 1, - "fill" => "none", - "d" => svg_path_rounded_rect( - graphic_x, - graphic_y, - block_x, - block_y, - 4 - ), - }) - - block_x = 0 - block_y += 10 - puts svg_extent_legend( - graphic_x + block_x, - graphic_y + block_y, - block_size - ) - block_y += block_size + 2 - - puts svg_extent_legend( - graphic_x + block_x, - graphic_y + block_y, - block_size, - "black", - "System", - count_by_identifier[nil], - 100.0 * (count_by_identifier[nil].to_f / space.pages.to_f) - ) - block_y += block_size + 2 - - identifiers.sort.each do |identifier, description| - puts svg_extent_legend( - graphic_x + block_x, - graphic_y + block_y, - block_size, - "#" + RGBHEX_COLORS_RANDOM[(identifier * COLOR_SPACING_PRIME) % RGBHEX_COLORS_RANDOM.size], - description, - count_by_identifier[identifier], - 100.0 * (count_by_identifier[identifier].to_f / space.pages.to_f) - ) - block_y += block_size + 2 - end - - puts svg_extent_legend( - graphic_x + block_x, - graphic_y + block_y, - block_size, - "white", - "Free space", - count_by_identifier[:free], - 100.0 * (count_by_identifier[:free].to_f / space.pages.to_f) - ) - - puts "" -end - - def space_lsn_age_illustrate(space) colors = ANSI_COLORS_HEATMAP - line_width = @options.illustration_line_width + width = @options.illustration_line_width # Calculate the minimum and maximum LSN in the space. This is pretty # inefficient as we end up scanning all pages twice. @@ -774,70 +521,57 @@ def space_lsn_age_illustrate(space) lsn_min = lsn_max = space.page(0).lsn space.each_page do |page_number, page| - if page.lsn != 0 - page_lsn[page_number] = page.lsn - lsn_min = page.lsn < lsn_min ? page.lsn : lsn_min - lsn_max = page.lsn > lsn_max ? page.lsn : lsn_max - end + next if page.lsn.zero? + + page_lsn[page_number] = page.lsn + lsn_min = [page.lsn, lsn_min].min + lsn_max = [page.lsn, lsn_max].max end lsn_delta = lsn_max - lsn_min puts - puts "%12s ╭%-#{line_width}s╮" % [ "Start Page", "─" * line_width ] + puts "%12s %-#{width}s " % ["", center(space.name, width)] + puts "%12s ╭%-#{width}s╮" % ["Start Page", "─" * width] start_page = 0 - page_lsn.each_slice(line_width) do |slice| - puts "%12i │%-#{line_width}s│" % [ + page_lsn.each_slice(width) do |slice| + puts "%12i │%-#{width}s│" % [ start_page, - slice.inject("") { |line, lsn| + slice.inject("") do |line, lsn| if lsn - age_ratio = (lsn - lsn_min).to_f / lsn_delta.to_f + age_ratio = (lsn - lsn_min).to_f / lsn_delta color = colors[(age_ratio * colors.size.to_f).floor] line += ansi_color(color, filled_block(1.0, nil)) else line += " " end line - }, + end, ] - start_page += line_width + start_page += width end - puts "%12s ╰%-#{line_width}s╯" % [ "", "─" * line_width ] + puts "%12s ╰%-#{width}s╯" % ["", "─" * width] - lsn_legend = "<" + ("─" * (colors.size - 2)) + ">" + _, lsn_freq = page_lsn.compact.histogram(colors.size, min: lsn_min, max: lsn_max) + lsn_freq_delta = lsn_freq.max - lsn_freq.min - begin - # Try to optionally replace the boring lsn_legend with a histogram of - # page age distribution. If histogram/array is not available, move on. - - require 'histogram/array' - - lsn_bins, lsn_freq = page_lsn.select { |lsn| !lsn.nil? }. - histogram(colors.size, :min => lsn_min, :max => lsn_max) - - lsn_freq_delta = lsn_freq.max - lsn_freq.min - - lsn_legend = "" - lsn_freq.each do |freq| - freq_norm = freq / lsn_freq_delta - if freq_norm > 0.0 - lsn_legend << filled_block(freq_norm) - else - # Avoid the "empty" block used for 0.0. - lsn_legend << " " - end - end - rescue LoadError - # That's okay! Leave the legend boring. + lsn_age_histogram = "".dup + lsn_freq.each do |freq| + freq_norm = freq / lsn_freq_delta + lsn_age_histogram << (freq_norm > 0.0 ? filled_block(freq_norm) : " ") end puts - puts "Legend (%s = 1 page):" % [filled_block(1.0, nil)] + puts "LSN Age Histogram (%s = ~%d pages):" % [ + filled_block(1.0, nil), + (space.pages.to_f / colors.size).round, + ] puts " %12s %s %-12s" % [ "Min LSN", - lsn_legend, - "Max LSN" ] + lsn_age_histogram, + "Max LSN", + ] puts " %12i %s %-12i" % [ lsn_min, colors.map { |c| ansi_color(c, filled_block(1.0, nil)) }.join, @@ -845,126 +579,6 @@ def space_lsn_age_illustrate(space) ] end -def space_lsn_age_illustrate_svg(space) - colors = RGBHEX_COLORS_HEATMAP - line_width = @options.illustration_line_width - block_size = @options.illustration_block_size - - puts "" - puts "" - - # Calculate the minimum and maximum LSN in the space. This is pretty - # inefficient as we end up scanning all pages twice. - page_lsn = Array.new(space.pages) - - lsn_min = lsn_max = space.page(0).lsn - space.each_page do |page_number, page| - if page.lsn != 0 - page_lsn[page_number] = page.lsn - lsn_min = page.lsn < lsn_min ? page.lsn : lsn_min - lsn_max = page.lsn > lsn_max ? page.lsn : lsn_max - end - end - lsn_delta = lsn_max - lsn_min - - graphic_x = 48 - graphic_y = 16 - - block_x = 0 - block_y = 0 - - puts svg("text", { - "y" => graphic_y - 3, - "x" => graphic_x - 7, - "font-family" => "monospace", - "font-size" => block_size, - "font-weight" => "bold", - "text-anchor" => "end", - }, "Page") - - start_page = 0 - page_lsn.each_slice(line_width) do |slice| - block_x = 0 - slice.each do |lsn| - rgbhex = "" - if lsn - age_ratio = (lsn - lsn_min).to_f / lsn_delta.to_f - color = colors[(age_ratio * colors.size.to_f).floor] - end - puts svg("rect", { - "y" => graphic_y + block_y, - "x" => graphic_x + block_x, - "width" => block_size, - "height" => block_size, - "fill" => color ? "#" + color : "black", - }) - block_x += block_size - end - puts svg("text", { - "y" => graphic_y + block_y + block_size, - "x" => graphic_x - 7, - "font-family" => "monospace", - "font-size" => block_size, - "text-anchor" => "end", - }, start_page) - block_y += block_size - start_page += line_width - end - - puts svg("path", { - "stroke" => "black", - "stroke-width" => 1, - "fill" => "none", - "d" => svg_path_rounded_rect( - graphic_x, - graphic_y, - line_width * block_size, - block_y, - 4 - ), - }) - - block_x = 0 - block_y += 16 - puts svg("text", { - "y" => graphic_y + block_y + block_size - 4, - "x" => graphic_x + block_x, - "font-family" => "monospace", - "font-size" => block_size, - "text-anchor" => "start", - }, lsn_min) - color_width = ((64.0 * block_size.to_f) / colors.size.to_f).round - colors.each do |color| - puts svg("rect", { - "y" => graphic_y + block_y + block_size, - "x" => graphic_x + block_x, - "width" => color_width, - "height" => block_size, - "fill" => "#" + color, - }) - block_x += color_width - end - puts svg("text", { - "y" => graphic_y + block_y + block_size - 4, - "x" => graphic_x + block_x, - "font-family" => "monospace", - "font-size" => block_size, - "text-anchor" => "end", - }, lsn_max) - - puts svg("text", { - "y" => graphic_y + block_y + block_size - 4, - "x" => graphic_x + (block_x / 2), - "font-family" => "monospace", - "font-weight" => "bold", - "font-size" => block_size, - "text-anchor" => "middle", - }, "LSN Age") - - - puts "\n" -end - def print_inode_summary(inode) puts "INODE fseg_id=%d, pages=%d, frag=%d, full=%d, not_full=%d, free=%d" % [ inode.fseg_id, @@ -977,6 +591,7 @@ def print_inode_summary(inode) end def print_inode_detail(inode) + # rubocop:disable Layout/LineLength puts "INODE fseg_id=%d, pages=%d, frag=%d pages (%s), full=%d extents (%s), not_full=%d extents (%s) (%d/%d pages used), free=%d extents (%s)" % [ inode.fseg_id, inode.total_pages, @@ -991,6 +606,7 @@ def print_inode_detail(inode) inode.free.length, inode.free.each.to_a.map { |x| "#{x.start_page}-#{x.end_page}" }.join(", "), ] + # rubocop:enable Layout/LineLength end def space_inodes_fseg_id(space) @@ -1011,6 +627,18 @@ def space_inodes_detail(space) end end +def space_sdi_construct_json(space) + space.sdi.each_object.map do |object| + { type: object.dd_object_type, id: object.id, object: object.data } + end +end + +def space_sdi_json_dump(space) + raise "Space does not have SDI data; is it older than MySQL 8.0?" unless space.sdi.valid? + + puts JSON.pretty_generate(space_sdi_construct_json(space)) +end + def page_account(innodb_system, space, page_number) puts "Accounting for page #{page_number}:" @@ -1021,13 +649,11 @@ def page_account(innodb_system, space, page_number) page = space.page(page_number) page_type = Innodb::Page::PAGE_TYPE[page.type] - puts " Page type is %s (%s, %s)." % [ + puts " Page type is %s (%s)." % [ page.type, page_type[:description], - page_type[:usage], ] - xdes = space.xdes_for_page(page_number) puts " Extent descriptor for pages %d-%d is at page %d, offset %d." % [ xdes.start_page, @@ -1044,25 +670,23 @@ def page_account(innodb_system, space, page_number) xdes_status = xdes.page_status(page_number) puts " Page is marked as %s in extent descriptor." % [ - xdes_status[:free] ? 'free' : 'used' + xdes_status[:free] ? "free" : "used", ] space.each_xdes_list do |name, list| - if list.include? xdes - puts " Extent is in #{name} list of space." - end + puts " Extent is in #{name} list of space." if list.include?(xdes) end page_inode = nil space.each_inode do |inode| inode.each_list do |name, list| - if list.include? xdes + if list.include?(xdes) page_inode = inode puts " Extent is in #{name} list of fseg #{inode.fseg_id}." end end - if inode.frag_array.include? page_number + if inode.frag_array.include?(page_number) # rubocop:disable Style/Next page_inode = inode puts " Page is in fragment array of fseg %d." % [ inode.fseg_id, @@ -1072,42 +696,30 @@ def page_account(innodb_system, space, page_number) space.each_index do |index| index.each_fseg do |fseg_name, fseg| - if page_inode == fseg - puts " Fseg is in #{fseg_name} fseg of index #{index.id}." - puts " Index root is page #{index.root.offset}." - if innodb_system - table_name, index_name = innodb_system.table_and_index_name_by_id(index.id) - if table_name and index_name - puts " Index is #{table_name}.#{index_name}." - end - end + next unless page_inode == fseg + + puts " Fseg is in #{fseg_name} fseg of index #{index.id}." + puts " Index root is page #{index.root.offset}." + if innodb_system + dd_index = innodb_system.data_dictionary.indexes.find(innodb_index_id: index.id) + puts " Index is #{dd_index.table.name}.#{dd_index.name}." if dd_index end end end - if space.system_space? - if page_inode == space.trx_sys.fseg - puts " Fseg is trx_sys." - end - - if page_inode == space.trx_sys.doublewrite[:fseg] - puts " Fseg is doublewrite buffer." - end + if space.system_space? # rubocop:disable Style/GuardClause + puts " Fseg is trx_sys." if page_inode == space.trx_sys.fseg + puts " Fseg is doublewrite buffer." if page_inode == space.trx_sys.doublewrite[:fseg] - if innodb_system - innodb_system.data_dictionary.each_data_dictionary_index do |table_name, index_name, index| - index.each_fseg do |fseg_name, fseg| - if page_inode == fseg - puts " Index is #{table_name}.#{index_name} of data dictionary." - end - end + innodb_system.data_dictionary&.each_data_dictionary_index do |table_name, index_name, index| + index.each_fseg do |_fseg_name, fseg| + puts " Index is #{table_name}.#{index_name} of data dictionary." if page_inode == fseg end end space.trx_sys.rsegs.each_with_index do |rseg_slot, index| - if page.fil_header[:space_id] == rseg_slot[:space_id] && - page.fil_header[:offset] == rseg_slot[:page_number] - puts " Page is a rollback segment in slot #{index}." + if page.fil_header.space_id == rseg_slot.space_id && page.fil_header.offset == rseg_slot.page_number + puts " Page is a rollback segment in slot #{index}." end end end @@ -1121,10 +733,9 @@ def page_validate_index(page) puts "done." directory_offsets = page.each_directory_offset.to_a - record_offsets = records.map { |rec| rec.offset } + record_offsets = records.map(&:offset) - invalid_directory_entries = - directory_offsets.select { |n| ! record_offsets.include?(n) } + invalid_directory_entries = directory_offsets.reject { |n| record_offsets.include?(n) } unless invalid_directory_entries.empty? page_is_valid = false @@ -1138,9 +749,7 @@ def page_validate_index(page) end # Read all records corresponding to valid directory entries. - directory_records = directory_offsets. - select { |o| !invalid_directory_entries.include?(o) }. - map { |o| page.record(o) } + directory_records = directory_offsets.reject { |o| invalid_directory_entries.include?(o) }.map { |o| page.record(o) } misordered_directory_entries = [] prev = nil @@ -1149,13 +758,13 @@ def page_validate_index(page) prev = rec next end - if rec.compare_key(prev.key.map { |v| v[:value]}) == 1 + if rec.compare_key(prev.key.map { |v| v[:value] }) == 1 page_is_valid = false misordered_directory_entries << { - :slot => page.offset_is_directory_slot?(rec.offset), - :offset => rec.offset, - :key => rec.key_string, - :prev_key => prev.key_string, + slot: page.offset_is_directory_slot?(rec.offset), + offset: rec.offset, + key: rec.key_string, + prev_key: prev.key_string, } end prev = rec @@ -1179,12 +788,12 @@ def page_validate_index(page) prev = rec next end - if rec.compare_key(prev.key.map { |v| v[:value]}) == 1 + if rec.compare_key(prev.key.map { |v| v[:value] }) == 1 page_is_valid = false misordered_records << { - :offset => rec.offset, - :key => rec.key_string, - :prev_key => prev.key_string, + offset: rec.offset, + key: rec.key_string, + prev_key: prev.key_string, } end prev = rec @@ -1203,8 +812,7 @@ def page_validate_index(page) page_is_valid end - -def page_validate(innodb_system, space, page_number) +def page_validate(_innodb_system, space, page_number) page_is_valid = true puts "Validating page %d..." % [page_number] @@ -1219,11 +827,11 @@ def page_validate(innodb_system, space, page_number) puts " header %10d (0x%08x), type %s" % [ page.checksum, page.checksum, - page.checksum_type ? page.checksum_type : "unknown", + page.checksum_type || "unknown", ] puts " trailer %10d (0x%08x)" % [ - page.checksum_trailer, - page.checksum_trailer, + page.fil_trailer.checksum, + page.fil_trailer.checksum, ] puts " Calculated checksums:" puts " crc32 %10d (0x%08x)" % [ @@ -1246,12 +854,12 @@ def page_validate(innodb_system, space, page_number) ] puts " Low 32 bits of LSN:" puts " header %10d (0x%08x)" % [ - page.lsn_low32_header, - page.lsn_low32_header, + page.fil_header.lsn_low32, + page.fil_header.lsn_low32, ] puts " trailer %10d (0x%08x)" % [ - page.lsn_low32_trailer, - page.lsn_low32_trailer, + page.fil_trailer.lsn_low32, + page.fil_trailer.lsn_low32, ] end @@ -1265,16 +873,14 @@ def page_validate(innodb_system, space, page_number) ] end if page.misplaced_space? - puts " Space's ID %d does not match page's stored space ID %d." % [ + puts " Space ID %d does not match page stored space ID %d." % [ page.space.space_id, page.space_id, ] end end - if page.type == :INDEX && !page_validate_index(page) - page_is_valid = false - end + page_is_valid = false if page.is_a?(Innodb::Page::Index) && !page_validate_index(page) puts "Page %d appears to be %s!" % [ page_number, @@ -1282,24 +888,21 @@ def page_validate(innodb_system, space, page_number) ] end -def page_directory_summary(space, page) - if page.type != :INDEX - usage 1, "Page must be an index page" - end +def page_directory_summary(_space, page) + usage(1, "Page must be an index page") unless page.is_a?(Innodb::Page::Index) - puts "%-8s%-8s%-14s%-8s%s" % [ - "slot", - "offset", - "type", - "owned", - "key", + puts "%-8s%-8s%-14s%-8s%s" % %w[ + slot + offset + type + owned + key ] page.directory.each_with_index do |offset, slot| record = page.record(offset) - key = if [:conventional, :node_pointer].include? record.header[:type] - "(%s)" % record.key_string - end + key = %i[conventional node_pointer].include?(record.header[:type]) ? "(%s)" % record.key_string : "" + puts "%-8i%-8i%-14s%-8i%s" % [ slot, offset, @@ -1310,58 +913,63 @@ def page_directory_summary(space, page) end end -def page_records(space, page) +def page_records(_space, page) page.each_record do |record| puts "Record %i: %s" % [ record.offset, record.string, ] - puts if record.header[:type] == :conventional end end def page_illustrate(page) width = 64 - unknown_page_content = page.type == :INDEX && page.record_describer.nil? + unknown_page_content = page.is_a?(Innodb::Page::Index) && page.record_describer.nil? blocks = Array.new(page.size, unknown_page_content ? "▞" : " ") identifiers = {} identifier_sort = 0 count_by_identifier = Hash.new(0) - page.each_region.sort { |a,b| a[:offset] <=> b[:offset]}.each do |region| - region[:length].times do |n| + page.each_region.sort_by(&:offset).each do |region| + region.length.times do |n| identifier = nil fraction = 0.0 - if region[:name] != :garbage - if n == region[:length] - 1 - fraction = 0.5 - else - fraction = 1.0 - end - identifier = region[:name].hash.abs + if region.name != :garbage + fraction = n == region.length - 1 ? 0.5 : 1.0 + identifier = region.name.hash.abs unless identifiers[identifier] # Prefix an integer <0123> on each name so that the legend can be # sorted by the appearance of each region in the page. identifiers[identifier] = "<%04i>%s" % [ identifier_sort, - region[:info] + region.info, ] identifier_sort += 1 end end - blocks[region[:offset] + n] = filled_block(fraction, identifier, BLOCK_CHARS_H) + blocks[region.offset + n] = filled_block(fraction, identifier, BLOCK_CHARS_H) count_by_identifier[identifier] += 1 end end puts - puts "%12s ╭%-#{width}s╮" % [ "Offset", "─" * width ] + puts "%12s %-#{width}s " % ["", center("Page #{page.offset} (#{page.type})", width)] + puts "%12s ╭%-#{width}s╮" % ["Offset", "─" * width] offset = 0 + skipped_lines = 0 blocks.each_slice(width) do |slice| - puts "%12i │%-s│" % [offset, slice.join] + if slice.any? { |s| s != " " } + if skipped_lines.positive? + puts "%12s │%-#{width}s│" % ["...", ""] + skipped_lines = 0 + end + puts "%12i │%-s│" % [offset, slice.join] + else + skipped_lines += 1 + end offset += width end - puts "%12s ╰%-#{width}s╯" % [ "", "─" * width ] + puts "%12s ╰%-#{width}s╯" % ["", "─" * width] puts puts "Legend (%s = 1 byte):" % [filled_block(1.0, nil)] @@ -1370,26 +978,26 @@ def page_illustrate(page) "Bytes", "Ratio", ] - identifiers.sort { |a,b| a[1] <=> b[1] }.each do |identifier, description| + identifiers.sort { |a, b| a[1] <=> b[1] }.each do |identifier, description| puts " %s %-30s %8i %7.2f%%" % [ filled_block(1.0, identifier), description.gsub(/^<\d+>/, ""), count_by_identifier[identifier], - 100.0 * (count_by_identifier[identifier].to_f / page.size.to_f), + 100.0 * (count_by_identifier[identifier].to_f / page.size), ] end puts " %s %-30s %8i %7.2f%%" % [ filled_block(0.0, nil), "Garbage", count_by_identifier[nil], - 100.0 * (count_by_identifier[nil].to_f / page.size.to_f), + 100.0 * (count_by_identifier[nil].to_f / page.size), ] - free_space = page.size - count_by_identifier.inject(0) { |sum,(k,v)| sum + v } + free_space = page.size - count_by_identifier.inject(0) { |sum, (_k, v)| sum + v } puts " %s %-30s %8i %7.2f%%" % [ unknown_page_content ? "▞" : " ", unknown_page_content ? "Unknown (no data dictionary)" : "Free", free_space, - 100.0 * (free_space.to_f / page.size.to_f), + 100.0 * (free_space.to_f / page.size), ] if unknown_page_content @@ -1403,21 +1011,19 @@ def page_illustrate(page) end def record_dump(page, record_offset) - unless record = page.record(record_offset) - raise "Record at offset #{record_offset} not found" - end + record = page.record(record_offset) + raise "Record at offset #{record_offset} not found" unless record record.dump +rescue IOError + raise "Record could not be read at offset #{record_offset}; is it a valid record offset?" end def record_history(page, record_offset) - unless page.leaf? - raise "Record is not located on a leaf page; no history available" - end + raise "Record is not located on a leaf page; no history available" unless page.leaf? - unless record = page.record(record_offset) - raise "Record at offset #{record_offset} not found" - end + record = page.record(record_offset) + raise "Record at offset #{record_offset} not found" unless record puts "%-14s%-20s%s" % [ "Transaction", @@ -1435,29 +1041,23 @@ def record_history(page, record_offset) end def index_fseg_lists(index, fseg_name) - unless index.fseg(fseg_name) - raise "File segment '#{fseg_name}' doesn't exist" - end + raise "File segment '#{fseg_name}' doesn't exist" unless index.fseg(fseg_name) print_lists(index.each_fseg_list(index.fseg(fseg_name))) end def index_fseg_list_iterate(index, fseg_name, list_name) - unless fseg = index.fseg(fseg_name) - raise "File segment '#{fseg_name}' doesn't exist" - end + fseg = index.fseg(fseg_name) + raise "File segment '#{fseg_name}' doesn't exist" unless fseg - unless list = fseg.list(list_name) - raise "List '#{list_name}' doesn't exist" - end + list = fseg.list(list_name) + raise "List '#{list_name}' doesn't exist" unless list print_xdes_list(index.space, list) end def index_fseg_frag_pages(index, fseg_name) - unless index.fseg(fseg_name) - raise "File segment '#{fseg_name}' doesn't exist" - end + raise "File segment '#{fseg_name}' doesn't exist" unless index.fseg(fseg_name) print_index_page_summary(index.each_fseg_frag_page(index.fseg(fseg_name))) end @@ -1472,17 +1072,17 @@ def index_recurse(index) page.records, page.record_space, ] - if page.level == 0 + if page.level.zero? page.each_record do |record| puts "%sRECORD: (%s) → (%s)" % [ - " " * (depth+1), + " " * (depth + 1), record.key_string, record.row_string, ] end end end, - lambda do |parent_page, child_page, child_min_key, depth| + lambda do |_parent_page, child_page, child_min_key, depth| puts "%sNODE POINTER RECORD ≥ (%s) → #%i" % [ " " * depth, child_min_key.map { |r| "%s=%s" % [r[:name], r[:value].inspect] }.join(", "), @@ -1493,13 +1093,13 @@ def index_recurse(index) end def index_record_offsets(index) - puts "%-20s%-20s" % [ - "page_offset", - "record_offset", + puts "%-20s%-20s" % %w[ + page_offset + record_offset ] index.recurse( - lambda do |page, depth| - if page.level == 0 + lambda do |page, _depth| + if page.level.zero? page.each_record do |record| puts "%-20i%-20i" % [ page.offset, @@ -1508,7 +1108,7 @@ def index_record_offsets(index) end end end, - lambda { |*x| } + ->(*_) {} ) end @@ -1528,13 +1128,13 @@ def index_digraph(index) child_key.join(", "), ] end - puts " %spage_%i [ shape = \"record\"; label = \"%s\"; ];" % [ + puts " %spage_%i [ shape = 'record'; label = '%s'; ];" % [ " " * depth, page.offset, label, ] end, - lambda do |parent_page, child_page, child_key, depth| + lambda do |parent_page, child_page, _child_key, depth| puts " %spage_%i:dir_%i → page_%i:page:nw;" % [ " " * depth, parent_page.offset, @@ -1547,14 +1147,14 @@ def index_digraph(index) end def index_level_summary(index, level) - puts "%-8s%-8s%-8s%-8s%-8s%-8s%-8s" % [ - "page", - "index", - "level", - "data", - "free", - "records", - "min_key", + puts "%-8s%-8s%-8s%-8s%-8s%-8s%-8s" % %w[ + page + index + level + data + free + records + min_key ] index.each_page_at_level(level) do |page| @@ -1571,20 +1171,19 @@ def index_level_summary(index, level) end def undo_history_summary(innodb_system) - history = innodb_system.history.each_history_list - history_list = history.select { |history| history.list.length > 0 } - - puts "%-8s%-8s%-14s%-20s%s" % [ - "Page", - "Offset", - "Transaction", - "Type", - "Table", + history_list = innodb_system.history.each_history_list.reject { |h| h.list.empty? } + + puts "%-8s%-8s%-14s%-20s%s" % %w[ + Page + Offset + Transaction + Type + Table ] history_list.each do |history| history.each_undo_record do |undo| - table_name = innodb_system.table_name_by_id(undo.table_id) + table_name = innodb_system.data_dictionary.tables(innodb_table_id: undo.table_id) puts "%-8s%-8s%-14s%-20s%s" % [ undo.page, undo.offset, @@ -1609,20 +1208,22 @@ def usage(exit_code, message = nil) exit exit_code end - print <<'END_OF_USAGE' + # rubocop:disable Layout/HeredocIndentation + print < Invocation examples: - innodb_space -s ibdata1 [-T tname [-I iname]] [options] - Use ibdata1 as the system tablespace and load the tname table (and the - iname index for modes that require it) from data located in the system - tablespace data dictionary. This will automatically generate a record - describer for any indexes. + innodb_space -s ibdata1 [-T table-name [-I index-name [-R record-offset]]] [options] + Use ibdata1 as the system tablespace and load the table-name table (and + the index-name index for modes that require it) from data located in the + system tablespace data dictionary. This will automatically generate a + record describer for any indexes using the data dictionary. - innodb_space -f tname.ibd [-r ./desc.rb -d DescClass] [options] - Use the tname.ibd table (and the DescClass describer where required). + innodb_space -f file-name.ibd [-r ./describer.rb -d DescriberClass] [options] + Use the file-name.ibd tablespace file (and the DescriberClass describer + where required) to read the tablespace structures or indexes. The following options are supported: @@ -1636,16 +1237,27 @@ The following options are supported: --system-space-file, -s Load the system tablespace file or files : Either a single file e.g. - "ibdata1", a comma-delimited list of files e.g. "ibdata1,ibdata1", or a + 'ibdata1', a comma-delimited list of files e.g. 'ibdata1,ibdata1', or a directory name. If a directory name is provided, it will be scanned for all - files named "ibdata?" which will then be sorted alphabetically and used to + files named 'ibdata?' which will then be sorted alphabetically and used to load the system tablespace. - --table-name, -T - Use the table name . + If using the --system-space-file option, the following options may also + be used: + + --table-name, -T + Use the table name . + + --index-name, -I + Use the index name . - --index-name, -I - Use the index name . + --system-space-tables, -x + Allow opening tables from the system space to support system spaces with + tables created without innodb-file-per-table enabled. + + --data-directory, -D + Open per-table tablespace files from rather than from the + directory where the system-space-file is located. --space-file, -f Load the tablespace file . @@ -1653,6 +1265,9 @@ The following options are supported: --page, -p Operate on the page . + --record, -R + Operate on the record located at within the index page. + --level, -l Operate on the level . @@ -1663,7 +1278,7 @@ The following options are supported: Operate on the file segment (fseg) . --require, -r - Use Ruby's "require" to load the file . This is useful for loading + Use Ruby's 'require' to load the file . This is useful for loading classes with record describers. --describer, -d @@ -1675,25 +1290,22 @@ The following modes are supported: Print a summary of all spaces in the system. data-dictionary-tables - Print all records in the SYS_TABLES data dictionary table. + Print the contents of the table list from the data dictionary. data-dictionary-columns - Print all records in the SYS_COLUMNS data dictionary table. + Print the contents of the column list from the data dictionary. data-dictionary-indexes - Print all records in the SYS_INDEXES data dictionary table. - - data-dictionary-fields - Print all records in the SYS_FIELDS data dictionary table. + Print the contents of the index list from the data dictionary. space-summary Summarize all pages within a tablespace. A starting page number can be provided with the --page/-p argument. space-index-pages-summary - Summarize all "INDEX" pages within a tablespace. This is useful to analyze - page fill rates and record counts per page. In addition to "INDEX" pages, - "ALLOCATED" pages are also printed and assumed to be completely empty. + Summarize all 'INDEX' pages within a tablespace. This is useful to analyze + page fill rates and record counts per page. In addition to 'INDEX' pages, + 'ALLOCATED' pages are also printed and assumed to be completely empty. A starting page number can be provided with the --page/-p argument. space-index-fseg-pages-summary @@ -1702,7 +1314,7 @@ The following modes are supported: space-index-pages-free-plot Use Ruby's gnuplot module to produce a scatterplot of page free space for - all "INDEX" and "ALLOCATED" pages in a tablespace. More aesthetically + all 'INDEX' and 'ALLOCATED' pages in a tablespace. More aesthetically pleasing plots can be produced with space-index-pages-summary output, but this is a quick and easy way to produce a passable plot. A starting page number can be provided with the --page/-p argument. @@ -1734,20 +1346,11 @@ The following modes are supported: color and Unicode box drawing characters to show page usage throughout the space. - space-extents-illustrate-svg - Iterate through all extents, illustrating the extent usage in SVG format - printed to stdout to show page usage throughout the space. - space-lsn-age-illustrate Iterate through all pages, producing a heat map colored by the page LSN using ANSI color and Unicode box drawing characters, allowing the user to get an overview of page modification recency. - space-lsn-age-illustrate-svg - Iterate through all pages, producing a heat map colored by the page LSN - producing SVG format output, allowing the user to get an overview of page - modification recency. - space-inodes-fseg-id Iterate through all inodes, printing only the FSEG ID. @@ -1757,6 +1360,10 @@ The following modes are supported: space-inodes-detail Iterate through all inodes, printing a detailed report of each FSEG. + space-sdi-json-dump + Dump the contents of any SDI (serialized dictionary information) from a + space, in JSON format, similar to ibd2sdi. + index-recurse Recurse an index, starting at the root (which must be provided in the first --page/-p argument), printing the node pages, node pointers (links), leaf @@ -1777,23 +1384,23 @@ The following modes are supported: index-fseg-internal-lists index-fseg-leaf-lists - Print a summary of all lists in an index file segment. Index root page must - be provided with --page/-p. + Print a summary of all lists in an index file segment. Index must be specified + by name with --index-name/-I or by root page number with --page/-p. index-fseg-internal-list-iterate index-fseg-leaf-list-iterate Iterate the file segment list (whose name is provided in the first --list/-L - argument) for internal or leaf pages for a given index (whose root page - is provided in the first --page/-p argument). The lists used for each - index are "full", "not_full", and "free". + argument) for internal or leaf pages for a given index (specified by name with + --index-name/-I or by root page number with --page/-p). The lists used for each + index are 'full', 'not_full', and 'free'. index-fseg-internal-frag-pages index-fseg-leaf-frag-pages - Print a summary of all fragment pages in an index file segment. Index root - page must be provided with --page/-p. + Print a summary of all fragment pages in an index file segment. Index must be + specified by name with --index-name/-I or by root page number with --page/-p. page-dump - Dump the contents of a page, using the Ruby pp ("pretty-print") module. + Dump the contents of a page, using the Ruby pp ('pretty-print') module. page-account Account for a page's usage in FSEGs. @@ -1827,16 +1434,39 @@ The following modes are supported: A record offset must be provided with -R/--record. END_OF_USAGE + # rubocop:enable Layout/HeredocIndentation exit exit_code end -Signal.trap("INT") { exit } -Signal.trap("PIPE") { exit } - -@options = OpenStruct.new +%w[INT PIPE].each do |name| + Signal.trap(name) { exit } if Signal.list.include?(name) +end + +InnodbSpaceOptions = Struct.new( + :trace, + :system_space_file, + :system_space_tables, + :data_directory, + :space_file, + :table_name, + :index_name, + :page, + :record, + :level, + :list, + :fseg_id, + :describer, + :illustration_line_width, + :illustration_block_size, + keyword_init: true +) + +@options = InnodbSpaceOptions.new @options.trace = 0 @options.system_space_file = nil +@options.system_space_tables = false +@options.data_directory = nil @options.space_file = nil @options.table_name = nil @options.index_name = nil @@ -1849,10 +1479,13 @@ Signal.trap("PIPE") { exit } @options.illustration_line_width = 64 @options.illustration_block_size = 8 +# rubocop:disable Layout/SpaceInsideArrayLiteralBrackets getopt_options = [ [ "--help", "-?", GetoptLong::NO_ARGUMENT ], [ "--trace", "-t", GetoptLong::NO_ARGUMENT ], [ "--system-space-file", "-s", GetoptLong::REQUIRED_ARGUMENT ], + [ "--system-space-tables", "-x", GetoptLong::NO_ARGUMENT ], + [ "--data-directory", "-D", GetoptLong::REQUIRED_ARGUMENT ], [ "--space-file", "-f", GetoptLong::REQUIRED_ARGUMENT ], [ "--table-name", "-T", GetoptLong::REQUIRED_ARGUMENT ], [ "--index-name", "-I", GetoptLong::REQUIRED_ARGUMENT ], @@ -1866,6 +1499,7 @@ getopt_options = [ [ "--illustration-line-width", GetoptLong::REQUIRED_ARGUMENT ], [ "--illustration-block-size", GetoptLong::REQUIRED_ARGUMENT ], ] +# rubocop:enable Layout/SpaceInsideArrayLiteralBrackets getopt = GetoptLong.new(*getopt_options) @@ -1877,6 +1511,10 @@ getopt.each do |opt, arg| @options.trace += 1 when "--system-space-file" @options.system_space_file = arg.split(",") + when "--system-space-tables" + @options.system_space_tables = true + when "--data-directory" + @options.data_directory = arg when "--space-file" @options.space_file = arg.split(",") when "--table-name" @@ -1904,18 +1542,31 @@ getopt.each do |opt, arg| end end -unless @options.system_space_file or @options.space_file - usage 1, "System space file (-s) or space file (-f) must be specified" +# rubocop:disable Style/IfUnlessModifier + +unless @options.system_space_file || @options.space_file + usage(1, "Either the --system-space-file (-s) or --space-file (-f) must be specified") end -if @options.system_space_file and @options.space_file - usage 1, "Only one of system space or space file may be specified" +if @options.system_space_file && @options.space_file + usage(1, "Only one of --system-space-file (-s) or --space-file (-f) may be specified") end -if @options.trace > 1 - BufferCursor.trace! +system_space_options = + @options.table_name || @options.index_name || @options.system_space_tables || @options.data_directory + +if !@options.system_space_file && system_space_options + usage( + 1, + %{ + The --table-name (-T), --index-name (-I), --system-space-tables (-x), and --data-directory (-D) + options can only be used with the --system-space-file (-s) option + }.squish + ) end +BufferCursor.trace! if @options.trace > 1 + # A few globals that we'll try to populate from the command-line arguments. innodb_system = nil space = nil @@ -1923,12 +1574,12 @@ index = nil page = nil if @options.system_space_file - innodb_system = Innodb::System.new(@options.system_space_file) + innodb_system = Innodb::System.new(@options.system_space_file, data_directory: @options.data_directory) end -if innodb_system and @options.table_name - table_tablespace = innodb_system.space_by_table_name(@options.table_name) - space = table_tablespace || innodb_system.system_space +if innodb_system && @options.table_name + space = innodb_system.space_by_table_name(@options.table_name) + raise "Couldn't load space for table #{@options.table_name}" unless space elsif @options.space_file space = Innodb::Space.new(@options.space_file) else @@ -1936,24 +1587,17 @@ else end if @options.describer - describer = eval(@options.describer) - unless describer - describer = Innodb::RecordDescriber.const_get(@options.describer) - end + describer = eval(@options.describer) # rubocop:disable Security/Eval + describer ||= Innodb::RecordDescriber.const_get(@options.describer) space.record_describer = describer.new end -if innodb_system and @options.table_name and @options.index_name +if innodb_system && @options.table_name && @options.index_name index = innodb_system.index_by_name(@options.table_name, @options.index_name) - if @options.page - page = space.page(@options.page) - else - page = index.root - end + page = @options.page ? space.page(@options.page) : index.root elsif @options.page - if page = space.page(@options.page) and page.type == :INDEX and page.root? - index = space.index(@options.page) - end + page = space.page(@options.page) + index = space.index(@options.page) if page.is_a?(Innodb::Page::Index) && page&.root? end # The non-option argument on the command line is the mode (usually the last, @@ -1961,54 +1605,82 @@ end mode = ARGV.shift unless mode - usage 1, "At least one mode must be provided" + usage(1, "At least one mode must be provided") end -if /^(system-|data-dictionary-)/.match(mode) and !innodb_system - usage 1, "System tablespace must be specified using -s/--system-space-file" +if /^(system-|data-dictionary-)/.match(mode) && !innodb_system + usage(1, "System tablespace must be specified using --system-space-file (-s)") end -if /^space-/.match(mode) and !space - usage 1, "Tablespace must be specified using either -f/--space-file or a combination of -s/--system-space-file and -T/--table" +if /^space-/.match(mode) && !space + usage( + 1, + %{ + Tablespace must be specified using either --space-file (-f) + or a combination of --system-space-file (-s) and --table (-T) + }.squish + ) +end + +if /^index-/.match(mode) && !index + usage( + 1, + %{ + Index must be specified using a combination of either --space-file (-f) and --page (-p) + or --system-space-file (-s), --table-name (-T), and --index-name (-I) + }.squish + ) end -if /^index-/.match(mode) and !index - usage 1, "Index must be specified using a combination of either -f/--space-file and -p/--page or -s/--system-space-file, -T/--table-name, and -I/--index-name" +if /^page-/.match(mode) && !page + usage(1, "Page number must be specified using --page (-p)") end -if /^page-/.match(mode) and !page - usage 1, "Page number must be specified using -p/--page" +if /^record-/.match(mode) && !@options.record + usage(1, "Record offset must be specified using --record (-R)") end -if /^record-/.match(mode) and !@options.record - usage 1, "Record offset must be specified using -R/--record" +if /^record-/.match(mode) && !page + usage( + 1, + %{ + An index page must be available when using --record (-R); specify either + --page (-p) or --table-name (-T) and --index-name (-I) for the index root page. + } + ) end -if /-list-iterate$/.match(mode) and !@options.list - usage 1, "List name must be specified using -L/--list" +if /^record-/.match(mode) && !page.is_a?(Innodb::Page::Index) + usage(1, "Mode #{mode} may be used only with index pages") end -if /-level-/.match(mode) and !@options.level - usage 1, "Level must be specified using -l/--level" +if /-list-iterate$/.match(mode) && !@options.list + usage(1, "List name must be specified using --list (-L)") end -if [ - "index-recurse", - "index-record-offsets", - "index-digraph", - "index-level-summary", -].include?(mode) and !index.record_describer - usage 1, "Record describer must be specified using -d/--describer" +if /-level-/.match(mode) && !@options.level + usage(1, "Level must be specified using --level (-l)") end -if ["space-index-fseg-pages-summary"].include?(mode) and !@options.fseg_id - usage 1, "File segment id must be specified using -F/--fseg-id" +if %w[ + index-recurse + index-record-offsets + index-digraph + index-level-summary +].include?(mode) && !index.record_describer + usage(1, "Record describer must be specified using --describer (-d)") end -if @options.trace > 0 - BufferCursor.trace! +if %w[ + space-index-fseg-pages-summary +].include?(mode) && !@options.fseg_id + usage(1, "File segment id must be specified using --fseg-id (-F)") end +# rubocop:enable Style/IfUnlessModifier + +BufferCursor.trace! if @options.trace.positive? + case mode when "system-spaces" system_spaces(innodb_system) @@ -2018,8 +1690,6 @@ when "data-dictionary-columns" data_dictionary_columns(innodb_system) when "data-dictionary-indexes" data_dictionary_indexes(innodb_system) -when "data-dictionary-fields" - data_dictionary_fields(innodb_system) when "space-summary" space_summary(space, @options.page || 0) when "space-index-pages-summary" @@ -2027,8 +1697,7 @@ when "space-index-pages-summary" when "space-index-fseg-pages-summary" space_index_fseg_pages_summary(space, @options.fseg_id) when "space-index-pages-free-plot" - file_name = space.name.sub(".ibd", "").sub(/[^a-zA-Z0-9_]/, "_") - space_index_pages_free_plot(space, file_name, @options.page || 0) + space_index_pages_free_plot(space, @options.page || 0) when "space-page-type-regions" space_page_type_regions(space, @options.page || 0) when "space-page-type-summary" @@ -2043,18 +1712,16 @@ when "space-extents" space_extents(space) when "space-extents-illustrate" space_extents_illustrate(space) -when "space-extents-illustrate-svg" - space_extents_illustrate_svg(space) when "space-lsn-age-illustrate" space_lsn_age_illustrate(space) -when "space-lsn-age-illustrate-svg" - space_lsn_age_illustrate_svg(space) when "space-inodes-fseg-id" space_inodes_fseg_id(space) when "space-inodes-summary" space_inodes_summary(space) when "space-inodes-detail" space_inodes_detail(space) +when "space-sdi-json-dump" + space_sdi_json_dump(space) when "index-recurse" index_recurse(index) when "index-record-offsets" diff --git a/examples/describer/.rubocop.yml b/examples/describer/.rubocop.yml new file mode 100644 index 00000000..91c714e7 --- /dev/null +++ b/examples/describer/.rubocop.yml @@ -0,0 +1,5 @@ +inherit_from: + - ../../.rubocop.yml +Naming/ClassAndModuleCamelCase: + Enabled: false + diff --git a/examples/describer/employees_db.rb b/examples/describer/employees_db.rb index e8de723e..ba199f5f 100644 --- a/examples/describer/employees_db.rb +++ b/examples/describer/employees_db.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require "innodb/record_describer" # CREATE TABLE employees ( diff --git a/examples/describer/hello_world.rb b/examples/describer/hello_world.rb index e7155007..1b678679 100644 --- a/examples/describer/hello_world.rb +++ b/examples/describer/hello_world.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require "innodb/record_describer" # CREATE TABLE hello_world ( diff --git a/examples/describer/simple_describer.rb b/examples/describer/simple_describer.rb index 713fd1f6..b623b688 100644 --- a/examples/describer/simple_describer.rb +++ b/examples/describer/simple_describer.rb @@ -1,4 +1,4 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true class SimpleDescriber < Innodb::RecordDescriber type :clustered diff --git a/innodb_ruby.gemspec b/innodb_ruby.gemspec index 932f4bc4..3a0bef6d 100644 --- a/innodb_ruby.gemspec +++ b/innodb_ruby.gemspec @@ -1,73 +1,36 @@ -lib = File.expand_path('../lib/', __FILE__) -$:.unshift lib unless $:.include?(lib) +# frozen_string_literal: true + +lib = File.expand_path("lib", __dir__) +$LOAD_PATH.unshift lib unless $LOAD_PATH.include?(lib) require "innodb/version" Gem::Specification.new do |s| - s.name = 'innodb_ruby' + s.name = "innodb_ruby" s.version = Innodb::VERSION - s.date = Time.now.strftime("%Y-%m-%d") - s.summary = 'InnoDB data file parser' - s.license = 'BSD-3-Clause' - s.description = 'Library for parsing InnoDB data files in Ruby' + s.summary = "InnoDB data file parser" + s.license = "BSD-3-Clause" + s.description = "Library for parsing InnoDB data files in Ruby" s.authors = [ - 'Jeremy Cole', - 'Davi Arnaut', - ] - s.email = 'jeremy@jcole.us' - s.homepage = 'https://github.com/jeremycole/innodb_ruby' - s.files = [ - 'LICENSE', - 'AUTHORS.md', - 'README.md', - 'lib/innodb.rb', - 'lib/innodb/checksum.rb', - 'lib/innodb/data_dictionary.rb', - 'lib/innodb/data_type.rb', - 'lib/innodb/field.rb', - 'lib/innodb/fseg_entry.rb', - 'lib/innodb/history.rb', - 'lib/innodb/history_list.rb', - 'lib/innodb/ibuf_bitmap.rb', - 'lib/innodb/ibuf_index.rb', - 'lib/innodb/index.rb', - 'lib/innodb/inode.rb', - 'lib/innodb/list.rb', - 'lib/innodb/lsn.rb', - 'lib/innodb/log.rb', - 'lib/innodb/log_block.rb', - 'lib/innodb/log_group.rb', - 'lib/innodb/log_record.rb', - 'lib/innodb/log_reader.rb', - 'lib/innodb/page.rb', - 'lib/innodb/page/blob.rb', - 'lib/innodb/page/fsp_hdr_xdes.rb', - 'lib/innodb/page/ibuf_bitmap.rb', - 'lib/innodb/page/index.rb', - 'lib/innodb/page/index_compressed.rb', - 'lib/innodb/page/inode.rb', - 'lib/innodb/page/sys.rb', - 'lib/innodb/page/sys_data_dictionary_header.rb', - 'lib/innodb/page/sys_ibuf_header.rb', - 'lib/innodb/page/sys_rseg_header.rb', - 'lib/innodb/page/trx_sys.rb', - 'lib/innodb/page/undo_log.rb', - 'lib/innodb/record.rb', - 'lib/innodb/record_describer.rb', - 'lib/innodb/space.rb', - 'lib/innodb/stats.rb', - 'lib/innodb/system.rb', - 'lib/innodb/undo_log.rb', - 'lib/innodb/undo_record.rb', - 'lib/innodb/util/buffer_cursor.rb', - 'lib/innodb/util/read_bits_at_offset.rb', - 'lib/innodb/version.rb', - 'lib/innodb/xdes.rb', - ] - s.executables = [ - 'innodb_log', - 'innodb_space', + "Jeremy Cole", + "Davi Arnaut", ] + s.email = "jeremy@jcole.us" + s.homepage = "https://github.com/jeremycole/innodb_ruby" + s.files = Dir.glob("{bin,lib}/**/*") + %w[LICENSE AUTHORS.md README.md] + s.executables = %w[innodb_log innodb_space] + + s.required_ruby_version = ">= 2.6" + + s.add_dependency("bigdecimal", "~> 3.1.8") + s.add_dependency("bindata", ">= 1.4.5", "< 3.0") + s.add_dependency("csv", "~> 3.3") + s.add_dependency("digest-crc", "~> 0.4", ">= 0.4.1") + s.add_dependency("getoptlong", "~> 0.2.1") + s.add_dependency("histogram", "~> 0.2") - s.add_runtime_dependency('bindata', '~> 1.4', '>= 1.4.5') - s.add_runtime_dependency('digest-crc', '~> 0.4', '>= 0.4.1') + s.add_development_dependency("gnuplot", "~> 2.6.0") + s.add_development_dependency("rspec", "~> 3.11.0") + s.add_development_dependency("rubocop", "~> 1.18") + s.add_development_dependency("rubocop-rspec", "~> 2.4") + s.metadata["rubygems_mfa_required"] = "false" end diff --git a/lib/innodb.rb b/lib/innodb.rb index cbd6724b..62ea5248 100644 --- a/lib/innodb.rb +++ b/lib/innodb.rb @@ -1,29 +1,40 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true # A set of classes for parsing and working with InnoDB data files. module Innodb - @@debug = false + @debug = false def self.debug? - @@debug == true + @debug == true end def self.debug=(value) - @@debug = value + @debug = value end end -require "pp" -require "enumerator" require "digest/crc32c" require "innodb/util/buffer_cursor" require "innodb/util/read_bits_at_offset" +require "innodb/util/hex_format" require "innodb/version" require "innodb/stats" require "innodb/checksum" +require "innodb/mysql_collation" +require "innodb/mysql_collations" +require "innodb/mysql_type" require "innodb/record_describer" +require "innodb/sys_data_dictionary" +require "innodb/sdi" +require "innodb/sdi/sdi_object" +require "innodb/sdi/table" +require "innodb/sdi/table_column" +require "innodb/sdi/table_index" +require "innodb/sdi/table_index_element" +require "innodb/sdi/tablespace" +require "innodb/sdi_data_dictionary" require "innodb/data_dictionary" require "innodb/page" require "innodb/page/blob" @@ -32,8 +43,26 @@ def self.debug=(value) require "innodb/page/inode" require "innodb/page/index" require "innodb/page/trx_sys" +require "innodb/page/sdi" +require "innodb/page/sdi_blob" require "innodb/page/sys" require "innodb/page/undo_log" +require "innodb/data_type" +require "innodb/data_type/bit" +require "innodb/data_type/blob" +require "innodb/data_type/character" +require "innodb/data_type/date" +require "innodb/data_type/datetime" +require "innodb/data_type/decimal" +require "innodb/data_type/enum" +require "innodb/data_type/floating_point" +require "innodb/data_type/innodb_roll_pointer" +require "innodb/data_type/innodb_transaction_id" +require "innodb/data_type/integer" +require "innodb/data_type/set" +require "innodb/data_type/time" +require "innodb/data_type/timestamp" +require "innodb/data_type/year" require "innodb/record" require "innodb/field" require "innodb/space" diff --git a/lib/innodb/checksum.rb b/lib/innodb/checksum.rb index f6682bc1..02abd234 100644 --- a/lib/innodb/checksum.rb +++ b/lib/innodb/checksum.rb @@ -1,31 +1,33 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true -class Innodb::Checksum - MAX = 0xFFFFFFFF.freeze - MASK1 = 1463735687.freeze - MASK2 = 1653893711.freeze +module Innodb + class Checksum + MAX = 0xFFFFFFFF + MASK1 = 1_463_735_687 + MASK2 = 1_653_893_711 - # This is derived from ut_fold_ulint_pair in include/ut0rnd.ic in the - # InnoDB source code. Since Ruby's Bignum class is *much* slower than its - # Integer class, we mask back to 32 bits to keep things from overflowing - # and being promoted to Bignum. - def self.fold_pair(n1, n2) - (((((((n1 ^ n2 ^ MASK2) << 8) & MAX) + n1) & MAX) ^ MASK1) + n2) & MAX - end + # This is derived from ut_fold_ulint_pair in include/ut0rnd.ic in the + # InnoDB source code. Since Ruby's Bignum class is *much* slower than its + # Integer class, we mask back to 32 bits to keep things from overflowing + # and being promoted to Bignum. + def self.fold_pair(num1, num2) + (((((((num1 ^ num2 ^ MASK2) << 8) & MAX) + num1) & MAX) ^ MASK1) + num2) & MAX + end - # Iterate through the provided enumerator, which is expected to return a - # Integer (or something coercible to it), and "fold" them together to produce - # a single value. - def self.fold_enumerator(enumerator) - fold = 0 - enumerator.each do |byte| - fold = fold_pair(fold, byte) + # Iterate through the provided enumerator, which is expected to return a + # Integer (or something coercible to it), and "fold" them together to produce + # a single value. + def self.fold_enumerator(enumerator) + fold = 0 + enumerator.each do |byte| + fold = fold_pair(fold, byte) + end + fold end - fold - end - # A simple helper (and example) to fold a provided string. - def self.fold_string(string) - fold_enumerator(string.bytes) + # A simple helper (and example) to fold a provided string. + def self.fold_string(string) + fold_enumerator(string.bytes) + end end end diff --git a/lib/innodb/data_dictionary.rb b/lib/innodb/data_dictionary.rb index e22af061..b401f0ae 100644 --- a/lib/innodb/data_dictionary.rb +++ b/lib/innodb/data_dictionary.rb @@ -1,689 +1,47 @@ -# -*- encoding : utf-8 -*- - -# A class representing InnoDB's data dictionary, which contains metadata about -# tables, columns, and indexes. -class Innodb::DataDictionary - # A record describer for SYS_TABLES clustered records. - class SYS_TABLES_PRIMARY < Innodb::RecordDescriber - type :clustered - key "NAME", "VARCHAR(100)", :NOT_NULL - row "ID", :BIGINT, :UNSIGNED, :NOT_NULL - row "N_COLS", :INT, :UNSIGNED, :NOT_NULL - row "TYPE", :INT, :UNSIGNED, :NOT_NULL - row "MIX_ID", :BIGINT, :UNSIGNED, :NOT_NULL - row "MIX_LEN", :INT, :UNSIGNED, :NOT_NULL - row "CLUSTER_NAME", "VARCHAR(100)", :NOT_NULL - row "SPACE", :INT, :UNSIGNED, :NOT_NULL - end - - # A record describer for SYS_TABLES secondary key on ID. - class SYS_TABLES_ID < Innodb::RecordDescriber - type :secondary - key "ID", :BIGINT, :UNSIGNED, :NOT_NULL - row "NAME", "VARCHAR(100)", :NOT_NULL - end - - # A record describer for SYS_COLUMNS clustered records. - class SYS_COLUMNS_PRIMARY < Innodb::RecordDescriber - type :clustered - key "TABLE_ID", :BIGINT, :UNSIGNED, :NOT_NULL - key "POS", :INT, :UNSIGNED, :NOT_NULL - row "NAME", "VARCHAR(100)", :NOT_NULL - row "MTYPE", :INT, :UNSIGNED, :NOT_NULL - row "PRTYPE", :INT, :UNSIGNED, :NOT_NULL - row "LEN", :INT, :UNSIGNED, :NOT_NULL - row "PREC", :INT, :UNSIGNED, :NOT_NULL - end - - # A record describer for SYS_INDEXES clustered records. - class SYS_INDEXES_PRIMARY < Innodb::RecordDescriber - type :clustered - key "TABLE_ID", :BIGINT, :UNSIGNED, :NOT_NULL - key "ID", :BIGINT, :UNSIGNED, :NOT_NULL - row "NAME", "VARCHAR(100)", :NOT_NULL - row "N_FIELDS", :INT, :UNSIGNED, :NOT_NULL - row "TYPE", :INT, :UNSIGNED, :NOT_NULL - row "SPACE", :INT, :UNSIGNED, :NOT_NULL - row "PAGE_NO", :INT, :UNSIGNED, :NOT_NULL - end - - # A record describer for SYS_FIELDS clustered records. - class SYS_FIELDS_PRIMARY < Innodb::RecordDescriber - type :clustered - key "INDEX_ID", :BIGINT, :UNSIGNED, :NOT_NULL - key "POS", :INT, :UNSIGNED, :NOT_NULL - row "COL_NAME", "VARCHAR(100)", :NOT_NULL - end - - # A hash of hashes of table name and index name to describer - # class. - DATA_DICTIONARY_RECORD_DESCRIBERS = { - :SYS_TABLES => { - :PRIMARY => SYS_TABLES_PRIMARY, - :ID => SYS_TABLES_ID - }, - :SYS_COLUMNS => { :PRIMARY => SYS_COLUMNS_PRIMARY }, - :SYS_INDEXES => { :PRIMARY => SYS_INDEXES_PRIMARY }, - :SYS_FIELDS => { :PRIMARY => SYS_FIELDS_PRIMARY }, - } - - # A hash of MySQL's internal type system to the stored - # values for those types, and the "external" SQL type. - MYSQL_TYPE = { - # :DECIMAL => { :value => 0, :type => :DECIMAL }, - :TINY => { :value => 1, :type => :TINYINT }, - :SHORT => { :value => 2, :type => :SMALLINT }, - :LONG => { :value => 3, :type => :INT }, - :FLOAT => { :value => 4, :type => :FLOAT }, - :DOUBLE => { :value => 5, :type => :DOUBLE }, - # :NULL => { :value => 6, :type => nil }, - :TIMESTAMP => { :value => 7, :type => :TIMESTAMP }, - :LONGLONG => { :value => 8, :type => :BIGINT }, - :INT24 => { :value => 9, :type => :MEDIUMINT }, - # :DATE => { :value => 10, :type => :DATE }, - :TIME => { :value => 11, :type => :TIME }, - :DATETIME => { :value => 12, :type => :DATETIME }, - :YEAR => { :value => 13, :type => :YEAR }, - :NEWDATE => { :value => 14, :type => :DATE }, - :VARCHAR => { :value => 15, :type => :VARCHAR }, - :BIT => { :value => 16, :type => :BIT }, - :NEWDECIMAL => { :value => 246, :type => :CHAR }, - # :ENUM => { :value => 247, :type => :ENUM }, - # :SET => { :value => 248, :type => :SET }, - :TINY_BLOB => { :value => 249, :type => :TINYBLOB }, - :MEDIUM_BLOB => { :value => 250, :type => :MEDIUMBLOB }, - :LONG_BLOB => { :value => 251, :type => :LONGBLOB }, - :BLOB => { :value => 252, :type => :BLOB }, - # :VAR_STRING => { :value => 253, :type => :VARCHAR }, - :STRING => { :value => 254, :type => :CHAR }, - :GEOMETRY => { :value => 255, :type => :GEOMETRY }, - } - - # A hash of MYSQL_TYPE keys by value :value key. - MYSQL_TYPE_BY_VALUE = MYSQL_TYPE.inject({}) { |h, (k, v)| h[v[:value]] = k; h } - - # A hash of InnoDB's internal type system to the values - # stored for each type. - COLUMN_MTYPE = { - :VARCHAR => 1, - :CHAR => 2, - :FIXBINARY => 3, - :BINARY => 4, - :BLOB => 5, - :INT => 6, - :SYS_CHILD => 7, - :SYS => 8, - :FLOAT => 9, - :DOUBLE => 10, - :DECIMAL => 11, - :VARMYSQL => 12, - :MYSQL => 13, - } - - # A hash of COLUMN_MTYPE keys by value. - COLUMN_MTYPE_BY_VALUE = COLUMN_MTYPE.inject({}) { |h, (k, v)| h[v] = k; h } - - # A hash of InnoDB "precise type" bitwise flags. - COLUMN_PRTYPE_FLAG = { - :NOT_NULL => 256, - :UNSIGNED => 512, - :BINARY => 1024, - :LONG_TRUE_VARCHAR => 4096, - } - - # A hash of COLUMN_PRTYPE keys by value. - COLUMN_PRTYPE_FLAG_BY_VALUE = COLUMN_PRTYPE_FLAG.inject({}) { |h, (k, v)| h[v] = k; h } - - # The bitmask to extract the MySQL internal type - # from the InnoDB "precise type". - COLUMN_PRTYPE_MYSQL_TYPE_MASK = 0xFF - - # A hash of InnoDB's index type flags. - INDEX_TYPE_FLAG = { - :CLUSTERED => 1, - :UNIQUE => 2, - :UNIVERSAL => 4, - :IBUF => 8, - :CORRUPT => 16, - :FTS => 32, - } - - # A hash of INDEX_TYPE_FLAG keys by value. - INDEX_TYPE_FLAG_BY_VALUE = INDEX_TYPE_FLAG.inject({}) { |h, (k, v)| h[v] = k; h } - - # Return the "external" SQL type string (such as "VARCHAR" or - # "INT") given the stored mtype and prtype from the InnoDB - # data dictionary. Note that not all types are extractable - # into fully defined SQL types due to the lossy nature of - # the MySQL-to-InnoDB interface regarding types. - def self.mtype_prtype_to_type_string(mtype, prtype, len, prec) - mysql_type = prtype & COLUMN_PRTYPE_MYSQL_TYPE_MASK - internal_type = MYSQL_TYPE_BY_VALUE[mysql_type] - external_type = MYSQL_TYPE[internal_type][:type] - - case external_type - when :VARCHAR - # One-argument: length. - "%s(%i)" % [external_type, len] - when :FLOAT, :DOUBLE - # Two-argument: length and precision. - "%s(%i,%i)" % [external_type, len, prec] - when :CHAR - if COLUMN_MTYPE_BY_VALUE[mtype] == :MYSQL - # When the mtype is :MYSQL, the column is actually - # stored as VARCHAR despite being a CHAR. This is - # done for CHAR columns having multi-byte character - # sets in order to limit size. Note that such data - # are still space-padded to at least len. - "VARCHAR(%i)" % [len] - else - "CHAR(%i)" % [len] - end - when :DECIMAL - # The DECIMAL type is designated as DECIMAL(M,D) - # however the M and D definitions are not stored - # in the InnoDB data dictionary. We need to define - # the column as something which will extract the - # raw bytes in order to read the column, but we - # can't figure out the right decimal type. The - # len stored here is actually the on-disk storage - # size. - "CHAR(%i)" % [len] - else - external_type - end - end - - # Return a full data type given an mtype and prtype, such - # as ["VARCHAR(10)", :NOT_NULL] or [:INT, :UNSIGNED]. - def self.mtype_prtype_to_data_type(mtype, prtype, len, prec) - data_type = [] - - if type = mtype_prtype_to_type_string(mtype, prtype, len, prec) - data_type << type - else - raise "Unsupported type (mtype #{mtype}, prtype #{prtype})" - end - - if prtype & COLUMN_PRTYPE_FLAG[:NOT_NULL] != 0 - data_type << :NOT_NULL - end - - if prtype & COLUMN_PRTYPE_FLAG[:UNSIGNED] != 0 - data_type << :UNSIGNED - end - - data_type - end - - attr_reader :system_space - - def initialize(system_space) - @system_space = system_space - end - - # A helper method to reach inside the system space and retrieve - # the data dictionary index locations from the data dictionary - # header. - def data_dictionary_indexes - system_space.data_dictionary_page.data_dictionary_header[:indexes] - end - - def data_dictionary_index_ids - if @data_dictionary_index_ids - return @data_dictionary_index_ids - end - - @data_dictionary_index_ids = {} - data_dictionary_indexes.each do |table, indexes| - indexes.each do |index, root_page_number| - if root_page = system_space.page(root_page_number) - @data_dictionary_index_ids[root_page.index_id] = { - :table => table, - :index => index - } +# frozen_string_literal: true + +require "innodb/data_dictionary/tablespaces" +require "innodb/data_dictionary/tables" + +module Innodb + class DataDictionary + attr_reader :tablespaces + attr_reader :tables + attr_reader :indexes + attr_reader :columns + + def initialize + @tablespaces = Tablespaces.new + @tables = Tables.new + @indexes = Indexes.new + @columns = Columns.new + end + + def inspect + format("#<%s: %i tablespaces, %i tables, %i indexes, %i columns>", + self.class.name, + tablespaces.count, + tables.count, + indexes.count, + columns.count) + end + + def refresh + tables.each do |table| + table.indexes.each do |index| + indexes.add(index) end - end - end - - @data_dictionary_index_ids - end - - def data_dictionary_table?(table_name) - DATA_DICTIONARY_RECORD_DESCRIBERS.include?(table_name.to_sym) - end - - def data_dictionary_index?(table_name, index_name) - return unless data_dictionary_table?(table_name) - DATA_DICTIONARY_RECORD_DESCRIBERS[table_name.to_sym].include?(index_name.to_sym) - end - - def data_dictionary_index_describer(table_name, index_name) - return unless data_dictionary_index?(table_name, index_name) - - DATA_DICTIONARY_RECORD_DESCRIBERS[table_name.to_sym][index_name.to_sym].new - end - - # Return an Innodb::Index object initialized to the - # internal data dictionary index with an appropriate - # record describer so that records can be recursed. - def data_dictionary_index(table_name, index_name) - unless table_entry = data_dictionary_indexes[table_name] - raise "Unknown data dictionary table #{table_name}" - end - - unless index_root_page = table_entry[index_name] - raise "Unknown data dictionary index #{table_name}.#{index_name}" - end - - # If we have a record describer for this index, load it. - record_describer = data_dictionary_index_describer(table_name, index_name) - - system_space.index(index_root_page, record_describer) - end - - # Iterate through all data dictionary indexes, yielding the - # table name, index name, and root page number. - def each_data_dictionary_index_root_page_number - unless block_given? - return enum_for(:each_data_dictionary_index_root_page_number) - end - - data_dictionary_indexes.each do |table_name, indexes| - indexes.each do |index_name, root_page_number| - yield table_name, index_name, root_page_number - end - end - - nil - end - - # Iterate through all data dictionary indexes, yielding the table - # name, index name, and the index itself as an Innodb::Index. - def each_data_dictionary_index - unless block_given? - return enum_for(:each_data_dictionary_index) - end - - data_dictionary_indexes.each do |table_name, indexes| - indexes.each do |index_name, root_page_number| - yield table_name, index_name, - data_dictionary_index(table_name, index_name) - end - end - - nil - end - - # Iterate through records from a data dictionary index yielding each record - # as a Innodb::Record object. - def each_record_from_data_dictionary_index(table, index) - unless block_given? - return enum_for(:each_index, table, index) - end - - data_dictionary_index(table, index).each_record do |record| - yield record - end - - nil - end - - # Iterate through the records in the SYS_TABLES data dictionary table. - def each_table - unless block_given? - return enum_for(:each_table) - end - - each_record_from_data_dictionary_index(:SYS_TABLES, :PRIMARY) do |record| - yield record.fields - end - - nil - end - - # Iterate through the records in the SYS_COLUMNS data dictionary table. - def each_column - unless block_given? - return enum_for(:each_column) - end - - each_record_from_data_dictionary_index(:SYS_COLUMNS, :PRIMARY) do |record| - yield record.fields - end - - nil - end - - # Iterate through the records in the SYS_INDEXES dictionary table. - def each_index - unless block_given? - return enum_for(:each_index) - end - - each_record_from_data_dictionary_index(:SYS_INDEXES, :PRIMARY) do |record| - yield record.fields - end - - nil - end - - # Iterate through the records in the SYS_FIELDS data dictionary table. - def each_field - unless block_given? - return enum_for(:each_field) - end - - each_record_from_data_dictionary_index(:SYS_FIELDS, :PRIMARY) do |record| - yield record.fields - end - - nil - end - - # A helper to iterate the method provided and return the first record - # where the record's field matches the provided value. - def object_by_field(method, field, value) - send(method).select { |o| o[field] == value }.first - end - - # A helper to iterate the method provided and return the first record - # where the record's fields f1 and f2 match the provided values v1 and v2. - def object_by_two_fields(method, f1, v1, f2, v2) - send(method).select { |o| o[f1] == v1 && o[f2] == v2 }.first - end - - # Lookup a table by table ID. - def table_by_id(table_id) - object_by_field(:each_table, - "ID", table_id) - end - # Lookup a table by table name. - def table_by_name(table_name) - object_by_field(:each_table, - "NAME", table_name) - end - - # Lookup a table by space ID. - def table_by_space_id(space_id) - object_by_field(:each_table, - "SPACE", space_id) - end - - # Lookup a column by table name and column name. - def column_by_name(table_name, column_name) - unless table = table_by_name(table_name) - return nil - end - - object_by_two_fields(:each_column, - "TABLE_ID", table["ID"], - "NAME", column_name) - end - - # Lookup an index by index ID. - def index_by_id(index_id) - object_by_field(:each_index, - "ID", index_id) - end - - # Lookup an index by table name and index name. - def index_by_name(table_name, index_name) - unless table = table_by_name(table_name) - return nil - end - - object_by_two_fields(:each_index, - "TABLE_ID", table["ID"], - "NAME", index_name) - end - - # Iterate through indexes by space ID. - def each_index_by_space_id(space_id) - unless block_given? - return enum_for(:each_index_by_space_id, space_id) - end - - each_index do |record| - yield record if record["SPACE"] == space_id - end - - nil - end - - # Iterate through all indexes in a table by table ID. - def each_index_by_table_id(table_id) - unless block_given? - return enum_for(:each_index_by_table_id, table_id) - end - - each_index do |record| - yield record if record["TABLE_ID"] == table_id - end - - nil - end - - # Iterate through all indexes in a table by table name. - def each_index_by_table_name(table_name) - unless block_given? - return enum_for(:each_index_by_table_name, table_name) - end - - unless table = table_by_name(table_name) - raise "Table #{table_name} not found" - end - - each_index_by_table_id(table["ID"]) do |record| - yield record - end - - nil - end - - # Iterate through all fields in an index by index ID. - def each_field_by_index_id(index_id) - unless block_given? - return enum_for(:each_field_by_index_id, index_id) - end - - each_field do |record| - yield record if record["INDEX_ID"] == index_id - end - - nil - end - - # Iterate through all fields in an index by index name. - def each_field_by_index_name(table_name, index_name) - unless block_given? - return enum_for(:each_field_by_name, table_name, index_name) - end - - unless index = index_by_name(table_name, index_name) - raise "Index #{index_name} for table #{table_name} not found" - end - - each_field_by_index_id(index["ID"]) do |record| - yield record - end - - nil - end - - # Iterate through all columns in a table by table ID. - def each_column_by_table_id(table_id) - unless block_given? - return enum_for(:each_column_by_table_id, table_id) - end - - each_column do |record| - yield record if record["TABLE_ID"] == table_id - end - - nil - end - - # Iterate through all columns in a table by table name. - def each_column_by_table_name(table_name) - unless block_given? - return enum_for(:each_column_by_table_name, table_name) - end - - unless table = table_by_name(table_name) - raise "Table #{table_name} not found" - end - - each_column_by_table_id(table["ID"]) do |record| - yield record - end - - nil - end - - # Iterate through all columns in an index by table name and index name. - def each_column_in_index_by_name(table_name, index_name) - unless block_given? - return enum_for(:each_column_in_index_by_name, table_name, index_name) - end - - each_field_by_index_name(table_name, index_name) do |record| - yield column_by_name(table_name, record["COL_NAME"]) - end - - nil - end - - # Iterate through all columns not in an index by table name and index name. - # This is useful when building index descriptions for secondary indexes. - def each_column_not_in_index_by_name(table_name, index_name) - unless block_given? - return enum_for(:each_column_not_in_index_by_name, table_name, index_name) - end - - columns_in_index = {} - each_column_in_index_by_name(table_name, index_name) do |record| - columns_in_index[record["NAME"]] = 1 - end - - each_column_by_table_name(table_name) do |record| - yield record unless columns_in_index.include?(record["NAME"]) - end - - nil - end - - # Return the name of the clustered index (usually "PRIMARY", but not always) - # for a given table name. - def clustered_index_name_by_table_name(table_name) - unless table_record = table_by_name(table_name) - raise "Table #{table_name} not found" - end - - if index_record = object_by_two_fields(:each_index, - "TABLE_ID", table_record["ID"], - "TYPE", 3) - index_record["NAME"] - end - end - - # Produce a Innodb::RecordDescriber-compatible column description - # given a type (:key, :row) and data dictionary SYS_COLUMNS record. - def _make_column_description(type, record) - { - :type => type, - :name => record["NAME"], - :description => self.class.mtype_prtype_to_data_type( - record["MTYPE"], - record["PRTYPE"], - record["LEN"], - record["PREC"] - ) - } - end - - # Iterate through Innodb::RecordDescriber-compatible column descriptions - # for a given index by table name and index name. - def each_column_description_by_index_name(table_name, index_name) - unless block_given? - return enum_for(:each_column_description_by_index_name, table_name, index_name) - end - - unless index = index_by_name(table_name, index_name) - raise "Index #{index_name} for table #{table_name} not found" - end - - columns_in_index = {} - each_column_in_index_by_name(table_name, index_name) do |record| - columns_in_index[record["NAME"]] = 1 - yield _make_column_description(:key, record) - end - - if index["TYPE"] & INDEX_TYPE_FLAG[:CLUSTERED] != 0 - each_column_by_table_name(table_name) do |record| - unless columns_in_index.include?(record["NAME"]) - yield _make_column_description(:row, record) + table.columns.each do |column| + columns.add(column) end end - else - clustered_index_name = clustered_index_name_by_table_name(table_name) - - each_column_in_index_by_name(table_name, clustered_index_name) do |record| - yield _make_column_description(:row, record) - end - end - - nil - end - # Return an Innodb::RecordDescriber object describing records for a given - # index by table name and index name. - def record_describer_by_index_name(table_name, index_name) - if data_dictionary_index?(table_name, index_name) - return data_dictionary_index_describer(table_name, index_name) + nil end - unless index = index_by_name(table_name, index_name) - raise "Index #{index_name} for table #{table_name} not found" - end - - describer = Innodb::RecordDescriber.new - - if index["TYPE"] & INDEX_TYPE_FLAG[:CLUSTERED] != 0 - describer.type :clustered - else - describer.type :secondary + def populated? + tablespaces.any? || tables.any? end - - each_column_description_by_index_name(table_name, index_name) do |column| - case column[:type] - when :key - describer.key column[:name], *column[:description] - when :row - describer.row column[:name], *column[:description] - end - end - - describer end - - # Return an Innodb::RecordDescriber object describing the records - # in a given index by index ID. - def record_describer_by_index_id(index_id) - if dd_index = data_dictionary_index_ids[index_id] - return data_dictionary_index_describer(dd_index[:table], dd_index[:index]) - end - - unless index = index_by_id(index_id) - raise "Index #{index_id} not found" - end - - unless table = table_by_id(index["TABLE_ID"]) - raise "Table #{INDEX["TABLE_ID"]} not found" - end - - record_describer_by_index_name(table["NAME"], index["NAME"]) - end - end diff --git a/lib/innodb/data_dictionary/column.rb b/lib/innodb/data_dictionary/column.rb new file mode 100644 index 00000000..8439811f --- /dev/null +++ b/lib/innodb/data_dictionary/column.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Innodb + class DataDictionary + class Column + attr_reader :name + attr_reader :description + attr_reader :table + + def initialize(name:, description:, table:) + @name = name + @description = description + @table = table + end + end + end +end diff --git a/lib/innodb/data_dictionary/columns.rb b/lib/innodb/data_dictionary/columns.rb new file mode 100644 index 00000000..b9b154e2 --- /dev/null +++ b/lib/innodb/data_dictionary/columns.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require "innodb/data_dictionary/object_store" +require "innodb/data_dictionary/column" + +module Innodb + class DataDictionary + class Columns < ObjectStore + def initialize + super(allowed_type: Column) + end + end + end +end diff --git a/lib/innodb/data_dictionary/index.rb b/lib/innodb/data_dictionary/index.rb new file mode 100644 index 00000000..6200728a --- /dev/null +++ b/lib/innodb/data_dictionary/index.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require "innodb/data_dictionary/index_column_references" + +module Innodb + class DataDictionary + class Index + attr_reader :name + attr_reader :type + attr_reader :tablespace + attr_reader :root_page_number + attr_reader :table + attr_reader :innodb_index_id + attr_reader :column_references + + def initialize(name:, type:, table:, tablespace:, root_page_number:, innodb_index_id: nil) + @name = name + @type = type + @table = table + @tablespace = tablespace + @root_page_number = root_page_number + @innodb_index_id = innodb_index_id + @column_references = IndexColumnReferences.new + end + + def record_describer + describer = Innodb::RecordDescriber.new + + describer.type(type) + + column_references.each do |column_reference| + case column_reference.usage + when :key + describer.key(column_reference.column.name, *column_reference.column.description) + when :row + describer.row(column_reference.column.name, *column_reference.column.description) + end + end + + describer + end + end + end +end diff --git a/lib/innodb/data_dictionary/index_column_reference.rb b/lib/innodb/data_dictionary/index_column_reference.rb new file mode 100644 index 00000000..c5930424 --- /dev/null +++ b/lib/innodb/data_dictionary/index_column_reference.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require "forwardable" + +module Innodb + class DataDictionary + class IndexColumnReference + extend Forwardable + + attr_reader :column + attr_reader :usage + attr_reader :index + + def_delegators :column, :name, :description, :table + + def initialize(column:, usage:, index:) + @column = column + @usage = usage + @index = index + end + end + end +end diff --git a/lib/innodb/data_dictionary/index_column_references.rb b/lib/innodb/data_dictionary/index_column_references.rb new file mode 100644 index 00000000..28393b53 --- /dev/null +++ b/lib/innodb/data_dictionary/index_column_references.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require "innodb/data_dictionary/object_store" +require "innodb/data_dictionary/index_column_reference" + +module Innodb + class DataDictionary + class IndexColumnReferences < ObjectStore + def initialize + super(allowed_type: IndexColumnReference) + end + end + end +end diff --git a/lib/innodb/data_dictionary/indexes.rb b/lib/innodb/data_dictionary/indexes.rb new file mode 100644 index 00000000..f71b023b --- /dev/null +++ b/lib/innodb/data_dictionary/indexes.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require "innodb/data_dictionary/object_store" +require "innodb/data_dictionary/index" + +module Innodb + class DataDictionary + class Indexes < ObjectStore + def initialize + super(allowed_type: Index) + end + end + end +end diff --git a/lib/innodb/data_dictionary/object_store.rb b/lib/innodb/data_dictionary/object_store.rb new file mode 100644 index 00000000..7d381dad --- /dev/null +++ b/lib/innodb/data_dictionary/object_store.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require "forwardable" + +module Innodb + class DataDictionary + class ObjectStore + extend Forwardable + def_delegators :@objects, :[], :first, :each, :empty?, :any?, :count + + class ObjectTypeError < RuntimeError; end + + attr_reader :allowed_type + attr_reader :objects + + def initialize(allowed_type: Object) + @allowed_type = allowed_type + @objects = [] + end + + def add(new_object) + raise ObjectTypeError unless new_object.is_a?(@allowed_type) + + @objects.push(new_object) + + new_object + end + + def make(**attributes) + add(@allowed_type.new(**attributes)) + end + + def by(**attributes) + @objects.select { |o| attributes.all? { |k, v| o.send(k) == v } } + end + + def find(**attributes) + by(**attributes).first + end + end + end +end diff --git a/lib/innodb/data_dictionary/table.rb b/lib/innodb/data_dictionary/table.rb new file mode 100644 index 00000000..7cc8cc2e --- /dev/null +++ b/lib/innodb/data_dictionary/table.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require "innodb/data_dictionary/columns" +require "innodb/data_dictionary/indexes" + +module Innodb + class DataDictionary + class Table + attr_reader :name + attr_reader :tablespace + attr_reader :innodb_table_id + attr_reader :columns + attr_reader :indexes + + def initialize(name:, tablespace: nil, innodb_table_id: nil) + @name = name + @tablespace = tablespace + @innodb_table_id = innodb_table_id + @columns = Columns.new + @indexes = Indexes.new + end + + def clustered_index + indexes.find(type: :clustered) + end + end + end +end diff --git a/lib/innodb/data_dictionary/tables.rb b/lib/innodb/data_dictionary/tables.rb new file mode 100644 index 00000000..4d314303 --- /dev/null +++ b/lib/innodb/data_dictionary/tables.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require "innodb/data_dictionary/object_store" +require "innodb/data_dictionary/table" + +module Innodb + class DataDictionary + class Tables < ObjectStore + def initialize + super(allowed_type: Table) + end + end + end +end diff --git a/lib/innodb/data_dictionary/tablespace.rb b/lib/innodb/data_dictionary/tablespace.rb new file mode 100644 index 00000000..137b3596 --- /dev/null +++ b/lib/innodb/data_dictionary/tablespace.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Innodb + class DataDictionary + class Tablespace + attr_reader :name + attr_reader :innodb_space_id + + def initialize(name:, innodb_space_id:) + @name = name + @innodb_space_id = innodb_space_id + end + end + end +end diff --git a/lib/innodb/data_dictionary/tablespaces.rb b/lib/innodb/data_dictionary/tablespaces.rb new file mode 100644 index 00000000..64df7a73 --- /dev/null +++ b/lib/innodb/data_dictionary/tablespaces.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require "innodb/data_dictionary/object_store" +require "innodb/data_dictionary/tablespace" + +module Innodb + class DataDictionary + class Tablespaces < ObjectStore + def initialize + super(allowed_type: Tablespace) + end + end + end +end diff --git a/lib/innodb/data_type.rb b/lib/innodb/data_type.rb index 2f0054a4..1964bcfb 100644 --- a/lib/innodb/data_type.rb +++ b/lib/innodb/data_type.rb @@ -1,416 +1,121 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true -require "stringio" -require "bigdecimal" -require "date" +require "csv" -class Innodb::DataType +module Innodb + class DataType + class InvalidSpecificationError < StandardError; end - # MySQL's Bit-Value Type (BIT). - class BitType - attr_reader :name, :width + # A hash of page types to specialized classes to handle them. Normally + # subclasses will register themselves in this list. + @specialized_classes = {} - def initialize(base_type, modifiers, properties) - nbits = modifiers.fetch(0, 1) - raise "Unsupported width for BIT type." unless nbits >= 0 and nbits <= 64 - @width = (nbits + 7) / 8 - @name = Innodb::DataType.make_name(base_type, modifiers, properties) + class << self + attr_reader :specialized_classes end - def value(data) - "0b%b" % BinData::const_get("Uint%dbe" % (@width * 8)).read(data) - end - end - - class IntegerType - attr_reader :name, :width - - def initialize(base_type, modifiers, properties) - @width = base_type_width_map[base_type] - @unsigned = properties.include?(:UNSIGNED) - @name = Innodb::DataType.make_name(base_type, modifiers, properties) - end - - def base_type_width_map - { - :BOOL => 1, - :BOOLEAN => 1, - :TINYINT => 1, - :SMALLINT => 2, - :MEDIUMINT => 3, - :INT => 4, - :INT6 => 6, - :BIGINT => 8, - } - end - - def value(data) - nbits = @width * 8 - @unsigned ? get_uint(data, nbits) : get_int(data, nbits) - end - - def get_uint(data, nbits) - BinData::const_get("Uint%dbe" % nbits).read(data) + def self.register_specialization(data_type, specialized_class) + @specialized_classes[data_type] = specialized_class end - def get_int(data, nbits) - BinData::const_get("Int%dbe" % nbits).read(data) ^ (-1 << (nbits - 1)) + def self.specialization_for(data_type) + # This needs to intentionally use Innodb::Page because we need to register + # in the class instance variable in *that* class. + Innodb::DataType.register_specialization(data_type, self) end - end - - class FloatType - attr_reader :name, :width - - def initialize(base_type, modifiers, properties) - @width = 4 - @name = Innodb::DataType.make_name(base_type, modifiers, properties) - end - - # Read a little-endian single-precision floating-point number. - def value(data) - BinData::FloatLe.read(data) - end - end - - class DoubleType - attr_reader :name, :width - def initialize(base_type, modifiers, properties) - @width = 8 - @name = Innodb::DataType.make_name(base_type, modifiers, properties) + def self.specialization_for?(data_type) + Innodb::DataType.specialized_classes.include?(data_type) end - # Read a little-endian double-precision floating-point number. - def value(data) - BinData::DoubleLe.read(data) + def self.ceil_to(value, multiple) + ((value + (multiple - 1)) / multiple) * multiple end - end - - # MySQL's Fixed-Point Type (DECIMAL), stored in InnoDB as a binary string. - class DecimalType - attr_reader :name, :width - # The value is stored as a sequence of signed big-endian integers, each - # representing up to 9 digits of the integral and fractional parts. The - # first integer of the integral part and/or the last integer of the - # fractional part might be compressed (or packed) and are of variable - # length. The remaining integers (if any) are uncompressed and 32 bits - # wide. - MAX_DIGITS_PER_INTEGER = 9 - BYTES_PER_DIGIT = [0, 1, 1, 2, 2, 3, 3, 4, 4, 4] - - def initialize(base_type, modifiers, properties) - precision, scale = sanity_check(modifiers) - integral = precision - scale - @uncomp_integral = integral / MAX_DIGITS_PER_INTEGER - @uncomp_fractional = scale / MAX_DIGITS_PER_INTEGER - @comp_integral = integral - (@uncomp_integral * MAX_DIGITS_PER_INTEGER) - @comp_fractional = scale - (@uncomp_fractional * MAX_DIGITS_PER_INTEGER) - @width = @uncomp_integral * 4 + BYTES_PER_DIGIT[@comp_integral] + - @comp_fractional * 4 + BYTES_PER_DIGIT[@comp_fractional] - @name = Innodb::DataType.make_name(base_type, modifiers, properties) + module HasNumericModifiers + def coerce_modifiers(modifiers) + modifiers = modifiers&.split(",") if modifiers.is_a?(String) + modifiers&.map(&:to_i) + end end - def value(data) - # Strings representing the integral and fractional parts. - intg, frac = "", "" - - stream = StringIO.new(data) - mask = sign_mask(stream) - - intg << get_digits(stream, mask, @comp_integral) - - (1 .. @uncomp_integral).each do - intg << get_digits(stream, mask, MAX_DIGITS_PER_INTEGER) + module HasStringListModifiers + def coerce_modifiers(modifiers) + CSV.parse_line(modifiers, quote_char: "'")&.map(&:to_s) end - (1 .. @uncomp_fractional).each do - frac << get_digits(stream, mask, MAX_DIGITS_PER_INTEGER) + def formatted_modifiers + CSV.generate_line(modifiers, quote_char: "'", force_quotes: true, row_sep: "") end - - frac << get_digits(stream, mask, @comp_fractional) - frac = "0" if frac.empty? - - # Convert to something resembling a string representation. - str = mask.to_s.chop + intg + '.' + frac - - BigDecimal(str).to_s('F') - end - - private - - # Ensure width specification (if any) is compliant. - def sanity_check(modifiers) - raise "Invalid width specification" unless modifiers.size <= 2 - precision = modifiers.fetch(0, 10) - raise "Unsupported precision for DECIMAL type" unless - precision >= 1 and precision <= 65 - scale = modifiers.fetch(1, 0) - raise "Unsupported scale for DECIMAL type" unless - scale >= 0 and scale <= 30 and scale <= precision - [precision, scale] - end - - # The sign is encoded in the high bit of the first byte/digit. The byte - # might be part of a larger integer, so apply the bit-flipper and push - # back the byte into the stream. - def sign_mask(stream) - byte = BinData::Uint8.read(stream) - sign = byte & 0x80 - byte.assign(byte ^ 0x80) - stream.rewind - byte.write(stream) - stream.rewind - (sign == 0) ? -1 : 0 - end - - # Return a string representing an integer with a specific number of digits. - def get_digits(stream, mask, digits) - nbits = BYTES_PER_DIGIT[digits] * 8 - return "" unless nbits > 0 - value = (BinData::const_get("Int%dbe" % nbits).read(stream) ^ mask) - # Preserve leading zeros. - ("%0" + digits.to_s + "d") % value - end - end - - # Fixed-length character type. - class CharacterType - attr_reader :name, :width - - def initialize(base_type, modifiers, properties) - @width = modifiers.fetch(0, 1) - @name = Innodb::DataType.make_name(base_type, modifiers, properties) - end - - def value(data) - # The SQL standard defines that CHAR fields should have end-spaces - # stripped off. - data.sub(/[ ]+$/, "") - end - end - - class VariableCharacterType - attr_reader :name, :width - - def initialize(base_type, modifiers, properties) - @width = modifiers[0] - raise "Invalid width specification" unless modifiers.size == 1 - @name = Innodb::DataType.make_name(base_type, modifiers, properties) - end - - def value(data) - # The SQL standard defines that VARCHAR fields should have end-spaces - # stripped off. - data.sub(/[ ]+$/, "") - end - end - - # Fixed-length binary type. - class BinaryType - attr_reader :name, :width - - def initialize(base_type, modifiers, properties) - @width = modifiers.fetch(0, 1) - @name = Innodb::DataType.make_name(base_type, modifiers, properties) - end - end - - class VariableBinaryType - attr_reader :name, :width - - def initialize(base_type, modifiers, properties) - @width = modifiers[0] - raise "Invalid width specification" unless modifiers.size == 1 - @name = Innodb::DataType.make_name(base_type, modifiers, properties) - end - end - - class BlobType - attr_reader :name - - def initialize(base_type, modifiers, properties) - @name = Innodb::DataType.make_name(base_type, modifiers, properties) end - end - class YearType - attr_reader :name, :width + attr_reader :type_name + attr_reader :modifiers + attr_reader :properties - def initialize(base_type, modifiers, properties) - @width = 1 - @display_width = modifiers.fetch(0, 4) - @name = Innodb::DataType.make_name(base_type, modifiers, properties) + def initialize(type_name, modifiers = nil, properties = nil) + @type_name = type_name + @modifiers = Array(coerce_modifiers(modifiers)) + @properties = Array(properties) end - def value(data) - year = BinData::Uint8.read(data) - return (year % 100).to_s if @display_width != 4 - return (year + 1900).to_s if year != 0 - "0000" + def variable? + false end - end - class TimeType - attr_reader :name, :width - - def initialize(base_type, modifiers, properties) - @width = 3 - @name = Innodb::DataType.make_name(base_type, modifiers, properties) + def blob? + false end def value(data) - time = BinData::Int24be.read(data) ^ (-1 << 23) - sign = "-" if time < 0 - time = time.abs - "%s%02d:%02d:%02d" % [sign, time / 10000, (time / 100) % 100, time % 100] - end - end - - class DateType - attr_reader :name, :width - - def initialize(base_type, modifiers, properties) - @width = 3 - @name = Innodb::DataType.make_name(base_type, modifiers, properties) + data end - def value(data) - date = BinData::Int24be.read(data) ^ (-1 << 23) - day = date & 0x1f - month = (date >> 5) & 0xf - year = date >> 9 - "%04d-%02d-%02d" % [year, month, day] + def coerce_modifiers(modifiers) + modifiers end - end - - class DatetimeType - attr_reader :name, :width - def initialize(base_type, modifiers, properties) - @width = 8 - @name = Innodb::DataType.make_name(base_type, modifiers, properties) + def formatted_modifiers + modifiers.join(",") end - def value(data) - datetime = BinData::Int64be.read(data) ^ (-1 << 63) - date = datetime / 1000000 - year, month, day = [date / 10000, (date / 100) % 100, date % 100] - time = datetime - (date * 1000000) - hour, min, sec = [time / 10000, (time / 100) % 100, time % 100] - "%04d-%02d-%02d %02d:%02d:%02d" % [year, month, day, hour, min, sec] + def format_type_name + [ + [ + type_name.to_s, + modifiers&.any? ? "(#{formatted_modifiers})" : nil, + ].compact.join, + *properties&.map { |p| p.to_s.sub("_", " ") }, + ].compact.join(" ") end - end - - class TimestampType - attr_reader :name, :width - def initialize(base_type, modifiers, properties) - @width = 4 - @name = Innodb::DataType.make_name(base_type, modifiers, properties) + def name + @name ||= format_type_name end - # Returns the UTC timestamp as a value in 'YYYY-MM-DD HH:MM:SS' format. - def value(data) - timestamp = BinData::Uint32be.read(data) - return "0000-00-00 00:00:00" if timestamp.zero? - DateTime.strptime(timestamp.to_s, '%s').strftime "%Y-%m-%d %H:%M:%S" + def length + raise NotImplementedError end - end - - # - # Data types for InnoDB system columns. - # - # Transaction ID. - class TransactionIdType - attr_reader :name, :width + # Parse a data type definition and extract the base type and any modifiers. + def self.parse_type_name_and_modifiers(type_string) + matches = /^(?[a-zA-Z0-9_]+)(?:\((?.+)\))?(?\s+unsigned)?$/.match(type_string) + raise "Unparseable type #{type_string}" unless matches - def initialize(base_type, modifiers, properties) - @width = 6 - @name = Innodb::DataType.make_name(base_type, modifiers, properties) - end + type_name = matches[:type_name].upcase.to_sym + return [type_name, []] unless matches[:modifiers] - def read(c) - c.name("transaction_id") { c.get_uint48 } + # Use the CSV parser since it can understand quotes properly. + [type_name, matches[:modifiers]] end - end - - # Rollback data pointer. - class RollPointerType - extend ReadBitsAtOffset - - attr_reader :name, :width - def initialize(base_type, modifiers, properties) - @width = 7 - @name = Innodb::DataType.make_name(base_type, modifiers, properties) - end + def self.parse(type_string, properties = nil) + type_name, modifiers = parse_type_name_and_modifiers(type_string.to_s) - def self.parse_roll_pointer(roll_ptr) - { - :is_insert => read_bits_at_offset(roll_ptr, 1, 55) == 1, - :rseg_id => read_bits_at_offset(roll_ptr, 7, 48), - :undo_log => { - :page => read_bits_at_offset(roll_ptr, 32, 16), - :offset => read_bits_at_offset(roll_ptr, 16, 0), - } - } - end + type_class = Innodb::DataType.specialized_classes[type_name] + raise "Unrecognized type #{type_name}" unless type_class - def value(data) - roll_ptr = BinData::Uint56be.read(data) - self.class.parse_roll_pointer(roll_ptr) + type_class.new(type_name, modifiers, properties) end - - end - - # Maps base type to data type class. - TYPES = { - :BIT => BitType, - :BOOL => IntegerType, - :BOOLEAN => IntegerType, - :TINYINT => IntegerType, - :SMALLINT => IntegerType, - :MEDIUMINT => IntegerType, - :INT => IntegerType, - :INT6 => IntegerType, - :BIGINT => IntegerType, - :FLOAT => FloatType, - :DOUBLE => DoubleType, - :DECIMAL => DecimalType, - :NUMERIC => DecimalType, - :CHAR => CharacterType, - :VARCHAR => VariableCharacterType, - :BINARY => BinaryType, - :VARBINARY => VariableBinaryType, - :TINYBLOB => BlobType, - :BLOB => BlobType, - :MEDIUMBLOB => BlobType, - :LONGBLOB => BlobType, - :TINYTEXT => BlobType, - :TEXT => BlobType, - :MEDIUMTEXT => BlobType, - :LONGTEXT => BlobType, - :YEAR => YearType, - :TIME => TimeType, - :DATE => DateType, - :DATETIME => DatetimeType, - :TIMESTAMP => TimestampType, - :TRX_ID => TransactionIdType, - :ROLL_PTR => RollPointerType, - } - - def self.make_name(base_type, modifiers, properties) - name = base_type.to_s - name << '(' + modifiers.join(',') + ')' if not modifiers.empty? - name << " " - name << properties.join(' ') - name.strip - end - - def self.new(base_type, modifiers, properties) - raise "Data type '#{base_type}' is not supported" unless TYPES.key?(base_type) - TYPES[base_type].new(base_type, modifiers, properties) end end diff --git a/lib/innodb/data_type/bit.rb b/lib/innodb/data_type/bit.rb new file mode 100644 index 00000000..5a47a591 --- /dev/null +++ b/lib/innodb/data_type/bit.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Innodb + class DataType + # MySQL's Bit-Value Type (BIT). + class Bit < DataType + specialization_for :BIT + + include HasNumericModifiers + + DEFAULT_SIZE = 1 + SUPPORTED_SIZE_RANGE = (1..64).freeze + + def initialize(type_name, modifiers, properties) + super + + @size = @modifiers.fetch(0, DEFAULT_SIZE) + raise "Unsupported width for #{@type_name} type" unless SUPPORTED_SIZE_RANGE.include?(@size) + end + + def value(data) + "0b%b" % BinData.const_get("Uint%dbe" % Innodb::DataType.ceil_to(@size, 8)).read(data) + end + + def length + @length = Innodb::DataType.ceil_to(@size, 8) / 8 + end + end + end +end diff --git a/lib/innodb/data_type/blob.rb b/lib/innodb/data_type/blob.rb new file mode 100644 index 00000000..e4bfc94c --- /dev/null +++ b/lib/innodb/data_type/blob.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Innodb + class DataType + class Blob < DataType + specialization_for :TINYBLOB + specialization_for :BLOB + specialization_for :MEDIUMBLOB + specialization_for :LONGBLOB + specialization_for :TINYTEXT + specialization_for :TEXT + specialization_for :MEDIUMTEXT + specialization_for :LONGTEXT + specialization_for :JSON + specialization_for :GEOMETRY + + include HasNumericModifiers + + def variable? + true + end + + def blob? + true + end + end + end +end diff --git a/lib/innodb/data_type/character.rb b/lib/innodb/data_type/character.rb new file mode 100644 index 00000000..df1521d9 --- /dev/null +++ b/lib/innodb/data_type/character.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Innodb + class DataType + # Fixed-length character type. + class Character < DataType + specialization_for :CHAR + specialization_for :VARCHAR + specialization_for :BINARY + specialization_for :VARBINARY + + include HasNumericModifiers + + VALID_LENGTH_RANGE = (0..65_535).freeze # 1..255 characters, up to 4 bytes each + DEFAULT_LENGTH = 1 + + def initialize(type_name, modifiers, properties) + super + + @variable = false + @binary = false + + if %i[VARCHAR VARBINARY].include?(@type_name) + @variable = true + if @modifiers.empty? + raise InvalidSpecificationError, "Missing length specification for variable-length type #{@type_name}" + elsif @modifiers.size > 1 + raise InvalidSpecificationError, "Invalid length specification for variable-length type #{@type_name}" + end + end + + @binary = true if %i[BINARY VARBINARY].include?(@type_name) + + @length = @modifiers.fetch(0, DEFAULT_LENGTH) + return if VALID_LENGTH_RANGE.include?(@length) + + raise InvalidSpecificationError, "Length #{@length} out of range for #{@type_name}" + end + + def variable? + @variable + end + + def value(data) + # The SQL standard defines that CHAR fields should have end-spaces + # stripped off. + @binary ? data : data.sub(/ +$/, "") + end + + attr_reader :length + end + end +end diff --git a/lib/innodb/data_type/date.rb b/lib/innodb/data_type/date.rb new file mode 100644 index 00000000..2e92a873 --- /dev/null +++ b/lib/innodb/data_type/date.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Innodb + class DataType + class Date < DataType + specialization_for :DATE + + include HasNumericModifiers + + def value(data) + date = BinData::Int24be.read(data) ^ (-1 << 23) + day = date & 0x1f + month = (date >> 5) & 0xf + year = date >> 9 + "%04d-%02d-%02d" % [year, month, day] + end + + def length + 3 + end + end + end +end diff --git a/lib/innodb/data_type/datetime.rb b/lib/innodb/data_type/datetime.rb new file mode 100644 index 00000000..7b6b2a66 --- /dev/null +++ b/lib/innodb/data_type/datetime.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Innodb + class DataType + class Datetime < DataType + specialization_for :DATETIME + + include HasNumericModifiers + + def value(data) + datetime = BinData::Int64be.read(data) ^ (-1 << 63) + date = datetime / 1_000_000 + year = date / 10_000 + month = (date / 100) % 100 + day = date % 100 + time = datetime - (date * 1_000_000) + hour = time / 10_000 + min = (time / 100) % 100 + sec = time % 100 + "%04d-%02d-%02d %02d:%02d:%02d" % [year, month, day, hour, min, sec] + end + + def length + 8 + end + end + end +end diff --git a/lib/innodb/data_type/decimal.rb b/lib/innodb/data_type/decimal.rb new file mode 100644 index 00000000..e6ff6699 --- /dev/null +++ b/lib/innodb/data_type/decimal.rb @@ -0,0 +1,133 @@ +# frozen_string_literal: true + +require "bigdecimal" +require "stringio" + +module Innodb + class DataType + # MySQL's Fixed-Point Type (DECIMAL), stored in InnoDB as a binary string. + class Decimal < DataType + specialization_for :DECIMAL + specialization_for :NUMERIC + + include HasNumericModifiers + + # The value is stored as a sequence of signed big-endian integers, each + # representing up to 9 digits of the integral and fractional parts. The + # first integer of the integral part and/or the last integer of the + # fractional part might be compressed (or packed) and are of variable + # length. The remaining integers (if any) are uncompressed and 32 bits + # wide. + MAX_DIGITS_PER_INTEGER = 9 + BYTES_PER_DIGIT = [0, 1, 1, 2, 2, 3, 3, 4, 4, 4].freeze + + DEFAULT_PRECISION = 10 + VALID_PRECISION_RANGE = (1..65).freeze + + DEFAULT_SCALE = 0 + VALID_SCALE_RANGE = (0..30).freeze + + def self.length_attributes(precision, scale) + integral = precision - scale + + integral_count_full_parts = integral / MAX_DIGITS_PER_INTEGER + integral_first_part_length = integral - (integral_count_full_parts * MAX_DIGITS_PER_INTEGER) + + fractional_count_full_parts = scale / MAX_DIGITS_PER_INTEGER + fractional_first_part_length = scale - (fractional_count_full_parts * MAX_DIGITS_PER_INTEGER) + + integral_length = (integral_count_full_parts * 4) + BYTES_PER_DIGIT[integral_first_part_length] + fractional_length = (fractional_count_full_parts * 4) + BYTES_PER_DIGIT[fractional_first_part_length] + + { + length: integral_length + fractional_length, + integral: { + length: integral_length, + first_part_length: integral_first_part_length, + count_full_parts: integral_count_full_parts, + }, + fractional: { + length: fractional_length, + first_part_length: fractional_first_part_length, + count_full_parts: fractional_count_full_parts, + }, + } + end + + def initialize(type_name, modifiers, properties) + super + + raise "Invalid #{@type_name} specification: #{@modifiers}" unless @modifiers.size <= 2 + + @precision = @modifiers.fetch(0, DEFAULT_PRECISION) + @scale = @modifiers.fetch(1, DEFAULT_SCALE) + + unless VALID_PRECISION_RANGE.include?(@precision) + raise "Unsupported precision #{@precision} for #{@type_name} type" + end + + unless VALID_SCALE_RANGE.include?(@scale) && @scale <= @precision + raise "Unsupported scale #{@scale} for #{@type_name} type" + end + + @length_attributes = self.class.length_attributes(@precision, @scale) + end + + def length + @length_attributes[:length] + end + + def value(data) + # Strings representing the integral and fractional parts. + intg = "".dup + frac = "".dup + + stream = StringIO.new(data) + mask = sign_mask(stream) + + intg << get_digits(stream, mask, @length_attributes[:integral][:first_part_length]) + + @length_attributes[:integral][:count_full_parts].times do + intg << get_digits(stream, mask, MAX_DIGITS_PER_INTEGER) + end + + @length_attributes[:fractional][:count_full_parts].times do + frac << get_digits(stream, mask, MAX_DIGITS_PER_INTEGER) + end + + frac << get_digits(stream, mask, @length_attributes[:fractional][:first_part_length]) + frac = "0" if frac.empty? + + # Convert to something resembling a string representation. + str = "#{mask.to_s.chop}#{intg}.#{frac}" + + BigDecimal(str).to_s("F") + end + + private + + # The sign is encoded in the high bit of the first byte/digit. The byte + # might be part of a larger integer, so apply the bit-flipper and push + # back the byte into the stream. + def sign_mask(stream) + byte = BinData::Uint8.read(stream) + sign = byte & 0x80 + byte.assign(byte ^ 0x80) + stream.rewind + byte.write(stream) + stream.rewind + sign.zero? ? -1 : 0 + end + + # Return a string representing an integer with a specific number of digits. + def get_digits(stream, mask, digits) + nbits = BYTES_PER_DIGIT[digits] * 8 + return "" unless nbits.positive? + + value = (BinData.const_get("Int%dbe" % nbits).read(stream) ^ mask) + # Preserve leading zeros. + "%0#{digits}d" % value + end + end + end +end diff --git a/lib/innodb/data_type/enum.rb b/lib/innodb/data_type/enum.rb new file mode 100644 index 00000000..1ba497e1 --- /dev/null +++ b/lib/innodb/data_type/enum.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Innodb + class DataType + class Enum < DataType + specialization_for :ENUM + + include HasStringListModifiers + + attr_reader :values + + def initialize(type_name, modifiers, properties) + super + + @values = { 0 => "" } + @values.merge!(@modifiers.each_with_index.to_h { |s, i| [i + 1, s] }) + end + + def bit_length + @bit_length ||= Innodb::DataType.ceil_to(Math.log2(@values.length).ceil, 8) + end + + def value(data) + index = BinData.const_get("Int%dbe" % bit_length).read(data) + values[index] + end + + def length + bit_length / 8 + end + end + end +end diff --git a/lib/innodb/data_type/floating_point.rb b/lib/innodb/data_type/floating_point.rb new file mode 100644 index 00000000..9e7cedd9 --- /dev/null +++ b/lib/innodb/data_type/floating_point.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Innodb + class DataType + class FloatingPoint < DataType + specialization_for :FLOAT + specialization_for :DOUBLE + + include HasNumericModifiers + + def value(data) + case type_name + when :FLOAT + BinData::FloatLe.read(data) + when :DOUBLE + BinData::DoubleLe.read(data) + end + end + + def length + case type_name + when :FLOAT + 4 + when :DOUBLE + 8 + end + end + end + end +end diff --git a/lib/innodb/data_type/innodb_roll_pointer.rb b/lib/innodb/data_type/innodb_roll_pointer.rb new file mode 100644 index 00000000..9208d075 --- /dev/null +++ b/lib/innodb/data_type/innodb_roll_pointer.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Innodb + class DataType + # Rollback data pointer. + class InnodbRollPointer < DataType + specialization_for :ROLL_PTR + + extend ReadBitsAtOffset + + Pointer = Struct.new( + :is_insert, + :rseg_id, + :undo_log, + keyword_init: true + ) + + def self.parse_roll_pointer(roll_ptr) + Pointer.new( + is_insert: read_bits_at_offset(roll_ptr, 1, 55) == 1, + rseg_id: read_bits_at_offset(roll_ptr, 7, 48), + undo_log: Innodb::Page::Address.new( + page: read_bits_at_offset(roll_ptr, 32, 16), + offset: read_bits_at_offset(roll_ptr, 16, 0) + ) + ) + end + + def value(data) + roll_ptr = BinData::Uint56be.read(data) + self.class.parse_roll_pointer(roll_ptr) + end + + def length + 7 + end + end + end +end diff --git a/lib/innodb/data_type/innodb_transaction_id.rb b/lib/innodb/data_type/innodb_transaction_id.rb new file mode 100644 index 00000000..3559e783 --- /dev/null +++ b/lib/innodb/data_type/innodb_transaction_id.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Innodb + class DataType + # Transaction ID. + class InnodbTransactionId < DataType + specialization_for :TRX_ID + + def value(data) + BinData::Uint48be.read(data).to_i + end + + def length + 6 + end + end + end +end diff --git a/lib/innodb/data_type/integer.rb b/lib/innodb/data_type/integer.rb new file mode 100644 index 00000000..45c4d533 --- /dev/null +++ b/lib/innodb/data_type/integer.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Innodb + class DataType + class Integer < DataType + specialization_for :BOOL + specialization_for :BOOLEAN + specialization_for :TINYINT + specialization_for :SMALLINT + specialization_for :MEDIUMINT + specialization_for :INT + specialization_for :INT6 + specialization_for :BIGINT + + include HasNumericModifiers + + TYPE_BIT_LENGTH_MAP = { + BOOL: 8, + BOOLEAN: 8, + TINYINT: 8, + SMALLINT: 16, + MEDIUMINT: 24, + INT: 32, + INT6: 48, + BIGINT: 64, + }.freeze + + def initialize(type_name, modifiers, properties) + super + + @unsigned = properties&.include?(:UNSIGNED) + end + + def bit_length + @bit_length ||= TYPE_BIT_LENGTH_MAP[type_name] + end + + def unsigned? + @unsigned + end + + def value(data) + if unsigned? + BinData.const_get("Uint%dbe" % bit_length).read(data) + else + BinData.const_get("Int%dbe" % bit_length).read(data) ^ (-1 << (bit_length - 1)) + end + end + + def length + bit_length / 8 + end + end + end +end diff --git a/lib/innodb/data_type/set.rb b/lib/innodb/data_type/set.rb new file mode 100644 index 00000000..addeb3f0 --- /dev/null +++ b/lib/innodb/data_type/set.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Innodb + class DataType + class Set < DataType + specialization_for :SET + + include HasStringListModifiers + + attr_reader :values + + def initialize(type_name, modifiers, properties) + super + + @values = @modifiers.each_with_index.to_h { |s, i| [2**i, s] } + end + + def bit_length + @bit_length ||= Innodb::DataType.ceil_to(@values.length, 8) + end + + def value(data) + bitmap = BinData.const_get("Int%dbe" % bit_length).read(data) + (0...bit_length).map { |i| bitmap & (2**i) }.reject(&:zero?).map { |i| values[i] } + end + + def length + bit_length / 8 + end + end + end +end diff --git a/lib/innodb/data_type/time.rb b/lib/innodb/data_type/time.rb new file mode 100644 index 00000000..339a26e6 --- /dev/null +++ b/lib/innodb/data_type/time.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Innodb + class DataType + class Time < DataType + specialization_for :TIME + + include HasNumericModifiers + + def value(data) + time = BinData::Int24be.read(data) ^ (-1 << 23) + sign = "-" if time.negative? + time = time.abs + "%s%02d:%02d:%02d" % [sign, time / 10_000, (time / 100) % 100, time % 100] + end + + def length + 3 + end + end + end +end diff --git a/lib/innodb/data_type/timestamp.rb b/lib/innodb/data_type/timestamp.rb new file mode 100644 index 00000000..cbcd8458 --- /dev/null +++ b/lib/innodb/data_type/timestamp.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require "date" + +module Innodb + class DataType + class Timestamp < DataType + specialization_for :TIMESTAMP + + include HasNumericModifiers + + # Returns the UTC timestamp as a value in 'YYYY-MM-DD HH:MM:SS' format. + def value(data) + timestamp = BinData::Uint32be.read(data) + return "0000-00-00 00:00:00" if timestamp.zero? + + DateTime.strptime(timestamp.to_s, "%s").strftime "%Y-%m-%d %H:%M:%S" + end + + def length + 4 + end + end + end +end diff --git a/lib/innodb/data_type/year.rb b/lib/innodb/data_type/year.rb new file mode 100644 index 00000000..c7ad95d6 --- /dev/null +++ b/lib/innodb/data_type/year.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Innodb + class DataType + class Year < DataType + specialization_for :YEAR + + include HasNumericModifiers + + DEFAULT_DISPLAY_WIDTH = 4 + VALID_DISPLAY_WIDTHS = [2, 4].freeze + + def initialize(type_name, modifiers, properties) + super + + @display_width = modifiers.fetch(0, DEFAULT_DISPLAY_WIDTH) + return if VALID_DISPLAY_WIDTHS.include?(@display_width) + + raise InvalidSpecificationError, "Unsupported display width #{@display_width} for type #{type_name}" + end + + def value(data) + year = BinData::Uint8.read(data) + return (year % 100).to_s if @display_width != 4 + return (year + 1900).to_s if year != 0 + + "0000" + end + + def length + 1 + end + end + end +end diff --git a/lib/innodb/field.rb b/lib/innodb/field.rb index 683a2614..6771786c 100644 --- a/lib/innodb/field.rb +++ b/lib/innodb/field.rb @@ -1,114 +1,111 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true require "innodb/data_type" # A single field in an InnoDB record (within an INDEX page). This class # provides essential information to parse records, including the length # of the fixed-width and variable-width portion of the field. -class Innodb::Field - attr_reader :position, :name, :data_type, :nullable - - # Size of a reference to data stored externally to the page. - EXTERN_FIELD_SIZE = 20 - - def initialize(position, name, type_definition, *properties) - @position = position - @name = name - @nullable = properties.delete(:NOT_NULL) ? false : true - base_type, modifiers = parse_type_definition(type_definition.to_s) - @data_type = Innodb::DataType.new(base_type, modifiers, properties) - end +module Innodb + class Field + ExternReference = Struct.new( + :space_id, + :page_number, + :offset, + :length, # rubocop:disable Lint/StructNewOverride + keyword_init: true + ) + + attr_reader :position + attr_reader :name + attr_reader :data_type + attr_reader :nullable + + # Size of a reference to data stored externally to the page. + EXTERN_FIELD_SIZE = 20 + + def initialize(position, name, type_definition, *properties) + @position = position + @name = name + @nullable = !properties.delete(:NOT_NULL) + @data_type = Innodb::DataType.parse(type_definition, properties) + end - # Return whether this field can be NULL. - def nullable? - @nullable - end + # Return whether this field can be NULL. + def nullable? + @nullable + end - # Return whether this field is NULL. - def null?(record) - nullable? && record[:header][:nulls].include?(@name) - end + # Return whether this field is NULL. + def null?(record) + nullable? && record.header.nulls.include?(@name) + end - # Return whether a part of this field is stored externally (off-page). - def extern?(record) - record[:header][:externs].include?(@name) - end + # Return whether a part of this field is stored externally (off-page). + def extern?(record) + record.header.externs.include?(@name) + end - def variable? - @data_type.is_a? Innodb::DataType::BlobType or - @data_type.is_a? Innodb::DataType::VariableBinaryType or - @data_type.is_a? Innodb::DataType::VariableCharacterType - end + def variable? + @data_type.variable? + end - def blob? - @data_type.is_a? Innodb::DataType::BlobType - end + def fixed? + !variable? + end - # Return the actual length of this variable-length field. - def length(record) - if record[:header][:lengths].include?(@name) - len = record[:header][:lengths][@name] - raise "Fixed-length mismatch" unless variable? || len == @data_type.width - else - len = @data_type.width + def blob? + @data_type.blob? end - extern?(record) ? len - EXTERN_FIELD_SIZE : len - end - # Read an InnoDB encoded data field. - def read(cursor, field_length) - cursor.name(@data_type.name) { cursor.get_bytes(field_length) } - end + # Return the actual length of this variable-length field. + def length(record) + if record.header.lengths.include?(@name) + len = record.header.lengths[@name] + if fixed? && len != @data_type.length + raise "Fixed-length mismatch; #{len} vs #{@data_type.length} for #{@data_type.name}" + end + else + len = @data_type.length + end + extern?(record) ? len - EXTERN_FIELD_SIZE : len + end - def value_by_length(cursor, field_length) - if @data_type.respond_to?(:read) - cursor.name(@data_type.name) { @data_type.read(cursor) } - elsif @data_type.respond_to?(:value) + # Read an InnoDB encoded data field. + def read(cursor, field_length) + cursor.name(@data_type.name) { cursor.read_bytes(field_length) } + end + + def value_by_length(cursor, field_length) @data_type.value(read(cursor, field_length)) - else - read(cursor, field_length) end - end - # Read the data value (e.g. encoded in the data). - def value(cursor, record) - return :NULL if null?(record) - value_by_length(cursor, length(record)) - end + # Read the data value (e.g. encoded in the data). + def value(cursor, record) + return :NULL if null?(record) - # Read an InnoDB external pointer field. - def extern(cursor, record) - return nil if not extern?(record) - cursor.name(@name) { read_extern(cursor) } - end + value_by_length(cursor, length(record)) + end - private - - # Return an external reference field. An extern field contains the page - # address and the length of the externally stored part of the record data. - def get_extern_reference(cursor) - { - :space_id => cursor.name("space_id") { cursor.get_uint32 }, - :page_number => cursor.name("page_number") { cursor.get_uint32 }, - :offset => cursor.name("offset") { cursor.get_uint32 }, - :length => cursor.name("length") { cursor.get_uint64 & 0x3fffffff } - } - end + # Read an InnoDB external pointer field. + def extern(cursor, record) + return unless extern?(record) - def read_extern(cursor) - cursor.name("extern") { get_extern_reference(cursor) } - end + cursor.name(@name) { read_extern(cursor) } + end - # Parse a data type definition and extract the base type and any modifiers. - def parse_type_definition(type_string) - if matches = /^([a-zA-Z0-9_]+)(\(([0-9, ]+)\))?$/.match(type_string) - base_type = matches[1].upcase.to_sym - if matches[3] - modifiers = matches[3].sub(/[ ]/, "").split(/,/).map { |s| s.to_i } - else - modifiers = [] + private + + # Return an external reference field. An extern field contains the page + # address and the length of the externally stored part of the record data. + def read_extern(cursor) + cursor.name("extern") do |c| + ExternReference.new( + space_id: c.name("space_id") { c.read_uint32 }, + page_number: c.name("page_number") { c.read_uint32 }, + offset: c.name("offset") { c.read_uint32 }, + length: c.name("length") { c.read_uint64 & 0x3fffffff } + ) end - [base_type, modifiers] end end end diff --git a/lib/innodb/fseg_entry.rb b/lib/innodb/fseg_entry.rb index d07335c3..3285fc48 100644 --- a/lib/innodb/fseg_entry.rb +++ b/lib/innodb/fseg_entry.rb @@ -1,36 +1,32 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true # An InnoDB file segment entry, which appears in a few places, such as the # FSEG header of INDEX pages, and in the TRX_SYS pages. -class Innodb::FsegEntry - # The size (in bytes) of an FSEG entry, which contains a two 32-bit integers - # and a 16-bit integer. - SIZE = 4 + 4 + 2 +module Innodb + class FsegEntry + # The size (in bytes) of an FSEG entry, which contains a two 32-bit integers + # and a 16-bit integer. + SIZE = 4 + 4 + 2 - # Return the FSEG entry address, which points to an entry on an INODE page. - def self.get_entry_address(cursor) - { - :space_id => cursor.name("space_id") { cursor.get_uint32 }, - :page_number => cursor.name("page_number") { - Innodb::Page.maybe_undefined(cursor.get_uint32) - }, - :offset => cursor.name("offset") { cursor.get_uint16 }, - } - end - - # Return an INODE entry which represents this file segment. - def self.get_inode(space, cursor) - address = cursor.name("address") { get_entry_address(cursor) } - if address[:offset] == 0 - return nil + # Return the FSEG entry address, which points to an entry on an INODE page. + def self.get_entry_address(cursor) + { + space_id: cursor.name("space_id") { cursor.read_uint32 }, + page_number: cursor.name("page_number") { Innodb::Page.maybe_undefined(cursor.read_uint32) }, + offset: cursor.name("offset") { cursor.read_uint16 }, + } end - page = space.page(address[:page_number]) - if page.type != :INODE - return nil - end + # Return an INODE entry which represents this file segment. + def self.get_inode(space, cursor) + address = cursor.name("address") { get_entry_address(cursor) } + return nil if address[:offset].zero? - page.inode_at(page.cursor(address[:offset])) + page = space.page(address[:page_number]) + return nil unless page.type == :INODE + + page.inode_at(page.cursor(address[:offset])) + end end end diff --git a/lib/innodb/history.rb b/lib/innodb/history.rb index 140d964c..c1f02830 100644 --- a/lib/innodb/history.rb +++ b/lib/innodb/history.rb @@ -1,30 +1,30 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true # The global history of record versions implemented through undo logs. -class Innodb::History - def initialize(innodb_system) - @innodb_system = innodb_system - end - - # A helper to get to the trx_sys page in the Innodb::System. - def trx_sys - @innodb_system.system_space.trx_sys - end +module Innodb + class History + def initialize(innodb_system) + @innodb_system = innodb_system + end - # A helper to get to the history_list of a given space_id and page number. - def history_list(space_id, page_number) - @innodb_system.space(space_id).page(page_number).history_list - end + # A helper to get to the trx_sys page in the Innodb::System. + def trx_sys + @innodb_system.system_space.trx_sys + end - # Iterate through all history lists (one per rollback segment, nominally - # there are 128 rollback segments). - def each_history_list - unless block_given? - return enum_for(:each_history_list) + # A helper to get to the history_list of a given space_id and page number. + def history_list(space_id, page_number) + @innodb_system.space(space_id).page(page_number).history_list end - trx_sys.rsegs.each do |slot| - yield history_list(slot[:space_id], slot[:page_number]) + # Iterate through all history lists (one per rollback segment, nominally + # there are 128 rollback segments). + def each_history_list + return enum_for(:each_history_list) unless block_given? + + trx_sys.rsegs.each do |slot| + yield history_list(slot[:space_id], slot[:page_number]) + end end end end diff --git a/lib/innodb/history_list.rb b/lib/innodb/history_list.rb index 169a01a5..49295a2c 100644 --- a/lib/innodb/history_list.rb +++ b/lib/innodb/history_list.rb @@ -1,106 +1,102 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true # A single history list; this is a more intelligent wrapper around the basic # Innodb::List::History which is provided elsewhere. -class Innodb::HistoryList - attr_reader :list +module Innodb + class HistoryList + attr_reader :list - # Initialize from a provided Innodb::List::History. - def initialize(list) - @list = list - end - - class UndoRecordCursor - def initialize(history, undo_record, direction=:forward) - @history = history - @undo_record = undo_record + # Initialize from a provided Innodb::List::History. + def initialize(list) + @list = list + end - case undo_record - when :min - @undo_log_cursor = history.list.list_cursor(:min, direction) - if @undo_log = @undo_log_cursor.node - @undo_record_cursor = @undo_log.undo_record_cursor(:min, direction) - end - when :max - @undo_log_cursor = history.list.list_cursor(:max, direction) - if @undo_log = @undo_log_cursor.node - @undo_record_cursor = @undo_log.undo_record_cursor(:max, direction) + class UndoRecordCursor + def initialize(history, undo_record, direction = :forward) + @history = history + @undo_record = undo_record + + # rubocop:disable Style/IfUnlessModifier + case undo_record + when :min + @undo_log_cursor = history.list.list_cursor(:min, direction) + if (@undo_log = @undo_log_cursor.node) + @undo_record_cursor = @undo_log.undo_record_cursor(:min, direction) + end + when :max + @undo_log_cursor = history.list.list_cursor(:max, direction) + if (@undo_log = @undo_log_cursor.node) + @undo_record_cursor = @undo_log.undo_record_cursor(:max, direction) + end + else + raise "Not implemented" end - else - raise "Not implemented" + # rubocop:enable Style/IfUnlessModifier end - end - def undo_record - unless @undo_record_cursor - return nil - end + def undo_record + return nil unless @undo_record_cursor - if rec = @undo_record_cursor.undo_record - return rec - end + if (rec = @undo_record_cursor.undo_record) + return rec + end - case @direction - when :forward - next_undo_record - when :backward - prev_undo_record + case @direction + when :forward + next_undo_record + when :backward + prev_undo_record + end end - end - def move_cursor(page, undo_record) - @undo_log = page - @undo_log_cursor = @undo_log.undo_record_cursor(undo_record, @direction) - end - - def next_undo_record - if rec = @undo_record_cursor.undo_record - return rec + def move_cursor(page, undo_record) + @undo_log = page + @undo_log_cursor = @undo_log.undo_record_cursor(undo_record, @direction) end - if undo_log = @undo_log_cursor.node - @undo_log = undo_log - @undo_record_cursor = @undo_log.undo_record_cursor(:min, @direction) - end + def next_undo_record + if (rec = @undo_record_cursor.undo_record) + return rec + end - @undo_record_cursor.undo_record - end + if (undo_log = @undo_log_cursor.node) + @undo_log = undo_log + @undo_record_cursor = @undo_log.undo_record_cursor(:min, @direction) + end - def prev_undo_record - if rec = @undo_log_cursor.undo_record - return rec + @undo_record_cursor.undo_record end - if undo_log = @undo_log_cursor.node - @undo_log = undo_log - @undo_record_cursor = @undo_log.undo_record_cursor(:max, @direction) - end + def prev_undo_record + if (rec = @undo_log_cursor.undo_record) + return rec + end - @undo_record_cursor.undo_record - end + if (undo_log = @undo_log_cursor.node) + @undo_log = undo_log + @undo_record_cursor = @undo_log.undo_record_cursor(:max, @direction) + end - def each_undo_record - unless block_given? - return enum_for(:each_undo_record) + @undo_record_cursor.undo_record end - while rec = undo_record - yield rec + def each_undo_record + return enum_for(:each_undo_record) unless block_given? + + while (rec = undo_record) + yield rec + end end end - end - def undo_record_cursor(undo_record=:min, direction=:forward) - UndoRecordCursor.new(self, undo_record, direction) - end - - def each_undo_record - unless block_given? - return enum_for(:each_undo_record) + def undo_record_cursor(undo_record = :min, direction = :forward) + UndoRecordCursor.new(self, undo_record, direction) end - undo_record_cursor.each_undo_record do |rec| - yield rec + def each_undo_record(&block) + return enum_for(:each_undo_record) unless block_given? + + undo_record_cursor.each_undo_record(&block) end end end diff --git a/lib/innodb/ibuf_bitmap.rb b/lib/innodb/ibuf_bitmap.rb index b9390097..3282de88 100644 --- a/lib/innodb/ibuf_bitmap.rb +++ b/lib/innodb/ibuf_bitmap.rb @@ -1,49 +1,49 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true -class Innodb::IbufBitmap - BITS_PER_PAGE = 4 +module Innodb + class IbufBitmap + PageStatus = Struct.new(:free, :buffered, :ibuf, keyword_init: true) - BITMAP_BV_FREE = 1 + 2 - BITMAP_BV_BUFFERED = 4 - BITMAP_BV_IBUF = 8 + BITS_PER_PAGE = 4 - BITMAP_BV_ALL = - BITMAP_BV_FREE | - BITMAP_BV_BUFFERED | - BITMAP_BV_IBUF + BITMAP_BV_FREE = 1 + 2 + BITMAP_BV_BUFFERED = 4 + BITMAP_BV_IBUF = 8 - def initialize(page, cursor) - @page = page - @bitmap = read_bitmap(page, cursor) - end + BITMAP_BV_ALL = + BITMAP_BV_FREE | + BITMAP_BV_BUFFERED | + BITMAP_BV_IBUF - def size_bitmap - (@page.space.pages_per_bookkeeping_page * BITS_PER_PAGE) / 8 - end + def initialize(page, cursor) + @page = page + @bitmap = read_bitmap(cursor) + end - def read_bitmap(page, cursor) - cursor.name("ibuf_bitmap") do |c| - c.get_bytes(size_bitmap) + def size_bitmap + (@page.space.pages_per_bookkeeping_page * BITS_PER_PAGE) / 8 end - end - def each_page_status - unless block_given? - return enum_for(:each_page_status) + def read_bitmap(cursor) + cursor.name("ibuf_bitmap") { |c| c.read_bytes(size_bitmap) } end - bitmap = @bitmap.enum_for(:each_byte) - - bitmap.each_with_index do |byte, byte_index| - (0..1).each do |page_offset| - page_number = (byte_index * 2) + page_offset - page_bits = ((byte >> (page_offset * BITS_PER_PAGE)) & BITMAP_BV_ALL) - page_status = { - :free => (page_bits & BITMAP_BV_FREE), - :buffered => (page_bits & BITMAP_BV_BUFFERED != 0), - :ibuf => (page_bits & BITMAP_BV_IBUF != 0), - } - yield page_number, page_status + def each_page_status + return enum_for(:each_page_status) unless block_given? + + bitmap = @bitmap.enum_for(:each_byte) + + bitmap.each_with_index do |byte, byte_index| + (0..1).each do |page_offset| + page_number = (byte_index * 2) + page_offset + page_bits = ((byte >> (page_offset * BITS_PER_PAGE)) & BITMAP_BV_ALL) + page_status = PageStatus.new( + free: (page_bits & BITMAP_BV_FREE), + buffered: (page_bits & BITMAP_BV_BUFFERED != 0), + ibuf: (page_bits & BITMAP_BV_IBUF != 0) + ) + yield page_number, page_status + end end end end diff --git a/lib/innodb/ibuf_index.rb b/lib/innodb/ibuf_index.rb index 4b9853a0..b71d2a19 100644 --- a/lib/innodb/ibuf_index.rb +++ b/lib/innodb/ibuf_index.rb @@ -1,3 +1,7 @@ -class Innodb::IbufIndex - INDEX_ID = 0xFFFFFFFF00000000 +# frozen_string_literal: true + +module Innodb + class IbufIndex + INDEX_ID = 0xFFFFFFFF00000000 + end end diff --git a/lib/innodb/index.rb b/lib/innodb/index.rb index 7e1c112f..ffc5674f 100644 --- a/lib/innodb/index.rb +++ b/lib/innodb/index.rb @@ -1,362 +1,333 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true # An InnoDB index B-tree, given an Innodb::Space and a root page number. -class Innodb::Index - attr_reader :root - attr_reader :space - attr_accessor :record_describer +module Innodb + class Index + attr_reader :root + attr_reader :space + attr_accessor :record_describer - def initialize(space, root_page_number, record_describer=nil) - @space = space - @record_describer = record_describer || space.record_describer + FSEG_LIST_NAMES = %i[ + internal + leaf + ].freeze - @root = page(root_page_number) + def initialize(space, root_page_number, record_describer = nil) + @space = space + @record_describer = record_describer || space.record_describer - unless @root - raise "Page #{root_page_number} couldn't be read" - end + @root = page(root_page_number) - # The root page should be an index page. - unless @root.type == :INDEX - raise "Page #{root_page_number} is a #{@root.type} page, not an INDEX page" - end + raise "Page #{root_page_number} couldn't be read" unless @root - # The root page should be the only page at its level. - unless @root.prev.nil? && @root.next.nil? - raise "Page #{root_page_number} is a node page, but not appear to be the root; it has previous page and next page pointers" - end - end + # The root page should be an index page. + unless @root.is_a?(Innodb::Page::Index) + raise "Page #{root_page_number} is a #{@root.type} page, not an INDEX page" + end - def page(page_number) - page = @space.page(page_number) or - raise "Page #{page_number} couldn't be read" - page.record_describer = @record_describer - page - end + # The root page should be the only page at its level. + raise "Page #{root_page_number} does not appear to be an index root" if @root.prev || @root.next + end - # A helper function to access the index ID in the page header. - def id - @root.page_header[:index_id] - end + def page(page_number) + page = @space.page(page_number) + raise "Page #{page_number} couldn't be read" unless page - # Return the type of node that the given page represents in the index tree. - def node_type(page) - if @root.offset == page.offset - :root - elsif page.level == 0 - :leaf - else - :internal + page.record_describer = @record_describer + page end - end - # Internal method used by recurse. - def _recurse(parent_page, page_proc, link_proc, depth=0) - if page_proc && parent_page.type == :INDEX - page_proc.call(parent_page, depth) + # A helper function to access the index ID in the page header. + def id + @root.page_header.index_id end - parent_page.each_child_page do |child_page_number, child_min_key| - child_page = page(child_page_number) - child_page.record_describer = record_describer - if child_page.type == :INDEX - if link_proc - link_proc.call(parent_page, child_page, child_min_key, depth+1) - end - _recurse(child_page, page_proc, link_proc, depth+1) + # Return the type of node that the given page represents in the index tree. + def node_type(page) + if @root.offset == page.offset + :root + elsif page.level.zero? + :leaf + else + :internal end end - end - # Walk an index tree depth-first, calling procs for each page and link - # in the tree. - def recurse(page_proc, link_proc) - _recurse(@root, page_proc, link_proc) - end + # Internal method used by recurse. + def _recurse(parent_page, page_proc, link_proc, depth = 0) + page_proc.call(parent_page, depth) if page_proc && parent_page.is_a?(Innodb::Page::Index) - # Return the first leaf page in the index by walking down the left side - # of the B-tree until a page at the given level is encountered. - def min_page_at_level(level) - page = @root - record = @root.min_record - while record && page.level > level - page = page(record.child_page_number) - record = page.min_record - end - page if page.level == level - end + parent_page.each_child_page do |child_page_number, child_min_key| + child_page = page(child_page_number) + child_page.record_describer = record_describer + next unless child_page.is_a?(Innodb::Page::Index) - # Return the minimum record in the index. - def min_record - if min_page = min_page_at_level(0) - min_page.min_record + link_proc&.call(parent_page, child_page, child_min_key, depth + 1) + _recurse(child_page, page_proc, link_proc, depth + 1) + end end - end - # Return the last leaf page in the index by walking down the right side - # of the B-tree until a page at the given level is encountered. - def max_page_at_level(level) - page = @root - record = @root.max_record - while record && page.level > level - page = page(record.child_page_number) - record = page.max_record + # Walk an index tree depth-first, calling procs for each page and link + # in the tree. + def recurse(page_proc, link_proc) + _recurse(@root, page_proc, link_proc) end - page if page.level == level - end - # Return the maximum record in the index. - def max_record - if max_page = max_page_at_level(0) - max_page.max_record + # Return the first leaf page in the index by walking down the left side + # of the B-tree until a page at the given level is encountered. + def min_page_at_level(level) + page = @root + record = @root.min_record + while record && page.level > level + page = page(record.child_page_number) + record = page.min_record + end + page if page.level == level end - end - - # Return the file segment with the given name from the fseg header. - def fseg(name) - @root.fseg_header[name] - end - def field_names - record_describer.field_names - end - - # Iterate through all file segments in the index. - def each_fseg - unless block_given? - return enum_for(:each_fseg) + # Return the minimum record in the index. + def min_record + min_page_at_level(0)&.min_record end - [:internal, :leaf].each do |fseg_name| - yield fseg_name, fseg(fseg_name) + # Return the last leaf page in the index by walking down the right side + # of the B-tree until a page at the given level is encountered. + def max_page_at_level(level) + page = @root + record = @root.max_record + while record && page.level > level + page = page(record.child_page_number) + record = page.max_record + end + page if page.level == level end - end - # Iterate through all lists in a given fseg. - def each_fseg_list(fseg) - unless block_given? - return enum_for(:each_fseg_list, fseg) + # Return the maximum record in the index. + def max_record + max_page_at_level(0)&.max_record end - fseg.each_list do |list_name, list| - yield list_name, list + # Return the file segment with the given name from the fseg header. + def fseg(name) + @root.fseg_header[name] end - end - # Iterate through all frag pages in a given fseg. - def each_fseg_frag_page(fseg) - unless block_given? - return enum_for(:each_fseg_frag_page, fseg) + def field_names + record_describer.field_names end - fseg.frag_array_pages.each do |page_number| - yield page_number, page(page_number) - end - end + # Iterate through all file segments in the index. + def each_fseg + return enum_for(:each_fseg) unless block_given? - # Iterate through all pages at this level starting with the provided page. - def each_page_from(page) - unless block_given? - return enum_for(:each_page_from, page) + FSEG_LIST_NAMES.each do |fseg_name| + yield fseg_name, fseg(fseg_name) + end end - while page && page.type == :INDEX - yield page - break unless page.next - page = page(page.next) - end - end + # Iterate through all lists in a given fseg. + def each_fseg_list(fseg, &block) + return enum_for(:each_fseg_list, fseg) unless block_given? - # Iterate through all pages at the given level by finding the first page - # and following the next pointers in each page. - def each_page_at_level(level) - unless block_given? - return enum_for(:each_page_at_level, level) + fseg.each_list(&block) end - each_page_from(min_page_at_level(level)) { |page| yield page } - end - - # Iterate through all records on all leaf pages in ascending order. - def each_record - unless block_given? - return enum_for(:each_record) - end + # Iterate through all frag pages in a given fseg. + def each_fseg_frag_page(fseg) + return enum_for(:each_fseg_frag_page, fseg) unless block_given? - each_page_at_level(0) do |page| - page.each_record do |record| - yield record + fseg.frag_array_pages.each do |page_number| + yield page_number, page(page_number) end end - end - # Search for a record within the entire index, walking down the non-leaf - # pages until a leaf page is found, and then verifying that the record - # returned on the leaf page is an exact match for the key. If a matching - # record is not found, nil is returned (either because linear_search_in_page - # returns nil breaking the loop, or because compare_key returns non-zero). - def linear_search(key) - Innodb::Stats.increment :linear_search - - page = @root - - if Innodb.debug? - puts "linear_search: root=%i, level=%i, key=(%s)" % [ - page.offset, - page.level, - key.join(", "), - ] - end + # Iterate through all pages at this level starting with the provided page. + def each_page_from(page) + return enum_for(:each_page_from, page) unless block_given? - while rec = - page.linear_search_from_cursor(page.record_cursor(page.infimum.next), key) - if page.level > 0 - # If we haven't reached a leaf page yet, move down the tree and search - # again using linear search. - page = page(rec.child_page_number) - else - # We're on a leaf page, so return the page and record if there is a - # match. If there is no match, break the loop and cause nil to be - # returned. - return rec if rec.compare_key(key) == 0 - break + while page.is_a?(Innodb::Page::Index) + yield page + break unless page.next + + page = page(page.next) end end - end - - # Search for a record within the entire index like linear_search, but use - # the page directory to search while making as few record comparisons as - # possible. If a matching record is not found, nil is returned. - def binary_search(key) - Innodb::Stats.increment :binary_search - page = @root + # Iterate through all pages at the given level by finding the first page + # and following the next pointers in each page. + def each_page_at_level(level, &block) + return enum_for(:each_page_at_level, level) unless block_given? - if Innodb.debug? - puts "binary_search: root=%i, level=%i, key=(%s)" % [ - page.offset, - page.level, - key.join(", "), - ] + each_page_from(min_page_at_level(level), &block) end - # Remove supremum from the page directory, since nothing can be scanned - # linearly from there anyway. - while rec = page.binary_search_by_directory(page.directory[0...-1], key) - if page.level > 0 - # If we haven't reached a leaf page yet, move down the tree and search - # again using binary search. - page = page(rec.child_page_number) - else - # We're on a leaf page, so return the page and record if there is a - # match. If there is no match, break the loop and cause nil to be - # returned. - return rec if rec.compare_key(key) == 0 - break - end - end - end + # Iterate through all records on all leaf pages in ascending order. + def each_record(&block) + return enum_for(:each_record) unless block_given? - # A cursor to walk the index (cursor) forwards or backward starting with - # a given record, or the minimum (:min) or maximum (:max) record in the - # index. - class IndexCursor - def initialize(index, record, direction) - Innodb::Stats.increment :index_cursor_create - @index = index - @direction = direction - case record - when :min - # Start at the minimum record on the minimum page in the index. - @page = index.min_page_at_level(0) - @page_cursor = @page.record_cursor(:min, direction) - when :max - # Start at the maximum record on the maximum page in the index. - @page = index.max_page_at_level(0) - @page_cursor = @page.record_cursor(:max, direction) - else - # Start at the record provided. - @page = record.page - @page_cursor = @page.record_cursor(record.offset, direction) + each_page_at_level(0) do |page| + page.each_record(&block) end end - # Return the next record in the order defined when the cursor was created. - def record - if rec = @page_cursor.record - return rec + # Search for a record within the entire index, walking down the non-leaf + # pages until a leaf page is found, and then verifying that the record + # returned on the leaf page is an exact match for the key. If a matching + # record is not found, nil is returned (either because linear_search_in_page + # returns nil breaking the loop, or because compare_key returns non-zero). + def linear_search(key) + Innodb::Stats.increment :linear_search + + page = @root + + if Innodb.debug? + puts "linear_search: root=%i, level=%i, key=(%s)" % [ + page.offset, + page.level, + key.join(", "), + ] end - case @direction - when :forward - next_record - when :backward - prev_record + while (rec = page.linear_search_from_cursor(page.record_cursor(page.infimum.next), key)) + if page.level.positive? + # If we haven't reached a leaf page yet, move down the tree and search + # again using linear search. + page = page(rec.child_page_number) + else + # We're on a leaf page, so return the page and record if there is a + # match. If there is no match, break the loop and cause nil to be + # returned. + return rec if rec.compare_key(key).zero? + + break + end end end - # Iterate through all records in the cursor. - def each_record - unless block_given? - return enum_for(:each_record) + # Search for a record within the entire index like linear_search, but use + # the page directory to search while making as few record comparisons as + # possible. If a matching record is not found, nil is returned. + def binary_search(key) + Innodb::Stats.increment :binary_search + + page = @root + + if Innodb.debug? + puts "binary_search: root=%i, level=%i, key=(%s)" % [ + page.offset, + page.level, + key.join(", "), + ] end - while rec = record - yield rec + # Remove supremum from the page directory, since nothing can be scanned + # linearly from there anyway. + while (rec = page.binary_search_by_directory(page.directory[0...-1], key)) + if page.level.positive? + # If we haven't reached a leaf page yet, move down the tree and search + # again using binary search. + page = page(rec.child_page_number) + else + # We're on a leaf page, so return the page and record if there is a + # match. If there is no match, break the loop and cause nil to be + # returned. + return rec if rec.compare_key(key).zero? + + break + end end end - private - - # Move the cursor to a new starting position in a given page. - def move_cursor(page, record) - unless @page = @index.page(page) - raise "Failed to load page" + # A cursor to walk the index (cursor) forwards or backward starting with + # a given record, or the minimum (:min) or maximum (:max) record in the + # index. + class IndexCursor + def initialize(index, record, direction) + Innodb::Stats.increment :index_cursor_create + @index = index + @direction = direction + case record + when :min + # Start at the minimum record on the minimum page in the index. + @page = index.min_page_at_level(0) + @page_cursor = @page.record_cursor(:min, direction) + when :max + # Start at the maximum record on the maximum page in the index. + @page = index.max_page_at_level(0) + @page_cursor = @page.record_cursor(:max, direction) + else + # Start at the record provided. + @page = record.page + @page_cursor = @page.record_cursor(record.offset, direction) + end end - unless @page_cursor = @page.record_cursor(record, @direction) - raise "Failed to position cursor" + # Return the next record in the order defined when the cursor was created. + def record + if (rec = @page_cursor.record) + return rec + end + + case @direction + when :forward + next_record + when :backward + prev_record + end end - end - # Move to the next record in the forward direction and return it. - def next_record - Innodb::Stats.increment :index_cursor_next_record + # Iterate through all records in the cursor. + def each_record + return enum_for(:each_record) unless block_given? - if rec = @page_cursor.record - return rec + while (rec = record) + yield rec + end end - unless next_page = @page.next - return nil + private + + # Move the cursor to a new starting position in a given page. + def move_cursor(page, record) + raise "Failed to load page" unless (@page = @index.page(page)) + raise "Failed to position cursor" unless (@page_cursor = @page.record_cursor(record, @direction)) end - move_cursor(next_page, :min) + # Move to the next record in the forward direction and return it. + def next_record + Innodb::Stats.increment :index_cursor_next_record - @page_cursor.record - end + if (rec = @page_cursor.record) + return rec + end - # Move to the previous record in the backward direction and return it. - def prev_record - Innodb::Stats.increment :index_cursor_prev_record + return unless (next_page = @page.next) - if rec = @page_cursor.record - return rec - end + move_cursor(next_page, :min) - unless prev_page = @page.prev - return nil + @page_cursor.record end - move_cursor(prev_page, :max) + # Move to the previous record in the backward direction and return it. + def prev_record + Innodb::Stats.increment :index_cursor_prev_record + + if (rec = @page_cursor.record) + return rec + end + + return unless (prev_page = @page.prev) - @page_cursor.record + move_cursor(prev_page, :max) + + @page_cursor.record + end end - end - # Return an IndexCursor starting at the given record (an Innodb::Record, - # :min, or :max) and cursor in the direction given (:forward or :backward). - def cursor(record=:min, direction=:forward) - IndexCursor.new(self, record, direction) + # Return an IndexCursor starting at the given record (an Innodb::Record, + # :min, or :max) and cursor in the direction given (:forward or :backward). + def cursor(record = :min, direction = :forward) + IndexCursor.new(self, record, direction) + end end end diff --git a/lib/innodb/inode.rb b/lib/innodb/inode.rb index 276c86dc..fb9567e0 100644 --- a/lib/innodb/inode.rb +++ b/lib/innodb/inode.rb @@ -1,187 +1,186 @@ -# -*- encoding : utf-8 -*- - -class Innodb::Inode - # The number of "slots" (each representing one page) in the fragment array - # within each Inode entry. - FRAG_ARRAY_N_SLOTS = 32 # FSP_EXTENT_SIZE / 2 - - # The size (in bytes) of each slot in the fragment array. - FRAG_SLOT_SIZE = 4 - - # A magic number which helps determine if an Inode structure is in use - # and populated with valid data. - MAGIC_N_VALUE = 97937874 - - # The size (in bytes) of an Inode entry. - SIZE = (16 + (3 * Innodb::List::BASE_NODE_SIZE) + - (FRAG_ARRAY_N_SLOTS * FRAG_SLOT_SIZE)) - - # Read an array of page numbers (32-bit integers, which may be nil) from - # the provided cursor. - def self.page_number_array(size, cursor) - size.times.map do |n| - cursor.name("page[#{n}]") do |c| - Innodb::Page.maybe_undefined(c.get_uint32) +# frozen_string_literal: true + +require "forwardable" + +module Innodb + class Inode + extend Forwardable + + Header = Struct.new( + :offset, + :fseg_id, + :not_full_n_used, + :free, + :not_full, + :full, + :magic_n, + :frag_array, + keyword_init: true + ) + + # The number of "slots" (each representing one page) in the fragment array + # within each Inode entry. + FRAG_ARRAY_N_SLOTS = 32 # FSP_EXTENT_SIZE / 2 + + # The size (in bytes) of each slot in the fragment array. + FRAG_SLOT_SIZE = 4 + + # A magic number which helps determine if an Inode structure is in use + # and populated with valid data. + MAGIC_N_VALUE = 97_937_874 + + # The size (in bytes) of an Inode entry. + SIZE = (16 + (3 * Innodb::List::BASE_NODE_SIZE) + + (FRAG_ARRAY_N_SLOTS * FRAG_SLOT_SIZE)) + + LISTS = %i[ + free + not_full + full + ].freeze + + # Read an array of page numbers (32-bit integers, which may be nil) from + # the provided cursor. + def self.page_number_array(size, cursor) + size.times.map do |n| + cursor.name("page[#{n}]") do |c| + Innodb::Page.maybe_undefined(c.read_uint32) + end end end - end - # Construct a new Inode by reading an FSEG header from a cursor. - def self.new_from_cursor(space, cursor) - data = { - :offset => cursor.position, - :fseg_id => cursor.name("fseg_id") { - cursor.get_uint64 - }, - :not_full_n_used => cursor.name("not_full_n_used") { - cursor.get_uint32 - }, - :free => cursor.name("list[free]") { - Innodb::List::Xdes.new(space, Innodb::List.get_base_node(cursor)) - }, - :not_full => cursor.name("list[not_full]") { - Innodb::List::Xdes.new(space, Innodb::List.get_base_node(cursor)) - }, - :full => cursor.name("list[full]") { - Innodb::List::Xdes.new(space, Innodb::List.get_base_node(cursor)) - }, - :magic_n => cursor.name("magic_n") { - cursor.get_uint32 - }, - :frag_array => cursor.name("frag_array") { - page_number_array(FRAG_ARRAY_N_SLOTS, cursor) - }, - } - - Innodb::Inode.new(space, data) - end - - attr_accessor :space + # Construct a new Inode by reading an FSEG header from a cursor. + def self.new_from_cursor(space, cursor) + Innodb::Inode.new( + space, + Header.new( + offset: cursor.position, + fseg_id: cursor.name("fseg_id") { cursor.read_uint64 }, + not_full_n_used: cursor.name("not_full_n_used") { cursor.read_uint32 }, + free: cursor.name("list[free]") { Innodb::List::Xdes.new(space, Innodb::List.get_base_node(cursor)) }, + not_full: cursor.name("list[not_full]") { Innodb::List::Xdes.new(space, Innodb::List.get_base_node(cursor)) }, + full: cursor.name("list[full]") { Innodb::List::Xdes.new(space, Innodb::List.get_base_node(cursor)) }, + magic_n: cursor.name("magic_n") { cursor.read_uint32 }, + frag_array: cursor.name("frag_array") { page_number_array(FRAG_ARRAY_N_SLOTS, cursor) } + ) + ) + end - def initialize(space, data) - @space = space - @data = data - end + attr_accessor :space + attr_accessor :header - def offset; @data[:offset]; end - def fseg_id; @data[:fseg_id]; end - def not_full_n_used; @data[:not_full_n_used]; end - def free; @data[:free]; end - def not_full; @data[:not_full]; end - def full; @data[:full]; end - def magic_n; @data[:magic_n]; end - def frag_array; @data[:frag_array]; end - - def inspect - "<%s space=%s, fseg=%i>" % [ - self.class.name, - space.inspect, - fseg_id, - ] - end + def initialize(space, header) + @space = space + @header = header + end - # Helper method to determine if an Inode is in use. Inodes that are not in - # use have an fseg_id of 0. - def allocated? - fseg_id != 0 - end + def_delegator :header, :offset + def_delegator :header, :fseg_id + def_delegator :header, :not_full_n_used + def_delegator :header, :free + def_delegator :header, :not_full + def_delegator :header, :full + def_delegator :header, :magic_n + def_delegator :header, :frag_array + + def inspect + "<%s space=%s, fseg=%i>" % [ + self.class.name, + space.inspect, + fseg_id, + ] + end - # Helper method to return an array of only non-nil fragment pages. - def frag_array_pages - frag_array.select { |n| ! n.nil? } - end + # Helper method to determine if an Inode is in use. Inodes that are not in + # use have an fseg_id of 0. + def allocated? + fseg_id != 0 + end - # Helper method to count non-nil fragment pages. - def frag_array_n_used - frag_array.inject(0) { |n, i| n += 1 if i; n } - end + # Helper method to return an array of only non-nil fragment pages. + def frag_array_pages + frag_array.compact + end - # Calculate the total number of pages in use (not free) within this fseg. - def used_pages - frag_array_n_used + not_full_n_used + - (full.length * @space.pages_per_extent) - end + # Helper method to count non-nil fragment pages. + def frag_array_n_used + frag_array_pages.count + end - # Calculate the total number of pages within this fseg. - def total_pages - frag_array_n_used + - (free.length * @space.pages_per_extent) + - (not_full.length * @space.pages_per_extent) + - (full.length * @space.pages_per_extent) - end + # Calculate the total number of pages in use (not free) within this fseg. + def used_pages + frag_array_n_used + not_full_n_used + + (full.length * @space.pages_per_extent) + end - # Calculate the fill factor of this fseg, in percent. - def fill_factor - total_pages > 0 ? 100.0 * (used_pages.to_f / total_pages.to_f) : 0.0 - end + # Calculate the total number of pages within this fseg. + def total_pages + frag_array_n_used + + (free.length * @space.pages_per_extent) + + (not_full.length * @space.pages_per_extent) + + (full.length * @space.pages_per_extent) + end - # Return an array of lists within an fseg. - def lists - [:free, :not_full, :full] - end + # Calculate the fill factor of this fseg, in percent. + def fill_factor + total_pages.positive? ? 100.0 * (used_pages.to_f / total_pages) : 0.0 + end - # Return a list from the fseg, given its name as a symbol. - def list(name) - @data[name] if lists.include? name - end + # Return a list from the fseg, given its name as a symbol. + def list(name) + return unless LISTS.include?(name) - # Iterate through all lists, yielding the list name and the list itself. - def each_list - unless block_given? - return enum_for(:each_list) + header[name] end - lists.each do |name| - yield name, list(name) - end + # Iterate through all lists, yielding the list name and the list itself. + def each_list + return enum_for(:each_list) unless block_given? - nil - end + LISTS.each do |name| + yield name, list(name) + end - # Iterate through the fragment array followed by all lists, yielding the - # page number. This allows a convenient way to identify all pages that are - # part of this inode. - def each_page_number - unless block_given? - return enum_for(:each_page_number) + nil end - frag_array_pages.each do |page_number| - yield page_number - end + # Iterate through the fragment array followed by all lists, yielding the + # page number. This allows a convenient way to identify all pages that are + # part of this inode. + def each_page_number(&block) + return enum_for(:each_page_number) unless block_given? + + frag_array_pages.each(&block) - each_list do |fseg_name, fseg_list| - fseg_list.each do |xdes| - xdes.each_page_status do |page_number| - yield page_number + each_list do |_fseg_name, fseg_list| + fseg_list.each do |xdes| + xdes.each_page_status(&block) end end + + nil end - nil - end + # Iterate through the page as associated with this inode using the + # each_page_number method, and yield the page number and page. + def each_page + return enum_for(:each_page) unless block_given? - # Iterate through the page as associated with this inode using the - # each_page_number method, and yield the page number and page. - def each_page - unless block_given? - return enum_for(:each_page) - end + each_page_number do |page_number| + yield page_number, space.page(page_number) + end - each_page_number do |page_number| - yield page_number, space.page(page_number) + nil end - nil - end - - # Compare one Innodb::Inode to another. - def ==(other) - fseg_id == other.fseg_id if other - end + # Compare one Innodb::Inode to another. + def ==(other) + fseg_id == other.fseg_id if other + end - # Dump a summary of this object for debugging purposes. - def dump - pp @data + # Dump a summary of this object for debugging purposes. + def dump + pp header + end end end diff --git a/lib/innodb/list.rb b/lib/innodb/list.rb index bb2cfc67..8ef3b4f7 100644 --- a/lib/innodb/list.rb +++ b/lib/innodb/list.rb @@ -1,233 +1,246 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true # An abstract InnoDB "free list" or FLST (renamed to just "list" here as it # frequently is used for structures that aren't free lists). This class must # be sub-classed to provide an appropriate #object_from_address method. -class Innodb::List - # An "address", which consists of a page number and byte offset within the - # page. This points to the list "node" pointers (prev and next) of the - # node, not necessarily the node object. - ADDRESS_SIZE = 4 + 2 - - # Read a node address from a cursor. Return nil if the address is an end - # or "NULL" pointer (the page number is UINT32_MAX), or the address if - # valid. - def self.get_address(cursor) - page = cursor.name("page") { - Innodb::Page.maybe_undefined(cursor.get_uint32) - } - offset = cursor.name("offset") { cursor.get_uint16 } - if page - { - :page => page, - :offset => offset, - } +module Innodb + class List + BaseNode = Struct.new( + :length, # rubocop:disable Lint/StructNewOverride + :first, # rubocop:disable Lint/StructNewOverride + :last, + keyword_init: true + ) + + Node = Struct.new( + :prev, + :next, + keyword_init: true + ) + + # An "address", which consists of a page number and byte offset within the + # page. This points to the list "node" pointers (prev and next) of the + # node, not necessarily the node object. + ADDRESS_SIZE = 4 + 2 + + # Read a node address from a cursor. Return nil if the address is an end + # or "NULL" pointer (the page number is UINT32_MAX), or the address if + # valid. + def self.get_address(cursor) + page = cursor.name("page") { Innodb::Page.maybe_undefined(cursor.read_uint32) } + offset = cursor.name("offset") { cursor.read_uint16 } + + Innodb::Page::Address.new(page: page, offset: offset) if page end - end - - # A list node consists of two addresses: the "previous" node address, and - # the "next" node address. - NODE_SIZE = 2 * ADDRESS_SIZE - - # Read a node, consisting of two consecutive addresses (:prev and :next) - # from a cursor. Either address may be nil, indicating the end of the - # linked list. - def self.get_node(cursor) - { - :prev => cursor.name("prev") { get_address(cursor) }, - :next => cursor.name("next") { get_address(cursor) }, - } - end - # A list base node consists of a list length followed by two addresses: - # the "first" node address, and the "last" node address. - BASE_NODE_SIZE = 4 + (2 * ADDRESS_SIZE) - - # Read a base node, consisting of a list length followed by two addresses - # (:first and :last) from a cursor. Either address may be nil. An empty list - # has a :length of 0 and :first and :last are nil. A list with only a single - # item will have a :length of 1 and :first and :last will point to the same - # address. - def self.get_base_node(cursor) - { - :length => cursor.name("length") { cursor.get_uint32 }, - :first => cursor.name("first") { get_address(cursor) }, - :last => cursor.name("last") { get_address(cursor) }, - } - end + # A list node consists of two addresses: the "previous" node address, and + # the "next" node address. + NODE_SIZE = 2 * ADDRESS_SIZE + + # Read a node, consisting of two consecutive addresses (:prev and :next) + # from a cursor. Either address may be nil, indicating the end of the + # linked list. + def self.get_node(cursor) + Node.new( + prev: cursor.name("prev") { get_address(cursor) }, + next: cursor.name("next") { get_address(cursor) } + ) + end - def initialize(space, base) - @space = space - @base = base - end + # A list base node consists of a list length followed by two addresses: + # the "first" node address, and the "last" node address. + BASE_NODE_SIZE = 4 + (2 * ADDRESS_SIZE) + + # Read a base node, consisting of a list length followed by two addresses + # (:first and :last) from a cursor. Either address may be nil. An empty list + # has a :length of 0 and :first and :last are nil. A list with only a single + # item will have a :length of 1 and :first and :last will point to the same + # address. + def self.get_base_node(cursor) + BaseNode.new( + length: cursor.name("length") { cursor.read_uint32 }, + first: cursor.name("first") { get_address(cursor) }, + last: cursor.name("last") { get_address(cursor) } + ) + end - attr_reader :space - attr_reader :base + attr_reader :space + attr_reader :base - # Abstract #object_from_address method which must be implemented by - # sub-classes in order to return a useful object given an object address. - def object_from_address(address) - raise "#{self.class} must implement object_from_address" - end + def initialize(space, base) + @space = space + @base = base + end - # Return the object pointed to by the "previous" address pointer of the - # provided object. - def prev(object) - unless object.respond_to? :prev_address - raise "Class #{object.class} does not respond to prev_address" + # Abstract #object_from_address method which must be implemented by + # sub-classes in order to return a useful object given an object address. + def object_from_address(_address) + raise "#{self.class} must implement object_from_address" end - object_from_address(object.prev_address) - end + # Return the object pointed to by the "previous" address pointer of the + # provided object. + def prev(object) + raise "Class #{object.class} does not respond to prev_address" unless object.respond_to?(:prev_address) - # Return the object pointed to by the "next" address pointer of the - # provided object. - def next(object) - unless object.respond_to? :next_address - raise "Class #{object.class} does not respond to next_address" + object_from_address(object.prev_address) end - object_from_address(object.next_address) - end + # Return the object pointed to by the "next" address pointer of the + # provided object. + def next(object) + raise "Class #{object.class} does not respond to next_address" unless object.respond_to?(:next_address) - # Return the number of items in the list. - def length - @base[:length] - end + object_from_address(object.next_address) + end - # Return the first object in the list using the list base node "first" - # address pointer. - def first - object_from_address(@base[:first]) - end + # Return the number of items in the list. + def length + @base.length + end - # Return the first object in the list using the list base node "last" - # address pointer. - def last - object_from_address(@base[:last]) - end + # Is the list currently empty? + def empty? + length.zero? + end - # Return a list cursor for the list. - def list_cursor(node=:min, direction=:forward) - ListCursor.new(self, node, direction) - end + # Return the first object in the list using the list base node "first" + # address pointer. + def first + object_from_address(@base.first) + end - # Return whether the given item is present in the list. This depends on the - # item and the items in the list implementing some sufficient == method. - # This is implemented rather inefficiently by constructing an array and - # leaning on Array#include? to do the real work. - def include?(item) - each.to_a.include? item - end + # Return the first object in the list using the list base node "last" + # address pointer. + def last + object_from_address(@base.last) + end - # Iterate through all nodes in the list. - def each - unless block_given? - return enum_for(:each) + # Return a list cursor for the list. + def list_cursor(node = :min, direction = :forward) + ListCursor.new(self, node, direction) end - list_cursor.each_node do |node| - yield node + # Return whether the given item is present in the list. This depends on the + # item and the items in the list implementing some sufficient == method. + # This is implemented rather inefficiently by constructing an array and + # leaning on Array#include? to do the real work. + def include?(item) + each.to_a.include?(item) end - end - # A list iteration cursor used primarily by the Innodb::List #cursor method - # implicitly. Keeps its own state for iterating through lists efficiently. - class ListCursor - def initialize(list, node=:min, direction=:forward) - @initial = true - @list = list - @direction = direction - - case node - when :min - @node = @list.first - when :max - @node = @list.last - else - @node = node - end + # Iterate through all nodes in the list. + def each(&block) + return enum_for(:each) unless block_given? + + list_cursor.each_node(&block) end - def node - if @initial - @initial = false - return @node + # A list iteration cursor used primarily by the Innodb::List #cursor method + # implicitly. Keeps its own state for iterating through lists efficiently. + class ListCursor + def initialize(list, node = :min, direction = :forward) + @initial = true + @list = list + @direction = direction + @node = initial_node(node) end - case @direction - when :forward - next_node - when :backward - prev_node + def initial_node(node) + case node + when :min + @list.first + when :max + @list.last + else + node + end end - end - # Return the previous entry from the current position, and advance the - # cursor position to the returned entry. If the cursor is currently nil, - # return the last entry in the list and adjust the cursor position to - # that entry. - def prev_node - if node = @list.prev(@node) - @node = node + def node + if @initial + @initial = false + return @node + end + + case @direction + when :forward + next_node + when :backward + prev_node + end end - end - # Return the next entry from the current position, and advance the - # cursor position to the returned entry. If the cursor is currently nil, - # return the first entry in the list and adjust the cursor position to - # that entry. - def next_node - if node = @list.next(@node) - @node = node + def goto_node(node) + @node = node if node end - end - def each_node - unless block_given? - return enum_for(:each_node) + # Return the previous entry from the current position, and advance the + # cursor position to the returned entry. If the cursor is currently nil, + # return the last entry in the list and adjust the cursor position to + # that entry. + def prev_node + goto_node(@list.prev(@node)) end - while n = node - yield n + # Return the next entry from the current position, and advance the + # cursor position to the returned entry. If the cursor is currently nil, + # return the first entry in the list and adjust the cursor position to + # that entry. + def next_node + goto_node(@list.next(@node)) + end + + def each_node + return enum_for(:each_node) unless block_given? + + while (n = node) + yield n + end end end - end -end -# A list of extent descriptor entries. Objects returned by list methods -# will be Innodb::Xdes objects. -class Innodb::List::Xdes < Innodb::List - def object_from_address(address) - if address && page = @space.page(address[:page]) - Innodb::Xdes.new(page, page.cursor(address[:offset] - 8)) + # A list of extent descriptor entries. Objects returned by list methods + # will be Innodb::Xdes objects. + class Xdes < Innodb::List + def object_from_address(address) + return unless address + + page = @space.page(address.page) + return unless page + + Innodb::Xdes.new(page, page.cursor(address.offset - 8)) + end end - end -end -# A list of Inode pages. Objects returned by list methods will be -# Innodb::Page::Inode objects. -class Innodb::List::Inode < Innodb::List - def object_from_address(address) - if address && page = @space.page(address[:page]) - page + # A list of Inode pages. Objects returned by list methods will be + # Innodb::Page::Inode objects. + class Inode < Innodb::List + def object_from_address(address) + return unless address + + @space.page(address.page) + end end - end -end -class Innodb::List::UndoPage < Innodb::List - def object_from_address(address) - if address && page = @space.page(address[:page]) - page + class UndoPage < Innodb::List + def object_from_address(address) + return unless address + + @space.page(address.page) + end end - end -end -class Innodb::List::History < Innodb::List - def object_from_address(address) - if address && page = @space.page(address[:page]) - Innodb::UndoLog.new(page, address[:offset] - 34) + class History < Innodb::List + def object_from_address(address) + return unless address + + page = @space.page(address.page) + return unless page + + Innodb::UndoLog.new(page, address.offset - 34) + end end end end diff --git a/lib/innodb/log.rb b/lib/innodb/log.rb index 95cb05e3..cbf50052 100644 --- a/lib/innodb/log.rb +++ b/lib/innodb/log.rb @@ -1,126 +1,155 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true # An InnoDB transaction log file. -class Innodb::Log - # A map of the name and position of the blocks that form the log header. - LOG_HEADER_BLOCK_MAP = { - :LOG_FILE_HEADER => 0, - :LOG_CHECKPOINT_1 => 1, - :EMPTY => 2, - :LOG_CHECKPOINT_2 => 3, - } - - # Number of blocks in the log file header. - LOG_HEADER_BLOCKS = LOG_HEADER_BLOCK_MAP.size - - # The size in bytes of the log file header. - LOG_HEADER_SIZE = LOG_HEADER_BLOCKS * Innodb::LogBlock::BLOCK_SIZE - - # Maximum number of log group checkpoints. - LOG_CHECKPOINT_GROUPS = 32 - - # Open a log file. - def initialize(filename) - @file = File.open(filename) - @size = @file.stat.size - @blocks = (@size / Innodb::LogBlock::BLOCK_SIZE) - LOG_HEADER_BLOCKS - @capacity = @blocks * Innodb::LogBlock::BLOCK_SIZE - end +module Innodb + class Log + Header = Struct.new( + :group_id, + :start_lsn, + :file_no, + :created_by, + keyword_init: true + ) - # The size (in bytes) of the log. - attr_reader :size + Checkpoint = Struct.new( + :number, + :lsn, + :lsn_offset, + :buffer_size, + :archived_lsn, + :group_array, + :checksum_1, + :checksum_2, + :fsp_free_limit, + :fsp_magic, + keyword_init: true + ) - # The log capacity (in bytes). - attr_reader :capacity + CheckpointGroup = Struct.new( + :archived_file_no, + :archived_offset, + keyword_init: true + ) - # The number of blocks in the log. - attr_reader :blocks + CheckpointSet = Struct.new( + :checkpoint_1, + :checkpoint_2, + keyword_init: true + ) - # Get the raw byte buffer for a specific block by block offset. - def block_data(offset) - raise "Invalid block offset" unless (offset % Innodb::LogBlock::BLOCK_SIZE).zero? - @file.sysseek(offset) - @file.sysread(Innodb::LogBlock::BLOCK_SIZE) - end + # A map of the name and position of the blocks that form the log header. + LOG_HEADER_BLOCK_MAP = { + LOG_FILE_HEADER: 0, + LOG_CHECKPOINT_1: 1, + EMPTY: 2, + LOG_CHECKPOINT_2: 3, + }.freeze - # Get a cursor to a block in a given offset of the log. - def block_cursor(offset) - BufferCursor.new(block_data(offset), 0) - end + # Number of blocks in the log file header. + LOG_HEADER_BLOCKS = LOG_HEADER_BLOCK_MAP.size - # Return the log header. - def header - offset = LOG_HEADER_BLOCK_MAP[:LOG_FILE_HEADER] * Innodb::LogBlock::BLOCK_SIZE - @header ||= block_cursor(offset).name("header") do |c| - { - :group_id => c.name("group_id") { c.get_uint32 }, - :start_lsn => c.name("start_lsn") { c.get_uint64 }, - :file_no => c.name("file_no") { c.get_uint32 }, - :created_by => c.name("created_by") { c.get_string(32) } - } + # The size in bytes of the log file header. + LOG_HEADER_SIZE = LOG_HEADER_BLOCKS * Innodb::LogBlock::BLOCK_SIZE + + # Maximum number of log group checkpoints. + LOG_CHECKPOINT_GROUPS = 32 + + # Open a log file. + def initialize(filename) + @file = File.open(filename) + @size = @file.stat.size + @blocks = (@size / Innodb::LogBlock::BLOCK_SIZE) - LOG_HEADER_BLOCKS + @capacity = @blocks * Innodb::LogBlock::BLOCK_SIZE end - end - # Read a log checkpoint from the given cursor. - def read_checkpoint(c) - # Log archive related fields (e.g. group_array) are not currently in - # use or even read by InnoDB. However, for the sake of completeness, - # they are included. - { - :number => c.name("number") { c.get_uint64 }, - :lsn => c.name("lsn") { c.get_uint64 }, - :lsn_offset => c.name("lsn_offset") { c.get_uint32 }, - :buffer_size => c.name("buffer_size") { c.get_uint32 }, - :archived_lsn => c.name("archived_lsn") { c.get_uint64 }, - :group_array => - (0 .. LOG_CHECKPOINT_GROUPS - 1).map do |n| - c.name("group_array[#{n}]") do - { - :archived_file_no => c.name("archived_file_no") { c.get_uint32 }, - :archived_offset => c.name("archived_offset") { c.get_uint32 }, - } - end - end, - :checksum_1 => c.name("checksum_1") { c.get_uint32 }, - :checksum_2 => c.name("checksum_2") { c.get_uint32 }, - :fsp_free_limit => c.name("fsp_free_limit") { c.get_uint32 }, - :fsp_magic => c.name("fsp_magic") { c.get_uint32 }, - } - end + # The size (in bytes) of the log. + attr_reader :size - # Return the log checkpoints. - def checkpoint - offset1 = LOG_HEADER_BLOCK_MAP[:LOG_CHECKPOINT_1] * Innodb::LogBlock::BLOCK_SIZE - offset2 = LOG_HEADER_BLOCK_MAP[:LOG_CHECKPOINT_2] * Innodb::LogBlock::BLOCK_SIZE - @checkpoint ||= - { - :checkpoint_1 => block_cursor(offset1).name("checkpoint_1") do |cursor| - cp = read_checkpoint(cursor) - cp.delete(:group_array) - cp - end, - :checkpoint_2 => block_cursor(offset2).name("checkpoint_2") do |cursor| - cp = read_checkpoint(cursor) - cp.delete(:group_array) - cp - end - } - end + # The log capacity (in bytes). + attr_reader :capacity - # Return a log block with a given block index as an InnoDB::LogBlock object. - # Blocks are indexed after the log file header, starting from 0. - def block(block_index) - return nil unless block_index.between?(0, @blocks - 1) - offset = (LOG_HEADER_BLOCKS + block_index.to_i) * Innodb::LogBlock::BLOCK_SIZE - Innodb::LogBlock.new(block_data(offset)) - end + # The number of blocks in the log. + attr_reader :blocks + + # Get the raw byte buffer for a specific block by block offset. + def block_data(offset) + raise "Invalid block offset" unless (offset % Innodb::LogBlock::BLOCK_SIZE).zero? + + @file.sysseek(offset) + @file.sysread(Innodb::LogBlock::BLOCK_SIZE) + end + + # Get a cursor to a block in a given offset of the log. + def block_cursor(offset) + BufferCursor.new(block_data(offset), 0) + end + + # Return the log header. + def header + offset = LOG_HEADER_BLOCK_MAP[:LOG_FILE_HEADER] * Innodb::LogBlock::BLOCK_SIZE + @header ||= block_cursor(offset).name("header") do |c| + Header.new( + group_id: c.name("group_id") { c.read_uint32 }, + start_lsn: c.name("start_lsn") { c.read_uint64 }, + file_no: c.name("file_no") { c.read_uint32 }, + created_by: c.name("created_by") { c.read_string(32) } + ) + end + end + + # Read a log checkpoint from the given cursor. + def read_checkpoint(cursor) + # Log archive related fields (e.g. group_array) are not currently in + # use or even read by InnoDB. However, for the sake of completeness, + # they are included. + Checkpoint.new( + number: cursor.name("number") { cursor.read_uint64 }, + lsn: cursor.name("lsn") { cursor.read_uint64 }, + lsn_offset: cursor.name("lsn_offset") { cursor.read_uint32 }, + buffer_size: cursor.name("buffer_size") { cursor.read_uint32 }, + archived_lsn: cursor.name("archived_lsn") { cursor.read_uint64 }, + group_array: + (0..(LOG_CHECKPOINT_GROUPS - 1)).map do |n| + cursor.name("group_array[#{n}]") do + CheckpointGroup.new( + archived_file_no: cursor.name("archived_file_no") { cursor.read_uint32 }, + archived_offset: cursor.name("archived_offset") { cursor.read_uint32 } + ) + end + end, + checksum_1: cursor.name("checksum_1") { cursor.read_uint32 }, + checksum_2: cursor.name("checksum_2") { cursor.read_uint32 }, + fsp_free_limit: cursor.name("fsp_free_limit") { cursor.read_uint32 }, + fsp_magic: cursor.name("fsp_magic") { cursor.read_uint32 } + ) + end + + # Return the log checkpoints. + def checkpoint + offset1 = LOG_HEADER_BLOCK_MAP[:LOG_CHECKPOINT_1] * Innodb::LogBlock::BLOCK_SIZE + offset2 = LOG_HEADER_BLOCK_MAP[:LOG_CHECKPOINT_2] * Innodb::LogBlock::BLOCK_SIZE + @checkpoint ||= CheckpointSet.new( + checkpoint_1: block_cursor(offset1).name("checkpoint_1") { |c| read_checkpoint(c) }, + checkpoint_2: block_cursor(offset2).name("checkpoint_2") { |c| read_checkpoint(c) } + ) + end + + # Return a log block with a given block index as an InnoDB::LogBlock object. + # Blocks are indexed after the log file header, starting from 0. + def block(block_index) + return nil unless block_index.between?(0, @blocks - 1) + + offset = (LOG_HEADER_BLOCKS + block_index.to_i) * Innodb::LogBlock::BLOCK_SIZE + Innodb::LogBlock.new(block_data(offset)) + end - # Iterate through all log blocks, returning the block index and an - # InnoDB::LogBlock object for each block. - def each_block - (0...@blocks).each do |block_index| - current_block = block(block_index) - yield block_index, current_block if current_block + # Iterate through all log blocks, returning the block index and an + # InnoDB::LogBlock object for each block. + def each_block + (0...@blocks).each do |block_index| + current_block = block(block_index) + yield block_index, current_block if current_block + end end end end diff --git a/lib/innodb/log_block.rb b/lib/innodb/log_block.rb index be10b65d..38c58684 100644 --- a/lib/innodb/log_block.rb +++ b/lib/innodb/log_block.rb @@ -1,120 +1,129 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true + +require "forwardable" # An InnoDB transaction log block. -class Innodb::LogBlock - # Log blocks are fixed-length at 512 bytes in InnoDB. - BLOCK_SIZE = 512 +module Innodb + class LogBlock + extend Forwardable - # Offset of the header within the log block. - HEADER_OFFSET = 0 + Header = Struct.new( + :flush, + :block_number, + :data_length, + :first_rec_group, + :checkpoint_no, + keyword_init: true + ) - # The size of the block header. - HEADER_SIZE = 4 + 2 + 2 + 4 + Trailer = Struct.new( + :checksum, + keyword_init: true + ) - # Offset of the trailer within ths log block. - TRAILER_OFFSET = BLOCK_SIZE - 4 + # Log blocks are fixed-length at 512 bytes in InnoDB. + BLOCK_SIZE = 512 - # The size of the block trailer. - TRAILER_SIZE = 4 + # Offset of the header within the log block. + HEADER_OFFSET = 0 - # Offset of the start of data in the block. - DATA_OFFSET = HEADER_SIZE + # The size of the block header. + HEADER_SIZE = 4 + 2 + 2 + 4 - # Size of the space available for log records. - DATA_SIZE = BLOCK_SIZE - HEADER_SIZE - TRAILER_SIZE + # Offset of the trailer within ths log block. + TRAILER_OFFSET = BLOCK_SIZE - 4 - # Mask used to get the flush bit in the header. - HEADER_FLUSH_BIT_MASK = 0x80000000 + # The size of the block trailer. + TRAILER_SIZE = 4 - # Initialize a log block by passing in a 512-byte buffer containing the raw - # log block contents. - def initialize(buffer) - unless buffer.size == BLOCK_SIZE - raise "Log block buffer provided was not #{BLOCK_SIZE} bytes" - end + # Offset of the start of data in the block. + DATA_OFFSET = HEADER_SIZE - @buffer = buffer - end + # Size of the space available for log records. + DATA_SIZE = BLOCK_SIZE - HEADER_SIZE - TRAILER_SIZE - # Return an BufferCursor object positioned at a specific offset. - def cursor(offset) - BufferCursor.new(@buffer, offset) - end + # Mask used to get the flush bit in the header. + HEADER_FLUSH_BIT_MASK = 0x80000000 - # Return the log block header. - def header - @header ||= cursor(HEADER_OFFSET).name("header") do |c| - { - :flush => c.name("flush") { - c.peek { (c.get_uint32 & HEADER_FLUSH_BIT_MASK) > 0 } - }, - :block_number => c.name("block_number") { - c.get_uint32 & ~HEADER_FLUSH_BIT_MASK - }, - :data_length => c.name("data_length") { c.get_uint16 }, - :first_rec_group => c.name("first_rec_group") { c.get_uint16 }, - :checkpoint_no => c.name("checkpoint_no") { c.get_uint32 }, - } - end - end + # Initialize a log block by passing in a 512-byte buffer containing the raw + # log block contents. + def initialize(buffer) + raise "Log block buffer provided was not #{BLOCK_SIZE} bytes" unless buffer.size == BLOCK_SIZE - # Return a slice of actual block data (that is, excluding header and - # trailer) starting at the given offset. - def data(offset = DATA_OFFSET) - length = header[:data_length] + @buffer = buffer + end - if length == BLOCK_SIZE - length -= TRAILER_SIZE + # Return an BufferCursor object positioned at a specific offset. + def cursor(offset) + BufferCursor.new(@buffer, offset) end - if offset < DATA_OFFSET || offset > length - raise "Invalid block data offset" + # Return the log block header. + def header + @header ||= cursor(HEADER_OFFSET).name("header") do |c| + Header.new( + flush: c.name("flush") { c.peek { (c.read_uint32 & HEADER_FLUSH_BIT_MASK).positive? } }, + block_number: c.name("block_number") { c.read_uint32 & ~HEADER_FLUSH_BIT_MASK }, + data_length: c.name("data_length") { c.read_uint16 }, + first_rec_group: c.name("first_rec_group") { c.read_uint16 }, + checkpoint_no: c.name("checkpoint_no") { c.read_uint32 } + ) + end end - @buffer.slice(offset, length - offset) - end + def_delegator :header, :flush + def_delegator :header, :block_number + def_delegator :header, :data_length + def_delegator :header, :first_rec_group + def_delegator :header, :checkpoint_no + + # Return a slice of actual block data (that is, excluding header and + # trailer) starting at the given offset. + def data(offset = DATA_OFFSET) + length = data_length + length -= TRAILER_SIZE if length == BLOCK_SIZE - # Return the log block trailer. - def trailer - @trailer ||= cursor(TRAILER_OFFSET).name("trailer") do |c| - { - :checksum => c.name("checksum") { c.get_uint32 }, - } + raise "Invalid block data offset" if offset < DATA_OFFSET || offset > length + + @buffer.slice(offset, length - offset) end - end - # A helper function to return the checksum from the trailer, for - # easier access. - def checksum - trailer[:checksum] - end + # Return the log block trailer. + def trailer + @trailer ||= cursor(TRAILER_OFFSET).name("trailer") do |c| + Trailer.new(checksum: c.name("checksum") { c.read_uint32 }) + end + end - # Calculate the checksum of the block using InnoDB's log block - # checksum algorithm. - def calculate_checksum - cksum = 1 - shift = (0..24).cycle - cursor(0).each_byte_as_uint8(TRAILER_OFFSET) do |b| - cksum &= 0x7fffffff - cksum += b + (b << shift.next) + def_delegator :trailer, :checksum + + # Calculate the checksum of the block using InnoDB's log block + # checksum algorithm. + def calculate_checksum + csum = 1 + shift = (0..24).cycle + cursor(0).each_byte_as_uint8(TRAILER_OFFSET) do |b| + csum &= 0x7fffffff + csum += b + (b << shift.next) + end + csum end - cksum - end - # Is the block corrupt? Calculate the checksum of the block and compare to - # the stored checksum; return true or false. - def corrupt? - checksum != calculate_checksum - end + # Is the block corrupt? Calculate the checksum of the block and compare to + # the stored checksum; return true or false. + def corrupt? + checksum != calculate_checksum + end - # Dump the contents of a log block for debugging purposes. - def dump - puts - puts "header:" - pp header + # Dump the contents of a log block for debugging purposes. + def dump + puts + puts "header:" + pp header - puts - puts "trailer:" - pp trailer + puts + puts "trailer:" + pp trailer + end end end diff --git a/lib/innodb/log_group.rb b/lib/innodb/log_group.rb index d7a330b0..3b46ced1 100644 --- a/lib/innodb/log_group.rb +++ b/lib/innodb/log_group.rb @@ -1,84 +1,73 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true # Group of InnoDB logs files that make up the redo log. -class Innodb::LogGroup - - # Initialize group given a set of sorted log files. - def initialize(log_files) - @logs = log_files.map { |fn| Innodb::Log.new(fn) } - sizes = @logs.map { |log| log.size } - raise "Log file sizes do not match" unless sizes.uniq.size == 1 - end - - # Iterate through all logs. - def each_log - unless block_given? - return enum_for(:each_log) +module Innodb + class LogGroup + # Initialize group given a set of sorted log files. + def initialize(log_files) + @logs = log_files.map { |fn| Innodb::Log.new(fn) } + raise "Log file sizes do not match" unless @logs.map(&:size).uniq.size == 1 end - @logs.each do |log| - yield log - end - end + # Iterate through all logs. + def each_log(&block) + return enum_for(:each_log) unless block_given? - # Iterate through all blocks. - def each_block - unless block_given? - return enum_for(:each_block) + @logs.each(&block) end - each_log do |log| - log.each_block do |block_index, block| - yield block_index, block + # Iterate through all blocks. + def each_block(&block) + return enum_for(:each_block) unless block_given? + + each_log do |log| + log.each_block(&block) end end - end - # The number of log files in the group. - def logs - @logs.count - end + # The number of log files in the group. + def logs + @logs.count + end - # Returns the log at the given position in the log group. - def log(log_no) - @logs.at(log_no) - end + # Returns the log at the given position in the log group. + def log(log_no) + @logs.at(log_no) + end - # The size in byes of each and every log in the group. - def log_size - @logs.first.size - end + # The size in byes of each and every log in the group. + def log_size + @logs.first.size + end - # The size of the log group (in bytes) - def size - @logs.first.size * @logs.count - end + # The size of the log group (in bytes) + def size + @logs.first.size * @logs.count + end - # The log group capacity (in bytes). - def capacity - @logs.first.capacity * @logs.count - end + # The log group capacity (in bytes). + def capacity + @logs.first.capacity * @logs.count + end - # Returns the LSN coordinates of the data at the start of the log group. - def start_lsn - [@logs.first.header[:start_lsn], Innodb::Log::LOG_HEADER_SIZE] - end + # Returns the LSN coordinates of the data at the start of the log group. + def start_lsn + [@logs.first.header[:start_lsn], Innodb::Log::LOG_HEADER_SIZE] + end - # Returns the LSN coordinates of the most recent (highest) checkpoint. - def max_checkpoint_lsn - checkpoint = @logs.first.checkpoint.max_by{|f,v| v[:number]}.last - checkpoint.values_at(:lsn, :lsn_offset) - end + # Returns the LSN coordinates of the most recent (highest) checkpoint. + def max_checkpoint_lsn + @logs.first.checkpoint.max_by(&:number).to_h.values_at(:lsn, :lsn_offset) + end - # Returns a LogReader using the given LSN reference coordinates. - def reader(lsn_coord = start_lsn) - lsn_no, lsn_offset = lsn_coord - lsn = Innodb::LSN.new(lsn_no, lsn_offset) - Innodb::LogReader.new(lsn, self) - end + # Returns a LogReader using the given LSN reference coordinates. + def reader(lsn_coordinates = start_lsn) + Innodb::LogReader.new(Innodb::LSN.new(*lsn_coordinates), self) + end - # Parse and return a record at a given LSN. - def record(lsn_no) - reader.seek(lsn_no).record + # Parse and return a record at a given LSN. + def record(lsn_no) + reader.seek(lsn_no).record + end end end diff --git a/lib/innodb/log_reader.rb b/lib/innodb/log_reader.rb index 6d9cd44d..ce746537 100644 --- a/lib/innodb/log_reader.rb +++ b/lib/innodb/log_reader.rb @@ -1,116 +1,117 @@ -# -*- encoding : utf-8 -*- - -require "ostruct" +# frozen_string_literal: true # Representation of the log group as a seekable stream of log records. -class Innodb::LogReader - - # Whether to checksum blocks. - attr_accessor :checksum - - def initialize(lsn, group) - @group = group - @context = OpenStruct.new(:buffer => String.new, - :buffer_lsn => lsn.dup, :record_lsn => lsn.dup) - end +module Innodb + class LogReader + # Checksum failed exception. + class ChecksumError < RuntimeError; end + + # EOF reached exception. + class EOFError < EOFError; end + + Context = Struct.new( + :buffer, + :buffer_lsn, + :record_lsn, + keyword_init: true + ) + + # Whether to checksum blocks. + # TODO: Hmm, nothing seems to actually set this. + attr_accessor :checksum + + def initialize(lsn, group) + @group = group + @context = Context.new(buffer: String.new, buffer_lsn: lsn.dup, record_lsn: lsn.dup) + end - # Seek to record starting position. - def seek(lsn_no) - check_lsn_no(lsn_no) - @context.buffer = String.new - @context.buffer_lsn.reposition(lsn_no, @group) - @context.record_lsn = @context.buffer_lsn.dup - self - end + # Seek to record starting position. + def seek(lsn_no) + check_lsn_no(lsn_no) + @context.buffer = String.new + @context.buffer_lsn.reposition(lsn_no, @group) + @context.record_lsn = @context.buffer_lsn.dup + self + end - # Returns the current LSN starting position. - def tell - @context.record_lsn.no - end + # Returns the current LSN starting position. + def tell + @context.record_lsn.no + end - # Read a record. - def record - cursor = BufferCursor.new(self, 0) - record = Innodb::LogRecord.new - record.read(cursor) - record.lsn = reposition(cursor.position) - record - end + # Read a record. + def record + cursor = BufferCursor.new(self, 0) + record = Innodb::LogRecord.new + record.read(cursor) + record.lsn = reposition(cursor.position) + record + end - # Call the given block once for each record in the log until the - # end of the log (or a corrupted block) is reached. If the follow - # argument is true, retry. - def each_record(follow, wait=0.5) - begin + # Call the given block once for each record in the log until the + # end of the log (or a corrupted block) is reached. If the follow + # argument is true, retry. + def each_record(follow, wait = 0.5) loop { yield record } rescue EOFError, ChecksumError - sleep(wait) and retry if follow - end - end - - # Read a slice of log data (that is, log data used for records). - def slice(position, length) - buffer = @context.buffer - length = position + length - - if length > buffer.size - preload(length) + sleep(wait) && retry if follow end - buffer.slice(position, length - position) - end + # Read a slice of log data (that is, log data used for records). + def slice(position, length) + buffer = @context.buffer + length = position + length - # Checksum failed exception. - class ChecksumError < RuntimeError - end + preload(length) if length > buffer.size - # EOF reached exception. - class EOFError < EOFError - end + buffer.slice(position, length - position) + end - private + private - # Check if LSN points to where records may be located. - def check_lsn_no(lsn_no) - lsn = @context.record_lsn.dup - lsn.reposition(lsn_no, @group) - raise "LSN #{lsn_no} is out of bounds" unless lsn.record?(@group) - end - - # Reposition to the beginning of the next record. - def reposition(length) - start_lsn_no = @context.record_lsn.no - delta_lsn_no = @context.record_lsn.delta(length) - @context.record_lsn.advance(delta_lsn_no, @group) - @context.buffer.slice!(0, length) - [start_lsn_no, start_lsn_no + delta_lsn_no] - end + # Check if LSN points to where records may be located. + def check_lsn_no(lsn_no) + lsn = @context.record_lsn.dup + lsn.reposition(lsn_no, @group) + raise "LSN #{lsn_no} is out of bounds" unless lsn.record?(@group) + end - # Reads the log block at the given LSN position. - def get_block(lsn) - log_no, block_no, block_offset = lsn.location(@group) - [@group.log(log_no).block(block_no), block_offset] - end + # Reposition to the beginning of the next record. + def reposition(length) + start_lsn_no = @context.record_lsn.no + delta_lsn_no = @context.record_lsn.delta(length) + @context.record_lsn.advance(delta_lsn_no, @group) + @context.buffer.slice!(0, length) + [start_lsn_no, start_lsn_no + delta_lsn_no] + end - # Preload the log buffer with enough data to satisfy the requested amount. - def preload(size) - buffer = @context.buffer - buffer_lsn = @context.buffer_lsn - - # If reading for the first time, offset points to the start of the - # record (somewhere in the block). Otherwise, the block is read as - # a whole and offset points to the start of the next block to read. - while buffer.size < size - block, offset = get_block(buffer_lsn) - break if checksum && corrupt = block.corrupt? - data = offset == 0 ? block.data : block.data(offset) - data_length = block.header[:data_length] - buffer << data - buffer_lsn.advance(data_length - offset, @group) - break if data_length < Innodb::LogBlock::BLOCK_SIZE + # Reads the log block at the given LSN position. + def get_block(lsn) + log_no, block_no, block_offset = lsn.location(@group) + [@group.log(log_no).block(block_no), block_offset] end - raise ChecksumError, "Block is corrupted" if corrupt - raise EOFError, "End of log reached" if buffer.size < size + # Preload the log buffer with enough data to satisfy the requested amount. + def preload(size) + buffer = @context.buffer + buffer_lsn = @context.buffer_lsn + + # If reading for the first time, offset points to the start of the + # record (somewhere in the block). Otherwise, the block is read as + # a whole and offset points to the start of the next block to read. + while buffer.size < size + block, offset = get_block(buffer_lsn) + break if checksum && (corrupt = block.corrupt?) + + data = offset.zero? ? block.data : block.data(offset) + data_length = block.header[:data_length] + buffer << data + buffer_lsn.advance(data_length - offset, @group) + break if data_length < Innodb::LogBlock::BLOCK_SIZE + end + + raise ChecksumError, "Block is corrupted" if corrupt + raise EOFError, "End of log reached" if buffer.size < size + end end end diff --git a/lib/innodb/log_record.rb b/lib/innodb/log_record.rb index 69ca397a..6e3ee320 100644 --- a/lib/innodb/log_record.rb +++ b/lib/innodb/log_record.rb @@ -1,317 +1,366 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true # An InnoDB transaction log block. -class Innodb::LogRecord - # Start and end LSNs for this record. - attr_accessor :lsn +module Innodb + class LogRecord + Preamble = Struct.new( + :type, + :single_record, + :space, + :page_number, + keyword_init: true + ) - # The size (in bytes) of the record. - attr_reader :size + IndexFieldInfo = Struct.new( + :mtype, + :prtype, + :length, # rubocop:disable Lint/StructNewOverride + keyword_init: true + ) - attr_reader :preamble + Index = Struct.new( + :n_cols, + :n_uniq, + :cols, + keyword_init: true + ) - attr_reader :payload + # Start and end LSNs for this record. + attr_accessor :lsn - # InnoDB log record types. - RECORD_TYPES = - { - 1 => :MLOG_1BYTE, 2 => :MLOG_2BYTE, - 4 => :MLOG_4BYTE, 8 => :MLOG_8BYTE, - 9 => :REC_INSERT, 10 => :REC_CLUST_DELETE_MARK, - 11 => :REC_SEC_DELETE_MARK, 13 => :REC_UPDATE_IN_PLACE, - 14 => :REC_DELETE, 15 => :LIST_END_DELETE, - 16 => :LIST_START_DELETE, 17 => :LIST_END_COPY_CREATED, - 18 => :PAGE_REORGANIZE, 19 => :PAGE_CREATE, - 20 => :UNDO_INSERT, 21 => :UNDO_ERASE_END, - 22 => :UNDO_INIT, 23 => :UNDO_HDR_DISCARD, - 24 => :UNDO_HDR_REUSE, 25 => :UNDO_HDR_CREATE, - 26 => :REC_MIN_MARK, 27 => :IBUF_BITMAP_INIT, - 28 => :LSN, 29 => :INIT_FILE_PAGE, - 30 => :WRITE_STRING, 31 => :MULTI_REC_END, - 32 => :DUMMY_RECORD, 33 => :FILE_CREATE, - 34 => :FILE_RENAME, 35 => :FILE_DELETE, - 36 => :COMP_REC_MIN_MARK, 37 => :COMP_PAGE_CREATE, - 38 => :COMP_REC_INSERT, 39 => :COMP_REC_CLUST_DELETE_MARK, - 40 => :COMP_REC_SEC_DELETE_MARK, 41 => :COMP_REC_UPDATE_IN_PLACE, - 42 => :COMP_REC_DELETE, 43 => :COMP_LIST_END_DELETE, - 44 => :COMP_LIST_START_DELETE, 45 => :COMP_LIST_END_COPY_CREATE, - 46 => :COMP_PAGE_REORGANIZE, 47 => :FILE_CREATE2, - 48 => :ZIP_WRITE_NODE_PTR, 49 => :ZIP_WRITE_BLOB_PTR, - 50 => :ZIP_WRITE_HEADER, 51 => :ZIP_PAGE_COMPRESS, - } + # The size (in bytes) of the record. + attr_reader :size - # Types of undo log segments. - UNDO_TYPES = { 1 => :UNDO_INSERT, 2 => :UNDO_UPDATE } + attr_reader :preamble - def read(cursor) - origin = cursor.position - @preamble = read_preamble(cursor) - @payload = read_payload(@preamble[:type], cursor) - @size = cursor.position - origin - end + attr_reader :payload - # Dump the contents of the record. - def dump - pp({:lsn => lsn, :size => size, :content => @preamble.merge(@payload)}) - end + # InnoDB log record types. + RECORD_TYPES = { + 1 => :MLOG_1BYTE, + 2 => :MLOG_2BYTE, + 4 => :MLOG_4BYTE, + 8 => :MLOG_8BYTE, + 9 => :REC_INSERT, + 10 => :REC_CLUST_DELETE_MARK, + 11 => :REC_SEC_DELETE_MARK, + 13 => :REC_UPDATE_IN_PLACE, + 14 => :REC_DELETE, + 15 => :LIST_END_DELETE, + 16 => :LIST_START_DELETE, + 17 => :LIST_END_COPY_CREATED, + 18 => :PAGE_REORGANIZE, + 19 => :PAGE_CREATE, + 20 => :UNDO_INSERT, + 21 => :UNDO_ERASE_END, + 22 => :UNDO_INIT, + 23 => :UNDO_HDR_DISCARD, + 24 => :UNDO_HDR_REUSE, + 25 => :UNDO_HDR_CREATE, + 26 => :REC_MIN_MARK, + 27 => :IBUF_BITMAP_INIT, + 28 => :LSN, + 29 => :INIT_FILE_PAGE, + 30 => :WRITE_STRING, + 31 => :MULTI_REC_END, + 32 => :DUMMY_RECORD, + 33 => :FILE_CREATE, + 34 => :FILE_RENAME, + 35 => :FILE_DELETE, + 36 => :COMP_REC_MIN_MARK, + 37 => :COMP_PAGE_CREATE, + 38 => :COMP_REC_INSERT, + 39 => :COMP_REC_CLUST_DELETE_MARK, + 40 => :COMP_REC_SEC_DELETE_MARK, + 41 => :COMP_REC_UPDATE_IN_PLACE, + 42 => :COMP_REC_DELETE, + 43 => :COMP_LIST_END_DELETE, + 44 => :COMP_LIST_START_DELETE, + 45 => :COMP_LIST_END_COPY_CREATE, + 46 => :COMP_PAGE_REORGANIZE, + 47 => :FILE_CREATE2, + 48 => :ZIP_WRITE_NODE_PTR, + 49 => :ZIP_WRITE_BLOB_PTR, + 50 => :ZIP_WRITE_HEADER, + 51 => :ZIP_PAGE_COMPRESS, + }.freeze - # Single record flag is masked in the record type. - SINGLE_RECORD_MASK = 0x80 - RECORD_TYPE_MASK = 0x7f + # Types of undo log segments. + UNDO_TYPES = { + 1 => :UNDO_INSERT, + 2 => :UNDO_UPDATE, + }.freeze - # Return a preamble of the first record in this block. - def read_preamble(c) - type_and_flag = c.name("type") { c.get_uint8 } - type = type_and_flag & RECORD_TYPE_MASK - type = RECORD_TYPES[type] || type - # Whether this is a single record for a single page. - single_record = (type_and_flag & SINGLE_RECORD_MASK) > 0 - case type - when :MULTI_REC_END, :DUMMY_RECORD - { :type => type } - else - { - :type => type, - :single_record => single_record, - :space => c.name("space") { c.get_ic_uint32 }, - :page_number => c.name("page_number") { c.get_ic_uint32 }, - } + def read(cursor) + origin = cursor.position + @preamble = read_preamble(cursor) + @payload = read_payload(@preamble.type, cursor) + @size = cursor.position - origin end - end - # Read the index part of a log record for a compact record insert. - # Ref. mlog_parse_index - def read_index(c) - n_cols = c.name("n_cols") { c.get_uint16 } - n_uniq = c.name("n_uniq") { c.get_uint16 } - cols = n_cols.times.collect do - info = c.name("field_info") { c.get_uint16 } - { - :mtype => ((info + 1) & 0x7fff) <= 1 ? :BINARY : :FIXBINARY, - :prtype => (info & 0x8000) != 0 ? :NOT_NULL : nil, - :length => info & 0x7fff - } + # Dump the contents of the record. + def dump + pp({ lsn: lsn, size: size, preamble: @preamble, payload: @payload }) end - { - :n_cols => n_cols, - :n_uniq => n_uniq, - :cols => cols, - } - end - # Flag of whether an insert log record contains info and status. - INFO_AND_STATUS_MASK = 0x1 + # Single record flag is masked in the record type. + SINGLE_RECORD_MASK = 0x80 + RECORD_TYPE_MASK = 0x7f + + # Return a preamble of the first record in this block. + def read_preamble(cursor) + type_and_flag = cursor.name("type") { cursor.read_uint8 } + type = type_and_flag & RECORD_TYPE_MASK + type = RECORD_TYPES[type] || type + # Whether this is a single record for a single page. + single_record = (type_and_flag & SINGLE_RECORD_MASK).positive? + case type + when :MULTI_REC_END, :DUMMY_RECORD + Preamble.new(type: type) + else + Preamble.new( + type: type, + single_record: single_record, + space: cursor.name("space") { cursor.read_ic_uint32 }, + page_number: cursor.name("page_number") { cursor.read_ic_uint32 } + ) + end + end - # Read the insert record into page part of a insert log. - # Ref. page_cur_parse_insert_rec - def read_insert_record(c) - page_offset = c.name("page_offset") { c.get_uint16 } - end_seg_len = c.name("end_seg_len") { c.get_ic_uint32 } + # Read the index part of a log record for a compact record insert. + # Ref. mlog_parse_index + def read_index(cursor) + n_cols = cursor.name("n_cols") { cursor.read_uint16 } + n_uniq = cursor.name("n_uniq") { cursor.read_uint16 } + cols = n_cols.times.collect do + info = cursor.name("field_info") { cursor.read_uint16 } + IndexFieldInfo.new( + mtype: ((info + 1) & 0x7fff) <= 1 ? :BINARY : :FIXBINARY, + prtype: (info & 0x8000).zero? ? nil : :NOT_NULL, + length: info & 0x7fff + ) + end - if (end_seg_len & INFO_AND_STATUS_MASK) != 0 - info_and_status_bits = c.get_uint8 - origin_offset = c.get_ic_uint32 - mismatch_index = c.get_ic_uint32 + Index.new(n_cols: n_cols, n_uniq: n_uniq, cols: cols) end - { - :page_offset => page_offset, - :end_seg_len => end_seg_len >> 1, - :info_and_status_bits => info_and_status_bits, - :origin_offset => origin_offset, - :mismatch_index => mismatch_index, - :record => c.name("record") { c.get_bytes(end_seg_len >> 1) }, - } - end + # Flag of whether an insert log record contains info and status. + INFO_AND_STATUS_MASK = 0x1 - # Read the log record for an in-place update. - # Ref. btr_cur_parse_update_in_place - def read_update_in_place_record(c) - { - :flags => c.name("flags") { c.get_uint8 }, - :sys_fields => read_sys_fields(c), - :rec_offset => c.name("rec_offset") { c.get_uint16 }, - :update_index => read_update_index(c), - } - end + # Read the insert record into page part of a insert log. + # Ref. page_cur_parse_insert_rec + def read_insert_record(cursor) + page_offset = cursor.name("page_offset") { cursor.read_uint16 } + end_seg_len = cursor.name("end_seg_len") { cursor.read_ic_uint32 } - LENGTH_NULL = 0xFFFFFFFF + if (end_seg_len & INFO_AND_STATUS_MASK) != 0 + info_and_status_bits = cursor.read_uint8 + origin_offset = cursor.read_ic_uint32 + mismatch_index = cursor.read_ic_uint32 + end - # Read the update vector for an update log record. - # Ref. row_upd_index_parse - def read_update_index(c) - info_bits = c.name("info_bits") { c.get_uint8 } - n_fields = c.name("n_fields") { c.get_ic_uint32 } - fields = n_fields.times.collect do { - :field_no => c.name("field_no") { c.get_ic_uint32 }, - :len => len = c.name("len") { c.get_ic_uint32 }, - :data => c.name("data") { len != LENGTH_NULL ? c.get_bytes(len) : :NULL }, + page_offset: page_offset, + end_seg_len: end_seg_len >> 1, + info_and_status_bits: info_and_status_bits, + origin_offset: origin_offset, + mismatch_index: mismatch_index, + record: cursor.name("record") { cursor.read_bytes(end_seg_len >> 1) }, } end - { - :info_bits => info_bits, - :n_fields => n_fields, - :fields => fields, - } - end - - # Read system fields values in a log record. - # Ref. row_upd_parse_sys_vals - def read_sys_fields(c) - { - :trx_id_pos => c.name("trx_id_pos") { c.get_ic_uint32 }, - :roll_ptr => c.name("roll_ptr") { c.get_bytes(7) }, - :trx_id => c.name("trx_id") { c.get_ic_uint64 }, - } - end - # Read the log record for delete marking or unmarking of a clustered - # index record. - # Ref. btr_cur_parse_del_mark_set_clust_rec - def read_clust_delete_mark(c) - { - :flags => c.name("flags") { c.get_uint8 }, - :value => c.name("value") { c.get_uint8 }, - :sys_fields => c.name("sys_fields") { read_sys_fields(c) }, - :offset => c.name("offset") { c.get_uint16 }, - } - end - - def read_payload(type, c) - case type - when :MLOG_1BYTE, :MLOG_2BYTE, :MLOG_4BYTE - { - :page_offset => c.name("page_offset") { c.get_uint16 }, - :value => c.name("value") { c.get_ic_uint32 } - } - when :MLOG_8BYTE - { - :offset => c.name("offset") { c.get_uint16 }, - :value => c.name("value") { c.get_ic_uint64 } - } - when :UNDO_HDR_CREATE, :UNDO_HDR_REUSE + # Read the log record for an in-place update. + # Ref. btr_cur_parse_update_in_place + def read_update_in_place_record(cursor) { - :trx_id => c.name("trx_id") { c.get_ic_uint64 } + flags: cursor.name("flags") { cursor.read_uint8 }, + sys_fields: read_sys_fields(cursor), + rec_offset: cursor.name("rec_offset") { cursor.read_uint16 }, + update_index: read_update_index(cursor), } - when :UNDO_INSERT - { - :length => len = c.name("length") { c.get_uint16 }, - :value => c.name("value") { c.get_bytes(len) } - } - when :REC_INSERT - { - :record => c.name("record") { read_insert_record(c) } - } - when :COMP_REC_INSERT - { - :index => c.name("index") { read_index(c) }, - :record => c.name("record") { read_insert_record(c) } - } - when :COMP_REC_UPDATE_IN_PLACE - { - :index => c.name("index") { read_index(c) }, - :record => c.name("record") { read_update_in_place_record(c) } - } - when :REC_UPDATE_IN_PLACE - { - :record => c.name("record") { read_update_in_place_record(c) } - } - when :WRITE_STRING - { - :offset => c.name("offset") { c.get_uint16 }, - :length => length = c.name("length") { c.get_uint16 }, - :value => c.name("value") { c.get_bytes(length) }, - } - when :UNDO_INIT - { - :type => c.name("type") { UNDO_TYPES[c.get_ic_uint32] } - } - when :FILE_CREATE, :FILE_DELETE - { - :name_len => len = c.name("name_len") { c.get_uint16 }, - :name => c.name("name") { c.get_bytes(len) }, - } - when :FILE_CREATE2 - { - :flags => c.name("flags") { c.get_uint32 }, - :name_len => len = c.name("name_len") { c.get_uint16 }, - :name => c.name("name") { c.get_bytes(len) }, - } - when :FILE_RENAME - { - :old => { - :name_len => len = c.name("name_len") { c.get_uint16 }, - :name => c.name("name") { c.get_bytes(len) }, - }, - :new => { - :name_len => len = c.name("name_len") { c.get_uint16 }, - :name => c.name("name") { c.get_bytes(len) }, + end + + LENGTH_NULL = 0xFFFFFFFF + + # Read the update vector for an update log record. + # Ref. row_upd_index_parse + def read_update_index(cursor) + info_bits = cursor.name("info_bits") { cursor.read_uint8 } + n_fields = cursor.name("n_fields") { cursor.read_ic_uint32 } + fields = n_fields.times.collect do + { + field_no: cursor.name("field_no") { cursor.read_ic_uint32 }, + len: len = cursor.name("len") { cursor.read_ic_uint32 }, + data: cursor.name("data") { len == LENGTH_NULL ? :NULL : cursor.read_bytes(len) }, } - } - when :COMP_REC_CLUST_DELETE_MARK + end { - :index => c.name("index") { read_index(c) }, - :record => c.name("record") { read_clust_delete_mark(c) } + info_bits: info_bits, + n_fields: n_fields, + fields: fields, } - when :REC_CLUST_DELETE_MARK - { - :record => c.name("record") { read_clust_delete_mark(c) } - } - when :COMP_REC_SEC_DELETE_MARK - { - :index => c.name("index") { read_index(c) }, - :value => c.name("value") { c.get_uint8 }, - :offset => c.name("offset") { c.get_uint16 }, - } - when :REC_SEC_DELETE_MARK - { - :value => c.name("value") { c.get_uint8 }, - :offset => c.name("offset") { c.get_uint16 }, - } - when :REC_DELETE - { - :offset => c.name("offset") { c.get_uint16 }, - } - when :COMP_REC_DELETE - { - :index => c.name("index") { read_index(c) }, - :offset => c.name("offset") { c.get_uint16 }, - } - when :REC_MIN_MARK, :COMP_REC_MIN_MARK - { - :offset => c.name("offset") { c.get_uint16 }, - } - when :LIST_START_DELETE, :LIST_END_DELETE - { - :offset => c.name("offset") { c.get_uint16 }, - } - when :COMP_LIST_START_DELETE, :COMP_LIST_END_DELETE - { - :index => c.name("index") { read_index(c) }, - :offset => c.name("offset") { c.get_uint16 }, - } - when :LIST_END_COPY_CREATED - { - :length => len = c.name("length") { c.get_uint32 }, - :data => c.name("data") { c.get_bytes(len) } - } - when :COMP_LIST_END_COPY_CREATE + end + + # Read system fields values in a log record. + # Ref. row_upd_parse_sys_vals + def read_sys_fields(cursor) { - :index => c.name("index") { read_index(c) }, - :length => len = c.name("length") { c.get_uint32 }, - :data => c.name("data") { c.get_bytes(len) } + trx_id_pos: cursor.name("trx_id_pos") { cursor.read_ic_uint32 }, + roll_ptr: cursor.name("roll_ptr") { cursor.read_bytes(7) }, + trx_id: cursor.name("trx_id") { cursor.read_ic_uint64 }, } - when :COMP_PAGE_REORGANIZE + end + + # Read the log record for delete marking or unmarking of a clustered + # index record. + # Ref. btr_cur_parse_del_mark_set_clust_rec + def read_clust_delete_mark(cursor) { - :index => c.name("index") { read_index(c) }, + flags: cursor.name("flags") { cursor.read_uint8 }, + value: cursor.name("value") { cursor.read_uint8 }, + sys_fields: cursor.name("sys_fields") { read_sys_fields(cursor) }, + offset: cursor.name("offset") { cursor.read_uint16 }, } - when :DUMMY_RECORD, :MULTI_REC_END, :INIT_FILE_PAGE, - :IBUF_BITMAP_INIT, :PAGE_CREATE, :COMP_PAGE_CREATE, - :PAGE_REORGANIZE, :UNDO_ERASE_END, :UNDO_HDR_DISCARD - {} - else - raise "Unsupported log record type: #{type.to_s}" end + + # The bodies of the branches here are sometimes duplicates, but logically distinct. + # rubocop:disable Lint/DuplicateBranch + def read_payload(type, cursor) + case type + when :MLOG_1BYTE, :MLOG_2BYTE, :MLOG_4BYTE + { + page_offset: cursor.name("page_offset") { cursor.read_uint16 }, + value: cursor.name("value") { cursor.read_ic_uint32 }, + } + when :MLOG_8BYTE + { + offset: cursor.name("offset") { cursor.read_uint16 }, + value: cursor.name("value") { cursor.read_ic_uint64 }, + } + when :UNDO_HDR_CREATE, :UNDO_HDR_REUSE + { + trx_id: cursor.name("trx_id") { cursor.read_ic_uint64 }, + } + when :UNDO_INSERT + { + length: length = cursor.name("length") { cursor.read_uint16 }, + value: cursor.name("value") { cursor.read_bytes(length) }, + } + when :REC_INSERT + { + record: cursor.name("record") { read_insert_record(cursor) }, + } + when :COMP_REC_INSERT + { + index: cursor.name("index") { read_index(cursor) }, + record: cursor.name("record") { read_insert_record(cursor) }, + } + when :COMP_REC_UPDATE_IN_PLACE + { + index: cursor.name("index") { read_index(cursor) }, + record: cursor.name("record") { read_update_in_place_record(cursor) }, + } + when :REC_UPDATE_IN_PLACE + { + record: cursor.name("record") { read_update_in_place_record(cursor) }, + } + when :WRITE_STRING + { + offset: cursor.name("offset") { cursor.read_uint16 }, + length: length = cursor.name("length") { cursor.read_uint16 }, + value: cursor.name("value") { cursor.read_bytes(length) }, + } + when :UNDO_INIT + { + type: cursor.name("type") { UNDO_TYPES[cursor.read_ic_uint32] }, + } + when :FILE_CREATE, :FILE_DELETE + { + name_len: name_len = cursor.name("name_len") { cursor.read_uint16 }, + name: cursor.name("name") { cursor.read_bytes(name_len) }, + } + when :FILE_CREATE2 + { + flags: cursor.name("flags") { cursor.read_uint32 }, + name_len: name_len = cursor.name("name_len") { cursor.read_uint16 }, + name: cursor.name("name") { cursor.read_bytes(name_len) }, + } + when :FILE_RENAME + { + old: { + name_len: name_len = cursor.name("name_len") { cursor.read_uint16 }, + name: cursor.name("name") { cursor.read_bytes(name_len) }, + }, + new: { + name_len: name_len = cursor.name("name_len") { cursor.read_uint16 }, + name: cursor.name("name") { cursor.read_bytes(name_len) }, + }, + } + when :COMP_REC_CLUST_DELETE_MARK + { + index: cursor.name("index") { read_index(cursor) }, + record: cursor.name("record") { read_clust_delete_mark(cursor) }, + } + when :REC_CLUST_DELETE_MARK + { + record: cursor.name("record") { read_clust_delete_mark(cursor) }, + } + when :COMP_REC_SEC_DELETE_MARK + { + index: cursor.name("index") { read_index(cursor) }, + value: cursor.name("value") { cursor.read_uint8 }, + offset: cursor.name("offset") { cursor.read_uint16 }, + } + when :REC_SEC_DELETE_MARK + { + value: cursor.name("value") { cursor.read_uint8 }, + offset: cursor.name("offset") { cursor.read_uint16 }, + } + when :REC_DELETE + { + offset: cursor.name("offset") { cursor.read_uint16 }, + } + when :COMP_REC_DELETE + { + index: cursor.name("index") { read_index(cursor) }, + offset: cursor.name("offset") { cursor.read_uint16 }, + } + when :REC_MIN_MARK, :COMP_REC_MIN_MARK + { + offset: cursor.name("offset") { cursor.read_uint16 }, + } + when :LIST_START_DELETE, :LIST_END_DELETE + { + offset: cursor.name("offset") { cursor.read_uint16 }, + } + when :COMP_LIST_START_DELETE, :COMP_LIST_END_DELETE + { + index: cursor.name("index") { read_index(cursor) }, + offset: cursor.name("offset") { cursor.read_uint16 }, + } + when :LIST_END_COPY_CREATED + { + length: length = cursor.name("length") { cursor.read_uint32 }, + data: cursor.name("data") { cursor.read_bytes(length) }, + } + when :COMP_LIST_END_COPY_CREATE + { + index: cursor.name("index") { read_index(cursor) }, + length: length = cursor.name("length") { cursor.read_uint32 }, + data: cursor.name("data") { cursor.read_bytes(length) }, + } + when :COMP_PAGE_REORGANIZE + { + index: cursor.name("index") { read_index(cursor) }, + } + when :DUMMY_RECORD, :MULTI_REC_END, :INIT_FILE_PAGE, + :IBUF_BITMAP_INIT, :PAGE_CREATE, :COMP_PAGE_CREATE, + :PAGE_REORGANIZE, :UNDO_ERASE_END, :UNDO_HDR_DISCARD + {} + else + raise "Unsupported log record type: #{type}" + end + end + # rubocop:enable Lint/DuplicateBranch end end diff --git a/lib/innodb/lsn.rb b/lib/innodb/lsn.rb index ca7ce4f5..5d95fcd2 100644 --- a/lib/innodb/lsn.rb +++ b/lib/innodb/lsn.rb @@ -1,103 +1,108 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true # A Log Sequence Number and its byte offset into the log group. -class Innodb::LSN - # The Log Sequence Number. - attr_reader :lsn_no - - # Alias :lsn_no attribute. - alias_method :no, :lsn_no +module Innodb + class LSN + # The Log Sequence Number. + attr_reader :lsn_no + + # Alias :lsn_no attribute. + alias no lsn_no + + # Initialize coordinates. + def initialize(lsn, offset) + @lsn_no = lsn + @lsn_offset = offset + end - # Initialize coordinates. - def initialize(lsn, offset) - @lsn_no = lsn - @lsn_offset = offset - end + # Place LSN in a new position. + def reposition(new_lsn_no, group) + new_offset = offset_of(@lsn_no, @lsn_offset, new_lsn_no, group) + @lsn_no = new_lsn_no + @lsn_offset = new_offset - # Place LSN in a new position. - def reposition(new_lsn_no, group) - new_offset = offset_of(@lsn_no, @lsn_offset, new_lsn_no, group) - @lsn_no, @lsn_offset = [new_lsn_no, new_offset] - end + [@lsn_no, @lsn_offset] + end - # Advance by a given LSN amount. - def advance(count_lsn_no, group) - new_lsn_no = @lsn_no + count_lsn_no - reposition(new_lsn_no, group) - end + # Advance by a given LSN amount. + def advance(count_lsn_no, group) + new_lsn_no = @lsn_no + count_lsn_no + reposition(new_lsn_no, group) + end - # Returns the location coordinates of this LSN. - def location(group) - location_of(@lsn_offset, group) - end + # Returns the location coordinates of this LSN. + def location(group) + location_of(@lsn_offset, group) + end - # Returns the LSN delta for the given amount of data. - def delta(length) - fragment = (@lsn_no % LOG_BLOCK_SIZE) - LOG_BLOCK_HEADER_SIZE - raise "Invalid fragment #{fragment} for LSN #{@lsn_no}" unless - fragment.between?(0, LOG_BLOCK_DATA_SIZE - 1) - length + (fragment + length) / LOG_BLOCK_DATA_SIZE * LOG_BLOCK_FRAME_SIZE - end + # Returns the LSN delta for the given amount of data. + def delta(length) + fragment = (@lsn_no % LOG_BLOCK_SIZE) - LOG_BLOCK_HEADER_SIZE + raise "Invalid fragment #{fragment} for LSN #{@lsn_no}" unless fragment.between?(0, LOG_BLOCK_DATA_SIZE - 1) - # Whether LSN might point to log record data. - def record?(group) - data_offset?(@lsn_offset, group) - end + length + ((fragment + length) / LOG_BLOCK_DATA_SIZE * LOG_BLOCK_FRAME_SIZE) + end - private + # Whether LSN might point to log record data. + def record?(group) + data_offset?(@lsn_offset, group) + end - # Short alias for the size of a log file header. - LOG_HEADER_SIZE = Innodb::Log::LOG_HEADER_SIZE + private - # Short aliases for the sizes of the subparts of a log block. - LOG_BLOCK_SIZE = Innodb::LogBlock::BLOCK_SIZE - LOG_BLOCK_HEADER_SIZE = Innodb::LogBlock::HEADER_SIZE - LOG_BLOCK_TRAILER_SIZE = Innodb::LogBlock::TRAILER_SIZE - LOG_BLOCK_DATA_SIZE = Innodb::LogBlock::DATA_SIZE - LOG_BLOCK_FRAME_SIZE = LOG_BLOCK_HEADER_SIZE + LOG_BLOCK_TRAILER_SIZE + # Short alias for the size of a log file header. + LOG_HEADER_SIZE = Innodb::Log::LOG_HEADER_SIZE - # Returns the coordinates of the given offset. - def location_of(offset, group) - log_no, log_offset = offset.divmod(group.size) - block_no, block_offset = (log_offset - LOG_HEADER_SIZE).divmod(LOG_BLOCK_SIZE) - [log_no, block_no, block_offset] - end + # Short aliases for the sizes of the subparts of a log block. + LOG_BLOCK_SIZE = Innodb::LogBlock::BLOCK_SIZE + LOG_BLOCK_HEADER_SIZE = Innodb::LogBlock::HEADER_SIZE + LOG_BLOCK_TRAILER_SIZE = Innodb::LogBlock::TRAILER_SIZE + LOG_BLOCK_DATA_SIZE = Innodb::LogBlock::DATA_SIZE + LOG_BLOCK_FRAME_SIZE = LOG_BLOCK_HEADER_SIZE + LOG_BLOCK_TRAILER_SIZE - # Returns the offset of the given LSN within a log group. - def offset_of(lsn, offset, new_lsn, group) - log_size = group.log_size - group_capacity = group.capacity - - # Calculate the offset in LSN. - if new_lsn >= lsn - lsn_offset = new_lsn - lsn - else - lsn_offset = lsn - new_lsn - lsn_offset %= group_capacity - lsn_offset = group_capacity - lsn_offset + # Returns the coordinates of the given offset. + def location_of(offset, group) + log_no, log_offset = offset.divmod(group.size) + block_no, block_offset = (log_offset - LOG_HEADER_SIZE).divmod(LOG_BLOCK_SIZE) + [log_no, block_no, block_offset] end - # Transpose group size offset to a group capacity offset. - group_offset = offset - (LOG_HEADER_SIZE * (1 + offset / log_size)) + # Returns the offset of the given LSN within a log group. + def offset_of(lsn, offset, new_lsn, group) + log_size = group.log_size + group_capacity = group.capacity - offset = (lsn_offset + group_offset) % group_capacity + # Calculate the offset in LSN. + if new_lsn >= lsn + lsn_offset = new_lsn - lsn + else + lsn_offset = lsn - new_lsn + lsn_offset %= group_capacity + lsn_offset = group_capacity - lsn_offset + end - # Transpose group capacity offset to a group size offset. - offset + LOG_HEADER_SIZE * (1 + offset / (log_size - LOG_HEADER_SIZE)) - end + # Transpose group size offset to a group capacity offset. + group_offset = offset - (LOG_HEADER_SIZE * (1 + (offset / log_size))) - # Whether offset points to the data area of an existing log block. - def data_offset?(offset, group) - log_offset = offset % group.size - log_no, block_no, block_offset = location_of(offset, group) + offset = (lsn_offset + group_offset) % group_capacity - status ||= log_no > group.logs - status ||= log_offset <= LOG_HEADER_SIZE - status ||= block_no < 0 - status ||= block_no >= group.log(log_no).blocks - status ||= block_offset < Innodb::LogBlock::DATA_OFFSET - status ||= block_offset >= Innodb::LogBlock::TRAILER_OFFSET + # Transpose group capacity offset to a group size offset. + offset + (LOG_HEADER_SIZE * (1 + (offset / (log_size - LOG_HEADER_SIZE)))) + end + + # Whether offset points to the data area of an existing log block. + def data_offset?(offset, group) + log_offset = offset % group.size + log_no, block_no, block_offset = location_of(offset, group) - !status + status ||= log_no > group.logs + status ||= log_offset <= LOG_HEADER_SIZE + status ||= block_no.negative? + status ||= block_no >= group.log(log_no).blocks + status ||= block_offset < Innodb::LogBlock::DATA_OFFSET + status ||= block_offset >= Innodb::LogBlock::TRAILER_OFFSET + + !status + end end end diff --git a/lib/innodb/mysql_collation.rb b/lib/innodb/mysql_collation.rb new file mode 100644 index 00000000..f9d639da --- /dev/null +++ b/lib/innodb/mysql_collation.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Innodb + class MysqlCollation + class DuplicateIdError < StandardError; end + class DuplicateNameError < StandardError; end + + @collations = [] + @collations_by_id = {} + @collations_by_name = {} + + class << self + attr_reader :collations + end + + def self.add(kwargs) + raise DuplicateIdError if @collations_by_id.key?(kwargs[:id]) + raise DuplicateNameError if @collations_by_name.key?(kwargs[:name]) + + collation = new(**kwargs) + @collations.push(collation) + @collations_by_id[collation.id] = collation + @collations_by_name[collation.name] = collation + @all_fixed_ids = nil + collation + end + + def self.by_id(id) + @collations_by_id[id] + end + + def self.by_name(name) + @collations_by_name[name] + end + + def self.all_fixed_ids + @all_fixed_ids ||= Innodb::MysqlCollation.collations.select(&:fixed?).map(&:id).sort + end + + attr_reader :id + attr_reader :name + attr_reader :character_set_name + attr_reader :mbminlen + attr_reader :mbmaxlen + + def initialize(id:, name:, character_set_name:, mbminlen:, mbmaxlen:) + @id = id + @name = name + @character_set_name = character_set_name + @mbminlen = mbminlen + @mbmaxlen = mbmaxlen + end + + def fixed? + mbminlen == mbmaxlen + end + + def variable? + !fixed? + end + end +end diff --git a/lib/innodb/mysql_collations.rb b/lib/innodb/mysql_collations.rb new file mode 100644 index 00000000..19b08ecb --- /dev/null +++ b/lib/innodb/mysql_collations.rb @@ -0,0 +1,292 @@ +# frozen_string_literal: true + +# Generated at 2024-11-26 00:42:28 UTC using innodb_ruby_generate_mysql_collations. Do not edit! + +# rubocop:disable all +Innodb::MysqlCollation.add(id: 1, name: "big5_chinese_ci", character_set_name: "big5", mbminlen: 1, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 2, name: "latin2_czech_cs", character_set_name: "latin2", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 3, name: "dec8_swedish_ci", character_set_name: "dec8", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 4, name: "cp850_general_ci", character_set_name: "cp850", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 5, name: "latin1_german1_ci", character_set_name: "latin1", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 6, name: "hp8_english_ci", character_set_name: "hp8", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 7, name: "koi8r_general_ci", character_set_name: "koi8r", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 8, name: "latin1_swedish_ci", character_set_name: "latin1", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 9, name: "latin2_general_ci", character_set_name: "latin2", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 10, name: "swe7_swedish_ci", character_set_name: "swe7", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 11, name: "ascii_general_ci", character_set_name: "ascii", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 12, name: "ujis_japanese_ci", character_set_name: "ujis", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 13, name: "sjis_japanese_ci", character_set_name: "sjis", mbminlen: 1, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 14, name: "cp1251_bulgarian_ci", character_set_name: "cp1251", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 15, name: "latin1_danish_ci", character_set_name: "latin1", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 16, name: "hebrew_general_ci", character_set_name: "hebrew", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 17, name: "filename", character_set_name: "filename", mbminlen: 1, mbmaxlen: 5) +Innodb::MysqlCollation.add(id: 18, name: "tis620_thai_ci", character_set_name: "tis620", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 19, name: "euckr_korean_ci", character_set_name: "euckr", mbminlen: 1, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 20, name: "latin7_estonian_cs", character_set_name: "latin7", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 21, name: "latin2_hungarian_ci", character_set_name: "latin2", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 22, name: "koi8u_general_ci", character_set_name: "koi8u", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 23, name: "cp1251_ukrainian_ci", character_set_name: "cp1251", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 24, name: "gb2312_chinese_ci", character_set_name: "gb2312", mbminlen: 1, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 25, name: "greek_general_ci", character_set_name: "greek", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 26, name: "cp1250_general_ci", character_set_name: "cp1250", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 27, name: "latin2_croatian_ci", character_set_name: "latin2", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 28, name: "gbk_chinese_ci", character_set_name: "gbk", mbminlen: 1, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 29, name: "cp1257_lithuanian_ci", character_set_name: "cp1257", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 30, name: "latin5_turkish_ci", character_set_name: "latin5", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 31, name: "latin1_german2_ci", character_set_name: "latin1", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 32, name: "armscii8_general_ci", character_set_name: "armscii8", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 33, name: "utf8mb3_general_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 34, name: "cp1250_czech_cs", character_set_name: "cp1250", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 35, name: "ucs2_general_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 36, name: "cp866_general_ci", character_set_name: "cp866", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 37, name: "keybcs2_general_ci", character_set_name: "keybcs2", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 38, name: "macce_general_ci", character_set_name: "macce", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 39, name: "macroman_general_ci", character_set_name: "macroman", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 40, name: "cp852_general_ci", character_set_name: "cp852", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 41, name: "latin7_general_ci", character_set_name: "latin7", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 42, name: "latin7_general_cs", character_set_name: "latin7", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 43, name: "macce_bin", character_set_name: "macce", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 44, name: "cp1250_croatian_ci", character_set_name: "cp1250", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 45, name: "utf8mb4_general_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 46, name: "utf8mb4_bin", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 47, name: "latin1_bin", character_set_name: "latin1", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 48, name: "latin1_general_ci", character_set_name: "latin1", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 49, name: "latin1_general_cs", character_set_name: "latin1", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 50, name: "cp1251_bin", character_set_name: "cp1251", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 51, name: "cp1251_general_ci", character_set_name: "cp1251", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 52, name: "cp1251_general_cs", character_set_name: "cp1251", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 53, name: "macroman_bin", character_set_name: "macroman", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 54, name: "utf16_general_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 55, name: "utf16_bin", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 56, name: "utf16le_general_ci", character_set_name: "utf16le", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 57, name: "cp1256_general_ci", character_set_name: "cp1256", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 58, name: "cp1257_bin", character_set_name: "cp1257", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 59, name: "cp1257_general_ci", character_set_name: "cp1257", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 60, name: "utf32_general_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 61, name: "utf32_bin", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 62, name: "utf16le_bin", character_set_name: "utf16le", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 63, name: "binary", character_set_name: "binary", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 64, name: "armscii8_bin", character_set_name: "armscii8", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 65, name: "ascii_bin", character_set_name: "ascii", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 66, name: "cp1250_bin", character_set_name: "cp1250", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 67, name: "cp1256_bin", character_set_name: "cp1256", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 68, name: "cp866_bin", character_set_name: "cp866", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 69, name: "dec8_bin", character_set_name: "dec8", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 70, name: "greek_bin", character_set_name: "greek", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 71, name: "hebrew_bin", character_set_name: "hebrew", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 72, name: "hp8_bin", character_set_name: "hp8", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 73, name: "keybcs2_bin", character_set_name: "keybcs2", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 74, name: "koi8r_bin", character_set_name: "koi8r", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 75, name: "koi8u_bin", character_set_name: "koi8u", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 76, name: "utf8mb3_tolower_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 77, name: "latin2_bin", character_set_name: "latin2", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 78, name: "latin5_bin", character_set_name: "latin5", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 79, name: "latin7_bin", character_set_name: "latin7", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 80, name: "cp850_bin", character_set_name: "cp850", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 81, name: "cp852_bin", character_set_name: "cp852", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 82, name: "swe7_bin", character_set_name: "swe7", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 83, name: "utf8mb3_bin", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 84, name: "big5_bin", character_set_name: "big5", mbminlen: 1, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 85, name: "euckr_bin", character_set_name: "euckr", mbminlen: 1, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 86, name: "gb2312_bin", character_set_name: "gb2312", mbminlen: 1, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 87, name: "gbk_bin", character_set_name: "gbk", mbminlen: 1, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 88, name: "sjis_bin", character_set_name: "sjis", mbminlen: 1, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 89, name: "tis620_bin", character_set_name: "tis620", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 90, name: "ucs2_bin", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 91, name: "ujis_bin", character_set_name: "ujis", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 92, name: "geostd8_general_ci", character_set_name: "geostd8", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 93, name: "geostd8_bin", character_set_name: "geostd8", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 94, name: "latin1_spanish_ci", character_set_name: "latin1", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 95, name: "cp932_japanese_ci", character_set_name: "cp932", mbminlen: 1, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 96, name: "cp932_bin", character_set_name: "cp932", mbminlen: 1, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 97, name: "eucjpms_japanese_ci", character_set_name: "eucjpms", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 98, name: "eucjpms_bin", character_set_name: "eucjpms", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 99, name: "cp1250_polish_ci", character_set_name: "cp1250", mbminlen: 1, mbmaxlen: 1) +Innodb::MysqlCollation.add(id: 101, name: "utf16_unicode_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 102, name: "utf16_icelandic_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 103, name: "utf16_latvian_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 104, name: "utf16_romanian_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 105, name: "utf16_slovenian_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 106, name: "utf16_polish_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 107, name: "utf16_estonian_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 108, name: "utf16_spanish_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 109, name: "utf16_swedish_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 110, name: "utf16_turkish_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 111, name: "utf16_czech_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 112, name: "utf16_danish_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 113, name: "utf16_lithuanian_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 114, name: "utf16_slovak_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 115, name: "utf16_spanish2_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 116, name: "utf16_roman_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 117, name: "utf16_persian_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 118, name: "utf16_esperanto_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 119, name: "utf16_hungarian_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 120, name: "utf16_sinhala_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 121, name: "utf16_german2_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 122, name: "utf16_croatian_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 123, name: "utf16_unicode_520_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 124, name: "utf16_vietnamese_ci", character_set_name: "utf16", mbminlen: 2, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 128, name: "ucs2_unicode_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 129, name: "ucs2_icelandic_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 130, name: "ucs2_latvian_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 131, name: "ucs2_romanian_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 132, name: "ucs2_slovenian_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 133, name: "ucs2_polish_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 134, name: "ucs2_estonian_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 135, name: "ucs2_spanish_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 136, name: "ucs2_swedish_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 137, name: "ucs2_turkish_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 138, name: "ucs2_czech_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 139, name: "ucs2_danish_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 140, name: "ucs2_lithuanian_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 141, name: "ucs2_slovak_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 142, name: "ucs2_spanish2_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 143, name: "ucs2_roman_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 144, name: "ucs2_persian_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 145, name: "ucs2_esperanto_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 146, name: "ucs2_hungarian_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 147, name: "ucs2_sinhala_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 148, name: "ucs2_german2_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 149, name: "ucs2_croatian_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 150, name: "ucs2_unicode_520_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 151, name: "ucs2_vietnamese_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 159, name: "ucs2_general_mysql500_ci", character_set_name: "ucs2", mbminlen: 2, mbmaxlen: 2) +Innodb::MysqlCollation.add(id: 160, name: "utf32_unicode_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 161, name: "utf32_icelandic_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 162, name: "utf32_latvian_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 163, name: "utf32_romanian_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 164, name: "utf32_slovenian_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 165, name: "utf32_polish_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 166, name: "utf32_estonian_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 167, name: "utf32_spanish_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 168, name: "utf32_swedish_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 169, name: "utf32_turkish_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 170, name: "utf32_czech_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 171, name: "utf32_danish_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 172, name: "utf32_lithuanian_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 173, name: "utf32_slovak_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 174, name: "utf32_spanish2_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 175, name: "utf32_roman_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 176, name: "utf32_persian_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 177, name: "utf32_esperanto_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 178, name: "utf32_hungarian_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 179, name: "utf32_sinhala_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 180, name: "utf32_german2_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 181, name: "utf32_croatian_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 182, name: "utf32_unicode_520_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 183, name: "utf32_vietnamese_ci", character_set_name: "utf32", mbminlen: 4, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 192, name: "utf8mb3_unicode_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 193, name: "utf8mb3_icelandic_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 194, name: "utf8mb3_latvian_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 195, name: "utf8mb3_romanian_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 196, name: "utf8mb3_slovenian_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 197, name: "utf8mb3_polish_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 198, name: "utf8mb3_estonian_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 199, name: "utf8mb3_spanish_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 200, name: "utf8mb3_swedish_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 201, name: "utf8mb3_turkish_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 202, name: "utf8mb3_czech_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 203, name: "utf8mb3_danish_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 204, name: "utf8mb3_lithuanian_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 205, name: "utf8mb3_slovak_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 206, name: "utf8mb3_spanish2_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 207, name: "utf8mb3_roman_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 208, name: "utf8mb3_persian_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 209, name: "utf8mb3_esperanto_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 210, name: "utf8mb3_hungarian_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 211, name: "utf8mb3_sinhala_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 212, name: "utf8mb3_german2_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 213, name: "utf8mb3_croatian_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 214, name: "utf8mb3_unicode_520_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 215, name: "utf8mb3_vietnamese_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 223, name: "utf8mb3_general_mysql500_ci", character_set_name: "utf8mb3", mbminlen: 1, mbmaxlen: 3) +Innodb::MysqlCollation.add(id: 224, name: "utf8mb4_unicode_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 225, name: "utf8mb4_icelandic_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 226, name: "utf8mb4_latvian_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 227, name: "utf8mb4_romanian_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 228, name: "utf8mb4_slovenian_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 229, name: "utf8mb4_polish_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 230, name: "utf8mb4_estonian_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 231, name: "utf8mb4_spanish_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 232, name: "utf8mb4_swedish_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 233, name: "utf8mb4_turkish_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 234, name: "utf8mb4_czech_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 235, name: "utf8mb4_danish_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 236, name: "utf8mb4_lithuanian_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 237, name: "utf8mb4_slovak_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 238, name: "utf8mb4_spanish2_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 239, name: "utf8mb4_roman_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 240, name: "utf8mb4_persian_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 241, name: "utf8mb4_esperanto_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 242, name: "utf8mb4_hungarian_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 243, name: "utf8mb4_sinhala_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 244, name: "utf8mb4_german2_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 245, name: "utf8mb4_croatian_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 246, name: "utf8mb4_unicode_520_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 247, name: "utf8mb4_vietnamese_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 248, name: "gb18030_chinese_ci", character_set_name: "gb18030", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 249, name: "gb18030_bin", character_set_name: "gb18030", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 250, name: "gb18030_unicode_520_ci", character_set_name: "gb18030", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 255, name: "utf8mb4_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 256, name: "utf8mb4_de_pb_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 257, name: "utf8mb4_is_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 258, name: "utf8mb4_lv_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 259, name: "utf8mb4_ro_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 260, name: "utf8mb4_sl_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 261, name: "utf8mb4_pl_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 262, name: "utf8mb4_et_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 263, name: "utf8mb4_es_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 264, name: "utf8mb4_sv_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 265, name: "utf8mb4_tr_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 266, name: "utf8mb4_cs_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 267, name: "utf8mb4_da_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 268, name: "utf8mb4_lt_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 269, name: "utf8mb4_sk_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 270, name: "utf8mb4_es_trad_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 271, name: "utf8mb4_la_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 273, name: "utf8mb4_eo_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 274, name: "utf8mb4_hu_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 275, name: "utf8mb4_hr_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 277, name: "utf8mb4_vi_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 278, name: "utf8mb4_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 279, name: "utf8mb4_de_pb_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 280, name: "utf8mb4_is_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 281, name: "utf8mb4_lv_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 282, name: "utf8mb4_ro_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 283, name: "utf8mb4_sl_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 284, name: "utf8mb4_pl_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 285, name: "utf8mb4_et_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 286, name: "utf8mb4_es_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 287, name: "utf8mb4_sv_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 288, name: "utf8mb4_tr_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 289, name: "utf8mb4_cs_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 290, name: "utf8mb4_da_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 291, name: "utf8mb4_lt_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 292, name: "utf8mb4_sk_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 293, name: "utf8mb4_es_trad_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 294, name: "utf8mb4_la_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 296, name: "utf8mb4_eo_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 297, name: "utf8mb4_hu_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 298, name: "utf8mb4_hr_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 300, name: "utf8mb4_vi_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 303, name: "utf8mb4_ja_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 304, name: "utf8mb4_ja_0900_as_cs_ks", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 305, name: "utf8mb4_0900_as_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 306, name: "utf8mb4_ru_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 307, name: "utf8mb4_ru_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 308, name: "utf8mb4_zh_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 309, name: "utf8mb4_0900_bin", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 310, name: "utf8mb4_nb_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 311, name: "utf8mb4_nb_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 312, name: "utf8mb4_nn_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 313, name: "utf8mb4_nn_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 314, name: "utf8mb4_sr_latn_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 315, name: "utf8mb4_sr_latn_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 316, name: "utf8mb4_bs_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 317, name: "utf8mb4_bs_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 318, name: "utf8mb4_bg_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 319, name: "utf8mb4_bg_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 320, name: "utf8mb4_gl_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 321, name: "utf8mb4_gl_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 322, name: "utf8mb4_mn_cyrl_0900_ai_ci", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) +Innodb::MysqlCollation.add(id: 323, name: "utf8mb4_mn_cyrl_0900_as_cs", character_set_name: "utf8mb4", mbminlen: 1, mbmaxlen: 4) diff --git a/lib/innodb/mysql_type.rb b/lib/innodb/mysql_type.rb new file mode 100644 index 00000000..46298142 --- /dev/null +++ b/lib/innodb/mysql_type.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module Innodb + class MysqlType + attr_reader :mysql_field_type_value + attr_reader :sdi_column_type_value + attr_reader :type + attr_reader :handle_as + + def initialize(type:, mysql_field_type_value:, sdi_column_type_value:, handle_as: nil) + @mysql_field_type_value = mysql_field_type_value + @sdi_column_type_value = sdi_column_type_value + @type = type + @handle_as = handle_as || type + end + + # A hash of MySQL's internal type system to the stored + # values for those types, and the 'external' SQL type. + TYPES = [ + new(type: :DECIMAL, mysql_field_type_value: 0, sdi_column_type_value: 1), + new(type: :TINYINT, mysql_field_type_value: 1, sdi_column_type_value: 2), + new(type: :SMALLINT, mysql_field_type_value: 2, sdi_column_type_value: 3), + new(type: :INT, mysql_field_type_value: 3, sdi_column_type_value: 4), + new(type: :FLOAT, mysql_field_type_value: 4, sdi_column_type_value: 5), + new(type: :DOUBLE, mysql_field_type_value: 5, sdi_column_type_value: 6), + new(type: :TYPE_NULL, mysql_field_type_value: 6, sdi_column_type_value: 7), + new(type: :TIMESTAMP, mysql_field_type_value: 7, sdi_column_type_value: 8), + new(type: :BIGINT, mysql_field_type_value: 8, sdi_column_type_value: 9), + new(type: :MEDIUMINT, mysql_field_type_value: 9, sdi_column_type_value: 10), + new(type: :DATE, mysql_field_type_value: 10, sdi_column_type_value: 11), + new(type: :TIME, mysql_field_type_value: 11, sdi_column_type_value: 12), + new(type: :DATETIME, mysql_field_type_value: 12, sdi_column_type_value: 13), + new(type: :YEAR, mysql_field_type_value: 13, sdi_column_type_value: 14), + new(type: :DATE, mysql_field_type_value: 14, sdi_column_type_value: 15), + new(type: :VARCHAR, mysql_field_type_value: 15, sdi_column_type_value: 16), + new(type: :BIT, mysql_field_type_value: 16, sdi_column_type_value: 17), + new(type: :TIMESTAMP2, mysql_field_type_value: 17, sdi_column_type_value: 18), + new(type: :DATETIME2, mysql_field_type_value: 18, sdi_column_type_value: 19), + new(type: :TIME2, mysql_field_type_value: 19, sdi_column_type_value: 20), + new(type: :NEWDECIMAL, mysql_field_type_value: 246, sdi_column_type_value: 21, handle_as: :CHAR), + new(type: :ENUM, mysql_field_type_value: 247, sdi_column_type_value: 22), + new(type: :SET, mysql_field_type_value: 248, sdi_column_type_value: 23), + new(type: :TINYBLOB, mysql_field_type_value: 249, sdi_column_type_value: 24), + new(type: :MEDIUMBLOB, mysql_field_type_value: 250, sdi_column_type_value: 25), + new(type: :LONGBLOB, mysql_field_type_value: 251, sdi_column_type_value: 26), + new(type: :BLOB, mysql_field_type_value: 252, sdi_column_type_value: 27), + new(type: :VARCHAR, mysql_field_type_value: 253, sdi_column_type_value: 28), + new(type: :CHAR, mysql_field_type_value: 254, sdi_column_type_value: 29), + new(type: :GEOMETRY, mysql_field_type_value: 255, sdi_column_type_value: 30), + new(type: :JSON, mysql_field_type_value: 245, sdi_column_type_value: 31), + ].freeze + + # A hash of types by mysql_field_type_value. + TYPES_BY_MYSQL_FIELD_TYPE_VALUE = Innodb::MysqlType::TYPES.to_h { |t| [t.mysql_field_type_value, t] }.freeze + + # A hash of types by sdi_column_type_value. + TYPES_BY_SDI_COLUMN_TYPE_VALUE = Innodb::MysqlType::TYPES.to_h { |t| [t.sdi_column_type_value, t] }.freeze + + def self.by_mysql_field_type(value) + TYPES_BY_MYSQL_FIELD_TYPE_VALUE[value] + end + + def self.by_sdi_column_type(value) + TYPES_BY_SDI_COLUMN_TYPE_VALUE[value] + end + end +end diff --git a/lib/innodb/page.rb b/lib/innodb/page.rb index c2f9b02a..b4e7fca6 100644 --- a/lib/innodb/page.rb +++ b/lib/innodb/page.rb @@ -1,493 +1,560 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true + +require "forwardable" # A generic class for any type of page, which handles reading the common # FIL header and trailer, and can handle (via #parse) dispatching to a more # specialized class depending on page type (which comes from the FIL header). # A page being handled by Innodb::Page indicates that its type is not currently # handled by any more specialized class. -class Innodb::Page - # A hash of page types to specialized classes to handle them. Normally - # subclasses will register themselves in this list. - SPECIALIZED_CLASSES = {} - - # Load a page as a generic page in order to make the "fil" header accessible, - # and then attempt to hand off the page to a specialized class to be - # re-parsed if possible. If there is no specialized class for this type - # of page, return the generic object. - # - # This could be optimized to reach into the page buffer and efficiently - # extract the page type in order to avoid throwing away a generic - # Innodb::Page object when parsing every specialized page, but this is - # a bit cleaner, and we're not particularly performance sensitive. - def self.parse(space, buffer, page_number=nil) - # Create a page object as a generic page. - page = Innodb::Page.new(space, buffer, page_number) - - # If there is a specialized class available for this page type, re-create - # the page object using that specialized class. - if specialized_class = SPECIALIZED_CLASSES[page.type] - page = specialized_class.handle(page, space, buffer, page_number) - end - - page - end - - # Allow the specialized class to do something that isn't 'new' with this page. - def self.handle(page, space, buffer, page_number=nil) - self.new(space, buffer, page_number) - end - - # Initialize a page by passing in a buffer containing the raw page contents. - # The buffer size should match the space's page size. - def initialize(space, buffer, page_number=nil) - unless space && buffer - raise "Page can't be initialized from nil space or buffer (space: #{space}, buffer: #{buffer})" +module Innodb + class Page + extend Forwardable + + Address = Struct.new( + :page, + :offset, + keyword_init: true + ) + + FilHeader = Struct.new( + :checksum, + :offset, + :prev, + :next, + :lsn, + :type, + :flush_lsn, + :space_id, + keyword_init: true + ) + + class FilHeader + def lsn_low32 + lsn & 0xffffffff + end end - unless space.page_size == buffer.size - raise "Buffer size #{buffer.size} is different than space page size" + FilTrailer = Struct.new( + :checksum, + :lsn_low32, + keyword_init: true + ) + + Region = Struct.new( + :offset, + :length, # rubocop:disable Lint/StructNewOverride + :name, + :info, + keyword_init: true + ) + + # A hash of page types to specialized classes to handle them. Normally + # subclasses will register themselves in this list. + @specialized_classes = {} + + class << self + attr_reader :specialized_classes end - @space = space - @buffer = buffer - @page_number = page_number - end - - attr_reader :space - - # Return the page size, to eventually be able to deal with non-16kB pages. - def size - @size ||= @buffer.size - end - - # Return a simple string to uniquely identify this page within the space. - # Be careful not to call anything which would instantiate a BufferCursor - # so that we can use this method in cursor initialization. - def name - page_offset = BinData::Uint32be.read(@buffer.slice(4, 4)) - page_type = BinData::Uint16be.read(@buffer.slice(24, 2)) - "%i,%s" % [ - page_offset, - PAGE_TYPE_BY_VALUE[page_type], - ] - end - - # If no block is passed, return an BufferCursor object positioned at a - # specific offset. If a block is passed, create a cursor at the provided - # offset and yield it to the provided block one time, and then return the - # return value of the block. - def cursor(buffer_offset) - new_cursor = BufferCursor.new(@buffer, buffer_offset) - new_cursor.push_name("space[#{space.name}]") - new_cursor.push_name("page[#{name}]") - - if block_given? - # Call the block once and return its return value. - yield new_cursor - else - # Return the cursor itself. - new_cursor + def self.register_specialization(page_type, specialized_class) + @specialized_classes[page_type] = specialized_class end - end - - # Return the byte offset of the start of the "fil" header, which is at the - # beginning of the page. Included here primarily for completeness. - def pos_fil_header - 0 - end - # Return the size of the "fil" header, in bytes. - def size_fil_header - 4 + 4 + 4 + 4 + 8 + 2 + 8 + 4 - end + def self.specialization_for(page_type) + # This needs to intentionally use Innodb::Page because we need to register + # in the class instance variable in *that* class. + Innodb::Page.register_specialization(page_type, self) + end - # The start of the checksummed portion of the file header. - def pos_partial_page_header - pos_fil_header + 4 - end + def self.specialization_for?(page_type) + Innodb::Page.specialized_classes.include?(page_type) + end - # The size of the portion of the fil header that is included in the - # checksum. Exclude the following: - # :checksum (offset 4, size 4) - # :flush_lsn (offset 26, size 8) - # :space_id (offset 34, size 4) - def size_partial_page_header - size_fil_header - 4 - 8 - 4 - end + # Load a page as a generic page in order to make the "fil" header accessible, + # and then attempt to hand off the page to a specialized class to be + # re-parsed if possible. If there is no specialized class for this type + # of page, return the generic object. + # + # This could be optimized to reach into the page buffer and efficiently + # extract the page type in order to avoid throwing away a generic + # Innodb::Page object when parsing every specialized page, but this is + # a bit cleaner, and we're not particularly performance sensitive. + def self.parse(space, buffer, page_number = nil) + # Create a page object as a generic page. + page = Innodb::Page.new(space, buffer, page_number) + + # If there is a specialized class available for this page type, re-create + # the page object using that specialized class. + if (specialized_class = specialized_classes[page.type]) + page = specialized_class.handle(page, space, buffer, page_number) + end - # Return the byte offset of the start of the "fil" trailer, which is at - # the end of the page. - def pos_fil_trailer - size - size_fil_trailer - end + page + end - # Return the size of the "fil" trailer, in bytes. - def size_fil_trailer - 4 + 4 - end + # Allow the specialized class to do something that isn't 'new' with this page. + def self.handle(_page, space, buffer, page_number = nil) + new(space, buffer, page_number) + end - # Return the position of the "body" of the page, which starts after the FIL - # header. - def pos_page_body - pos_fil_header + size_fil_header - end + # Initialize a page by passing in a buffer containing the raw page contents. + # The buffer size should match the space's page size. + def initialize(space, buffer, page_number = nil) + raise "Page can't be initialized from nil space or buffer (space: #{space}, buffer: #{buffer})" unless buffer + raise "Buffer size #{buffer.size} is different than space page size" if space && space.page_size != buffer.size - # Return the size of the page body, excluding the header and trailer. - def size_page_body - size - size_fil_trailer - size_fil_header - end + @space = space + @buffer = buffer + @page_number = page_number + end - # InnoDB Page Type constants from include/fil0fil.h. - PAGE_TYPE = { - :ALLOCATED => { - :value => 0, - :description => "Freshly allocated", - :usage => "page type field has not been initialized", - }, - :UNDO_LOG => { - :value => 2, - :description => "Undo log", - :usage => "stores previous values of modified records", - }, - :INODE => { - :value => 3, - :description => "File segment inode", - :usage => "bookkeeping for file segments", - }, - :IBUF_FREE_LIST => { - :value => 4, - :description => "Insert buffer free list", - :usage => "bookkeeping for insert buffer free space management", - }, - :IBUF_BITMAP => { - :value => 5, - :description => "Insert buffer bitmap", - :usage => "bookkeeping for insert buffer writes to be merged", - }, - :SYS => { - :value => 6, - :description => "System internal", - :usage => "used for various purposes in the system tablespace", - }, - :TRX_SYS => { - :value => 7, - :description => "Transaction system header", - :usage => "bookkeeping for the transaction system in system tablespace", - }, - :FSP_HDR => { - :value => 8, - :description => "File space header", - :usage => "header page (page 0) for each tablespace file", - }, - :XDES => { - :value => 9, - :description => "Extent descriptor", - :usage => "header page for subsequent blocks of 16,384 pages", - }, - :BLOB => { - :value => 10, - :description => "Uncompressed BLOB", - :usage => "externally-stored uncompressed BLOB column data", - }, - :ZBLOB => { - :value => 11, - :description => "First compressed BLOB", - :usage => "externally-stored compressed BLOB column data, first page", - }, - :ZBLOB2 => { - :value => 12, - :description => "Subsequent compressed BLOB", - :usage => "externally-stored compressed BLOB column data, subsequent page", - }, - :INDEX => { - :value => 17855, - :description => "B+Tree index", - :usage => "table and index data stored in B+Tree structure", - }, - } - - PAGE_TYPE_BY_VALUE = PAGE_TYPE.inject({}) { |h, (k, v)| h[v[:value]] = k; h } - - # A helper to convert "undefined" values stored in previous and next pointers - # in the page header to nil. - def self.maybe_undefined(value) - value == 4294967295 ? nil : value - end + attr_reader :space - # Return the "fil" header from the page, which is common for all page types. - def fil_header - @fil_header ||= cursor(pos_fil_header).name("fil_header") do |c| - { - :checksum => c.name("checksum") { c.get_uint32 }, - :offset => c.name("offset") { c.get_uint32 }, - :prev => c.name("prev") { - Innodb::Page.maybe_undefined(c.get_uint32) - }, - :next => c.name("next") { - Innodb::Page.maybe_undefined(c.get_uint32) - }, - :lsn => c.name("lsn") { c.get_uint64 }, - :type => c.name("type") { PAGE_TYPE_BY_VALUE[c.get_uint16] }, - :flush_lsn => c.name("flush_lsn") { c.get_uint64 }, - :space_id => c.name("space_id") { c.get_uint32 }, - } + # Return the page size, to eventually be able to deal with non-16kB pages. + def size + @size ||= @buffer.size end - end - # Return the "fil" trailer from the page, which is common for all page types. - def fil_trailer - @fil_trailer ||= cursor(pos_fil_trailer).name("fil_trailer") do |c| - { - :checksum => c.name("checksum") { c.get_uint32 }, - :lsn_low32 => c.name("lsn_low32") { c.get_uint32 }, - } + def default_page_size? + size == Innodb::Space::DEFAULT_PAGE_SIZE end - end - # A helper function to return the checksum from the "fil" header, for easier - # access. - def checksum - fil_header[:checksum] - end + # Return a simple string to uniquely identify this page within the space. + # Be careful not to call anything which would instantiate a BufferCursor + # so that we can use this method in cursor initialization. + def name + page_offset = BinData::Uint32be.read(@buffer.slice(4, 4)) + page_type = BinData::Uint16be.read(@buffer.slice(24, 2)) + "%i,%s" % [ + page_offset, + PAGE_TYPE_BY_VALUE[page_type], + ] + end - # A helper function to return the checksum from the "fil" trailer, for easier - # access. - def checksum_trailer - fil_trailer[:checksum] - end + # If no block is passed, return an BufferCursor object positioned at a + # specific offset. If a block is passed, create a cursor at the provided + # offset and yield it to the provided block one time, and then return the + # return value of the block. + def cursor(buffer_offset) + new_cursor = BufferCursor.new(@buffer, buffer_offset) + new_cursor.push_name("space[#{space&.name || 'unknown'}]") + new_cursor.push_name("page[#{name}]") + + if block_given? + # Call the block once and return its return value. + yield new_cursor + else + # Return the cursor itself. + new_cursor + end + end - # A helper function to return the page offset from the "fil" header, for - # easier access. - def offset - fil_header[:offset] - end + # Return the byte offset of the start of the "fil" header, which is at the + # beginning of the page. Included here primarily for completeness. + def pos_fil_header + 0 + end - # A helper function to return the page number of the logical previous page - # (from the doubly-linked list from page to page) from the "fil" header, - # for easier access. - def prev - fil_header[:prev] - end + # Return the size of the "fil" header, in bytes. + def size_fil_header + 4 + 4 + 4 + 4 + 8 + 2 + 8 + 4 + end - # A helper function to return the page number of the logical next page - # (from the doubly-linked list from page to page) from the "fil" header, - # for easier access. - def next - fil_header[:next] - end + # The start of the checksummed portion of the file header. + def pos_partial_page_header + pos_fil_header + 4 + end - # A helper function to return the LSN from the page header, for easier access. - def lsn - fil_header[:lsn] - end + # The size of the portion of the fil header that is included in the + # checksum. Exclude the following: + # :checksum (offset 4, size 4) + # :flush_lsn (offset 26, size 8) + # :space_id (offset 34, size 4) + def size_partial_page_header + size_fil_header - 4 - 8 - 4 + end - # A helper function to return the low 32 bits of the LSN from the page header - # for use in comparing to the low 32 bits stored in the trailer. - def lsn_low32_header - fil_header[:lsn] & 0xffffffff - end + # Return the byte offset of the start of the "fil" trailer, which is at + # the end of the page. + def pos_fil_trailer + size - size_fil_trailer + end - # A helper function to return the low 32 bits of the LSN as stored in the page - # trailer. - def lsn_low32_trailer - fil_trailer[:lsn_low32] - end + # Return the size of the "fil" trailer, in bytes. + def size_fil_trailer + 4 + 4 + end - # A helper function to return the page type from the "fil" header, for easier - # access. - def type - fil_header[:type] - end + # Return the position of the "body" of the page, which starts after the FIL + # header. + def pos_page_body + pos_fil_header + size_fil_header + end - # A helper function to return the space ID from the "fil" header, for easier - # access. - def space_id - fil_header[:space_id] - end + # Return the size of the page body, excluding the header and trailer. + def size_page_body + size - size_fil_trailer - size_fil_header + end - # Iterate each byte of the FIL header. - def each_page_header_byte_as_uint8 - unless block_given? - return enum_for(:each_page_header_byte_as_uint8) + # InnoDB Page Type constants from include/fil0fil.h. + PAGE_TYPE = { + ALLOCATED: { + value: 0, + description: "Freshly allocated", + }, + UNDO_LOG: { + value: 2, + description: "Undo log", + }, + INODE: { + value: 3, + description: "File segment inode", + }, + IBUF_FREE_LIST: { + value: 4, + description: "Insert buffer free list", + }, + IBUF_BITMAP: { + value: 5, + description: "Insert buffer bitmap", + }, + SYS: { + value: 6, + description: "System internal", + }, + TRX_SYS: { + value: 7, + description: "Transaction system header", + }, + FSP_HDR: { + value: 8, + description: "File space header", + }, + XDES: { + value: 9, + description: "Extent descriptor", + }, + BLOB: { + value: 10, + description: "Uncompressed BLOB", + }, + ZBLOB: { + value: 11, + description: "First compressed BLOB", + }, + ZBLOB2: { + value: 12, + description: "Subsequent compressed BLOB", + }, + UNKNOWN: { + value: 13, + description: "Unknown", + }, + COMPRESSED: { + value: 14, + description: "Compressed", + }, + ENCRYPTED: { + value: 15, + description: "Encrypted", + }, + COMPRESSED_AND_ENCRYPTED: { + value: 16, + description: "Compressed and Encrypted", + }, + ENCRYPTED_RTREE: { + value: 17, + description: "Encrypted R-tree", + }, + SDI_BLOB: { + value: 18, + description: "Uncompressed SDI BLOB", + }, + SDI_ZBLOB: { + value: 19, + description: "Compressed SDI BLOB", + }, + LEGACY_DBLWR: { + value: 20, + description: "Legacy doublewrite buffer", + }, + RSEG_ARRAY: { + value: 21, + description: "Rollback Segment Array", + }, + LOB_INDEX: { + value: 22, + description: "Index of uncompressed LOB", + }, + LOB_DATA: { + value: 23, + description: "Data of uncompressed LOB", + }, + LOB_FIRST: { + value: 24, + description: "First page of an uncompressed LOB", + }, + ZLOB_FIRST: { + value: 25, + description: "First page of a compressed LOB", + }, + ZLOB_DATA: { + value: 26, + description: "Data of compressed LOB", + }, + ZLOB_INDEX: { + value: 27, + description: "Index of compressed LOB", + }, + ZLOB_FRAG: { + value: 28, + description: "Fragment of compressed LOB", + }, + ZLOB_FRAG_ENTRY: { + value: 29, + description: "Index of fragment for compressed LOB", + }, + SDI: { + value: 17_853, + description: "Serialized Dictionary Information", + }, + RTREE: { + value: 17_854, + description: "R-tree index", + }, + INDEX: { + value: 17_855, + description: "B+Tree index", + }, + }.freeze + + PAGE_TYPE_BY_VALUE = PAGE_TYPE.each_with_object({}) { |(k, v), h| h[v[:value]] = k } + + # A page number representing "undefined" values, (4294967295). + UNDEFINED_PAGE_NUMBER = (2**32) - 1 + + # A helper to check if a page number is the undefined page number. + def self.undefined?(page_number) + page_number == UNDEFINED_PAGE_NUMBER end - cursor(pos_partial_page_header). - each_byte_as_uint8(size_partial_page_header) do |byte| - yield byte + # A helper to convert "undefined" values stored in previous and next pointers + # in the page header to nil. + def self.maybe_undefined(page_number) + page_number unless undefined?(page_number) end - end - # Iterate each byte of the page body, except for the FIL header and - # the FIL trailer. - def each_page_body_byte_as_uint8 - unless block_given? - return enum_for(:each_page_body_byte_as_uint8) + def self.page_type_by_value(value) + PAGE_TYPE_BY_VALUE[value] || value end - cursor(pos_page_body). - each_byte_as_uint8(size_page_body) do |byte| - yield byte + # Return the "fil" header from the page, which is common for all page types. + def fil_header + @fil_header ||= cursor(pos_fil_header).name("fil_header") do |c| + FilHeader.new( + checksum: c.name("checksum") { c.read_uint32 }, + offset: c.name("offset") { c.read_uint32 }, + prev: c.name("prev") { Innodb::Page.maybe_undefined(c.read_uint32) }, + next: c.name("next") { Innodb::Page.maybe_undefined(c.read_uint32) }, + lsn: c.name("lsn") { c.read_uint64 }, + type: c.name("type") { Innodb::Page.page_type_by_value(c.read_uint16) }, + flush_lsn: c.name("flush_lsn") { c.read_uint64 }, + space_id: c.name("space_id") { c.read_uint32 } + ) + end end - end - # Calculate the checksum of the page using InnoDB's algorithm. - def checksum_innodb - unless size == 16384 - raise "Checksum calculation is only supported for 16 KiB pages" + # Return the "fil" trailer from the page, which is common for all page types. + def fil_trailer + @fil_trailer ||= cursor(pos_fil_trailer).name("fil_trailer") do |c| + FilTrailer.new( + checksum: c.name("checksum") { c.read_uint32 }, + lsn_low32: c.name("lsn_low32") { c.read_uint32 } + ) + end end - @checksum_innodb ||= begin - # Calculate the InnoDB checksum of the page header. - c_partial_header = Innodb::Checksum.fold_enumerator(each_page_header_byte_as_uint8) + def_delegator :fil_header, :checksum + def_delegator :fil_header, :offset + def_delegator :fil_header, :prev + def_delegator :fil_header, :next + def_delegator :fil_header, :lsn + def_delegator :fil_header, :type + def_delegator :fil_header, :space_id - # Calculate the InnoDB checksum of the page body. - c_page_body = Innodb::Checksum.fold_enumerator(each_page_body_byte_as_uint8) + # Iterate each byte of the FIL header. + def each_page_header_byte_as_uint8(&block) + return enum_for(:each_page_header_byte_as_uint8) unless block_given? - # Add the two checksums together, and mask the result back to 32 bits. - (c_partial_header + c_page_body) & Innodb::Checksum::MAX + cursor(pos_partial_page_header).each_byte_as_uint8(size_partial_page_header, &block) end - end - def checksum_innodb? - checksum == checksum_innodb - end + # Iterate each byte of the page body, except for the FIL header and + # the FIL trailer. + def each_page_body_byte_as_uint8(&block) + return enum_for(:each_page_body_byte_as_uint8) unless block_given? - # Calculate the checksum of the page using the CRC32c algorithm. - def checksum_crc32 - unless size == 16384 - raise "Checksum calculation is only supported for 16 KiB pages" + cursor(pos_page_body).each_byte_as_uint8(size_page_body, &block) end - @checksum_crc32 ||= begin - # Calculate the CRC32c of the page header. - crc_partial_header = Digest::CRC32c.new - each_page_header_byte_as_uint8 do |byte| - crc_partial_header << byte.chr + # Calculate the checksum of the page using InnoDB's algorithm. + def checksum_innodb + raise "Checksum calculation is only supported for 16 KiB pages" unless default_page_size? + + @checksum_innodb ||= begin + # Calculate the InnoDB checksum of the page header. + c_partial_header = Innodb::Checksum.fold_enumerator(each_page_header_byte_as_uint8) + + # Calculate the InnoDB checksum of the page body. + c_page_body = Innodb::Checksum.fold_enumerator(each_page_body_byte_as_uint8) + + # Add the two checksums together, and mask the result back to 32 bits. + (c_partial_header + c_page_body) & Innodb::Checksum::MAX end + end + + def checksum_innodb? + checksum == checksum_innodb + end - # Calculate the CRC32c of the page body. - crc_page_body = Digest::CRC32c.new - each_page_body_byte_as_uint8 do |byte| - crc_page_body << byte.chr + # Calculate the checksum of the page using the CRC32c algorithm. + def checksum_crc32 + raise "Checksum calculation is only supported for 16 KiB pages" unless default_page_size? + + @checksum_crc32 ||= begin + # Calculate the CRC32c of the page header. + crc_partial_header = Digest::CRC32c.new + each_page_header_byte_as_uint8 do |byte| + crc_partial_header << byte.chr + end + + # Calculate the CRC32c of the page body. + crc_page_body = Digest::CRC32c.new + each_page_body_byte_as_uint8 do |byte| + crc_page_body << byte.chr + end + + # Bitwise XOR the two checksums together. + crc_partial_header.checksum ^ crc_page_body.checksum end + end - # Bitwise XOR the two checksums together. - crc_partial_header.checksum ^ crc_page_body.checksum + def checksum_crc32? + checksum == checksum_crc32 end - end - def checksum_crc32? - checksum == checksum_crc32 - end + # Is the page checksum correct? + def checksum_valid? + checksum_crc32? || checksum_innodb? + end - # Is the page checksum correct? - def checksum_valid? - checksum_crc32? || checksum_innodb? - end + # Is the page checksum incorrect? + def checksum_invalid? + !checksum_valid? + end - # Is the page checksum incorrect? - def checksum_invalid? - !checksum_valid? - end + def checksum_type + return :crc32 if checksum_crc32? + return :innodb if checksum_innodb? - def checksum_type - case - when checksum_crc32? - :crc32 - when checksum_innodb? - :innodb + nil end - end - # Is the LSN stored in the header different from the one stored in the - # trailer? - def torn? - lsn_low32_header != lsn_low32_trailer - end + # Is the LSN stored in the header different from the one stored in the + # trailer? + def torn? + fil_header.lsn_low32 != fil_trailer.lsn_low32 + end - # Is the page in the doublewrite buffer? - def in_doublewrite_buffer? - space && space.system_space? && space.doublewrite_page?(offset) - end + # Is the page in the doublewrite buffer? + def in_doublewrite_buffer? + space&.system_space? && space.doublewrite_page?(offset) + end - # Is the space ID stored in the header different from that of the space - # provided when initializing this page? - def misplaced_space? - space && (space_id != space.space_id) - end + # Is the space ID stored in the header different from that of the space + # provided when initializing this page? + def misplaced_space? + space && (space_id != space.space_id) + end - # Is the page number stored in the header different from the page number - # which was supposed to be read? - def misplaced_offset? - offset != @page_number - end + # Is the page number stored in the header different from the page number + # which was supposed to be read? + def misplaced_offset? + offset != @page_number + end - # Is the page misplaced in the wrong file or by offset in the file? - def misplaced? - !in_doublewrite_buffer? && (misplaced_space? || misplaced_offset?) - end + # Is the page misplaced in the wrong file or by offset in the file? + def misplaced? + !in_doublewrite_buffer? && (misplaced_space? || misplaced_offset?) + end - # Is the page corrupt, either due to data corruption, tearing, or in the - # wrong place? - def corrupt? - checksum_invalid? || torn? || misplaced? - end + # Is the page corrupt, either due to data corruption, tearing, or in the + # wrong place? + def corrupt? + checksum_invalid? || torn? || misplaced? + end - def each_region - unless block_given? - return enum_for(:each_region) + # Is this an extent descriptor page (either FSP_HDR or XDES)? + def extent_descriptor? + type == :FSP_HDR || type == :XDES end - yield({ - :offset => pos_fil_header, - :length => size_fil_header, - :name => :fil_header, - :info => "FIL Header", - }) + def each_region + return enum_for(:each_region) unless block_given? - yield({ - :offset => pos_fil_trailer, - :length => size_fil_trailer, - :name => :fil_trailer, - :info => "FIL Trailer", - }) + yield Region.new( + offset: pos_fil_header, + length: size_fil_header, + name: :fil_header, + info: "FIL Header" + ) - nil - end + yield Region.new( + offset: pos_fil_trailer, + length: size_fil_trailer, + name: :fil_trailer, + info: "FIL Trailer" + ) - # Implement a custom inspect method to avoid irb printing the contents of - # the page buffer, since it's very large and mostly not interesting. - def inspect - if fil_header - "#<%s: size=%i, space_id=%i, offset=%i, type=%s, prev=%s, next=%s, checksum_valid?=%s (%s), torn?=%s, misplaced?=%s>" % [ - self.class, - size, - fil_header[:space_id], - fil_header[:offset], - fil_header[:type], - fil_header[:prev] || "nil", - fil_header[:next] || "nil", - checksum_valid?, - checksum_type ? checksum_type : "unknown", - torn?, - misplaced?, - ] - else - "#<#{self.class}>" + nil end - end - # Dump the contents of a page for debugging purposes. - def dump - puts "#{self}:" - puts + def inspect_header_fields + return nil unless fil_header + + %i[ + size + space_id + offset + type + prev + next + checksum_valid? + checksum_type + torn? + misplaced? + ].map { |m| "#{m}=#{send(m).inspect}" }.join(", ") + end + + # Implement a custom inspect method to avoid irb printing the contents of + # the page buffer, since it's very large and mostly not interesting. + def inspect + "#<#{self.class} #{inspect_header_fields || '(page header unavailable)'}>" + end - puts "fil header:" - pp fil_header - puts + # Dump the contents of a page for debugging purposes. + def dump + puts "#{self}:" + puts - puts "fil trailer:" - pp fil_trailer - puts + puts "fil header:" + pp fil_header + puts + + puts "fil trailer:" + pp fil_trailer + puts + end end end diff --git a/lib/innodb/page/blob.rb b/lib/innodb/page/blob.rb index 2db61ab6..3dfc4a17 100644 --- a/lib/innodb/page/blob.rb +++ b/lib/innodb/page/blob.rb @@ -1,86 +1,79 @@ -# -*- encoding : utf-8 -*- - -class Innodb::Page::Blob < Innodb::Page - def pos_blob_header - pos_page_body - end - - def size_blob_header - 4 + 4 - end - - def pos_blob_data - pos_blob_header + size_blob_header - end - - def blob_header - cursor(pos_blob_header).name("blob_header") do |c| - { - :length => c.name("length") { c.get_uint32 }, - :next => c.name("next") { Innodb::Page.maybe_undefined(c.get_uint32) }, - } - end - end - - def blob_data - cursor(pos_blob_data).name("blob_data") do |c| - c.get_bytes(blob_header[:length]) - end - end - - def dump_hex(string) - slice_size = 16 - bytes = string.split("").map { |s| s.ord } - string.split("").each_slice(slice_size).each_with_index do |slice_bytes, slice_count| - puts "%08i %-23s %-23s |%-16s|" % [ - (slice_count * slice_size), - slice_bytes[0..8].map { |n| "%02x" % n.ord }.join(" "), - slice_bytes[8..16].map { |n| "%02x" % n.ord }.join(" "), - slice_bytes.join(""), - ] +# frozen_string_literal: true + +module Innodb + class Page + class Blob < Page + specialization_for :BLOB + + def pos_blob_header + pos_page_body + end + + def size_blob_header + 4 + 4 + end + + def pos_blob_data + pos_blob_header + size_blob_header + end + + def blob_header + cursor(pos_blob_header).name("blob_header") do |c| + { + length: c.name("length") { c.read_uint32 }, + next: c.name("next") { Innodb::Page.maybe_undefined(c.read_uint32) }, + } + end + end + + def blob_data + cursor(pos_blob_data).name("blob_data") do |c| + c.read_bytes(blob_header[:length]) + end + end + + def next_blob_page + return unless blob_header[:next] + + space.page(blob_header[:next]) + end + + def each_region(&block) + return enum_for(:each_region) unless block_given? + + super + + yield Region.new( + offset: pos_blob_header, + length: size_blob_header, + name: :blob_header, + info: "Blob Header" + ) + + yield Region.new( + offset: pos_blob_data, + length: blob_header[:length], + name: :blob_data, + info: "Blob Data" + ) + + nil + end + + # Dump the contents of a page for debugging purposes. + def dump + super + + puts "blob header:" + pp blob_header + puts + + puts "blob data:" + HexFormat.puts(blob_data) + puts + + puts + end end end - - def each_region - unless block_given? - return enum_for(:each_region) - end - - super do |region| - yield region - end - - yield({ - :offset => pos_blob_header, - :length => size_blob_header, - :name => :blob_header, - :info => "Blob Header", - }) - - yield({ - :offset => pos_blob_data, - :length => blob_header[:length], - :name => :blob_data, - :info => "Blob Data", - }) - - nil - end - - # Dump the contents of a page for debugging purposes. - def dump - super - - puts "blob header:" - pp blob_header - puts - - puts "blob data:" - dump_hex(blob_data) - puts - - puts - end end - -Innodb::Page::SPECIALIZED_CLASSES[:BLOB] = Innodb::Page::Blob diff --git a/lib/innodb/page/fsp_hdr_xdes.rb b/lib/innodb/page/fsp_hdr_xdes.rb index aeb4c3fe..16f24a51 100644 --- a/lib/innodb/page/fsp_hdr_xdes.rb +++ b/lib/innodb/page/fsp_hdr_xdes.rb @@ -1,4 +1,4 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true require "innodb/list" require "innodb/xdes" @@ -12,186 +12,271 @@ # # The basic structure of FSP_HDR and XDES pages is: FIL header, FSP header, # an array of 256 XDES entries, empty (unused) space, and FIL trailer. -class Innodb::Page::FspHdrXdes < Innodb::Page - extend ReadBitsAtOffset +module Innodb + class Page + class FspHdrXdes < Page + extend ReadBitsAtOffset - # A value added to the adjusted exponent stored in the page size field of - # the flags in the FSP header. - FLAGS_PAGE_SIZE_SHIFT = 9 + specialization_for :FSP_HDR + specialization_for :XDES - def self.shift_page_size(page_size_shifted) - if page_size_shifted != 0 - (1 << (FLAGS_PAGE_SIZE_SHIFT + page_size_shifted)) - end - end + Flags = Struct.new( + :system_page_size, + :compressed, + :page_size, + :post_antelope, + :atomic_blobs, + :data_directory, + :value, + keyword_init: true + ) - # Decode the "flags" field in the FSP header, returning a hash of useful - # decodings of the flags (based on MySQl 5.6 definitions). The flags are: - # - # Offset Size Description - # 0 1 Post-Antelope Flag. - # 1 4 Compressed Page Size (zip_size). This is stored as a - # power of 2, minus 9. Since 0 is reserved to mean "not - # compressed", the minimum value is 1, thus making the - # smallest page size 1024 (2 ** (9 + 1)). - # 5 1 Atomic Blobs Flag. - # 6 4 System Page Size (innodb_page_size, UNIV_PAGE_SIZE). - # The setting of the system page size when the tablespace - # was created, stored in the same format as the compressed - # page size above. - # 10 1 Data Directory Flag. - # - def self.decode_flags(flags) - system_page_size = - shift_page_size(read_bits_at_offset(flags, 4, 6)) || - Innodb::Space::DEFAULT_PAGE_SIZE - compressed_page_size = shift_page_size(read_bits_at_offset(flags, 4, 1)) - - { - :system_page_size => system_page_size, - :compressed => compressed_page_size ? false : true, - :page_size => compressed_page_size || system_page_size, - :post_antelope => read_bits_at_offset(flags, 1, 0) == 1, - :atomic_blobs => read_bits_at_offset(flags, 1, 5) == 1, - :data_directory => read_bits_at_offset(flags, 1, 10) == 1, - :value => flags, - } - end + Header = Struct.new( + :space_id, + :unused, + :size, # rubocop:disable Lint/StructNewOverride + :free_limit, + :flags, + :frag_n_used, + :free, + :free_frag, + :full_frag, + :first_unused_seg, + :full_inodes, + :free_inodes, + keyword_init: true + ) - # The FSP header immediately follows the FIL header. - def pos_fsp_header - pos_page_body - end + EncryptionHeader = Struct.new( + :magic, + :master_key_id, + :key, + :iv, + :server_uuid, + :checksum, + keyword_init: true + ) - # The FSP header contains six 32-bit integers, one 64-bit integer, and 5 - # list base nodes. - def size_fsp_header - ((4 * 6) + (1 * 8) + (5 * Innodb::List::BASE_NODE_SIZE)) - end + SdiHeader = Struct.new( + :version, + :root_page_number, + keyword_init: true + ) - # The XDES entry array immediately follows the FSP header. - def pos_xdes_array - pos_fsp_header + size_fsp_header - end + # A value added to the adjusted exponent stored in the page size field of + # the flags in the FSP header. + FLAGS_PAGE_SIZE_SHIFT = 9 - # The number of entries in the XDES array. Defined as page size divided by - # extent size. - def entries_in_xdes_array - size / space.pages_per_extent - end + def self.shift_page_size(page_size_shifted) + (1 << (FLAGS_PAGE_SIZE_SHIFT + page_size_shifted)) if page_size_shifted != 0 + end - def size_xdes_entry - @size_xdes_entry ||= Innodb::Xdes.new(self, cursor(pos_xdes_array)).size_entry - end + # Decode the "flags" field in the FSP header, returning a hash of useful + # decodings of the flags (based on MySQl 5.6 definitions). The flags are: + # + # Offset Size Description + # 0 1 Post-Antelope Flag. + # 1 4 Compressed Page Size (zip_size). This is stored as a + # power of 2, minus 9. Since 0 is reserved to mean "not + # compressed", the minimum value is 1, thus making the + # smallest page size 1024 (2 ** (9 + 1)). + # 5 1 Atomic Blobs Flag. + # 6 4 System Page Size (innodb_page_size, UNIV_PAGE_SIZE). + # The setting of the system page size when the tablespace + # was created, stored in the same format as the compressed + # page size above. + # 10 1 Data Directory Flag. + # + def self.decode_flags(flags) + system_page_size = + shift_page_size(read_bits_at_offset(flags, 4, 6)) || + Innodb::Space::DEFAULT_PAGE_SIZE + compressed_page_size = shift_page_size(read_bits_at_offset(flags, 4, 1)) - def size_xdes_array - entries_in_xdes_array * size_xdes_entry - end + Flags.new( + system_page_size: system_page_size, + compressed: compressed_page_size ? false : true, + page_size: compressed_page_size || system_page_size, + post_antelope: read_bits_at_offset(flags, 1, 0) == 1, + atomic_blobs: read_bits_at_offset(flags, 1, 5) == 1, + data_directory: read_bits_at_offset(flags, 1, 10) == 1, + value: flags + ) + end - # Read the FSP (filespace) header, which contains a few counters and flags, - # as well as list base nodes for each list maintained in the filespace. - def fsp_header - @fsp_header ||= cursor(pos_fsp_header).name("fsp") do |c| - { - :space_id => c.name("space_id") { c.get_uint32 }, - :unused => c.name("unused") { c.get_uint32 }, - :size => c.name("size") { c.get_uint32 }, - :free_limit => c.name("free_limit") { c.get_uint32 }, - :flags => c.name("flags") { - self.class.decode_flags(c.get_uint32) - }, - :frag_n_used => c.name("frag_n_used") { c.get_uint32 }, - :free => c.name("list[free]") { - Innodb::List::Xdes.new(@space, Innodb::List.get_base_node(c)) - }, - :free_frag => c.name("list[free_frag]") { - Innodb::List::Xdes.new(@space, Innodb::List.get_base_node(c)) - }, - :full_frag => c.name("list[full_frag]") { - Innodb::List::Xdes.new(@space, Innodb::List.get_base_node(c)) - }, - :first_unused_seg => c.name("first_unused_seg") { c.get_uint64 }, - :full_inodes => c.name("list[full_inodes]") { - Innodb::List::Inode.new(@space, Innodb::List.get_base_node(c)) - }, - :free_inodes => c.name("list[free_inodes]") { - Innodb::List::Inode.new(@space, Innodb::List.get_base_node(c)) - }, - } - end - end + # The FSP header immediately follows the FIL header. + def pos_fsp_header + pos_page_body + end - # Iterate through all lists in the file space. - def each_list - unless block_given? - return enum_for(:each_list) - end + # The FSP header contains six 32-bit integers, one 64-bit integer, and 5 + # list base nodes. + def size_fsp_header + ((4 * 6) + (1 * 8) + (5 * Innodb::List::BASE_NODE_SIZE)) + end - fsp_header.each do |key, value| - yield key, value if value.is_a?(Innodb::List) - end - end + # The XDES entry array immediately follows the FSP header. + def pos_xdes_array + pos_fsp_header + size_fsp_header + end - # Iterate through all XDES entries in order. This is useful for debugging, - # but each of these entries is actually a node in some other list. The state - # field in the XDES entry indicates which type of list it is present in, - # although not necessarily which list (e.g. :fseg). - def each_xdes - unless block_given? - return enum_for(:each_xdes) - end + # The number of entries in the XDES array. Defined as page size divided by + # extent size. + def entries_in_xdes_array + size / space.pages_per_extent + end - cursor(pos_xdes_array).name("xdes_array") do |c| - entries_in_xdes_array.times do |n| - yield Innodb::Xdes.new(self, c) + def size_xdes_entry + @size_xdes_entry ||= Innodb::Xdes.new(self, cursor(pos_xdes_array)).size_entry end - end - end - def each_region - unless block_given? - return enum_for(:each_region) - end + def size_xdes_array + entries_in_xdes_array * size_xdes_entry + end - super do |region| - yield region - end + def pos_encryption_header + pos_xdes_array + size_xdes_array + end - yield({ - :offset => pos_fsp_header, - :length => size_fsp_header, - :name => :fsp_header, - :info => "FSP Header", - }) - - each_xdes do |xdes| - state = xdes.state || "unused" - yield({ - :offset => xdes.offset, - :length => size_xdes_entry, - :name => "xdes_#{state}".to_sym, - :info => "Extent Descriptor (#{state})", - }) - end + def size_encryption_header + 3 + 4 + (32 * 2) + 36 + 4 + 4 + end - nil - end + def pos_sdi_header + pos_encryption_header + size_encryption_header + end + + def size_sdi_header + 8 + end + + # Read the FSP (filespace) header, which contains a few counters and flags, + # as well as list base nodes for each list maintained in the filespace. + def fsp_header + @fsp_header ||= cursor(pos_fsp_header).name("fsp") do |c| + Header.new( + space_id: c.name("space_id") { c.read_uint32 }, + unused: c.name("unused") { c.read_uint32 }, + size: c.name("size") { c.read_uint32 }, + free_limit: c.name("free_limit") { c.read_uint32 }, + flags: c.name("flags") { self.class.decode_flags(c.read_uint32) }, + frag_n_used: c.name("frag_n_used") { c.read_uint32 }, + free: c.name("list[free]") { Innodb::List::Xdes.new(@space, Innodb::List.get_base_node(c)) }, + free_frag: c.name("list[free_frag]") { Innodb::List::Xdes.new(@space, Innodb::List.get_base_node(c)) }, + full_frag: c.name("list[full_frag]") { Innodb::List::Xdes.new(@space, Innodb::List.get_base_node(c)) }, + first_unused_seg: c.name("first_unused_seg") { c.read_uint64 }, + full_inodes: c.name("list[full_inodes]") { Innodb::List::Inode.new(@space, Innodb::List.get_base_node(c)) }, + free_inodes: c.name("list[free_inodes]") { Innodb::List::Inode.new(@space, Innodb::List.get_base_node(c)) } + ) + end + end + + # Iterate through all lists in the file space. + def each_list + return enum_for(:each_list) unless block_given? + + fsp_header.to_h.each do |key, value| + yield key, value if value.is_a?(Innodb::List) + end + end + + # Iterate through all XDES entries in order. This is useful for debugging, + # but each of these entries is actually a node in some other list. The state + # field in the XDES entry indicates which type of list it is present in, + # although not necessarily which list (e.g. :fseg). + def each_xdes + return enum_for(:each_xdes) unless block_given? + + cursor(pos_xdes_array).name("xdes_array") do |c| + entries_in_xdes_array.times do + yield Innodb::Xdes.new(self, c) + end + end + end + + def encryption_header + @encryption_header ||= cursor(pos_encryption_header).name("encryption_header") do |c| + EncryptionHeader.new( + magic: c.name("magic") { c.read_bytes(3) }, + master_key_id: c.name("master_key_id") { c.read_uint32 }, + key: c.name("key") { c.read_bytes(32) }, + iv: c.name("iv") { c.read_bytes(32) }, + server_uuid: c.name("server_uuid") { c.read_string(36) }, + checksum: c.name("checksum") { c.read_uint32 } + ) + end + end - # Dump the contents of a page for debugging purposes. - def dump - super + def sdi_header + @sdi_header ||= cursor(pos_sdi_header).name("sdi_header") do |c| + SdiHeader.new( + version: c.name("version") { c.read_uint32 }, + root_page_number: c.name("root_page_number") { c.read_uint32 } + ) + end + end + + def each_region(&block) + return enum_for(:each_region) unless block_given? + + super + + yield Region.new( + offset: pos_fsp_header, + length: size_fsp_header, + name: :fsp_header, + info: "FSP Header" + ) + + each_xdes do |xdes| + state = xdes.state || "unused" + yield Region.new( + offset: xdes.offset, + length: size_xdes_entry, + name: :"xdes_#{state}", + info: "Extent Descriptor (#{state})" + ) + end + + yield Region.new( + offset: pos_encryption_header, + length: size_encryption_header, + name: :encryption_header, + info: "Encryption Header" + ) + + yield Region.new( + offset: pos_sdi_header, + length: size_sdi_header, + name: :sdi_header, + info: "SDI Header" + ) - puts "fsp header:" - pp fsp_header - puts + nil + end + + # Dump the contents of a page for debugging purposes. + def dump + super + + puts "fsp header:" + pp fsp_header + puts + + puts "xdes entries:" + each_xdes do |xdes| + pp xdes + end + puts - puts "xdes entries:" - each_xdes do |xdes| - pp xdes + puts "encryption header:" + pp encryption_header + puts + + puts "serialized dictionary information header:" + pp sdi_header + puts + end end - puts end end - -Innodb::Page::SPECIALIZED_CLASSES[:FSP_HDR] = Innodb::Page::FspHdrXdes -Innodb::Page::SPECIALIZED_CLASSES[:XDES] = Innodb::Page::FspHdrXdes diff --git a/lib/innodb/page/ibuf_bitmap.rb b/lib/innodb/page/ibuf_bitmap.rb index 454f2e9d..c5ef25c4 100644 --- a/lib/innodb/page/ibuf_bitmap.rb +++ b/lib/innodb/page/ibuf_bitmap.rb @@ -1,47 +1,47 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true -class Innodb::Page::IbufBitmap < Innodb::Page - extend ReadBitsAtOffset +module Innodb + class Page + class IbufBitmap < Page + extend ReadBitsAtOffset - def pos_ibuf_bitmap - pos_page_body - end + specialization_for :IBUF_BITMAP - def size_ibuf_bitmap - (Innodb::IbufBitmap::BITS_PER_PAGE * space.pages_per_bookkeeping_page) / 8 - end + def pos_ibuf_bitmap + pos_page_body + end - def ibuf_bitmap - Innodb::IbufBitmap.new(self, cursor(pos_ibuf_bitmap)) - end + def size_ibuf_bitmap + (Innodb::IbufBitmap::BITS_PER_PAGE * space.pages_per_bookkeeping_page) / 8 + end - def each_region - unless block_given? - return enum_for(:each_region) - end + def ibuf_bitmap + Innodb::IbufBitmap.new(self, cursor(pos_ibuf_bitmap)) + end - super do |region| - yield region - end + def each_region(&block) + return enum_for(:each_region) unless block_given? - yield({ - :offset => pos_ibuf_bitmap, - :length => size_ibuf_bitmap, - :name => :ibuf_bitmap, - :info => "Insert Buffer Bitmap", - }) + super - nil - end + yield Region.new( + offset: pos_ibuf_bitmap, + length: size_ibuf_bitmap, + name: :ibuf_bitmap, + info: "Insert Buffer Bitmap" + ) - def dump - super + nil + end - puts "ibuf bitmap:" - ibuf_bitmap.each_page_status do |page_number, page_status| - puts " Page %i: %s" % [page_number, page_status.inspect] + def dump + super + + puts "ibuf bitmap:" + ibuf_bitmap.each_page_status do |page_number, page_status| + puts " Page %i: %s" % [page_number, page_status.inspect] + end + end end end end - -Innodb::Page::SPECIALIZED_CLASSES[:IBUF_BITMAP] = Innodb::Page::IbufBitmap diff --git a/lib/innodb/page/index.rb b/lib/innodb/page/index.rb index aeffbf46..11e55a64 100644 --- a/lib/innodb/page/index.rb +++ b/lib/innodb/page/index.rb @@ -1,4 +1,6 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true + +require "forwardable" require "innodb/fseg_entry" @@ -10,1085 +12,1114 @@ # header, fixed-width system records (infimum and supremum), user records # (the actual data) which grow ascending by offset, free space, the page # directory which grows descending by offset, and the FIL trailer. -class Innodb::Page::Index < Innodb::Page - # The size (in bytes) of the "next" pointer in each record header. - RECORD_NEXT_SIZE = 2 - - # The size (in bytes) of the bit-packed fields in each record header for - # "redundant" record format. - RECORD_REDUNDANT_BITS_SIZE = 4 - - # Masks for 1-byte record end-offsets within "redundant" records. - RECORD_REDUNDANT_OFF1_OFFSET_MASK = 0x7f - RECORD_REDUNDANT_OFF1_NULL_MASK = 0x80 - - # Masks for 2-byte record end-offsets within "redundant" records. - RECORD_REDUNDANT_OFF2_OFFSET_MASK = 0x3fff - RECORD_REDUNDANT_OFF2_NULL_MASK = 0x8000 - RECORD_REDUNDANT_OFF2_EXTERN_MASK = 0x4000 - - # The size (in bytes) of the bit-packed fields in each record header for - # "compact" record format. - RECORD_COMPACT_BITS_SIZE = 3 - - # Maximum number of fields. - RECORD_MAX_N_SYSTEM_FIELDS = 3 - RECORD_MAX_N_FIELDS = 1024 - 1 - RECORD_MAX_N_USER_FIELDS = RECORD_MAX_N_FIELDS - RECORD_MAX_N_SYSTEM_FIELDS * 2 - - # Page direction values possible in the page_header's :direction field. - PAGE_DIRECTION = { - 1 => :left, # Inserts have been in descending order. - 2 => :right, # Inserts have been in ascending order. - 3 => :same_rec, # Unused by InnoDB. - 4 => :same_page, # Unused by InnoDB. - 5 => :no_direction, # Inserts have been in random order. - } - - # Record types used in the :type field of the record header. - RECORD_TYPES = { - 0 => :conventional, # A normal user record in a leaf page. - 1 => :node_pointer, # A node pointer in a non-leaf page. - 2 => :infimum, # The system "infimum" record. - 3 => :supremum, # The system "supremum" record. - } - - # This record is the minimum record at this level of the B-tree. - RECORD_INFO_MIN_REC_FLAG = 1 - - # This record has been marked as deleted. - RECORD_INFO_DELETED_FLAG = 2 - - # The size (in bytes) of the record pointers in each page directory slot. - PAGE_DIR_SLOT_SIZE = 2 - - # The minimum number of records "owned" by each record with an entry in - # the page directory. - PAGE_DIR_SLOT_MIN_N_OWNED = 4 - - # The maximum number of records "owned" by each record with an entry in - # the page directory. - PAGE_DIR_SLOT_MAX_N_OWNED = 8 - - # Return the byte offset of the start of the "index" page header, which - # immediately follows the "fil" header. - def pos_index_header - pos_page_body - end - - # The size of the "index" header. - def size_index_header - 2 + 2 + 2 + 2 + 2 + 2 + 2 + 2 + 2 + 8 + 2 + 8 - end - - # Return the byte offset of the start of the "fseg" header, which immediately - # follows the "index" header. - def pos_fseg_header - pos_index_header + size_index_header - end +module Innodb + class Page + class Index < Page + extend Forwardable + + specialization_for :INDEX + + RecordHeader = Struct.new( + :length, # rubocop:disable Lint/StructNewOverride + :next, + :type, + :heap_number, + :n_owned, + :info_flags, + :offset_size, + :n_fields, + :nulls, + :lengths, + :externs, + keyword_init: true + ) + + class RecordHeader + # This record is the minimum record at this level of the B-tree. + RECORD_INFO_MIN_REC_FLAG = 1 + + # This record has been marked as deleted. + RECORD_INFO_DELETED_FLAG = 2 + + def min_rec? + (info_flags & RECORD_INFO_MIN_REC_FLAG) != 0 + end - # The size of the "fseg" header. - def size_fseg_header - 2 * Innodb::FsegEntry::SIZE - end + def deleted? + (info_flags & RECORD_INFO_DELETED_FLAG) != 0 + end + end - # Return the size of the header for each record. - def size_record_header - case page_header[:format] - when :compact - RECORD_NEXT_SIZE + RECORD_COMPACT_BITS_SIZE - when :redundant - RECORD_NEXT_SIZE + RECORD_REDUNDANT_BITS_SIZE - end - end + SystemRecord = Struct.new( + :offset, + :header, + :next, + :data, + :length, # rubocop:disable Lint/StructNewOverride + keyword_init: true + ) + + UserRecord = Struct.new( + :type, + :format, + :offset, + :header, + :next, + :key, + :row, + :sys, + :child_page_number, + :transaction_id, + :roll_pointer, + :length, # rubocop:disable Lint/StructNewOverride + keyword_init: true + ) + + FieldDescriptor = Struct.new( + :name, + :type, + :value, + :extern, + keyword_init: true + ) + + FsegHeader = Struct.new( + :leaf, + :internal, + keyword_init: true + ) + + PageHeader = Struct.new( + :n_dir_slots, + :heap_top, + :n_heap_format, + :n_heap, + :format, + :garbage_offset, + :garbage_size, + :last_insert_offset, + :direction, + :n_direction, + :n_recs, + :max_trx_id, + :level, + :index_id, + keyword_init: true + ) + + # The size (in bytes) of the "next" pointer in each record header. + RECORD_NEXT_SIZE = 2 + + # The size (in bytes) of the bit-packed fields in each record header for + # "redundant" record format. + RECORD_REDUNDANT_BITS_SIZE = 4 + + # Masks for 1-byte record end-offsets within "redundant" records. + RECORD_REDUNDANT_OFF1_OFFSET_MASK = 0x7f + RECORD_REDUNDANT_OFF1_NULL_MASK = 0x80 + + # Masks for 2-byte record end-offsets within "redundant" records. + RECORD_REDUNDANT_OFF2_OFFSET_MASK = 0x3fff + RECORD_REDUNDANT_OFF2_NULL_MASK = 0x8000 + RECORD_REDUNDANT_OFF2_EXTERN_MASK = 0x4000 + + # The size (in bytes) of the bit-packed fields in each record header for + # "compact" record format. + RECORD_COMPACT_BITS_SIZE = 3 + + # Maximum number of fields. + RECORD_MAX_N_SYSTEM_FIELDS = 3 + RECORD_MAX_N_FIELDS = 1024 - 1 + RECORD_MAX_N_USER_FIELDS = RECORD_MAX_N_FIELDS - (RECORD_MAX_N_SYSTEM_FIELDS * 2) + + # Page direction values possible in the page_header's :direction field. + PAGE_DIRECTION = { + 1 => :left, # Inserts have been in descending order. + 2 => :right, # Inserts have been in ascending order. + 3 => :same_rec, # Unused by InnoDB. + 4 => :same_page, # Unused by InnoDB. + 5 => :no_direction, # Inserts have been in random order. + }.freeze + + # Record types used in the :type field of the record header. + RECORD_TYPES = { + 0 => :conventional, # A normal user record in a leaf page. + 1 => :node_pointer, # A node pointer in a non-leaf page. + 2 => :infimum, # The system "infimum" record. + 3 => :supremum, # The system "supremum" record. + }.freeze + + # The size (in bytes) of the record pointers in each page directory slot. + PAGE_DIR_SLOT_SIZE = 2 + + # The minimum number of records "owned" by each record with an entry in + # the page directory. + PAGE_DIR_SLOT_MIN_N_OWNED = 4 + + # The maximum number of records "owned" by each record with an entry in + # the page directory. + PAGE_DIR_SLOT_MAX_N_OWNED = 8 + + attr_writer :record_describer + + # Return the byte offset of the start of the "index" page header, which + # immediately follows the "fil" header. + def pos_index_header + pos_page_body + end - # The size of the additional data structures in the header of the system - # records, which is just 1 byte in redundant format to store the offset - # of the end of the field. This is needed specifically here since we need - # to be able to calculate the fixed positions of these system records. - def size_mum_record_header_additional - case page_header[:format] - when :compact - 0 # No additional data is stored in compact format. - when :redundant - 1 # A 1-byte offset for 1 field is stored in redundant format. - end - end + # The size of the "index" header. + def size_index_header + 2 + 2 + 2 + 2 + 2 + 2 + 2 + 2 + 2 + 8 + 2 + 8 + end - # The size of the data from the supremum or infimum records. - def size_mum_record - 8 - end + # Return the byte offset of the start of the "fseg" header, which immediately + # follows the "index" header. + def pos_fseg_header + pos_index_header + size_index_header + end - # Return the byte offset of the start of the "origin" of the infimum record, - # which is always the first record in the singly-linked record chain on any - # page, and represents a record with a "lower value than any possible user - # record". The infimum record immediately follows the page header. - def pos_infimum - pos_records + - size_record_header + - size_mum_record_header_additional - end + # The size of the "fseg" header. + def size_fseg_header + 2 * Innodb::FsegEntry::SIZE + end - # Return the byte offset of the start of the "origin" of the supremum record, - # which is always the last record in the singly-linked record chain on any - # page, and represents a record with a "higher value than any possible user - # record". The supremum record immediately follows the infimum record. - def pos_supremum - pos_infimum + - size_record_header + - size_mum_record_header_additional + - size_mum_record - end + # Return the size of the header for each record. + def size_record_header + case page_header[:format] + when :compact + RECORD_NEXT_SIZE + RECORD_COMPACT_BITS_SIZE + when :redundant + RECORD_NEXT_SIZE + RECORD_REDUNDANT_BITS_SIZE + end + end - # Return the byte offset of the start of records within the page (the - # position immediately after the page header). - def pos_records - size_fil_header + - size_index_header + - size_fseg_header - end + # The size of the additional data structures in the header of the system + # records, which is just 1 byte in redundant format to store the offset + # of the end of the field. This is needed specifically here since we need + # to be able to calculate the fixed positions of these system records. + def size_mum_record_header_additional + case page_header[:format] + when :compact + 0 # No additional data is stored in compact format. + when :redundant + 1 # A 1-byte offset for 1 field is stored in redundant format. + end + end - # Return the byte offset of the start of the user records in a page, which - # immediately follows the supremum record. - def pos_user_records - pos_supremum + size_mum_record - end + # The size of the data from the supremum or infimum records. + def size_mum_record + 8 + end - # The position of the page directory, which starts at the "fil" trailer and - # grows backwards from there. - def pos_directory - pos_fil_trailer - end + # Return the byte offset of the start of the "origin" of the infimum record, + # which is always the first record in the singly-linked record chain on any + # page, and represents a record with a "lower value than any possible user + # record". The infimum record immediately follows the page header. + def pos_infimum + pos_records + + size_record_header + + size_mum_record_header_additional + end - # The amount of space consumed by the page header. - def header_space - # The end of the supremum system record is the beginning of the space - # available for user records. - pos_user_records - end + # Return the byte offset of the start of the "origin" of the supremum record, + # which is always the last record in the singly-linked record chain on any + # page, and represents a record with a "higher value than any possible user + # record". The supremum record immediately follows the infimum record. + def pos_supremum + pos_infimum + + size_record_header + + size_mum_record_header_additional + + size_mum_record + end - # The number of directory slots in use. - def directory_slots - page_header[:n_dir_slots] - end + # Return the byte offset of the start of records within the page (the + # position immediately after the page header). + def pos_records + size_fil_header + + size_index_header + + size_fseg_header + end - # The amount of space consumed by the page directory. - def directory_space - directory_slots * PAGE_DIR_SLOT_SIZE - end + # Return the byte offset of the start of the user records in a page, which + # immediately follows the supremum record. + def pos_user_records + pos_supremum + size_mum_record + end - # The amount of space consumed by the trailers in the page. - def trailer_space - size_fil_trailer - end + # The position of the page directory, which starts at the "fil" trailer and + # grows backwards from there. + def pos_directory + pos_fil_trailer + end - # Return the amount of free space in the page. - def free_space - page_header[:garbage_size] + - (size - size_fil_trailer - directory_space - page_header[:heap_top]) - end + # The amount of space consumed by the page header. + def header_space + # The end of the supremum system record is the beginning of the space + # available for user records. + pos_user_records + end - # Return the amount of used space in the page. - def used_space - size - free_space - end + # The number of directory slots in use. + def directory_slots + page_header[:n_dir_slots] + end - # Return the amount of space occupied by records in the page. - def record_space - used_space - header_space - directory_space - trailer_space - end + # The amount of space consumed by the page directory. + def directory_space + directory_slots * PAGE_DIR_SLOT_SIZE + end - # Return the actual bytes of the portion of the page which is used to - # store user records (eliminate the headers and trailer from the page). - def record_bytes - data(pos_user_records, page_header[:heap_top] - pos_user_records) - end + # The amount of space consumed by the trailers in the page. + def trailer_space + size_fil_trailer + end - # Return the "index" header. - def page_header - @page_header ||= cursor(pos_index_header).name("index") do |c| - index = { - :n_dir_slots => c.name("n_dir_slots") { c.get_uint16 }, - :heap_top => c.name("heap_top") { c.get_uint16 }, - :n_heap_format => c.name("n_heap_format") { c.get_uint16 }, - :garbage_offset => c.name("garbage_offset") { c.get_uint16 }, - :garbage_size => c.name("garbage_size") { c.get_uint16 }, - :last_insert_offset => c.name("last_insert_offset") { c.get_uint16 }, - :direction => c.name("direction") { PAGE_DIRECTION[c.get_uint16] }, - :n_direction => c.name("n_direction") { c.get_uint16 }, - :n_recs => c.name("n_recs") { c.get_uint16 }, - :max_trx_id => c.name("max_trx_id") { c.get_uint64 }, - :level => c.name("level") { c.get_uint16 }, - :index_id => c.name("index_id") { c.get_uint64 }, - } - index[:n_heap] = index[:n_heap_format] & (2**15-1) - index[:format] = (index[:n_heap_format] & 1<<15) == 0 ? - :redundant : :compact - index.delete :n_heap_format - - index - end - end + # Return the amount of free space in the page. + def free_space + page_header[:garbage_size] + + (size - size_fil_trailer - directory_space - page_header[:heap_top]) + end - # A helper function to return the index id. - def index_id - page_header && page_header[:index_id] - end + # Return the amount of used space in the page. + def used_space + size - free_space + end - # A helper function to return the page level from the "page" header, for - # easier access. - def level - page_header && page_header[:level] - end + # Return the amount of space occupied by records in the page. + def record_space + used_space - header_space - directory_space - trailer_space + end - # A helper function to return the number of records. - def records - page_header && page_header[:n_recs] - end + # A helper to calculate the amount of space consumed per record. + def space_per_record + page_header.n_recs.positive? ? (record_space.to_f / page_header.n_recs) : 0 + end - # A helper function to identify root index pages; they must be the only pages - # at their level. - def root? - self.prev.nil? && self.next.nil? - end + # Return the "index" header. + def page_header + @page_header ||= cursor(pos_index_header).name("index") do |c| + index = PageHeader.new( + n_dir_slots: c.name("n_dir_slots") { c.read_uint16 }, + heap_top: c.name("heap_top") { c.read_uint16 }, + n_heap_format: c.name("n_heap_format") { c.read_uint16 }, + garbage_offset: c.name("garbage_offset") { c.read_uint16 }, + garbage_size: c.name("garbage_size") { c.read_uint16 }, + last_insert_offset: c.name("last_insert_offset") { c.read_uint16 }, + direction: c.name("direction") { PAGE_DIRECTION[c.read_uint16] }, + n_direction: c.name("n_direction") { c.read_uint16 }, + n_recs: c.name("n_recs") { c.read_uint16 }, + max_trx_id: c.name("max_trx_id") { c.read_uint64 }, + level: c.name("level") { c.read_uint16 }, + index_id: c.name("index_id") { c.read_uint64 } + ) + + index.n_heap = index.n_heap_format & ((2**15) - 1) + index.format = (index.n_heap_format & (1 << 15)).zero? ? :redundant : :compact + + index + end + end - # A helper function to identify leaf index pages. - def leaf? - level == 0 - end + def_delegator :page_header, :index_id + def_delegator :page_header, :level + def_delegator :page_header, :n_recs, :records + def_delegator :page_header, :garbage_offset - # A helper function to return the offset to the first free record. - def garbage_offset - page_header && page_header[:garbage_offset] - end + # A helper function to identify root index pages; they must be the only pages + # at their level. + def root? + prev.nil? && self.next.nil? + end - # Return the "fseg" header. - def fseg_header - @fseg_header ||= cursor(pos_fseg_header).name("fseg") do |c| - { - :leaf => c.name("fseg[leaf]") { - Innodb::FsegEntry.get_inode(@space, c) - }, - :internal => c.name("fseg[internal]") { - Innodb::FsegEntry.get_inode(@space, c) - }, - } - end - end + # A helper function to identify leaf index pages. + def leaf? + level.zero? + end - # Return the header from a record. - def record_header(cursor) - origin = cursor.position - header = {} - cursor.backward.name("header") do |c| - case page_header[:format] - when :compact - # The "next" pointer is a relative offset from the current record. - header[:next] = c.name("next") { origin + c.get_sint16 } - - # Fields packed in a 16-bit integer (LSB first): - # 3 bits for type - # 13 bits for heap_number - bits1 = c.name("bits1") { c.get_uint16 } - header[:type] = RECORD_TYPES[bits1 & 0x07] - header[:heap_number] = (bits1 & 0xfff8) >> 3 - when :redundant - # The "next" pointer is an absolute offset within the page. - header[:next] = c.name("next") { c.get_uint16 } - - # Fields packed in a 24-bit integer (LSB first): - # 1 bit for offset_size (0 = 2 bytes, 1 = 1 byte) - # 10 bits for n_fields - # 13 bits for heap_number - bits1 = c.name("bits1") { c.get_uint24 } - header[:offset_size] = (bits1 & 1) == 0 ? 2 : 1 - header[:n_fields] = (bits1 & (((1 << 10) - 1) << 1)) >> 1 - header[:heap_number] = (bits1 & (((1 << 13) - 1) << 11)) >> 11 + # A helper to determine if an this page is part of an insert buffer index. + def ibuf_index? + index_id == Innodb::IbufIndex::INDEX_ID end - # Fields packed in an 8-bit integer (LSB first): - # 4 bits for n_owned - # 4 bits for flags - bits2 = c.name("bits2") { c.get_uint8 } - header[:n_owned] = bits2 & 0x0f - info = (bits2 & 0xf0) >> 4 - header[:min_rec] = (info & RECORD_INFO_MIN_REC_FLAG) != 0 - header[:deleted] = (info & RECORD_INFO_DELETED_FLAG) != 0 - - case page_header[:format] - when :compact - record_header_compact_additional(header, cursor) - when :redundant - record_header_redundant_additional(header, cursor) + # Return the "fseg" header. + def fseg_header + @fseg_header ||= cursor(pos_fseg_header).name("fseg") do |c| + FsegHeader.new( + leaf: c.name("fseg[leaf]") { Innodb::FsegEntry.get_inode(@space, c) }, + internal: c.name("fseg[internal]") { Innodb::FsegEntry.get_inode(@space, c) } + ) + end end - header[:length] = origin - cursor.position - end + # Return the header from a record. + def record_header(cursor) + origin = cursor.position + header = RecordHeader.new + cursor.backward.name("header") do |c| + case page_header.format + when :compact + # The "next" pointer is a relative offset from the current record. + header.next = c.name("next") { origin + c.read_sint16 } + + # Fields packed in a 16-bit integer (LSB first): + # 3 bits for type + # 13 bits for heap_number + bits1 = c.name("bits1") { c.read_uint16 } + header.type = RECORD_TYPES[bits1 & 0x07] + header.heap_number = (bits1 & 0xfff8) >> 3 + when :redundant + # The "next" pointer is an absolute offset within the page. + header.next = c.name("next") { c.read_uint16 } + + # Fields packed in a 24-bit integer (LSB first): + # 1 bit for offset_size (0 = 2 bytes, 1 = 1 byte) + # 10 bits for n_fields + # 13 bits for heap_number + bits1 = c.name("bits1") { c.read_uint24 } + header.offset_size = (bits1 & 1).zero? ? 2 : 1 + header.n_fields = (bits1 & (((1 << 10) - 1) << 1)) >> 1 + header.heap_number = (bits1 & (((1 << 13) - 1) << 11)) >> 11 + end - header - end + # Fields packed in an 8-bit integer (LSB first): + # 4 bits for n_owned + # 4 bits for flags + bits2 = c.name("bits2") { c.read_uint8 } + header.n_owned = bits2 & 0x0f + header.info_flags = (bits2 & 0xf0) >> 4 + + case page_header.format + when :compact + record_header_compact_additional(header, cursor) + when :redundant + record_header_redundant_additional(header, cursor) + end + + header.length = origin - cursor.position + end - # Read additional header information from a compact format record header. - def record_header_compact_additional(header, cursor) - case header[:type] - when :conventional, :node_pointer - # The variable-length part of the record header contains a - # bit vector indicating NULL fields and the length of each - # non-NULL variable-length field. - if record_format - header[:nulls] = cursor.name("nulls") { - record_header_compact_null_bitmap(cursor) - } - header[:lengths], header[:externs] = - cursor.name("lengths_and_externs") { - record_header_compact_variable_lengths_and_externs(cursor, - header[:nulls]) - } + header end - end - end - # Return an array indicating which fields are null. - def record_header_compact_null_bitmap(cursor) - fields = record_fields + # Read additional header information from a compact format record header. + def record_header_compact_additional(header, cursor) + case header.type + when :conventional, :node_pointer + # The variable-length part of the record header contains a + # bit vector indicating NULL fields and the length of each + # non-NULL variable-length field. + if record_format + header.nulls = cursor.name("nulls") { record_header_compact_null_bitmap(cursor) } + header.lengths, header.externs = cursor.name("lengths_and_externs") do + record_header_compact_variable_lengths_and_externs(cursor, header.nulls) + end + end + end + end - # The number of bits in the bitmap is the number of nullable fields. - size = fields.count { |f| f.nullable? } + # Return an array indicating which fields are null. + def record_header_compact_null_bitmap(cursor) + fields = record_fields - # There is no bitmap if there are no nullable fields. - return [] unless size > 0 + # The number of bits in the bitmap is the number of nullable fields. + size = fields.count(&:nullable?) - null_bit_array = cursor.get_bit_array(size).reverse! + # There is no bitmap if there are no nullable fields. + return [] unless size.positive? - # For every nullable field, select the ones which are actually null. - fields.inject([]) do |nulls, f| - nulls << f.name if f.nullable? && (null_bit_array.shift == 1) - nulls - end - end + # TODO: This is really ugly. + null_bit_array = cursor.read_bit_array(size).reverse! - # Return an array containing an array of the length of each variable-length - # field and an array indicating which fields are stored externally. - def record_header_compact_variable_lengths_and_externs(cursor, nulls) - fields = (record_format[:key] + record_format[:row]) + # For every nullable field, select the ones which are actually null. + fields.select { |f| f.nullable? && (null_bit_array.shift == 1) }.map(&:name) + end - lengths = {} - externs = [] + # Return an array containing an array of the length of each variable-length + # field and an array indicating which fields are stored externally. + def record_header_compact_variable_lengths_and_externs(cursor, nulls) + fields = (record_format[:key] + record_format[:row]) - # For each non-NULL variable-length field, the record header contains - # the length in one or two bytes. - fields.each do |f| - next if !f.variable? || nulls.include?(f.name) + lengths = {} + externs = [] - len = cursor.get_uint8 - ext = false + # For each non-NULL variable-length field, the record header contains + # the length in one or two bytes. + fields.each do |f| + next if !f.variable? || nulls.include?(f.name) - # Two bytes are used only if the length exceeds 127 bytes and the - # maximum length exceeds 255 bytes (or the field is a BLOB type). - if len > 127 && (f.blob? || f.data_type.width > 255) - ext = (0x40 & len) != 0 - len = ((len & 0x3f) << 8) + cursor.get_uint8 - end + len = cursor.read_uint8 + ext = false - lengths[f.name] = len - externs << f.name if ext - end + # Two bytes are used only if the length exceeds 127 bytes and the + # maximum length exceeds 255 bytes (or the field is a BLOB type). + if len > 127 && (f.blob? || f.data_type.length > 255) + ext = (0x40 & len) != 0 + len = ((len & 0x3f) << 8) + cursor.read_uint8 + end - return lengths, externs - end + lengths[f.name] = len + externs << f.name if ext + end - # Read additional header information from a redundant format record header. - def record_header_redundant_additional(header, cursor) - lengths, nulls, externs = [], [], [] - - field_offsets = record_header_redundant_field_end_offsets(header, cursor) - - this_field_offset = 0 - field_offsets.each do |n| - case header[:offset_size] - when 1 - next_field_offset = (n & RECORD_REDUNDANT_OFF1_OFFSET_MASK) - lengths << (next_field_offset - this_field_offset) - nulls << ((n & RECORD_REDUNDANT_OFF1_NULL_MASK) != 0) - externs << false - when 2 - next_field_offset = (n & RECORD_REDUNDANT_OFF2_OFFSET_MASK) - lengths << (next_field_offset - this_field_offset) - nulls << ((n & RECORD_REDUNDANT_OFF2_NULL_MASK) != 0) - externs << ((n & RECORD_REDUNDANT_OFF2_EXTERN_MASK) != 0) + [lengths, externs] end - this_field_offset = next_field_offset - end - # If possible, refer to fields by name rather than position for - # better formatting (i.e. pp). - if record_format - header[:lengths], header[:nulls], header[:externs] = {}, [], [] + # Read additional header information from a redundant format record header. + def record_header_redundant_additional(header, cursor) + lengths = [] + nulls = [] + externs = [] + + field_offsets = record_header_redundant_field_end_offsets(header, cursor) + + this_field_offset = 0 + field_offsets.each do |n| + case header.offset_size + when 1 + next_field_offset = (n & RECORD_REDUNDANT_OFF1_OFFSET_MASK) + lengths << (next_field_offset - this_field_offset) + nulls << ((n & RECORD_REDUNDANT_OFF1_NULL_MASK) != 0) + externs << false + when 2 + next_field_offset = (n & RECORD_REDUNDANT_OFF2_OFFSET_MASK) + lengths << (next_field_offset - this_field_offset) + nulls << ((n & RECORD_REDUNDANT_OFF2_NULL_MASK) != 0) + externs << ((n & RECORD_REDUNDANT_OFF2_EXTERN_MASK) != 0) + end + this_field_offset = next_field_offset + end - record_fields.each do |f| - header[:lengths][f.name] = lengths[f.position] - header[:nulls] << f.name if nulls[f.position] - header[:externs] << f.name if externs[f.position] + # If possible, refer to fields by name rather than position for + # better formatting (i.e. pp). + if record_format + header.lengths = {} + header.nulls = [] + header.externs = [] + + record_fields.each do |f| + header.lengths[f.name] = lengths[f.position] + header.nulls << f.name if nulls[f.position] + header.externs << f.name if externs[f.position] + end + else + header.lengths = lengths + header.nulls = nulls + header.externs = externs + end end - else - header[:lengths], header[:nulls], header[:externs] = lengths, nulls, externs - end - end - # Read field end offsets from the provided cursor for each field as counted - # by n_fields. - def record_header_redundant_field_end_offsets(header, cursor) - (0...header[:n_fields]).to_a.inject([]) do |offsets, n| - cursor.name("field_end_offset[#{n}]") { - offsets << cursor.get_uint_by_size(header[:offset_size]) - } - offsets - end - end + # Read field end offsets from the provided cursor for each field as counted + # by n_fields. + def record_header_redundant_field_end_offsets(header, cursor) + header.n_fields.times.map do |n| + cursor.name("field_end_offset[#{n}]") { cursor.read_uint_by_size(header.offset_size) } + end + end - # Parse and return simple fixed-format system records, such as InnoDB's - # internal infimum and supremum records. - def system_record(offset) - cursor(offset).name("record[#{offset}]") do |c| - header = c.peek { record_header(c) } - Innodb::Record.new(self, { - :offset => offset, - :header => header, - :next => header[:next], - :data => c.name("data") { c.get_bytes(size_mum_record) }, - :length => c.position - offset, - }) - end - end + # Parse and return simple fixed-format system records, such as InnoDB's + # internal infimum and supremum records. + def system_record(offset) + cursor(offset).name("record[#{offset}]") do |c| + header = c.peek { record_header(c) } + Innodb::Record.new( + self, + SystemRecord.new( + offset: offset, + header: header, + next: header.next, + data: c.name("data") { c.read_bytes(size_mum_record) }, + length: c.position - offset + ) + ) + end + end - # Return the infimum record on a page. - def infimum - @infimum ||= system_record(pos_infimum) - end + # Return the infimum record on a page. + def infimum + @infimum ||= system_record(pos_infimum) + end - # Return the supremum record on a page. - def supremum - @supremum ||= system_record(pos_supremum) - end + # Return the supremum record on a page. + def supremum + @supremum ||= system_record(pos_supremum) + end - def record_describer=(o) - @record_describer = o - end + def make_record_describer + if space.innodb_system.data_dictionary && index_id && !ibuf_index? + @record_describer = space.innodb_system + .data_dictionary + .indexes + .find(innodb_index_id: index_id) + .record_describer + elsif space + @record_describer = space.record_describer + end + end - def record_describer - return @record_describer if @record_describer + def record_describer + @record_describer ||= make_record_describer + end - if space and space.innodb_system and index_id - @record_describer = - space.innodb_system.data_dictionary.record_describer_by_index_id(index_id) - elsif space - @record_describer = space.record_describer - end + # Return a set of field objects that describe the record. + def make_record_description + position = (0..RECORD_MAX_N_FIELDS).each + description = record_describer.description + fields = { type: description[:type], key: [], sys: [], row: [] } - @record_describer - end + description[:key].each do |field| + fields[:key] << Innodb::Field.new(position.next, field[:name], *field[:type]) + end - # Return a set of field objects that describe the record. - def make_record_description - position = (0..RECORD_MAX_N_FIELDS).each - description = record_describer.description - fields = {:type => description[:type], :key => [], :sys => [], :row => []} + # If this is a leaf page of the clustered index, read InnoDB's internal + # fields, a transaction ID and roll pointer. + if leaf? && fields[:type] == :clustered + [["DB_TRX_ID", :TRX_ID], ["DB_ROLL_PTR", :ROLL_PTR]].each do |name, type| + fields[:sys] << Innodb::Field.new(position.next, name, type, :NOT_NULL) + end + end - description[:key].each do |field| - fields[:key] << Innodb::Field.new(position.next, field[:name], *field[:type]) - end + # If this is a leaf page of the clustered index, or any page of a + # secondary index, read the non-key fields. + if (leaf? && fields[:type] == :clustered) || (fields[:type] == :secondary) + description[:row].each do |field| + fields[:row] << Innodb::Field.new(position.next, field[:name], *field[:type]) + end + end - # If this is a leaf page of the clustered index, read InnoDB's internal - # fields, a transaction ID and roll pointer. - if level == 0 && fields[:type] == :clustered - [["DB_TRX_ID", :TRX_ID,],["DB_ROLL_PTR", :ROLL_PTR]].each do |name, type| - fields[:sys] << Innodb::Field.new(position.next, name, type, :NOT_NULL) + fields end - end - # If this is a leaf page of the clustered index, or any page of a - # secondary index, read the non-key fields. - if (level == 0 && fields[:type] == :clustered) || (fields[:type] == :secondary) - description[:row].each do |field| - fields[:row] << Innodb::Field.new(position.next, field[:name], *field[:type]) + # Return (and cache) the record format provided by an external class. + def record_format + @record_format ||= make_record_description if record_describer end - end - fields - end - - # Return (and cache) the record format provided by an external class. - def record_format - if record_describer - @record_format ||= make_record_description() - end - end - - # Returns the (ordered) set of fields that describe records in this page. - def record_fields - if record_format - record_format.values_at(:key, :sys, :row).flatten.sort_by {|f| f.position} - end - end - - # Parse and return a record at a given offset. - def record(offset) - return nil unless offset - return infimum if offset == pos_infimum - return supremum if offset == pos_supremum - - cursor(offset).forward.name("record[#{offset}]") do |c| - # There is a header preceding the row itself, so back up and read it. - header = c.peek { record_header(c) } - - this_record = { - :format => page_header[:format], - :offset => offset, - :header => header, - :next => header[:next] == 0 ? nil : (header[:next]), - } - - if record_format - this_record[:type] = record_format[:type] - - # Used to indicate whether a field is part of key/row/sys. - fmap = [:key, :row, :sys].inject({}) do |h, k| - this_record[k] = [] - record_format[k].each { |f| h[f.position] = k } - h - end + # Returns the (ordered) set of fields that describe records in this page. + def record_fields + record_format.values_at(:key, :sys, :row).flatten.sort_by(&:position) if record_format + end - # Read the fields present in this record. - record_fields.each do |f| - p = fmap[f.position] - c.name("#{p.to_s}[#{f.name}]") do - this_record[p] << { - :name => f.name, - :type => f.data_type.name, - :value => f.value(c, this_record), - :extern => f.extern(c, this_record), - }.reject { |k, v| v.nil? } + # Parse and return a record at a given offset. + def record(offset) + return nil unless offset + return infimum if offset == pos_infimum + return supremum if offset == pos_supremum + + cursor(offset).forward.name("record[#{offset}]") do |c| + # There is a header preceding the row itself, so back up and read it. + header = c.peek { record_header(c) } + + this_record = UserRecord.new( + format: page_header.format, + offset: offset, + header: header, + next: header.next.zero? ? nil : header.next + ) + + if record_format + this_record.type = record_format[:type] + + # Used to indicate whether a field is part of key/row/sys. + # TODO: There's probably a better way to do this. + fmap = %i[key row sys].each_with_object({}) do |k, h| + this_record[k] = [] + record_format[k].each { |f| h[f.position] = k } + end + + # Read the fields present in this record. + record_fields.each do |f| + p = fmap[f.position] + c.name("#{p}[#{f.name}]") do + this_record[p] << FieldDescriptor.new( + name: f.name, + type: f.data_type.name, + value: f.value(c, this_record), + extern: f.extern(c, this_record) + ) + end + end + + # If this is a node (non-leaf) page, it will have a child page number + # (or "node pointer") stored as the last field. + this_record.child_page_number = c.name("child_page_number") { c.read_uint32 } unless leaf? + + this_record.length = c.position - offset + + # Add system field accessors for convenience. + this_record.sys.each do |f| + case f[:name] + when "DB_TRX_ID" + this_record.transaction_id = f[:value] + when "DB_ROLL_PTR" + this_record.roll_pointer = f[:value] + end + end end + + Innodb::Record.new(self, this_record) end + end - # If this is a node (non-leaf) page, it will have a child page number - # (or "node pointer") stored as the last field. - if level > 0 - # Read the node pointer in a node (non-leaf) page. - this_record[:child_page_number] = - c.name("child_page_number") { c.get_uint32 } + # Return an array of row offsets for all entries in the page directory. + def directory + @directory ||= cursor(pos_directory).backward.name("page_directory") do |c| + directory_slots.times.map { |n| c.name("slot[#{n}]") { c.read_uint16 } } end + end - this_record[:length] = c.position - offset + # Return the slot number of the provided offset in the page directory, or nil + # if the offset is not present in the page directory. + def offset_directory_slot(offset) + directory.index(offset) + end - # Add system field accessors for convenience. - this_record[:sys].each do |f| - case f[:name] - when "DB_TRX_ID" - this_record[:transaction_id] = f[:value] - when "DB_ROLL_PTR" - this_record[:roll_pointer] = f[:value] - end - end + # Return the slot number of the provided record in the page directory, or nil + # if the record is not present in the page directory. + def record_directory_slot(this_record) + offset_directory_slot(this_record.offset) end - Innodb::Record.new(self, this_record) - end - end + # Return the slot number for the page directory entry which "owns" the + # provided record. This will be either the record itself, or the nearest + # record with an entry in the directory and a value greater than the record. + def directory_slot_for_record(this_record) + slot = record_directory_slot(this_record) + return slot if slot + + search_cursor = record_cursor(this_record.next) + raise "Could not position cursor" unless search_cursor - # Return an array of row offsets for all entries in the page directory. - def directory - return @directory if @directory + while (rec = search_cursor.record) + slot = record_directory_slot(rec) + return slot if slot + end - @directory = [] - cursor(pos_directory).backward.name("page_directory") do |c| - directory_slots.times do |n| - @directory.push c.name("slot[#{n}]") { c.get_uint16 } + record_directory_slot(supremum) end - end - @directory - end + def each_directory_offset + return enum_for(:each_directory_offset) unless block_given? - # Return the slot number of the provided offset in the page directory, or nil - # if the offset is not present in the page directory. - def offset_is_directory_slot?(offset) - directory.index(offset) - end + directory.each do |offset| + yield offset unless [pos_infimum, pos_supremum].include?(offset) + end + end - # Return the slot number of the provided record in the page directory, or nil - # if the record is not present in the page directory. - def record_is_directory_slot?(this_record) - offset_is_directory_slot?(this_record.offset) - end + def each_directory_record + return enum_for(:each_directory_record) unless block_given? - # Return the slot number for the page directory entry which "owns" the - # provided record. This will be either the record itself, or the nearest - # record with an entry in the directory and a value greater than the record. - def directory_slot_for_record(this_record) - if slot = record_is_directory_slot?(this_record) - return slot - end + each_directory_offset do |offset| + yield record(offset) + end + end - unless search_cursor = record_cursor(this_record.next) - raise "Couldn't position cursor" - end + # A class for cursoring through records starting from an arbitrary point. + class RecordCursor + def initialize(page, offset, direction) + Innodb::Stats.increment :page_record_cursor_create - while rec = search_cursor.record - if slot = record_is_directory_slot?(rec) - return slot - end - end + @initial = true + @page = page + @direction = direction + @record = initial_record(offset) + end - return record_is_directory_slot?(supremum) - end + def initial_record(offset) + case offset + when :min + @page.min_record + when :max + @page.max_record + else + # Offset is a byte offset of a record (hopefully). + @page.record(offset) + end + end - def each_directory_offset - unless block_given? - return enum_for(:each_directory_offset) - end + # Return the next record, and advance the cursor. Return nil when the + # end of records (supremum) is reached. + def next_record + Innodb::Stats.increment :page_record_cursor_next_record - directory.each do |offset| - yield offset unless [pos_infimum, pos_supremum].include?(offset) - end - end + rec = @page.record(@record.next) - def each_directory_record - unless block_given? - return enum_for(:each_directory_record) - end + # The garbage record list's end is self-linked, so we must check for + # both supremum and the current record's offset. + if rec == @page.supremum || rec.offset == @record.offset + # We've reached the end of the linked list at supremum. + nil + else + @record = rec + end + end - each_directory_offset do |offset| - yield record(offset) - end - end + # Return the previous record, and advance the cursor. Return nil when the + # end of records (infimum) is reached. + def prev_record + Innodb::Stats.increment :page_record_cursor_prev_record - # A class for cursoring through records starting from an arbitrary point. - class RecordCursor - def initialize(page, offset, direction) - Innodb::Stats.increment :page_record_cursor_create - - @initial = true - @page = page - @direction = direction - case offset - when :min - @record = @page.min_record - when :max - @record = @page.max_record - else - # Offset is a byte offset of a record (hopefully). - @record = @page.record(offset) - end - end + slot = @page.directory_slot_for_record(@record) + raise "Could not find slot for record" unless slot - # Return the next record, and advance the cursor. Return nil when the - # end of records (supremum) is reached. - def next_record - Innodb::Stats.increment :page_record_cursor_next_record + search_cursor = @page.record_cursor(@page.directory[slot - 1]) + raise "Could not position search cursor" unless search_cursor - rec = @page.record(@record.next) + while (rec = search_cursor.record) && rec.offset != @record.offset + next unless rec.next == @record.offset - # The garbage record list's end is self-linked, so we must check for - # both supremum and the current record's offset. - if rec == @page.supremum || rec.offset == @record.offset - # We've reached the end of the linked list at supremum. - nil - else - @record = rec - end - end + return if rec == @page.infimum - # Return the previous record, and advance the cursor. Return nil when the - # end of records (infimum) is reached. - def prev_record - Innodb::Stats.increment :page_record_cursor_prev_record + return @record = rec + end + end - unless slot = @page.directory_slot_for_record(@record) - raise "Couldn't find slot for record" - end + # Return the next record in the order defined when the cursor was created. + def record + if @initial + @initial = false + return @record + end - unless search_cursor = @page.record_cursor(@page.directory[slot-1]) - raise "Couldn't position search cursor" - end + case @direction + when :forward + next_record + when :backward + prev_record + end + end - while rec = search_cursor.record and rec.offset != @record.offset - if rec.next == @record.offset - if rec == @page.infimum - return nil + # Iterate through all records in the cursor. + def each_record + return enum_for(:each_record) unless block_given? + + while (rec = record) + yield rec end - return @record = rec end end - end - # Return the next record in the order defined when the cursor was created. - def record - if @initial - @initial = false - return @record + # Return a RecordCursor starting at offset. + def record_cursor(offset = :min, direction = :forward) + RecordCursor.new(self, offset, direction) end - case @direction - when :forward - next_record - when :backward - prev_record + def record_if_exists(offset) + each_record do |rec| + return rec if rec.offset == offset + end end - end - # Iterate through all records in the cursor. - def each_record - unless block_given? - return enum_for(:each_record) + # Return the minimum record on this page. + def min_record + min = record(infimum.next) + min if min != supremum end - while rec = record - yield rec + # Return the maximum record on this page. + def max_record + # Since the records are only singly-linked in the forward direction, in + # order to do find the last record, we must create a cursor and walk + # backwards one step. + max_cursor = record_cursor(supremum.offset, :backward) + raise "Could not position cursor" unless max_cursor + + # Note the deliberate use of prev_record rather than record; we want + # to skip over supremum itself. + max = max_cursor.prev_record + max if max != infimum end - end - end - # Return a RecordCursor starting at offset. - def record_cursor(offset=:min, direction=:forward) - RecordCursor.new(self, offset, direction) - end - - def record_if_exists(offset) - each_record do |rec| - return rec if rec.offset == offset - end - end - - # Return the minimum record on this page. - def min_record - min = record(infimum.next) - min if min != supremum - end + # Search for a record within a single page, and return either a perfect + # match for the key, or the last record closest to they key but not greater + # than the key. (If an exact match is desired, compare_key must be used to + # check if the returned record matches. This makes the function useful for + # search in both leaf and non-leaf pages.) + def linear_search_from_cursor(search_cursor, key) + Innodb::Stats.increment :linear_search_from_cursor + + this_rec = search_cursor.record + + if Innodb.debug? + puts "linear_search_from_cursor: page=%i, level=%i, start=(%s)" % [ + offset, + level, + this_rec && this_rec.key_string, + ] + end - # Return the maximum record on this page. - def max_record - # Since the records are only singly-linked in the forward direction, in - # order to do find the last record, we must create a cursor and walk - # backwards one step. - unless max_cursor = record_cursor(supremum.offset, :backward) - raise "Couldn't position cursor" - end - # Note the deliberate use of prev_record rather than record; we want - # to skip over supremum itself. - max = max_cursor.prev_record - max if max != infimum - end + # Iterate through all records until finding either a matching record or + # one whose key is greater than the desired key. + while this_rec && (next_rec = search_cursor.record) + Innodb::Stats.increment :linear_search_from_cursor_record_scans + + if Innodb.debug? + puts "linear_search_from_cursor: page=%i, level=%i, current=(%s)" % [ + offset, + level, + this_rec.key_string, + ] + end - # Search for a record within a single page, and return either a perfect - # match for the key, or the last record closest to they key but not greater - # than the key. (If an exact match is desired, compare_key must be used to - # check if the returned record matches. This makes the function useful for - # search in both leaf and non-leaf pages.) - def linear_search_from_cursor(search_cursor, key) - Innodb::Stats.increment :linear_search_from_cursor - - this_rec = search_cursor.record - - if Innodb.debug? - puts "linear_search_from_cursor: page=%i, level=%i, start=(%s)" % [ - offset, - level, - this_rec && this_rec.key_string, - ] - end + # If we reach supremum, return the last non-system record we got. + return this_rec if next_rec.header.type == :supremum - # Iterate through all records until finding either a matching record or - # one whose key is greater than the desired key. - while this_rec && next_rec = search_cursor.record - Innodb::Stats.increment :linear_search_from_cursor_record_scans - - if Innodb.debug? - puts "linear_search_from_cursor: page=%i, level=%i, current=(%s)" % [ - offset, - level, - this_rec && this_rec.key_string, - ] - end + return this_rec if this_rec.compare_key(key).negative? - # If we reach supremum, return the last non-system record we got. - return this_rec if next_rec.header[:type] == :supremum + # The desired key is either an exact match for this_rec or is greater + # than it but less than next_rec. If this is a non-leaf page, that + # will mean that the record will fall on the leaf page this node + # pointer record points to, if it exists at all. + return this_rec if !this_rec.compare_key(key).negative? && next_rec.compare_key(key).negative? - if this_rec.compare_key(key) < 0 - return this_rec - end + this_rec = next_rec + end - if (this_rec.compare_key(key) >= 0) && - (next_rec.compare_key(key) < 0) - # The desired key is either an exact match for this_rec or is greater - # than it but less than next_rec. If this is a non-leaf page, that - # will mean that the record will fall on the leaf page this node - # pointer record points to, if it exists at all. - return this_rec + this_rec end - this_rec = next_rec - end - - this_rec - end - - # Search or a record within a single page using the page directory to limit - # the number of record comparisons required. Once the last page directory - # entry closest to but not greater than the key is found, fall back to - # linear search using linear_search_from_cursor to find the closest record - # whose key is not greater than the desired key. (If an exact match is - # desired, the returned record must be checked in the same way as the above - # linear_search_from_cursor function.) - def binary_search_by_directory(dir, key) - Innodb::Stats.increment :binary_search_by_directory - - return nil if dir.empty? - - # Split the directory at the mid-point (using integer math, so the division - # is rounding down). Retrieve the record that sits at the mid-point. - mid = ((dir.size-1) / 2) - rec = record(dir[mid]) - - if Innodb.debug? - puts "binary_search_by_directory: page=%i, level=%i, dir.size=%i, dir[%i]=(%s)" % [ - offset, - level, - dir.size, - mid, - rec.key_string, - ] - end - - # The mid-point record was the infimum record, which is not comparable with - # compare_key, so we need to just linear scan from here. If the mid-point - # is the beginning of the page there can't be many records left to check - # anyway. - if rec.header[:type] == :infimum - return linear_search_from_cursor(record_cursor(rec.next), key) - end + # Search or a record within a single page using the page directory to limit + # the number of record comparisons required. Once the last page directory + # entry closest to but not greater than the key is found, fall back to + # linear search using linear_search_from_cursor to find the closest record + # whose key is not greater than the desired key. (If an exact match is + # desired, the returned record must be checked in the same way as the above + # linear_search_from_cursor function.) + def binary_search_by_directory(dir, key) + Innodb::Stats.increment :binary_search_by_directory + + return if dir.empty? + + # Split the directory at the mid-point (using integer math, so the division + # is rounding down). Retrieve the record that sits at the mid-point. + mid = ((dir.size - 1) / 2) + rec = record(dir[mid]) + return unless rec + + if Innodb.debug? + puts "binary_search_by_directory: page=%i, level=%i, dir.size=%i, dir[%i]=(%s)" % [ + offset, + level, + dir.size, + mid, + rec.key_string, + ] + end - # Compare the desired key to the mid-point record's key. - case rec.compare_key(key) - when 0 - # An exact match for the key was found. Return the record. - Innodb::Stats.increment :binary_search_by_directory_exact_match - rec - when +1 - # The mid-point record's key is less than the desired key. - if dir.size > 2 - # There are more entries remaining from the directory, recurse again - # using binary search on the right half of the directory, which - # represents values greater than or equal to the mid-point record's - # key. - Innodb::Stats.increment :binary_search_by_directory_recurse_right - binary_search_by_directory(dir[mid...dir.size], key) - else - next_rec = record(dir[mid+1]) - next_key = next_rec && next_rec.compare_key(key) - if dir.size == 1 || next_key == -1 || next_key == 0 - # This is the last entry remaining from the directory, or our key is - # greater than rec and less than rec+1's key. Use linear search to - # find the record starting at rec. - Innodb::Stats.increment :binary_search_by_directory_linear_search - linear_search_from_cursor(record_cursor(rec.offset), key) - elsif next_key == +1 - Innodb::Stats.increment :binary_search_by_directory_linear_search - linear_search_from_cursor(record_cursor(next_rec.offset), key) - else - nil + # The mid-point record was the infimum record, which is not comparable with + # compare_key, so we need to just linear scan from here. If the mid-point + # is the beginning of the page there can't be many records left to check + # anyway. + return linear_search_from_cursor(record_cursor(rec.next), key) if rec.header.type == :infimum + + # Compare the desired key to the mid-point record's key. + case rec.compare_key(key) + when 0 + # An exact match for the key was found. Return the record. + Innodb::Stats.increment :binary_search_by_directory_exact_match + rec + when +1 + # The mid-point record's key is less than the desired key. + if dir.size > 2 + # There are more entries remaining from the directory, recurse again + # using binary search on the right half of the directory, which + # represents values greater than or equal to the mid-point record's + # key. + Innodb::Stats.increment :binary_search_by_directory_recurse_right + binary_search_by_directory(dir[mid...dir.size], key) + else + next_rec = record(dir[mid + 1]) + next_key = next_rec&.compare_key(key) + if dir.size == 1 || next_key == -1 || next_key.zero? + # This is the last entry remaining from the directory, or our key is + # greater than rec and less than rec+1's key. Use linear search to + # find the record starting at rec. + Innodb::Stats.increment :binary_search_by_directory_linear_search + linear_search_from_cursor(record_cursor(rec.offset), key) + elsif next_key == +1 + Innodb::Stats.increment :binary_search_by_directory_linear_search + linear_search_from_cursor(record_cursor(next_rec.offset), key) + end + end + when -1 + # The mid-point record's key is greater than the desired key. + if dir.size == 1 + # If this is the last entry remaining from the directory, we didn't + # find anything workable. + Innodb::Stats.increment :binary_search_by_directory_empty_result + nil + else + # Recurse on the left half of the directory, which represents values + # less than the mid-point record's key. + Innodb::Stats.increment :binary_search_by_directory_recurse_left + binary_search_by_directory(dir[0...mid], key) + end end end - when -1 - # The mid-point record's key is greater than the desired key. - if dir.size == 1 - # If this is the last entry remaining from the directory, we didn't - # find anything workable. - Innodb::Stats.increment :binary_search_by_directory_empty_result - nil - else - # Recurse on the left half of the directory, which represents values - # less than the mid-point record's key. - Innodb::Stats.increment :binary_search_by_directory_recurse_left - binary_search_by_directory(dir[0...mid], key) - end - end - end - # Iterate through all records. - def each_record - unless block_given? - return enum_for(:each_record) - end + # Iterate through all records. + def each_record + return enum_for(:each_record) unless block_given? - c = record_cursor(:min) + c = record_cursor(:min) - while rec = c.record - yield rec - end - - nil - end - - # Iterate through all records in the garbage list. - def each_garbage_record - unless block_given? - return enum_for(:each_garbage_record) - end - - if garbage_offset == 0 - return nil - end + while (rec = c.record) + yield rec + end - c = record_cursor(garbage_offset) + nil + end - while rec = c.record - yield rec - end + # Iterate through all records in the garbage list. + def each_garbage_record + return enum_for(:each_garbage_record) unless block_given? + return if garbage_offset.zero? - nil - end + c = record_cursor(garbage_offset) - # Iterate through all child pages of a node (non-leaf) page, which are - # stored as records with the child page number as the last field in the - # record. - def each_child_page - return nil if level == 0 + while (rec = c.record) + yield rec + end - unless block_given? - return enum_for(:each_child_page) - end + nil + end - each_record do |rec| - yield rec.child_page_number, rec.key - end + # Iterate through all child pages of a node (non-leaf) page, which are + # stored as records with the child page number as the last field in the + # record. + def each_child_page + return if leaf? - nil - end + return enum_for(:each_child_page) unless block_given? - def each_region - unless block_given? - return enum_for(:each_region) - end + each_record do |rec| + yield rec.child_page_number, rec.key + end - super do |region| - yield region - end + nil + end - yield({ - :offset => pos_index_header, - :length => size_index_header, - :name => :index_header, - :info => "Index Header", - }) - - yield({ - :offset => pos_fseg_header, - :length => size_fseg_header, - :name => :fseg_header, - :info => "File Segment Header", - }) - - yield({ - :offset => pos_infimum - 5, - :length => size_mum_record + 5, - :name => :infimum, - :info => "Infimum", - }) - - yield({ - :offset => pos_supremum - 5, - :length => size_mum_record + 5, - :name => :supremum, - :info => "Supremum", - }) - - directory_slots.times do |n| - yield({ - :offset => pos_directory - (n * 2), - :length => 2, - :name => :directory, - :info => "Page Directory", - }) - end + def each_region(&block) + return enum_for(:each_region) unless block_given? + + super + + yield Region.new( + offset: pos_index_header, + length: size_index_header, + name: :index_header, + info: "Index Header" + ) + + yield Region.new( + offset: pos_fseg_header, + length: size_fseg_header, + name: :fseg_header, + info: "File Segment Header" + ) + + yield Region.new( + offset: pos_infimum - 5, + length: size_mum_record + 5, + name: :infimum, + info: "Infimum" + ) + + yield Region.new( + offset: pos_supremum - 5, + length: size_mum_record + 5, + name: :supremum, + info: "Supremum" + ) + + directory_slots.times do |n| + yield Region.new( + offset: pos_directory - (n * 2), + length: 2, + name: :directory, + info: "Page Directory" + ) + end - each_garbage_record do |record| - yield({ - :offset => record.offset - record.header[:length], - :length => record.length + record.header[:length], - :name => :garbage, - :info => "Garbage", - }) - end + each_garbage_record do |record| + yield Region.new( + offset: record.offset - record.header.length, + length: record.length + record.header.length, + name: :garbage, + info: "Garbage" + ) + end - each_record do |record| - yield({ - :offset => record.offset - record.header[:length], - :length => record.header[:length], - :name => :record_header, - :info => "Record Header", - }) - - yield({ - :offset => record.offset, - :length => record.length || 1, - :name => :record_data, - :info => "Record Data", - }) - end + each_record do |record| + yield Region.new( + offset: record.offset - record.header.length, + length: record.header.length, + name: :record_header, + info: "Record Header" + ) + + yield Region.new( + offset: record.offset, + length: record.length || 1, + name: :record_data, + info: "Record Data" + ) + end - nil - end + nil + end - # Dump the contents of a page for debugging purposes. - def dump - super - - puts "page header:" - pp page_header - puts - - puts "fseg header:" - pp fseg_header - puts - - puts "sizes:" - puts " %-15s%5i" % [ "header", header_space ] - puts " %-15s%5i" % [ "trailer", trailer_space ] - puts " %-15s%5i" % [ "directory", directory_space ] - puts " %-15s%5i" % [ "free", free_space ] - puts " %-15s%5i" % [ "used", used_space ] - puts " %-15s%5i" % [ "record", record_space ] - puts " %-15s%5.2f" % [ - "per record", - (page_header[:n_recs] > 0) ? (record_space / page_header[:n_recs]) : 0 - ] - puts - - puts "page directory:" - pp directory - puts - - puts "system records:" - pp infimum.record - pp supremum.record - puts - - puts "garbage records:" - each_garbage_record do |rec| - pp rec.record - puts - end - puts + # Dump the contents of a page for debugging purposes. + def dump + super + + puts "page header:" + pp page_header + puts + + puts "fseg header:" + pp fseg_header + puts + + puts "sizes:" + puts " %-15s%5i" % ["header", header_space] + puts " %-15s%5i" % ["trailer", trailer_space] + puts " %-15s%5i" % ["directory", directory_space] + puts " %-15s%5i" % ["free", free_space] + puts " %-15s%5i" % ["used", used_space] + puts " %-15s%5i" % ["record", record_space] + puts " %-15s%5.2f" % ["per record", space_per_record] + puts + + puts "page directory:" + pp directory + puts + + puts "system records:" + pp infimum.record + pp supremum.record + puts + + if ibuf_index? + puts "(records not dumped due to this being an insert buffer index)" + elsif !record_describer + puts "(records not dumped due to missing record describer or data dictionary)" + else + puts "garbage records:" + each_garbage_record do |rec| + pp rec.record + puts + end + puts - puts "records:" - each_record do |rec| - pp rec.record - puts + puts "records:" + each_record do |rec| + pp rec.record + puts + end + end + puts + end end - puts end end - -Innodb::Page::SPECIALIZED_CLASSES[:INDEX] = Innodb::Page::Index diff --git a/lib/innodb/page/index_compressed.rb b/lib/innodb/page/index_compressed.rb index e60345f3..f0f44fcb 100644 --- a/lib/innodb/page/index_compressed.rb +++ b/lib/innodb/page/index_compressed.rb @@ -1,46 +1,46 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true # This is horribly incomplete and broken. InnoDB compression does not # currently work in innodb_ruby. Patches are welcome! # (Hint hint, nudge nudge, Facebook developers!) -class Innodb::Page::Index::Compressed < Innodb::Page::Index - # The number of directory slots in use. - def directory_slots - page_header[:n_heap] - 2 - end +module Innodb + class Page + class IndexCompressed < Index + specialization_for({ type: :INDEX, compressed: true }) - def directory - super.map { |n| n & 0x3fff } - end + # The number of directory slots in use. + def directory_slots + page_header[:n_heap] - 2 + end - def uncompressed_columns_size - if level == 0 - if record_format && record_format[:type] == :clustered - 6 + 7 # Transaction ID + Roll Pointer - else - 0 + def directory + super.map { |n| n & 0x3fff } end - else - 4 # Node pointer for non-leaf pages - end - end - # Return the amount of free space in the page. - def free_space - free_space_start = size - size_fil_trailer - directory_space - - (uncompressed_columns_size * (page_header[:n_heap] - 2)) - puts "Free space start == %04x" % [offset * size + free_space_start] - c = cursor(free_space_start).backward - zero_bytes = 0 - while (b = c.get_uint8) == 0 - zero_bytes += 1 + def uncompressed_columns_size + if leaf? + if record_format && record_format[:type] == :clustered + 6 + 7 # Transaction ID + Roll Pointer + else + 0 + end + else + 4 # Node pointer for non-leaf pages + end + end + + # Return the amount of free space in the page. + def free_space + free_space_start = + size - size_fil_trailer - directory_space - (uncompressed_columns_size * (page_header.n_heap - 2)) + puts "Free space start == %04x" % [(offset * size) + free_space_start] + c = cursor(free_space_start).backward + zero_bytes = 0 + zero_bytes += 1 while c.read_uint8.zero? + zero_bytes + # page_header[:garbage] + (size - size_fil_trailer - directory_space - page_header[:heap_top]) + end end - zero_bytes - #page_header[:garbage] + - # (size - size_fil_trailer - directory_space - page_header[:heap_top]) end end - -Innodb::Page::SPECIALIZED_CLASSES[{:type => :INDEX, :compressed => true}] = Innodb::Page::Index::Compressed - diff --git a/lib/innodb/page/inode.rb b/lib/innodb/page/inode.rb index fc5f2b98..e98172bb 100644 --- a/lib/innodb/page/inode.rb +++ b/lib/innodb/page/inode.rb @@ -1,139 +1,130 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true require "innodb/list" # A specialized class for handling INODE pages, which contain index FSEG (file # segment) information. This allows all extents and individual pages assigned # to each index to be found. -class Innodb::Page::Inode < Innodb::Page - # Return the byte offset of the list node, which immediately follows the - # FIL header. - def pos_list_entry - pos_page_body - end - - # Return the size of the list node. - def size_list_entry - Innodb::List::NODE_SIZE - end - - # Return the byte offset of the Inode array in the page, which immediately - # follows the list entry. - def pos_inode_array - pos_list_entry + size_list_entry - end - - # The number of Inode entries that fit on a page. - def inodes_per_page - (size - pos_inode_array - 10) / Innodb::Inode::SIZE - end +module Innodb + class Page + class Inode < Page + specialization_for :INODE + + # Return the byte offset of the list node, which immediately follows the + # FIL header. + def pos_list_entry + pos_page_body + end - def size_inode_array - inodes_per_page * Innodb::Inode::SIZE - end + # Return the size of the list node. + def size_list_entry + Innodb::List::NODE_SIZE + end - # Return the list entry. - def list_entry - cursor(pos_list_entry).name("list") do |c| - Innodb::List.get_node(c) - end - end + # Return the byte offset of the Inode array in the page, which immediately + # follows the list entry. + def pos_inode_array + pos_list_entry + size_list_entry + end - # Return the "previous" address pointer from the list entry. This is used - # by Innodb::List::Inode to iterate through Inode lists. - def prev_address - list_entry[:prev] - end + # The number of Inode entries that fit on a page. + def inodes_per_page + (size - pos_inode_array - 10) / Innodb::Inode::SIZE + end - # Return the "next" address pointer from the list entry. This is used - # by Innodb::List::Inode to iterate through Inode lists. - def next_address - list_entry[:next] - end + def size_inode_array + inodes_per_page * Innodb::Inode::SIZE + end - # Read a single Inode entry from the provided byte offset by creating a - # cursor and reading the inode using the inode method. - def inode_at(cursor) - cursor.name("inode[#{cursor.position}]") { |c| Innodb::Inode.new_from_cursor(@space, c) } - end + # Return the list entry. + def list_entry + cursor(pos_list_entry).name("list") { |c| Innodb::List.get_node(c) } + end - # Iterate through all Inodes in the inode array. - def each_inode - unless block_given? - return enum_for(:each_inode) - end + # Return the "previous" address pointer from the list entry. This is used + # by Innodb::List::Inode to iterate through Inode lists. + def prev_address + list_entry.prev + end - inode_cursor = cursor(pos_inode_array) - inodes_per_page.times do |n| - inode_cursor.name("inode[#{n}]") do |c| - this_inode = Innodb::Inode.new_from_cursor(@space, c) - yield this_inode + # Return the "next" address pointer from the list entry. This is used + # by Innodb::List::Inode to iterate through Inode lists. + def next_address + list_entry.next end - end - end - # Iterate through all allocated inodes in the inode array. - def each_allocated_inode - unless block_given? - return enum_for(:each_allocated_inode) - end + # Read a single Inode entry from the provided byte offset by creating a + # cursor and reading the inode using the inode method. + def inode_at(cursor) + cursor.name("inode[#{cursor.position}]") { |c| Innodb::Inode.new_from_cursor(@space, c) } + end - each_inode do |this_inode| - yield this_inode if this_inode.allocated? - end - end + # Iterate through all Inodes in the inode array. + def each_inode + return enum_for(:each_inode) unless block_given? - def each_region - unless block_given? - return enum_for(:each_region) - end + inode_cursor = cursor(pos_inode_array) + inodes_per_page.times do |n| + inode_cursor.name("inode[#{n}]") do |c| + yield Innodb::Inode.new_from_cursor(@space, c) + end + end + end - super do |region| - yield region - end + # Iterate through all allocated inodes in the inode array. + def each_allocated_inode + return enum_for(:each_allocated_inode) unless block_given? - yield({ - :offset => pos_list_entry, - :length => size_list_entry, - :name => :list_entry, - :info => "Inode List Entry", - }) - - each_inode do |inode| - if inode.allocated? - yield({ - :offset => inode.offset, - :length => Innodb::Inode::SIZE, - :name => :inode_used, - :info => "Inode (used)", - }) - else - yield({ - :offset => inode.offset, - :length => Innodb::Inode::SIZE, - :name => :inode_free, - :info => "Inode (free)", - }) + each_inode do |this_inode| + yield this_inode if this_inode.allocated? + end end - end - nil - end + def each_region(&block) + return enum_for(:each_region) unless block_given? + + super + + yield Region.new( + offset: pos_list_entry, + length: size_list_entry, + name: :list_entry, + info: "Inode List Entry" + ) + + each_inode do |inode| + if inode.allocated? + yield Region.new( + offset: inode.offset, + length: Innodb::Inode::SIZE, + name: :inode_used, + info: "Inode (used)" + ) + else + yield Region.new( + offset: inode.offset, + length: Innodb::Inode::SIZE, + name: :inode_free, + info: "Inode (free)" + ) + end + end + + nil + end - # Dump the contents of a page for debugging purposes. - def dump - super + # Dump the contents of a page for debugging purposes. + def dump + super - puts "list entry:" - pp list_entry - puts + puts "list entry:" + pp list_entry + puts - puts "inodes:" - each_inode do |inode| - inode.dump + puts "inodes:" + each_inode(&:dump) + puts + end end - puts end end - -Innodb::Page::SPECIALIZED_CLASSES[:INODE] = Innodb::Page::Inode diff --git a/lib/innodb/page/sdi.rb b/lib/innodb/page/sdi.rb new file mode 100644 index 00000000..28af1dc7 --- /dev/null +++ b/lib/innodb/page/sdi.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Innodb + class Page + # SDI (Serialized Dictionary Information) pages are actually INDEX pages and store data dictionary + # information in an InnoDB index structure in the typical way. However they use a fixed definition + # for the (unnamed except for in-memory as "SDI_") index, since there would, logically, + # be nowhere else to store the definition of this index. + class Sdi < Index + specialization_for :SDI + + # Every SDI index has the same structure, equivalent to the following SQL: + # + # CREATE TABLE `SDI_` ( + # `type` INT UNSIGNED NOT NULL, + # `id` BIGINT UNSIGNED NOT NULL, + # `uncompressed_len` INT UNSIGNED NOT NULL, + # `compressed_len` INT UNSIGNED NOT NULL, + # `data` LONGBLOB NOT NULL, + # PRIMARY KEY (`type`, `id`) + # ) + # + class RecordDescriber < Innodb::RecordDescriber + type :clustered + key "type", :INT, :UNSIGNED, :NOT_NULL + key "id", :BIGINT, :UNSIGNED, :NOT_NULL + row "uncompressed_len", :INT, :UNSIGNED, :NOT_NULL + row "compressed_len", :INT, :UNSIGNED, :NOT_NULL + row "data", :BLOB, :NOT_NULL + end + + def make_record_describer + RecordDescriber.new + end + end + end +end diff --git a/lib/innodb/page/sdi_blob.rb b/lib/innodb/page/sdi_blob.rb new file mode 100644 index 00000000..35d0252e --- /dev/null +++ b/lib/innodb/page/sdi_blob.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Innodb + class Page + # SDI (Serialized Dictionary Information) BLOB pages are actually BLOB pages with a different page + # type number but otherwise the same structure. + class SdiBlob < Blob + specialization_for :SDI_BLOB + end + end +end diff --git a/lib/innodb/page/sys.rb b/lib/innodb/page/sys.rb index 984e4281..486b2ef7 100644 --- a/lib/innodb/page/sys.rb +++ b/lib/innodb/page/sys.rb @@ -1,4 +1,4 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true require "innodb/page/sys_rseg_header" require "innodb/page/sys_data_dictionary_header" @@ -7,20 +7,18 @@ # Another layer of indirection for pages of type SYS, as they have multiple # uses within InnoDB. We'll override the self.handle method and check the # page offset to decide which type of SYS page this is. -class Innodb::Page::Sys < Innodb::Page - def self.handle(page, space, buffer, page_number=nil) - case - when page.offset == 3 - Innodb::Page::SysIbufHeader.new(space, buffer, page_number) - when page.offset == 7 - Innodb::Page::SysDataDictionaryHeader.new(space, buffer, page_number) - when space.rseg_page?(page.offset) - Innodb::Page::SysRsegHeader.new(space, buffer, page_number) - else - # We can't do anything better, so pass on the generic InnoDB::Page. - page +module Innodb + class Page + class Sys < Page + specialization_for :SYS + + def self.handle(page, space, buffer, page_number = nil) + return Innodb::Page::SysIbufHeader.new(space, buffer, page_number) if page.offset == 3 + return Innodb::Page::SysDataDictionaryHeader.new(space, buffer, page_number) if page.offset == 7 + return Innodb::Page::SysRsegHeader.new(space, buffer, page_number) if space.rseg_page?(page.offset) + + page + end end end end - -Innodb::Page::SPECIALIZED_CLASSES[:SYS] = Innodb::Page::Sys diff --git a/lib/innodb/page/sys_data_dictionary_header.rb b/lib/innodb/page/sys_data_dictionary_header.rb index a9e4d2f5..1fe0601d 100644 --- a/lib/innodb/page/sys_data_dictionary_header.rb +++ b/lib/innodb/page/sys_data_dictionary_header.rb @@ -1,70 +1,92 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true -class Innodb::Page::SysDataDictionaryHeader < Innodb::Page - # The position of the data dictionary header within the page. - def pos_data_dictionary_header - pos_page_body - end +module Innodb + class Page + class SysDataDictionaryHeader < Page + Header = Struct.new( + :max_row_id, + :max_table_id, + :max_index_id, + :max_space_id, + :unused_mix_id_low, + :indexes, + :unused_space, + :fseg, + keyword_init: true + ) - # The size of the data dictionary header. - def size_data_dictionary_header - ((8 * 3) + (4 * 7) + 4 + Innodb::FsegEntry::SIZE) - end + # The position of the data dictionary header within the page. + def pos_data_dictionary_header + pos_page_body + end - # Parse the data dictionary header from the page. - def data_dictionary_header - cursor(pos_data_dictionary_header).name("data_dictionary_header") do |c| - { - :max_row_id => c.name("max_row_id") { c.get_uint64 }, - :max_table_id => c.name("max_table_id") { c.get_uint64 }, - :max_index_id => c.name("max_index_id") { c.get_uint64 }, - :max_space_id => c.name("max_space_id") { c.get_uint32 }, - :unused_mix_id_low => c.name("unused_mix_id_low") { c.get_uint32 }, - :indexes => c.name("indexes") {{ - :SYS_TABLES => c.name("SYS_TABLES") {{ - :PRIMARY => c.name("PRIMARY") { c.get_uint32 }, - :ID => c.name("ID") { c.get_uint32 }, - }}, - :SYS_COLUMNS => c.name("SYS_COLUMNS") {{ - :PRIMARY => c.name("PRIMARY") { c.get_uint32 }, - }}, - :SYS_INDEXES => c.name("SYS_INDEXES") {{ - :PRIMARY => c.name("PRIMARY") { c.get_uint32 }, - }}, - :SYS_FIELDS => c.name("SYS_FIELDS") {{ - :PRIMARY => c.name("PRIMARY") { c.get_uint32 }, - }} - }}, - :unused_space => c.name("unused_space") { c.get_bytes(4) }, - :fseg => c.name("fseg") { Innodb::FsegEntry.get_inode(@space, c) }, - } - end - end + # The size of the data dictionary header. + def size_data_dictionary_header + ((8 * 3) + (4 * 7) + 4 + Innodb::FsegEntry::SIZE) + end - def each_region - unless block_given? - return enum_for(:each_region) - end + # Parse the data dictionary header from the page. + def data_dictionary_header + cursor(pos_data_dictionary_header).name("data_dictionary_header") do |c| + Header.new( + max_row_id: c.name("max_row_id") { c.read_uint64 }, + max_table_id: c.name("max_table_id") { c.read_uint64 }, + max_index_id: c.name("max_index_id") { c.read_uint64 }, + max_space_id: c.name("max_space_id") { c.read_uint32 }, + unused_mix_id_low: c.name("unused_mix_id_low") { c.read_uint32 }, + indexes: c.name("indexes") do + { + "SYS_TABLES" => c.name("SYS_TABLES") do + { + "PRIMARY" => c.name("PRIMARY") { c.read_uint32 }, + "ID" => c.name("ID") { c.read_uint32 }, + } + end, + "SYS_COLUMNS" => c.name("SYS_COLUMNS") do + { + "PRIMARY" => c.name("PRIMARY") { c.read_uint32 }, + } + end, + "SYS_INDEXES" => c.name("SYS_INDEXES") do + { + "PRIMARY" => c.name("PRIMARY") { c.read_uint32 }, + } + end, + "SYS_FIELDS" => c.name("SYS_FIELDS") do + { + "PRIMARY" => c.name("PRIMARY") { c.read_uint32 }, + } + end, + } + end, + unused_space: c.name("unused_space") { c.read_bytes(4) }, + fseg: c.name("fseg") { Innodb::FsegEntry.get_inode(@space, c) } + ) + end + end - super do |region| - yield region - end + def each_region(&block) + return enum_for(:each_region) unless block_given? - yield({ - :offset => pos_data_dictionary_header, - :length => size_data_dictionary_header, - :name => :data_dictionary_header, - :info => "Data Dictionary Header", - }) + super - nil - end + yield Region.new( + offset: pos_data_dictionary_header, + length: size_data_dictionary_header, + name: :data_dictionary_header, + info: "Data Dictionary Header" + ) - def dump - super + nil + end - puts - puts "data_dictionary header:" - pp data_dictionary_header + def dump + super + + puts + puts "data_dictionary header:" + pp data_dictionary_header + end + end end end diff --git a/lib/innodb/page/sys_ibuf_header.rb b/lib/innodb/page/sys_ibuf_header.rb index a82d80a7..791f9c0c 100644 --- a/lib/innodb/page/sys_ibuf_header.rb +++ b/lib/innodb/page/sys_ibuf_header.rb @@ -1,45 +1,48 @@ -# -*- encoding : utf-8 -*- - -class Innodb::Page::SysIbufHeader < Innodb::Page - def pos_ibuf_header - pos_page_body - end - - def size_ibuf_header - Innodb::FsegEntry::SIZE - end - - def ibuf_header - cursor(pos_ibuf_header).name("ibuf_header") do |c| - { - :fseg => c.name("fseg") { - Innodb::FsegEntry.get_inode(space, c) - } - } - end - end - - def each_region - unless block_given? - return enum_for(:each_region) - end - - super do |region| - yield region +# frozen_string_literal: true + +module Innodb + class Page + class SysIbufHeader < Page + Header = Struct.new( + :fseg, + keyword_init: true + ) + + def pos_ibuf_header + pos_page_body + end + + def size_ibuf_header + Innodb::FsegEntry::SIZE + end + + def ibuf_header + cursor(pos_ibuf_header).name("ibuf_header") do |c| + Header.new( + fseg: c.name("fseg") { Innodb::FsegEntry.get_inode(space, c) } + ) + end + end + + def each_region(&block) + return enum_for(:each_region) unless block_given? + + super + + yield Region.new( + offset: pos_ibuf_header, + length: size_ibuf_header, + name: :ibuf_header, + info: "Insert Buffer Header" + ) + end + + def dump + super + + puts "ibuf header:" + pp ibuf_header + end end - - yield({ - :offset => pos_ibuf_header, - :length => size_ibuf_header, - :name => :ibuf_header, - :info => "Insert Buffer Header", - }) - end - - def dump - super - - puts "ibuf header:" - pp ibuf_header end end diff --git a/lib/innodb/page/sys_rseg_header.rb b/lib/innodb/page/sys_rseg_header.rb index f5411e6c..0321319b 100644 --- a/lib/innodb/page/sys_rseg_header.rb +++ b/lib/innodb/page/sys_rseg_header.rb @@ -1,99 +1,105 @@ -# -*- encoding : utf-8 -*- - -class Innodb::Page::SysRsegHeader < Innodb::Page - # The number of undo log slots in the page. - UNDO_SEGMENT_SLOTS = 1024 - - # The position of the rollback segment header within the page. - def pos_rseg_header - pos_page_body - end - - # The size of the rollback segment header. - def size_rseg_header - 4 + 4 + Innodb::List::BASE_NODE_SIZE + Innodb::FsegEntry::SIZE - end +# frozen_string_literal: true + +module Innodb + class Page + class SysRsegHeader < Page + Header = Struct.new( + :max_size, + :history_size, + :history_list, + :fseg, + keyword_init: true + ) + + # The number of undo log slots in the page. + UNDO_SEGMENT_SLOTS = 1024 + + # The position of the rollback segment header within the page. + def pos_rseg_header + pos_page_body + end - def pos_undo_segment_array - pos_rseg_header + size_rseg_header - end + # The size of the rollback segment header. + def size_rseg_header + 4 + 4 + Innodb::List::BASE_NODE_SIZE + Innodb::FsegEntry::SIZE + end - def size_undo_segment_slot - 4 - end + def pos_undo_segment_array + pos_rseg_header + size_rseg_header + end - # Parse the rollback segment header from the page. - def rseg_header - cursor(pos_rseg_header).name("rseg_header") do |c| - { - :max_size => c.name("max_size") { c.get_uint32 }, - :history_size => c.name("history_size") { c.get_uint32 }, - :history_list => c.name("history_list") { - Innodb::List::History.new(@space, Innodb::List.get_base_node(c)) - }, - :fseg => c.name("fseg") { Innodb::FsegEntry.get_inode(@space, c) }, - } - end - end + def size_undo_segment_slot + 4 + end - def history_list - Innodb::HistoryList.new(rseg_header[:history_list]) - end + # Parse the rollback segment header from the page. + def rseg_header + cursor(pos_rseg_header).name("rseg_header") do |c| + Header.new( + max_size: c.name("max_size") { c.read_uint32 }, + history_size: c.name("history_size") { c.read_uint32 }, + history_list: c.name("history_list") do + Innodb::List::History.new(@space, Innodb::List.get_base_node(c)) + end, + fseg: c.name("fseg") { Innodb::FsegEntry.get_inode(@space, c) } + ) + end + end - def each_undo_segment - unless block_given? - return enum_for(:each_undo_segment) - end + def history_list + Innodb::HistoryList.new(rseg_header.history_list) + end - cursor(pos_undo_segment_array).name("undo_segment_array") do |c| - (0...UNDO_SEGMENT_SLOTS).each do |slot| - page_number = c.name("slot[#{slot}]") { - Innodb::Page.maybe_undefined(c.get_uint32) - } - yield slot, page_number + def each_undo_segment + return enum_for(:each_undo_segment) unless block_given? + + cursor(pos_undo_segment_array).name("undo_segment_array") do |c| + (0...UNDO_SEGMENT_SLOTS).each do |slot| + page_number = c.name("slot[#{slot}]") do + Innodb::Page.maybe_undefined(c.read_uint32) + end + yield slot, page_number + end + end end - end - end - def each_region - unless block_given? - return enum_for(:each_region) - end + def each_region(&block) + return enum_for(:each_region) unless block_given? - super do |region| - yield region - end + super - yield({ - :offset => pos_rseg_header, - :length => size_rseg_header, - :name => :rseg_header, - :info => "Rollback Segment Header", - }) - - (0...UNDO_SEGMENT_SLOTS).each do |slot| - yield({ - :offset => pos_undo_segment_array + (slot * size_undo_segment_slot), - :length => size_undo_segment_slot, - :name => :undo_segment_slot, - :info => "Undo Segment Slot", - }) - end + yield Region.new( + offset: pos_rseg_header, + length: size_rseg_header, + name: :rseg_header, + info: "Rollback Segment Header" + ) - nil - end + (0...UNDO_SEGMENT_SLOTS).each do |slot| + yield Region.new( + offset: pos_undo_segment_array + (slot * size_undo_segment_slot), + length: size_undo_segment_slot, + name: :undo_segment_slot, + info: "Undo Segment Slot" + ) + end - def dump - super + nil + end + + def dump + super - puts - puts "rollback segment header:" - pp rseg_header + puts + puts "rollback segment header:" + pp rseg_header - puts - puts "undo segment array:" - each_undo_segment do |slot, page_number| - puts " #{slot}: #{page_number}" + puts + puts "undo segment array:" + each_undo_segment do |slot, page_number| + puts " #{slot}: #{page_number}" + end + end end end end diff --git a/lib/innodb/page/trx_sys.rb b/lib/innodb/page/trx_sys.rb index 69ab09c9..1864ba04 100644 --- a/lib/innodb/page/trx_sys.rb +++ b/lib/innodb/page/trx_sys.rb @@ -1,4 +1,6 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true + +require "forwardable" # A specialized class for TRX_SYS pages, which contain various information # about the transaction system within InnoDB. Only one TRX_SYS page exists in @@ -9,207 +11,227 @@ # empty space, master binary log information, empty space, local binary # log information, empty space, doublewrite information (repeated twice), # empty space, and FIL trailer. -class Innodb::Page::TrxSys < Innodb::Page - # The TRX_SYS header immediately follows the FIL header. - def pos_trx_sys_header - pos_page_body - end +module Innodb + class Page + class TrxSys < Page + extend Forwardable + + specialization_for :TRX_SYS + + RsegSlot = Struct.new( + :offset, + :space_id, + :page_number, + keyword_init: true + ) + + MysqlLogInfo = Struct.new( + :magic_n, + :offset, + :name, + keyword_init: true + ) + + DoublewritePageInfo = Struct.new( + :magic_n, + :page_number, + keyword_init: true + ) + + DoublewriteInfo = Struct.new( + :fseg, + :page_info, + :space_id_stored, + keyword_init: true + ) + + Header = Struct.new( + :trx_id, + :fseg, + :rsegs, + :binary_log, + :master_log, + :doublewrite, + keyword_init: true + ) + + # The TRX_SYS header immediately follows the FIL header. + def pos_trx_sys_header + pos_page_body + end - def size_trx_sys_header - 8 + Innodb::FsegEntry::SIZE - end + def size_trx_sys_header + 8 + Innodb::FsegEntry::SIZE + end - def pos_rsegs_array - pos_trx_sys_header + size_trx_sys_header - end + def pos_rsegs_array + pos_trx_sys_header + size_trx_sys_header + end - def size_mysql_log_info - 4 + 8 + 100 - end + def size_mysql_log_info + 4 + 8 + 100 + end - # The master's binary log information is located 2000 bytes from the end of - # the page. - def pos_mysql_master_log_info - size - 2000 - end + # The master's binary log information is located 2000 bytes from the end of + # the page. + def pos_mysql_master_log_info + size - 2_000 + end - # The local binary log information is located 1000 bytes from the end of - # the page. - def pos_mysql_binary_log_info - size - 1000 - end + # The local binary log information is located 1000 bytes from the end of + # the page. + def pos_mysql_binary_log_info + size - 1_000 + end - # The doublewrite buffer information is located 200 bytes from the end of - # the page. - def pos_doublewrite_info - size - 200 - end + # The doublewrite buffer information is located 200 bytes from the end of + # the page. + def pos_doublewrite_info + size - 200 + end - def size_doublewrite_info - Innodb::FsegEntry::SIZE + (2 * (4 + 4 + 4)) + 4 - end + def size_doublewrite_info + Innodb::FsegEntry::SIZE + (2 * (4 + 4 + 4)) + 4 + end - # A magic number present in each MySQL binary log information structure, - # which helps identify whether the structure is populated or not. - MYSQL_LOG_MAGIC_N = 873422344 - - N_RSEGS = 128 - - def rsegs_array(cursor) - @rsegs_array ||= (0...N_RSEGS).to_a.inject([]) do |a, n| - cursor.name("slot[#{n}]") do |c| - slot = { - :offset => c.position, - :space_id => c.name("space_id") { - Innodb::Page.maybe_undefined(c.get_uint32) - }, - :page_number => c.name("page_number") { - Innodb::Page.maybe_undefined(c.get_uint32) - }, - } - if slot[:space_id] && slot[:page_number] - a << slot + # A magic number present in each MySQL binary log information structure, + # which helps identify whether the structure is populated or not. + MYSQL_LOG_MAGIC_N = 873_422_344 + + # A magic number present in each doublewrite buffer information structure, + # which helps identify whether the structure is populated or not. + DOUBLEWRITE_MAGIC_N = 536_853_855 + + # A magic number present in the overall doublewrite buffer structure, + # which identifies whether the space id is stored. + DOUBLEWRITE_SPACE_ID_STORED_MAGIC_N = 1_783_657_386 + + N_RSEGS = 128 + + def rsegs_array(cursor) + @rsegs_array ||= N_RSEGS.times.each_with_object([]) do |n, a| + cursor.name("slot[#{n}]") do |c| + slot = RsegSlot.new( + offset: c.position, + space_id: c.name("space_id") { Innodb::Page.maybe_undefined(c.read_uint32) }, + page_number: c.name("page_number") { Innodb::Page.maybe_undefined(c.read_uint32) } + ) + a << slot if slot.space_id && slot.page_number + end end end - a - end - end - # Read a MySQL binary log information structure from a given position. - def mysql_log_info(cursor, offset) - cursor.peek(offset) do |c| - if c.name("magic_n") { c.get_uint32 } == MYSQL_LOG_MAGIC_N - { - :offset => c.name("offset") { c.get_uint64 }, - :name => c.name("name") { c.get_bytes(100) }, - } + # Read a MySQL binary log information structure from a given position. + def mysql_log_info(cursor, offset) + cursor.peek(offset) do |c| + magic_n = c.name("magic_n") { c.read_uint32 } == MYSQL_LOG_MAGIC_N + break unless magic_n + + MysqlLogInfo.new( + magic_n: magic_n, + offset: c.name("offset") { c.read_uint64 }, + name: c.name("name") { c.read_bytes(100) } + ) + end end - end - end - # A magic number present in each doublewrite buffer information structure, - # which helps identify whether the structure is populated or not. - DOUBLEWRITE_MAGIC_N = 536853855 - - # Read a single doublewrite buffer information structure from a given cursor. - def doublewrite_page_info(cursor) - { - :magic_n => cursor.name("magic_n") { cursor.get_uint32 }, - :page_number => [ - cursor.name("page[0]") { cursor.get_uint32 }, - cursor.name("page[1]") { cursor.get_uint32 }, - ], - } - end + # Read a single doublewrite buffer information structure from a given cursor. + def doublewrite_page_info(cursor) + magic_n = cursor.name("magic_n") { cursor.read_uint32 } - # A magic number present in the overall doublewrite buffer structure, - # which identifies whether the space id is stored. - DOUBLEWRITE_SPACE_ID_STORED_MAGIC_N = 1783657386 - - # Read the overall doublewrite buffer structures - def doublewrite_info(cursor) - cursor.peek(pos_doublewrite_info) do |c_doublewrite| - c_doublewrite.name("doublewrite") do |c| - { - :fseg => c.name("fseg") { Innodb::FsegEntry.get_inode(@space, c) }, - :page_info => [ - c.name("group[0]") { doublewrite_page_info(c) }, - c.name("group[1]") { doublewrite_page_info(c) }, - ], - :space_id_stored => - (c.name("space_id_stored") { c.get_uint32 } == - DOUBLEWRITE_SPACE_ID_STORED_MAGIC_N), - } + DoublewritePageInfo.new( + magic_n: magic_n, + page_number: [0, 1].map { |n| cursor.name("page[#{n}]") { cursor.read_uint32 } } + ) end - end - end - # Read the TRX_SYS headers and other information. - def trx_sys - @trx_sys ||= cursor(pos_trx_sys_header).name("trx_sys") do |c| - { - :trx_id => c.name("trx_id") { c.get_uint64 }, - :fseg => c.name("fseg") { - Innodb::FsegEntry.get_inode(@space, c) - }, - :rsegs => c.name("rsegs") { - rsegs_array(c) - }, - :binary_log => c.name("binary_log") { - mysql_log_info(c, pos_mysql_binary_log_info) - }, - :master_log => c.name("master_log") { - mysql_log_info(c, pos_mysql_master_log_info) - }, - :doublewrite => doublewrite_info(c), - } - end - end - - def trx_id; trx_sys[:trx_id]; end - def fseg; trx_sys[:fseg]; end - def rsegs; trx_sys[:rsegs]; end - def binary_log; trx_sys[:binary_log]; end - def master_log; trx_sys[:master_log]; end - def doublewrite; trx_sys[:doublewrite]; end - - def each_region - unless block_given? - return enum_for(:each_region) - end + # Read the overall doublewrite buffer structures + def doublewrite_info(cursor) + cursor.peek(pos_doublewrite_info) do |c_doublewrite| + c_doublewrite.name("doublewrite") do |c| + DoublewriteInfo.new( + fseg: c.name("fseg") { Innodb::FsegEntry.get_inode(@space, c) }, + page_info: [0, 1].map { |n| c.name("group[#{n}]") { doublewrite_page_info(c) } }, + space_id_stored: (c.name("space_id_stored") { c.read_uint32 } == DOUBLEWRITE_SPACE_ID_STORED_MAGIC_N) + ) + end + end + end - super do |region| - yield region - end + # Read the TRX_SYS headers and other information. + def trx_sys + @trx_sys ||= cursor(pos_trx_sys_header).name("trx_sys") do |c| + Header.new( + trx_id: c.name("trx_id") { c.read_uint64 }, + fseg: c.name("fseg") { Innodb::FsegEntry.get_inode(@space, c) }, + rsegs: c.name("rsegs") { rsegs_array(c) }, + binary_log: c.name("binary_log") { mysql_log_info(c, pos_mysql_binary_log_info) }, + master_log: c.name("master_log") { mysql_log_info(c, pos_mysql_master_log_info) }, + doublewrite: doublewrite_info(c) + ) + end + end - yield({ - :offset => pos_trx_sys_header, - :length => size_trx_sys_header, - :name => :trx_sys_header, - :info => "Transaction System Header", - }) - - rsegs.each do |rseg| - yield({ - :offset => rseg[:offset], - :length => 4 + 4, - :name => :rseg, - :info => "Rollback Segment", - }) - end + def_delegator :trx_sys, :trx_id + def_delegator :trx_sys, :fseg + def_delegator :trx_sys, :rsegs + def_delegator :trx_sys, :binary_log + def_delegator :trx_sys, :master_log + def_delegator :trx_sys, :doublewrite + + def each_region(&block) + return enum_for(:each_region) unless block_given? + + super + + yield Region.new( + offset: pos_trx_sys_header, + length: size_trx_sys_header, + name: :trx_sys_header, + info: "Transaction System Header" + ) + + rsegs.each do |rseg| + yield Region.new( + offset: rseg[:offset], + length: 4 + 4, + name: :rseg, + info: "Rollback Segment" + ) + end - yield({ - :offset => pos_mysql_binary_log_info, - :length => size_mysql_log_info, - :name => :mysql_binary_log_info, - :info => "Binary Log Info", - }) - - yield({ - :offset => pos_mysql_master_log_info, - :length => size_mysql_log_info, - :name => :mysql_master_log_info, - :info => "Master Log Info", - }) - - yield({ - :offset => pos_doublewrite_info, - :length => size_doublewrite_info, - :name => :doublewrite_info, - :info => "Double Write Buffer Info", - }) - - nil - end + yield Region.new( + offset: pos_mysql_binary_log_info, + length: size_mysql_log_info, + name: :mysql_binary_log_info, + info: "Binary Log Info" + ) + + yield Region.new( + offset: pos_mysql_master_log_info, + length: size_mysql_log_info, + name: :mysql_master_log_info, + info: "Master Log Info" + ) + + yield Region.new( + offset: pos_doublewrite_info, + length: size_doublewrite_info, + name: :doublewrite_info, + info: "Double Write Buffer Info" + ) + + nil + end - # Dump the contents of a page for debugging purposes. - def dump - super + # Dump the contents of a page for debugging purposes. + def dump + super - puts "trx_sys:" - pp trx_sys - puts + puts "trx_sys:" + pp trx_sys + puts + end + end end end - -Innodb::Page::SPECIALIZED_CLASSES[:TRX_SYS] = Innodb::Page::TrxSys diff --git a/lib/innodb/page/undo_log.rb b/lib/innodb/page/undo_log.rb index 8024b225..f35b8d22 100644 --- a/lib/innodb/page/undo_log.rb +++ b/lib/innodb/page/undo_log.rb @@ -1,95 +1,109 @@ -# -*- encoding : utf-8 -*- - -class Innodb::Page::UndoLog < Innodb::Page - def pos_undo_page_header - pos_page_body - end - - def size_undo_page_header - 2 + 2 + 2 + Innodb::List::NODE_SIZE - end - - def pos_undo_segment_header - pos_undo_page_header + size_undo_page_header - end - - def size_undo_segment_header - 2 + 2 + Innodb::FsegEntry::SIZE + Innodb::List::BASE_NODE_SIZE - end - - def pos_undo_logs - pos_undo_segment_header + size_undo_segment_header - end - - UNDO_PAGE_TYPES = { - 1 => :insert, - 2 => :update, - } - - UNDO_SEGMENT_STATES = { - 1 => :active, - 2 => :cached, - 3 => :to_free, - 4 => :to_purge, - 5 => :prepared, - } - - def undo_page_header - @undo_page_header ||= - cursor(pos_undo_page_header).name("undo_page_header") do |c| - { - :type => c.name("type") { UNDO_PAGE_TYPES[c.get_uint16] }, - :latest_log_record_offset => c.name("latest_log_record_offset") { c.get_uint16 }, - :free_offset => c.name("free_offset") { c.get_uint16 }, - :page_list_node => c.name("page_list") { Innodb::List.get_node(c) }, - } +# frozen_string_literal: true + +module Innodb + class Page + class UndoLog < Page + specialization_for :UNDO_LOG + + PageHeader = Struct.new( + :type, + :latest_log_record_offset, + :free_offset, + :page_list_node, + keyword_init: true + ) + + SegmentHeader = Struct.new( + :state, + :last_log_offset, + :fseg, + :page_list, + keyword_init: true + ) + + def pos_undo_page_header + pos_page_body + end + + def size_undo_page_header + 2 + 2 + 2 + Innodb::List::NODE_SIZE + end + + def pos_undo_segment_header + pos_undo_page_header + size_undo_page_header + end + + def size_undo_segment_header + 2 + 2 + Innodb::FsegEntry::SIZE + Innodb::List::BASE_NODE_SIZE + end + + def pos_undo_logs + pos_undo_segment_header + size_undo_segment_header + end + + UNDO_PAGE_TYPES = { + 1 => :insert, + 2 => :update, + }.freeze + + UNDO_SEGMENT_STATES = { + 1 => :active, + 2 => :cached, + 3 => :to_free, + 4 => :to_purge, + 5 => :prepared, + }.freeze + + def undo_page_header + @undo_page_header ||= cursor(pos_undo_page_header).name("undo_page_header") do |c| + PageHeader.new( + type: c.name("type") { UNDO_PAGE_TYPES[c.read_uint16] }, + latest_log_record_offset: c.name("latest_log_record_offset") { c.read_uint16 }, + free_offset: c.name("free_offset") { c.read_uint16 }, + page_list_node: c.name("page_list") { Innodb::List.get_node(c) } + ) + end + end + + def prev_address + undo_page_header[:page_list_node][:prev] + end + + def next_address + undo_page_header[:page_list_node][:next] + end + + def undo_segment_header + @undo_segment_header ||= cursor(pos_undo_segment_header).name("undo_segment_header") do |c| + SegmentHeader.new( + state: c.name("state") { UNDO_SEGMENT_STATES[c.read_uint16] }, + last_log_offset: c.name("last_log_offset") { c.read_uint16 }, + fseg: c.name("fseg") { Innodb::FsegEntry.get_inode(@space, c) }, + page_list: c.name("page_list") { Innodb::List::UndoPage.new(@space, Innodb::List.get_base_node(c)) } + ) + end + end + + def undo_log(pos) + Innodb::UndoLog.new(self, pos) + end + + # Dump the contents of a page for debugging purposes. + def dump + super + + puts "undo page header:" + pp undo_page_header + puts + + puts "undo segment header:" + pp undo_segment_header + puts + + puts "last undo log:" + undo_log(undo_segment_header[:last_log_offset]).dump unless undo_segment_header[:last_log_offset].zero? + puts + end end end - - def prev_address - undo_page_header[:page_list_node][:prev] - end - - def next_address - undo_page_header[:page_list_node][:next] - end - - def undo_segment_header - @undo_segment_header ||= - cursor(pos_undo_segment_header).name("undo_segment_header") do |c| - { - :state => c.name("state") { UNDO_SEGMENT_STATES[c.get_uint16] }, - :last_log_offset => c.name("last_log_offset") { c.get_uint16 }, - :fseg => c.name("fseg") { Innodb::FsegEntry.get_inode(@space, c) }, - :page_list => c.name("page_list") { - Innodb::List::UndoPage.new(@space, Innodb::List.get_base_node(c)) - }, - } - end - end - - def undo_log(pos) - Innodb::UndoLog.new(self, pos) - end - - # Dump the contents of a page for debugging purposes. - def dump - super - - puts "undo page header:" - pp undo_page_header - puts - - puts "undo segment header:" - pp undo_segment_header - puts - - puts "last undo log:" - if undo_segment_header[:last_log_offset] != 0 - undo_log(undo_segment_header[:last_log_offset]).dump - end - puts - end end - -Innodb::Page::SPECIALIZED_CLASSES[:UNDO_LOG] = Innodb::Page::UndoLog diff --git a/lib/innodb/record.rb b/lib/innodb/record.rb index da8425d4..820c50f9 100644 --- a/lib/innodb/record.rb +++ b/lib/innodb/record.rb @@ -1,195 +1,166 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true -class Innodb::Record - attr_reader :page - attr_accessor :record +require "forwardable" - def initialize(page, record) - @page = page - @record = record - end - - def header - record[:header] - end - - def type - header[:type] - end - - def heap_number - header[:heap_number] - end +module Innodb + class Record + extend Forwardable - def n_owned - header[:n_owned] - end - - def deleted? - header[:deleted] - end - - def min_rec? - header[:min_rec] - end - - def offset - record[:offset] - end + attr_reader :page + attr_accessor :record - def length - record[:length] - end - - def next - record[:next] - end + def initialize(page, record) + @page = page + @record = record + end - def key - record[:key] - end + def_delegator :record, :header + def_delegator :record, :offset + def_delegator :record, :length + def_delegator :record, :next + def_delegator :record, :key + def_delegator :record, :row + def_delegator :record, :transaction_id + def_delegator :record, :roll_pointer + def_delegator :record, :child_page_number + + def_delegator :header, :type + def_delegator :header, :heap_number + def_delegator :header, :n_owned + def_delegator :header, :heap_number + def_delegator :header, :deleted? + def_delegator :header, :min_rec? + + def key_string + key&.map { |r| "%s=%s" % [r.name, r.value.inspect] }&.join(", ") + end - def key_string - key && key.map { |r| "%s=%s" % [r[:name], r[:value].inspect] }.join(", ") - end + def row_string + row&.map { |r| "%s=%s" % [r.name, r.value.inspect] }&.join(", ") + end - def row - record[:row] - end + def full_value_with_externs_for_field(field) + blob_value = field.value + extern_page = field.extern && page.space.page(field.extern.page_number) + while extern_page + blob_value += extern_page.blob_data + extern_page = extern_page.next_blob_page + end + blob_value + end - def row_string - row && row.map { |r| "%s=%s" % [r[:name], r[:value].inspect] }.join(", ") - end + def undo + return nil unless roll_pointer + return unless (innodb_system = @page.space.innodb_system) - def transaction_id - record[:transaction_id] - end + undo_page = innodb_system.system_space.page(roll_pointer.undo_log.page) + return unless undo_page - def roll_pointer - record[:roll_pointer] - end + new_undo_record = Innodb::UndoRecord.new(undo_page, roll_pointer.undo_log.offset) + new_undo_record.index_page = page + new_undo_record + end - def undo - return nil unless roll_pointer + def each_undo_record + return enum_for(:each_undo_record) unless block_given? - if innodb_system = @page.space.innodb_system - undo_space = innodb_system.system_space - if undo_page = undo_space.page(roll_pointer[:undo_log][:page]) - new_undo_record = Innodb::UndoRecord.new(undo_page, roll_pointer[:undo_log][:offset]) - new_undo_record.index_page = page - new_undo_record + undo_record = undo + while undo_record + yield undo_record + undo_record = undo_record.prev_by_history end - end - end - def each_undo_record - unless block_given? - return enum_for(:each_undo_record) + nil end - undo_record = undo - while undo_record - yield undo_record - undo_record = undo_record.prev_by_history - end - - nil - end - - def child_page_number - record[:child_page_number] - end - - def string - if child_page_number - "(%s) → #%s" % [key_string, child_page_number] - else - "(%s) → (%s)" % [key_string, row_string] + def string + if child_page_number + "(%s) → #%s" % [key_string, child_page_number] + else + "(%s) → (%s)" % [key_string, row_string] + end end - end - def uncached_fields - fields_hash = {} - [:key, :row].each do |group| - if record[group] - record[group].each do |column| - fields_hash[column[:name]] = column[:value] + def uncached_fields + fields_hash = {} + %i[key row].each do |group| + record[group]&.each do |column| + fields_hash[column.name] = column.value end end + fields_hash end - fields_hash - end - def fields - @fields ||= uncached_fields - end + def fields + @fields ||= uncached_fields + end + + # Compare two arrays of fields to determine if they are equal. This follows + # the same comparison rules as strcmp and others: + # 0 = a is equal to b + # -1 = a is less than b + # +1 = a is greater than b + def compare_key(other_key) + Innodb::Stats.increment :compare_key + + return 0 if other_key.nil? && key.nil? + return -1 if other_key.nil? || (!key.nil? && other_key.size < key.size) + return +1 if key.nil? || (!other_key.nil? && other_key.size > key.size) + + key.each_index do |i| + Innodb::Stats.increment :compare_key_field_comparison + return -1 if other_key[i] < key[i].value + return +1 if other_key[i] > key[i].value + end - # Compare two arrays of fields to determine if they are equal. This follows - # the same comparison rules as strcmp and others: - # 0 = a is equal to b - # -1 = a is less than b - # +1 = a is greater than b - def compare_key(other_key) - Innodb::Stats.increment :compare_key - - return 0 if other_key.nil? && key.nil? - return -1 if other_key.nil? || (!key.nil? && other_key.size < key.size) - return +1 if key.nil? || (!other_key.nil? && other_key.size > key.size) - - key.each_index do |i| - Innodb::Stats.increment :compare_key_field_comparison - return -1 if other_key[i] < key[i][:value] - return +1 if other_key[i] > key[i][:value] + 0 end - return 0 - end + def dump + puts "Record at offset %i" % offset + puts - def dump - puts "Record at offset %i" % offset - puts - - puts "Header:" - puts " %-20s: %i" % ["Next record offset", header[:next]] - puts " %-20s: %i" % ["Heap number", header[:heap_number]] - puts " %-20s: %s" % ["Type", header[:type]] - puts " %-20s: %s" % ["Deleted", header[:deleted]] - puts " %-20s: %s" % ["Length", header[:length]] - puts - - if page.leaf? - puts "System fields:" - puts " Transaction ID: %s" % transaction_id - puts " Roll Pointer:" - puts " Undo Log: page %i, offset %i" % [ - roll_pointer[:undo_log][:page], - roll_pointer[:undo_log][:offset], - ] - puts " Rollback Segment ID: %i" % roll_pointer[:rseg_id] - puts " Insert: %s" % roll_pointer[:is_insert] + puts "Header:" + puts " %-20s: %i" % ["Next record offset", header.next] + puts " %-20s: %i" % ["Heap number", header.heap_number] + puts " %-20s: %s" % ["Type", header.type] + puts " %-20s: %s" % ["Deleted", header.deleted?] + puts " %-20s: %s" % ["Length", header.length] puts - end - puts "Key fields:" - key.each do |field| - puts " %s: %s" % [ - field[:name], - field[:value].inspect, - ] - end - puts + if page.leaf? + puts "System fields:" + puts " Transaction ID: %s" % transaction_id + puts " Roll Pointer:" + puts " Undo Log: page %i, offset %i" % [ + roll_pointer.undo_log.page, + roll_pointer.undo_log.offset, + ] + puts " Rollback Segment ID: %i" % roll_pointer.rseg_id + puts " Insert: %s" % roll_pointer.is_insert + puts + end - if page.leaf? - puts "Non-key fields:" - row.each do |field| + puts "Key fields:" + key.each do |field| puts " %s: %s" % [ - field[:name], - field[:value].inspect, + field.name, + field.value.inspect, ] end puts - else - puts "Child page number: %i" % child_page_number + + if page.leaf? + puts "Non-key fields:" + row.each do |field| + puts " %s: %s" % [ + field.name, + field.value.inspect, + ] + end + else + puts "Child page number: %i" % child_page_number + end puts end end diff --git a/lib/innodb/record_describer.rb b/lib/innodb/record_describer.rb index 3f71ba4b..acbfa4d2 100644 --- a/lib/innodb/record_describer.rb +++ b/lib/innodb/record_describer.rb @@ -1,4 +1,4 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true # # A class to describe record layouts for InnoDB indexes. Designed to be usable @@ -54,87 +54,67 @@ # my_table_clustered.row "age", :INT, :UNSIGNED # -class Innodb::RecordDescriber - # Internal method to initialize the class's instance variable on access. - def self.static_description - @static_description ||= { - :type => nil, - :key => [], - :row => [] - } - end +module Innodb + class RecordDescriber + # Internal method to initialize the class's instance variable on access. + def self.static_description + @static_description ||= { type: nil, key: [], row: [] } + end - # A 'type' method to be used from the DSL. - def self.type(type) - static_description[:type] = type - end + # A 'type' method to be used from the DSL. + def self.type(type) + static_description[:type] = type + end - # An internal method wrapped with 'key' and 'row' helpers. - def self.add_static_field(group, name, type) - static_description[group] << { :name => name, :type => type } - end + # An internal method wrapped with 'key' and 'row' helpers. + def self.add_static_field(group, name, type) + static_description[group] << { name: name, type: type } + end - # A 'key' method to be used from the DSL. - def self.key(name, *type) - add_static_field :key, name, type - end + # A 'key' method to be used from the DSL. + def self.key(name, *type) + add_static_field :key, name, type + end - # A 'row' method to be used from the DSL. - def self.row(name, *type) - add_static_field :row, name, type - end + # A 'row' method to be used from the DSL. + def self.row(name, *type) + add_static_field :row, name, type + end - attr_accessor :description + attr_accessor :description - def initialize - @description = self.class.static_description.dup - @description[:key] = @description[:key].dup - @description[:row] = @description[:row].dup - end - - # Set the type of this record (:clustered or :secondary). - def type(type) - description[:type] = type - end + def initialize + @description = self.class.static_description.dup + @description[:key] = @description[:key].dup + @description[:row] = @description[:row].dup + end - # An internal method wrapped with 'key' and 'row' helpers. - def add_field(group, name, type) - description[group] << { :name => name, :type => type } - end + # Set the type of this record (:clustered or :secondary). + def type(type) + description[:type] = type + end - # Add a key column to the record description. - def key(name, *type) - add_field :key, name, type - end + # An internal method wrapped with 'key' and 'row' helpers. + def add_field(group, name, type) + description[group] << { name: name, type: type } + end - # Add a row (non-key) column to the record description. - def row(name, *type) - add_field :row, name, type - end + # Add a key column to the record description. + def key(name, *type) + add_field :key, name, type + end - def field_names - names = [] - [:key, :row].each do |group| - names += description[group].map { |n| n[:name] } + # Add a row (non-key) column to the record description. + def row(name, *type) + add_field :row, name, type end - names - end - def generate_class(name="Describer_#{object_id}") - str = "class #{name}\n" - str << " type %s\n" % [ - description[:type].inspect - ] - [:key, :row].each do |group| - description[group].each do |item| - str << " %s %s, %s\n" % [ - group, - item[:name].inspect, - item[:type].map { |s| s.inspect }.join(", "), - ] + def field_names + names = [] + %i[key row].each do |group| + names += description[group].map { |n| n[:name] } end + names end - str << "end\n" - str end end diff --git a/lib/innodb/sdi.rb b/lib/innodb/sdi.rb new file mode 100644 index 00000000..25f7a9a0 --- /dev/null +++ b/lib/innodb/sdi.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Innodb + class Sdi + # A hash of page types to specialized classes to handle them. Normally + # subclasses will register themselves in this list. + @specialized_classes = {} + + class << self + attr_reader :specialized_classes + + def register_specialization(id, specialized_class) + @specialized_classes[id] = specialized_class + end + + def parse_semicolon_value_list(data) + data&.split(";").to_h { |x| x.split("=") } + end + + def parse_se_private_data(data) + parse_semicolon_value_list(data) + end + + def parse_options(data) + parse_semicolon_value_list(data) + end + end + + attr_reader :space + + def initialize(space) + @space = space + end + + def sdi_header + @sdi_header ||= space.page(0).sdi_header + end + + def version + sdi_header[:version] + end + + def root_page_number + sdi_header[:root_page_number] + end + + def valid? + root_page_number != 0 + end + + def index + return unless valid? + + space.index(root_page_number) + end + + def each_object + return unless valid? + return enum_for(:each_object) unless block_given? + + index.each_record do |record| + yield SdiObject.from_record(record) + end + + nil + end + + def tables + each_object.select { |o| o.is_a?(Table) } + end + + def tablespaces + each_object.select { |o| o.is_a?(Tablespace) } + end + end +end diff --git a/lib/innodb/sdi/sdi_object.rb b/lib/innodb/sdi/sdi_object.rb new file mode 100644 index 00000000..da70681b --- /dev/null +++ b/lib/innodb/sdi/sdi_object.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require "zlib" +require "json" + +module Innodb + class Sdi + class SdiObject + class << self + def specialization_for(id) + Innodb::Sdi.register_specialization(id, self) + end + + def from_record(record) + type = record.key[0].value + id = record.key[1].value + # Ignore uncompressed_len = record.row[0].value + # Ignore compressed_len = record.row[1].value + data = record.full_value_with_externs_for_field(record.row[2]) + + type_handler = Innodb::Sdi.specialized_classes[type] + return unless type_handler + + parsed_data = JSON.parse(Zlib::Inflate.inflate(data)) + type_handler.new(type, id, parsed_data) + end + end + + attr_reader :type + attr_reader :id + attr_reader :data + + def initialize(type, id, data) + @type = type + @id = id + @data = data + end + + def mysqld_version_id + data["mysqld_version_id"] + end + + def dd_version + data["dd_version"] + end + + def sdi_version + data["sdi_version"] + end + + def dd_object_type + data["dd_object_type"] + end + + def dd_object + data["dd_object"] + end + + def name + dd_object["name"] + end + + def options + Innodb::Sdi.parse_options(data["options"]) + end + + def se_private_data + Innodb::Sdi.parse_se_private_data(dd_object["se_private_data"]) + end + end + end +end diff --git a/lib/innodb/sdi/table.rb b/lib/innodb/sdi/table.rb new file mode 100644 index 00000000..e9c068d2 --- /dev/null +++ b/lib/innodb/sdi/table.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Innodb + class Sdi + class Table < SdiObject + specialization_for 1 + + def name + format("%s/%s", dd_object["schema_ref"], dd_object["name"]) + end + + def columns + dd_object["columns"].map { |column| TableColumn.new(self, column) } + end + + def indexes + dd_object["indexes"].map { |index| TableIndex.new(self, index) } + end + + def space_id + indexes.first.space_id + end + + def find_index_by_name(name) + indexes.find { |index| index.name == name } + end + + def clustered_index + indexes.select { |i| %i[PRIMARY UNIQUE].include?(i.type) }.first + end + end + end +end diff --git a/lib/innodb/sdi/table_column.rb b/lib/innodb/sdi/table_column.rb new file mode 100644 index 00000000..c20595ce --- /dev/null +++ b/lib/innodb/sdi/table_column.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module Innodb + class Sdi + class TableColumn + HIDDEN_STATUSES = { + VISIBLE: 1, + HIDDEN_SE: 2, + HIDDEN_SQL: 3, + HIDDEN_USER: 4, + }.freeze + + HIDDEN_STATUSES_BY_VALUE = HIDDEN_STATUSES.invert + + attr_reader :table + attr_reader :data + + def initialize(table, data) + @table = table + @data = data + end + + def name + data["name"] + end + + def type + data["type"] + end + + def nullable? + data["is_nullable"] + end + + def zerofill? + data["is_zerofill"] + end + + def unsigned? + data["is_unsigned"] + end + + def auto_increment? + data["is_auto_increment"] + end + + def virtual? + data["is_virtual"] + end + + def explicit_collation? + data["is_explicit_collation"] + end + + def hidden_status + HIDDEN_STATUSES_BY_VALUE[data["hidden"]] + end + + def visible? + hidden_status != :VISIBLE + end + + def hidden? + !visible? + end + + def system? + %w[DB_TRX_ID DB_ROLL_PTR].include?(name) + end + + def description + [ + data["column_type_utf8"].sub(/ unsigned$/, ""), + unsigned? ? :UNSIGNED : nil, + nullable? ? nil : :NOT_NULL, + ].compact + end + + def se_private_data + Innodb::Sdi.parse_se_private_data(data["se_private_data"]) + end + + def table_id + se_private_data["table_id"].to_i + end + end + end +end diff --git a/lib/innodb/sdi/table_index.rb b/lib/innodb/sdi/table_index.rb new file mode 100644 index 00000000..7cfbc8c2 --- /dev/null +++ b/lib/innodb/sdi/table_index.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +module Innodb + class Sdi + class TableIndex + TYPES = { + 1 => :PRIMARY, + 2 => :UNIQUE, + 3 => :MULTIPLE, + 4 => :FULLTEXT, + 5 => :SPATIAL, + }.freeze + + ALGORITHMS = { + 1 => :SE_SPECIFIC, + 2 => :BTREE, + 3 => :RTREE, + 4 => :HASH, + 5 => :FULLTEXT, + }.freeze + + attr_reader :table + attr_reader :data + + def initialize(table, data) + @table = table + @data = data + end + + def name + data["name"] + end + + def hidden? + data["hidden"] + end + + def visible? + data["is_visible"] + end + + def generated? + data["is_generated"] + end + + def elements + data["elements"] + .map { |element| TableIndexElement.new(self, element) } + .sort_by(&:ordinal_position) + end + + def se_private_data + Innodb::Sdi.parse_se_private_data(data["se_private_data"]) + end + + def innodb_index_id + se_private_data["id"].to_i + end + + def root_page_number + se_private_data["root"].to_i + end + + def innodb_space_id + se_private_data["space_id"].to_i + end + + def innodb_table_id + se_private_data["table_id"].to_i + end + + def trx_id + se_private_data["trx_id"].to_i + end + + def options + Innodb::Sdi.parse_options(data["options"]) + end + + def type + TYPES[data["type"]] + end + + def primary? + type == :PRIMARY + end + + def clustered? + table.clustered_index.innodb_index_id == innodb_index_id + end + end + end +end diff --git a/lib/innodb/sdi/table_index_element.rb b/lib/innodb/sdi/table_index_element.rb new file mode 100644 index 00000000..9bf9b085 --- /dev/null +++ b/lib/innodb/sdi/table_index_element.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Innodb + class Sdi + class TableIndexElement + attr_reader :index + attr_reader :data + + def initialize(index, data) + @index = index + @data = data + end + + def ordinal_position + data["ordinal_position"] + end + + def length + data["length"] + end + + def order + data["order"] + end + + def hidden? + data["hidden"] + end + + def visible? + !hidden? + end + + def key? + visible? + end + + def row? + hidden? + end + + def type + return :sys if %w[DB_TRX_ID DB_ROLL_PTR].include?(column.name) + return :key if key? + + :row + end + + def column_opx + data["column_opx"] + end + + def column + index.table.columns[column_opx] + end + end + end +end diff --git a/lib/innodb/sdi/tablespace.rb b/lib/innodb/sdi/tablespace.rb new file mode 100644 index 00000000..cc8d054c --- /dev/null +++ b/lib/innodb/sdi/tablespace.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Innodb + class Sdi + class Tablespace < SdiObject + specialization_for 2 + + def space_id + se_private_data["id"].to_i + end + end + end +end diff --git a/lib/innodb/sdi_data_dictionary.rb b/lib/innodb/sdi_data_dictionary.rb new file mode 100644 index 00000000..2cb2aaa8 --- /dev/null +++ b/lib/innodb/sdi_data_dictionary.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +# A class representing MySQL's SDI-based data dictionary (used in MySQL +# versions starting in MySQL 8.0), which contains metadata about tables, +# columns, and indexes distributed in BLOBs of JSON stored in each InnoDB +# tablespace file. +module Innodb + class SdiDataDictionary + extend Forwardable + + attr_reader :innodb_system + + def_delegator :innodb_system, :data_dictionary + + def initialize(innodb_system) + @innodb_system = innodb_system + end + + def populate_data_dictionary_using_space_sdi(space) + sdi_tablespace = space.sdi.tablespaces.first + return unless sdi_tablespace + + innodb_space_id = sdi_tablespace.se_private_data["id"].to_i + new_tablespace = data_dictionary.tablespaces.make(name: sdi_tablespace.name, innodb_space_id: innodb_space_id) + + space.sdi.tables.each do |table| + new_table = data_dictionary.tables.make(name: table.name, tablespace: new_tablespace) + + table.columns.each do |column| + next if %w[DB_TRX_ID DB_ROLL_PTR].include?(column.name) + + new_table.columns.make(name: column.name, description: column.description, table: new_table) + end + + table.indexes.each do |index| + new_index = new_table.indexes.make( + name: index.name, + type: index.clustered? ? :clustered : :secondary, + table: new_table, + tablespace: new_tablespace, + root_page_number: index.root_page_number, + innodb_index_id: index.innodb_index_id + ) + + db_trx_id = Innodb::DataDictionary::Column.new( + name: "DB_TRX_ID", + description: %i[DB_TRX_ID], + table: new_table + ) + db_roll_ptr = Innodb::DataDictionary::Column.new( + name: "DB_ROLL_PTR", + description: %i[DB_ROLL_PTR], + table: new_table + ) + + index.elements.each do |element| + case element.column.name + when "DB_TRX_ID" + new_index.column_references.make(column: db_trx_id, usage: :sys, index: new_index) + when "DB_ROLL_PTR" + new_index.column_references.make(column: db_roll_ptr, usage: :sys, index: new_index) + else + new_index.column_references.make(column: new_table.columns.find(name: element.column.name), + usage: element.type, index: new_index) + end + end + end + end + + nil + end + + def populate_data_dictionary + data_dictionary.tablespaces.make(name: "innodb_system", innodb_space_id: 0) + + innodb_system.each_space do |space| + populate_data_dictionary_using_space_sdi(space) + end + end + end +end diff --git a/lib/innodb/space.rb b/lib/innodb/space.rb index 616f6875..d6ac35fb 100644 --- a/lib/innodb/space.rb +++ b/lib/innodb/space.rb @@ -1,528 +1,509 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true # An InnoDB space file, which can be either a multi-table ibdataN file # or a single-table "innodb_file_per_table" .ibd file. -class Innodb::Space - # InnoDB's default page size is 16KiB. - DEFAULT_PAGE_SIZE = 16384 - - # A map of InnoDB system space fixed-allocation pages. This can be used to - # check whether a space is a system space or not, as non-system spaces will - # not match this pattern. - SYSTEM_SPACE_PAGE_MAP = { - 0 => :FSP_HDR, - 1 => :IBUF_BITMAP, - 2 => :INODE, - 3 => :SYS, - 4 => :INDEX, - 5 => :TRX_SYS, - 6 => :SYS, - 7 => :SYS, - } - - class DataFile - attr_reader :file - attr_reader :size - attr_reader :offset - - def initialize(filename, offset) - @file = File.open(filename) - @size = @file.stat.size - @offset = offset - end - - def name - prefix = "" - if File.extname(file.path) == ".ibd" - prefix = File.basename(File.dirname(file.path)) + "/" +module Innodb + class Space + # InnoDB's default page size is 16KiB. + DEFAULT_PAGE_SIZE = 16 * 1024 + + # The default extent size is 1 MiB defined originally as 64 pages. + DEFAULT_EXTENT_SIZE = 64 * DEFAULT_PAGE_SIZE + + # A map of InnoDB system space fixed-allocation pages. This can be used to + # check whether a space is a system space or not, as non-system spaces will + # not match this pattern. + SYSTEM_SPACE_PAGE_MAP = { + 0 => :FSP_HDR, + 1 => :IBUF_BITMAP, + 2 => :INODE, + 3 => :SYS, + 4 => :INDEX, + 5 => :TRX_SYS, + 6 => :SYS, + 7 => :SYS, + }.freeze + + XDES_LISTS = %i[ + free + free_frag + full_frag + ].freeze + + # An array of Innodb::Inode list names. + INODE_LISTS = %i[ + full_inodes + free_inodes + ].freeze + + class DataFile + attr_reader :file + attr_reader :size + attr_reader :offset + + def initialize(filename, offset) + @file = File.open(filename) + @size = @file.stat.size + @offset = offset end - prefix + File.basename(file.path) + def name + file.path + end end - end - # Open a space file, optionally providing the page size to use. Pages - # that aren't 16 KiB may not be supported well. - def initialize(filenames) - filenames = [filenames] unless filenames.is_a?(Array) + # Open a space file, optionally providing the page size to use. Pages + # that aren't 16 KiB may not be supported well. + def initialize(filenames, innodb_system: nil) + filenames = [filenames] unless filenames.is_a?(Array) - @data_files = [] - @size = 0 - filenames.each do |filename| - file = DataFile.new(filename, @size) - @size += file.size - @data_files << file - end - - @system_page_size = fsp_flags[:system_page_size] - @page_size = fsp_flags[:page_size] - @compressed = fsp_flags[:compressed] + @data_files = [] + @size = 0 + filenames.each do |filename| + file = DataFile.new(filename, @size) + @size += file.size + @data_files << file + end - @pages = (@size / @page_size) - @innodb_system = nil - @record_describer = nil - end + @system_page_size = fsp_flags.system_page_size + @page_size = fsp_flags.page_size + @compressed = fsp_flags.compressed - # The Innodb::System to which this space belongs, if any. - attr_accessor :innodb_system + @pages = (@size / @page_size) + @innodb_system = innodb_system + @record_describer = nil + end - # An object which can be used to describe records found in pages within - # this space. - attr_accessor :record_describer + # The Innodb::System to which this space belongs, if any. + attr_accessor :innodb_system - # The system default page size (in bytes), equivalent to UNIV_PAGE_SIZE. - attr_reader :system_page_size + # An object which can be used to describe records found in pages within + # this space. + attr_accessor :record_describer - # The size (in bytes) of each page in the space. - attr_reader :page_size + # The system default page size (in bytes), equivalent to UNIV_PAGE_SIZE. + attr_reader :system_page_size - # The size (in bytes) of the space - attr_reader :size + # The size (in bytes) of each page in the space. + attr_reader :page_size - # The number of pages in the space. - attr_reader :pages + # The size (in bytes) of the space + attr_reader :size - # Return a string which can uniquely identify this space. Be careful not - # to do anything which could instantiate a BufferCursor so that we can use - # this method in cursor initialization. - def name - @name ||= @data_files.map { |f| f.name }.join(",") - end + # The number of pages in the space. + attr_reader :pages - def inspect - "<%s file=%s, page_size=%i, pages=%i>" % [ - self.class.name, - name.inspect, - page_size, - pages, - ] - end + # Return a string which can uniquely identify this space. Be careful not + # to do anything which could instantiate a BufferCursor so that we can use + # this method in cursor initialization. + def name + @name ||= @data_files.map(&:name).join(",") + end - # Read the FSP header "flags" field by byte offset within the space file. - # This is useful in order to initialize the page size, as we can't properly - # read the FSP_HDR page before we know its size. - def raw_fsp_header_flags - # A simple sanity check. The FIL header should be initialized in page 0, - # to offset 0 and page type :FSP_HDR (8). - page_offset = BinData::Uint32be.read(read_at_offset(4, 4)).to_i - page_type = BinData::Uint16be.read(read_at_offset(24, 2)).to_i - unless page_offset == 0 && Innodb::Page::PAGE_TYPE_BY_VALUE[page_type] == :FSP_HDR - raise "Something is very wrong; Page 0 does not seem to be type FSP_HDR; got page type %i but expected %i" % [ - page_type, - Innodb::Page::PAGE_TYPE[:FSP_HDR][:value], + def inspect + "<%s file=%s, page_size=%i, pages=%i>" % [ + self.class.name, + name.inspect, + page_size, + pages, ] end - # Another sanity check. The Space ID should be the same in both the FIL - # and FSP headers. - fil_space = BinData::Uint32be.read(read_at_offset(34, 4)).to_i - fsp_space = BinData::Uint32be.read(read_at_offset(38, 4)).to_i - unless fil_space == fsp_space - raise "Something is very wrong; FIL and FSP header Space IDs don't match: FIL is %i but FSP is %i" % [ - fil_space, - fsp_space, - ] + # Read the FSP header "flags" field by byte offset within the space file. + # This is useful in order to initialize the page size, as we can't properly + # read the FSP_HDR page before we know its size. + def raw_fsp_header_flags + # A simple sanity check. The FIL header should be initialized in page 0, + # to offset 0 and page type :FSP_HDR (8). + page_offset = BinData::Uint32be.read(read_at_offset(4, 4)).to_i + page_type = BinData::Uint16be.read(read_at_offset(24, 2)).to_i + unless page_offset.zero? && Innodb::Page::PAGE_TYPE_BY_VALUE[page_type] == :FSP_HDR + raise "Something is very wrong; Page 0 does not seem to be type FSP_HDR; got page type %i but expected %i" % [ + page_type, + Innodb::Page::PAGE_TYPE[:FSP_HDR][:value], + ] + end + + # Another sanity check. The Space ID should be the same in both the FIL + # and FSP headers. + fil_space = BinData::Uint32be.read(read_at_offset(34, 4)).to_i + fsp_space = BinData::Uint32be.read(read_at_offset(38, 4)).to_i + unless fil_space == fsp_space + raise "Something is very wrong; FIL and FSP header Space IDs do not match: FIL is %i but FSP is %i" % [ + fil_space, + fsp_space, + ] + end + + # Well, we're as sure as we can be. Read the flags field and decode it. + flags_value = BinData::Uint32be.read(read_at_offset(54, 4)) + Innodb::Page::FspHdrXdes.decode_flags(flags_value) end - # Well, we're as sure as we can be. Read the flags field and decode it. - flags_value = BinData::Uint32be.read(read_at_offset(54, 4)) - Innodb::Page::FspHdrXdes.decode_flags(flags_value) - end + # The FSP header flags, decoded. If the page size has not been initialized, + # reach into the raw bytes of the FSP_HDR page and attempt to decode the + # flags field that way. + def fsp_flags + return fsp.flags if @page_size - # The FSP header flags, decoded. If the page size has not been initialized, - # reach into the raw bytes of the FSP_HDR page and attempt to decode the - # flags field that way. - def fsp_flags - if @page_size - return fsp[:flags] - else raw_fsp_header_flags end - end - # The number of pages per extent. - def pages_per_extent - # Note that uncompressed tables and compressed tables using the same page - # size will have a different number of pages per "extent" because InnoDB - # compression uses the FSP_EXTENT_SIZE define (which is then based on the - # UNIV_PAGE_SIZE define, which may be based on the innodb_page_size system - # variable) for compressed tables rather than something based on the actual - # compressed page size. - # - # For this reason, an "extent" differs in size as follows (the maximum page - # size supported for compressed tables is the innodb_page_size): - # - # innodb_page_size | innodb compression | - # page size | extent size | pages | page size | extent size | pages | - # 16384 | 1 MiB | 64 | 16384 | 1 MiB | 64 | - # | 8192 | 512 KiB | 64 | - # | 4096 | 256 KiB | 64 | - # | 2048 | 128 KiB | 64 | - # | 1024 | 64 KiB | 64 | - # 8192 | 1 MiB | 128 | 8192 | 1 MiB | 128 | - # | 4096 | 512 KiB | 128 | - # | 2048 | 256 KiB | 128 | - # | 1024 | 128 KiB | 128 | - # 4096 | 1 MiB | 256 | 4096 | 1 MiB | 256 | - # | 2048 | 512 KiB | 256 | - # | 1024 | 256 KiB | 256 | - # - - 1048576 / system_page_size - end + # The number of pages per extent. + def pages_per_extent + # Note that uncompressed tables and compressed tables using the same page + # size will have a different number of pages per "extent" because InnoDB + # compression uses the FSP_EXTENT_SIZE define (which is then based on the + # UNIV_PAGE_SIZE define, which may be based on the innodb_page_size system + # variable) for compressed tables rather than something based on the actual + # compressed page size. + # + # For this reason, an "extent" differs in size as follows (the maximum page + # size supported for compressed tables is the innodb_page_size): + # + # innodb_page_size | innodb compression | + # page size | extent size | pages | page size | extent size | pages | + # 16384 | 1 MiB | 64 | 16384 | 1 MiB | 64 | + # | 8192 | 512 KiB | 64 | + # | 4096 | 256 KiB | 64 | + # | 2048 | 128 KiB | 64 | + # | 1024 | 64 KiB | 64 | + # 8192 | 1 MiB | 128 | 8192 | 1 MiB | 128 | + # | 4096 | 512 KiB | 128 | + # | 2048 | 256 KiB | 128 | + # | 1024 | 128 KiB | 128 | + # 4096 | 1 MiB | 256 | 4096 | 1 MiB | 256 | + # | 2048 | 512 KiB | 256 | + # | 1024 | 256 KiB | 256 | + # - # The size (in bytes) of an extent. - def extent_size - pages_per_extent * page_size - end - - # The number of pages per FSP_HDR/XDES/IBUF_BITMAP page. This is crudely - # mapped to the page size, and works for pages down to 1KiB. - def pages_per_bookkeeping_page - page_size - end + DEFAULT_EXTENT_SIZE / system_page_size + end - # The FSP_HDR/XDES page which will contain the XDES entry for a given page. - def xdes_page_for_page(page_number) - page_number - (page_number % pages_per_bookkeeping_page) - end + # The size (in bytes) of an extent. + def extent_size + pages_per_extent * page_size + end - # The IBUF_BITMAP page which will contain the bitmap entry for a given page. - def ibuf_bitmap_page_for_page(page_number) - page_number - (page_number % pages_per_bookkeeping_page) + 1 - end + # The number of pages per FSP_HDR/XDES/IBUF_BITMAP page. This is crudely + # mapped to the page size, and works for pages down to 1KiB. + def pages_per_bookkeeping_page + page_size + end - # The XDES entry offset for a given page within its FSP_HDR/XDES page's - # XDES array. - def xdes_entry_for_page(page_number) - relative_page_number = page_number - xdes_page_for_page(page_number) - relative_page_number / pages_per_extent - end + # The FSP_HDR/XDES page which will contain the XDES entry for a given page. + def xdes_page_for_page(page_number) + page_number - (page_number % pages_per_bookkeeping_page) + end - # Return the Innodb::Xdes entry which represents a given page. - def xdes_for_page(page_number) - xdes_array = page(xdes_page_for_page(page_number)).each_xdes.to_a - xdes_array[xdes_entry_for_page(page_number)] - end + # The IBUF_BITMAP page which will contain the bitmap entry for a given page. + def ibuf_bitmap_page_for_page(page_number) + page_number - (page_number % pages_per_bookkeeping_page) + 1 + end - def data_file_for_offset(offset) - @data_files.each do |file| - return file if offset < file.size - offset -= file.size + # The XDES entry offset for a given page within its FSP_HDR/XDES page's + # XDES array. + def xdes_entry_for_page(page_number) + relative_page_number = page_number - xdes_page_for_page(page_number) + relative_page_number / pages_per_extent end - nil - end - # Get the raw byte buffer of size bytes at offset in the file. - def read_at_offset(offset, size) - return nil unless offset < @size && (offset + size) <= @size - data_file = data_file_for_offset(offset) - data_file.file.seek(offset - data_file.offset) - data_file.file.read(size) - end + # Return the Innodb::Xdes entry which represents a given page. + def xdes_for_page(page_number) + xdes_array = page(xdes_page_for_page(page_number)).each_xdes.to_a + xdes_array[xdes_entry_for_page(page_number)] + end - # Get the raw byte buffer for a specific page by page number. - def page_data(page_number) - read_at_offset(page_number * page_size, page_size) - end + def data_file_for_offset(offset) + @data_files.each do |file| + return file if offset < file.size - # Get an Innodb::Page object for a specific page by page number. - def page(page_number) - data = page_data(page_number) - if data - Innodb::Page.parse(self, data, page_number) - else + offset -= file.size + end nil end - end - - # Determine whether this space looks like a system space. If the initial - # pages in the space match the SYSTEM_SPACE_PAGE_MAP, it is likely to be - # a system space. - def system_space? - SYSTEM_SPACE_PAGE_MAP.each do |page_number, type| - # We can't use page() here, because system_space? need to be used - # in the Innodb::Page::Sys.parse to determine what type of page - # is being looked at. Using page() would cause us to keep recurse - # infinitely. Use Innodb::Page.new instead to load the page as - # simply as possible. - test_page = Innodb::Page.new(self, page_data(page_number)) - return false unless test_page.type == type - end - true - end - # Return the page number for the space's FSP_HDR page. - def page_fsp_hdr - 0 - end + # Get the raw byte buffer of size bytes at offset in the file. + def read_at_offset(offset, size) + return nil unless offset < @size && (offset + size) <= @size - # Get (and cache) the FSP header from the FSP_HDR page. - def fsp - @fsp ||= page(page_fsp_hdr).fsp_header - end + data_file = data_file_for_offset(offset) + data_file.file.seek(offset - data_file.offset) + data_file.file.read(size) + end - def space_id - fsp[:space_id] - end + # Get the raw byte buffer for a specific page by page number. + def page_data(page_number) + read_at_offset(page_number * page_size, page_size) + end - # Return the page number for the space's TRX_SYS page. - def page_trx_sys - 5 - end + # Get an Innodb::Page object for a specific page by page number. + def page(page_number) + data = page_data(page_number) + Innodb::Page.parse(self, data, page_number) if data + end - # Get the Innodb::Page::TrxSys page for a system space. - def trx_sys - page(page_trx_sys) if system_space? - end + # Determine whether this space looks like a system space. If the initial + # pages in the space match the SYSTEM_SPACE_PAGE_MAP, it is likely to be + # a system space. + def system_space? + SYSTEM_SPACE_PAGE_MAP.each do |page_number, type| + # We can't use page() here, because system_space? need to be used + # in the Innodb::Page::Sys.parse to determine what type of page + # is being looked at. Using page() would cause us to keep recurse + # infinitely. Use Innodb::Page.new instead to load the page as + # simply as possible. + test_page = Innodb::Page.new(self, page_data(page_number)) + return false unless test_page.type == type + end + true + end - def rseg_page?(page_number) - if trx_sys - rseg_match = trx_sys.rsegs.select { |rseg| - rseg[:space_id] == 0 && rseg[:page_number] == page_number - } + # Return the page number for the space's FSP_HDR page. + def page_fsp_hdr + 0 + end - ! rseg_match.empty? + # Get (and cache) the FSP header from the FSP_HDR page. + def fsp + @fsp ||= page(page_fsp_hdr).fsp_header end - end - # Return the page number for the space's SYS data dictionary header. - def page_sys_data_dictionary - 7 - end + def space_id + fsp[:space_id] + end - # Get the Innodb::Page::SysDataDictionaryHeader page for a system space. - def data_dictionary_page - page(page_sys_data_dictionary) if system_space? - end + def checked_page_class!(page, expected_class) + return page if page.instance_of?(expected_class) - # Get an Innodb::List object for a specific list by list name. - def list(name) - if xdes_lists.include?(name) || inode_lists.include?(name) - fsp[name] + raise "Page #{page.offset} is not the correct type, found: #{page.class}, expected: #{expected_class}" end - end - - # Get an Innodb::Index object for a specific index by root page number. - def index(root_page_number, record_describer=nil) - Innodb::Index.new(self, root_page_number, - record_describer || @record_describer) - end - # Iterate through all root page numbers for indexes in the space. - def each_index_root_page_number - unless block_given? - return enum_for(:each_index_root_page_number) + # Return the page number for the space's TRX_SYS page. + def page_trx_sys + 5 end - if innodb_system - # Retrieve the index root page numbers from the data dictionary. - innodb_system.data_dictionary.each_index_by_space_id(space_id) do |record| - yield record["PAGE_NO"] - end - else - # Guess that the index root pages will be present starting at page 3, - # and walk forward until we find a non-root page. This should work fine - # for IBD files, if they haven't added indexes online. - (3...@pages).each do |page_number| - page = page(page_number) - if page.type == :INDEX && page.root? - yield page_number - end - end + # Get the Innodb::Page::TrxSys page for a system space. + def trx_sys + raise "Transaction System is only available in system spaces" unless system_space? + + checked_page_class!(page(page_trx_sys), Innodb::Page::TrxSys) end - nil - end + def rseg_page?(page_number) + return false unless trx_sys - # Iterate through all indexes in the space. - def each_index - unless block_given? - return enum_for(:each_index) + trx_sys.rsegs.any? { |rseg| rseg.space_id.zero? && rseg.page_number == page_number } end - each_index_root_page_number do |page_number| - yield index(page_number) + # Return the page number for the space's SYS data dictionary header. + def page_sys_data_dictionary + 7 end - nil - end + # Get the Innodb::Page::SysDataDictionaryHeader page for a system space. + def data_dictionary_page + raise "Data Dictionary is only available in system spaces" unless system_space? - # An array of Innodb::Inode list names. - def inode_lists - [:full_inodes, :free_inodes] - end + checked_page_class!(page(page_sys_data_dictionary), Innodb::Page::SysDataDictionaryHeader) + end - # Iterate through Innodb::Inode lists in the space. - def each_inode_list - unless block_given? - return enum_for(:each_inode_list) + # Get an Innodb::List object for a specific list by list name. + def list(name) + fsp[name] if XDES_LISTS.include?(name) || INODE_LISTS.include?(name) end - inode_lists.each do |name| - yield name, list(name) + def sdi + @sdi ||= Innodb::Sdi.new(self) + end + + def sdi? + sdi.valid? end - end - # Iterate through Innodb::Inode objects in the space. - def each_inode - unless block_given? - return enum_for(:each_inode) + # Get an Innodb::Index object for a specific index by root page number. + def index(root_page_number, record_describer = nil) + Innodb::Index.new(self, root_page_number, record_describer || @record_describer) end - each_inode_list.each do |name, list| - list.each do |page| - page.each_allocated_inode do |inode| - yield inode + # Iterate through all root page numbers for indexes in the space. + def each_index_root_page_number + return enum_for(:each_index_root_page_number) unless block_given? + + if innodb_system&.data_dictionary&.populated? + # Retrieve the index root page numbers from the data dictionary. + # TODO: An efficient way to handle this? + tablespace = innodb_system.data_dictionary.tablespaces.find(innodb_space_id: space_id) + innodb_system.data_dictionary.tables.each do |table| + table.indexes.by(tablespace: tablespace).each do |index| + yield index.root_page_number + end + end + else + # Guess that the index root pages will be present starting at page 3, + # and walk forward until we find a non-root page. This should work fine + # for IBD files, if they haven't added indexes online. + (3...@pages).each do |page_number| + page = page(page_number) + yield page_number if page.is_a?(Innodb::Page::Index) && page.root? end end + + nil end - end - # Return an Inode by fseg_id. Iterates through the inode list, but it - # normally is fairly small, so should be relatively efficient. - def inode(fseg_id) - each_inode.select { |inode| inode.fseg_id == fseg_id }.first - end + # Iterate through all indexes in the space. + def each_index + return enum_for(:each_index) unless block_given? - # Iterate through the page numbers in the doublewrite buffer. - def each_doublewrite_page_number - return nil unless system_space? + each_index_root_page_number do |page_number| + yield index(page_number) + end - unless block_given? - return enum_for(:each_doublewrite_page_number) + nil end - trx_sys.doublewrite[:page_info][0][:page_number].each do |start_page| - (start_page...(start_page+pages_per_extent)).each do |page_number| - yield page_number + # Iterate through Innodb::Inode lists in the space. + def each_inode_list + return enum_for(:each_inode_list) unless block_given? + + INODE_LISTS.each do |name| + yield name, list(name) end end - end - # Return true if a page is in the doublewrite buffer. - def doublewrite_page?(page_number) - return false unless system_space? - @doublewrite_pages ||= each_doublewrite_page_number.to_a - @doublewrite_pages.include?(page_number) - end + # Iterate through Innodb::Inode objects in the space. + def each_inode(&block) + return enum_for(:each_inode) unless block_given? - # Iterate through all pages in a space, returning the page number and an - # Innodb::Page object for each one. - def each_page(start_page=0) - unless block_given? - return enum_for(:each_page, start_page) + each_inode_list do |_name, list| + list.each do |page| + page.each_allocated_inode(&block) + end + end end - (start_page...@pages).each do |page_number| - current_page = page(page_number) - yield page_number, current_page if current_page + # Return an Inode by fseg_id. Iterates through the inode list, but it + # normally is fairly small, so should be relatively efficient. + def inode(fseg_id) + each_inode.select { |inode| inode.fseg_id == fseg_id }.first end - end - # An array of Innodb::Xdes list names. - def xdes_lists - [:free, :free_frag, :full_frag] - end + # Iterate through the page numbers in the doublewrite buffer. + def each_doublewrite_page_number(&block) + return nil unless system_space? + return enum_for(:each_doublewrite_page_number) unless block_given? - # Iterate through Innodb::Xdes lists in the space. - def each_xdes_list - unless block_given? - return enum_for(:each_xdes_list) + trx_sys.doublewrite[:page_info][0][:page_number].each do |start_page| + (start_page...(start_page + pages_per_extent)).each(&block) + end end - xdes_lists.each do |name| - yield name, list(name) - end - end + # Return true if a page is in the doublewrite buffer. + def doublewrite_page?(page_number) + return false unless system_space? - # An array of all FSP/XDES page numbers for the space. - def each_xdes_page_number - unless block_given? - return enum_for(:each_xdes_page_number) + @doublewrite_pages ||= each_doublewrite_page_number.to_a + @doublewrite_pages.include?(page_number) end - 0.step(pages - 1, pages_per_bookkeeping_page).each do |n| - yield n - end - end + # Iterate through all pages in a space, returning the page number and an + # Innodb::Page object for each one. + def each_page(start_page = 0) + return enum_for(:each_page, start_page) unless block_given? - # Iterate through all FSP_HDR/XDES pages, returning an Innodb::Page object - # for each one. - def each_xdes_page - unless block_given? - return enum_for(:each_xdes_page) + (start_page...@pages).each do |page_number| + current_page = page(page_number) + yield page_number, current_page if current_page + end end - each_xdes_page_number do |page_number| - current_page = page(page_number) - yield current_page if current_page and [:FSP_HDR, :XDES].include?(current_page.type) - end - end + # Iterate through Innodb::Xdes lists in the space. + def each_xdes_list + return enum_for(:each_xdes_list) unless block_given? - # Iterate through all extent descriptors for the space, returning an - # Innodb::Xdes object for each one. - def each_xdes - unless block_given? - return enum_for(:each_xdes) + XDES_LISTS.each do |name| + yield name, list(name) + end end - each_xdes_page do |xdes_page| - xdes_page.each_xdes do |xdes| - # Only return initialized XDES entries; :state will be nil for extents - # that have not been allocated yet. - yield xdes if xdes.xdes[:state] - end + # An array of all FSP/XDES page numbers for the space. + def each_xdes_page_number(&block) + return enum_for(:each_xdes_page_number) unless block_given? + + 0.step(pages - 1, pages_per_bookkeeping_page).each(&block) end - end - # Iterate through all pages, yielding the page number, page object, - # and page status. - def each_page_status(start_page=0) - unless block_given? - return enum_for(:each_page_with_status, start_page) + # Iterate through all extent descriptor pages, returning an Innodb::Page object + # for each one. + def each_xdes_page + return enum_for(:each_xdes_page) unless block_given? + + each_xdes_page_number do |page_number| + current_page = page(page_number) + yield current_page if current_page&.extent_descriptor? + end end - each_xdes do |xdes| - xdes.each_page_status do |page_number, page_status| - next if page_number < start_page - next if page_number >= @pages + # Iterate through all extent descriptors for the space, returning an + # Innodb::Xdes object for each one. + def each_xdes + return enum_for(:each_xdes) unless block_given? - if this_page = page(page_number) - yield page_number, this_page, page_status + each_xdes_page do |xdes_page| + xdes_page.each_xdes do |xdes| + # Only return initialized XDES entries; :state will be nil for extents + # that have not been allocated yet. + yield xdes if xdes.xdes[:state] end end end - end - # A helper to produce a printable page type. - def type_for_page(page, page_status) - page_status[:free] ? "FREE (#{page.type})" : page.type - end + # Iterate through all pages, yielding the page number, page object, + # and page status. + def each_page_status(start_page = 0) + return enum_for(:each_page_status, start_page) unless block_given? + + each_xdes do |xdes| + xdes.each_page_status do |page_number, page_status| + next if page_number < start_page + next if page_number >= @pages - # Iterate through unique regions in the space by page type. This is useful - # to achieve an overall view of the space. - def each_page_type_region(start_page=0) - unless block_given? - return enum_for(:each_page_type_region, start_page) + if (this_page = page(page_number)) + yield page_number, this_page, page_status + end + end + end end - region = nil - each_page_status(start_page) do |page_number, page, page_status| - page_type = type_for_page(page, page_status) - if region && region[:type] == page_type - region[:end] = page_number - region[:count] += 1 - else - yield region if region - region = { - :start => page_number, - :end => page_number, - :type => page_type, - :count => 1, - } + # A helper to produce a printable page type. + def type_for_page(page, page_status) + page_status[:free] ? "FREE (#{page.type})" : page.type + end + + # Iterate through unique regions in the space by page type. This is useful + # to achieve an overall view of the space. + def each_page_type_region(start_page = 0) + return enum_for(:each_page_type_region, start_page) unless block_given? + + region = nil + each_page_status(start_page) do |page_number, page, page_status| + page_type = type_for_page(page, page_status) + if region && region[:type] == page_type + region[:end] = page_number + region[:count] += 1 + else + yield region if region + region = { + start: page_number, + end: page_number, + type: page_type, + count: 1, + } + end end + yield region if region end - yield region if region end end diff --git a/lib/innodb/stats.rb b/lib/innodb/stats.rb index db9b7f18..78e78f35 100644 --- a/lib/innodb/stats.rb +++ b/lib/innodb/stats.rb @@ -1,46 +1,44 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true # Collect stats globally within innodb_ruby for comparison purposes and for # correctness checking. -class Innodb::Stats - @@data = Hash.new(0) +module Innodb + class Stats + @data = Hash.new(0) - # Return the data hash directly. - def self.data - @@data - end - - # Increment a statistic by name (typically a symbol), optionally by a value - # provided. - def self.increment(name, value=1) - @@data[name] += value - end + class << self + attr_reader :data + end - # Get a statistic by name. - def self.get(name) - @@data[name] - end + # Increment a statistic by name (typically a symbol), optionally by a value + # provided. + def self.increment(name, value = 1) + @data[name] += value + end - # Reset all statistics. - def self.reset - @@data.clear - nil - end + # Get a statistic by name. + def self.get(name) + @data[name] + end - # Print a simple report of collected statistics, optionally to the IO object - # provided, or by default to STDOUT. - def self.print_report(io=STDOUT) - io.puts "%-50s%10s" % [ - "Statistic", - "Count", - ] - @@data.sort.each do |name, count| - io.puts "%-50s%10i" % [ - name, - count - ] + # Reset all statistics. + def self.reset + @data.clear + nil end - nil + # Print a simple report of collected statistics, optionally to the IO object + # provided, or by default to STDOUT. + def self.print_report(io = $stdout) + io.puts "%-50s%10s" % %w[Statistic Count] + @data.sort.each do |name, count| + io.puts "%-50s%10i" % [ + name, + count, + ] + end + + nil + end end end diff --git a/lib/innodb/sys_data_dictionary.rb b/lib/innodb/sys_data_dictionary.rb new file mode 100644 index 00000000..c61d4a1e --- /dev/null +++ b/lib/innodb/sys_data_dictionary.rb @@ -0,0 +1,310 @@ +# frozen_string_literal: true + +require "forwardable" + +# A class representing InnoDB's SYS_* data dictionary (used in MySQL +# versions prior to MySQL 8.0), which contains metadata about tables, +# columns, and indexes in internal InnoDB tables named SYS_*. +module Innodb + class SysDataDictionary + # rubocop:disable Layout/ExtraSpacing + SYSTEM_TABLES = [ + { + name: "SYS_TABLES", + columns: [ + { name: "NAME", description: ["VARCHAR(100)", :NOT_NULL] }, + { name: "ID", description: %i[BIGINT UNSIGNED NOT_NULL] }, + { name: "N_COLS", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "TYPE", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "MIX_ID", description: %i[BIGINT UNSIGNED NOT_NULL] }, + { name: "MIX_LEN", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "CLUSTER_NAME", description: ["VARCHAR(100)", :NOT_NULL] }, + { name: "SPACE", description: %i[INT UNSIGNED NOT_NULL] }, + ], + indexes: [ + { name: "PRIMARY", type: :clustered, column_names: ["NAME"] }, + { name: "ID", type: :secondary, column_names: ["ID"] }, + ], + }, + { + name: "SYS_COLUMNS", + columns: [ + { name: "TABLE_ID", description: %i[BIGINT UNSIGNED NOT_NULL] }, + { name: "POS", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "NAME", description: ["VARCHAR(100)", :NOT_NULL] }, + { name: "MTYPE", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "PRTYPE", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "LEN", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "PREC", description: %i[INT UNSIGNED NOT_NULL] }, + ], + indexes: [ + { name: "PRIMARY", type: :clustered, column_names: %w[TABLE_ID POS] }, + ], + }, + { + name: "SYS_INDEXES", + columns: [ + { name: "TABLE_ID", description: %i[BIGINT UNSIGNED NOT_NULL] }, + { name: "ID", description: %i[BIGINT UNSIGNED NOT_NULL] }, + { name: "NAME", description: ["VARCHAR(100)", :NOT_NULL] }, + { name: "N_FIELDS", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "TYPE", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "SPACE", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "PAGE_NO", description: %i[INT UNSIGNED NOT_NULL] }, + ], + indexes: [ + { name: "PRIMARY", type: :clustered, column_names: %w[TABLE_ID ID] }, + ], + }, + { + name: "SYS_FIELDS", + columns: [ + { name: "INDEX_ID", description: %i[BIGINT UNSIGNED NOT_NULL] }, + { name: "POS", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "COL_NAME", description: ["VARCHAR(100)", :NOT_NULL] }, + ], + indexes: [ + { name: "PRIMARY", type: :clustered, column_names: %w[INDEX_ID POS] }, + ], + }, + ].freeze + # rubocop:enable Layout/ExtraSpacing + + # A hash of InnoDB's internal type system to the values + # stored for each type. + COLUMN_MTYPE = { + VARCHAR: 1, + CHAR: 2, + FIXBINARY: 3, + BINARY: 4, + BLOB: 5, + INT: 6, + SYS_CHILD: 7, + SYS: 8, + FLOAT: 9, + DOUBLE: 10, + DECIMAL: 11, + VARMYSQL: 12, + MYSQL: 13, + }.freeze + + # A hash of COLUMN_MTYPE keys by value. + COLUMN_MTYPE_BY_VALUE = COLUMN_MTYPE.invert.freeze + + # A hash of InnoDB 'precise type' bitwise flags. + COLUMN_PRTYPE_FLAG = { + NOT_NULL: 256, + UNSIGNED: 512, + BINARY: 1024, + LONG_TRUE_VARCHAR: 4096, + }.freeze + + # A hash of COLUMN_PRTYPE keys by value. + COLUMN_PRTYPE_FLAG_BY_VALUE = COLUMN_PRTYPE_FLAG.invert.freeze + + # The bitmask to extract the MySQL internal type + # from the InnoDB 'precise type'. + COLUMN_PRTYPE_MYSQL_TYPE_MASK = 0xFF + + # A hash of InnoDB's index type flags. + INDEX_TYPE_FLAG = { + CLUSTERED: 1, + UNIQUE: 2, + UNIVERSAL: 4, + IBUF: 8, + CORRUPT: 16, + FTS: 32, + }.freeze + + # A hash of INDEX_TYPE_FLAG keys by value. + INDEX_TYPE_FLAG_BY_VALUE = INDEX_TYPE_FLAG.invert.freeze + + # Return the 'external' SQL type string (such as 'VARCHAR' or + # 'INT') given the stored mtype and prtype from the InnoDB + # data dictionary. Note that not all types are extractable + # into fully defined SQL types due to the lossy nature of + # the MySQL-to-InnoDB interface regarding types. + def self.mtype_prtype_to_type_string(mtype, prtype, len, prec) + mysql_type = prtype & COLUMN_PRTYPE_MYSQL_TYPE_MASK + internal_type = Innodb::MysqlType.by_mysql_field_type(mysql_type) + external_type = internal_type.handle_as + + case external_type + when :VARCHAR + # One-argument: length. + "%s(%i)" % [external_type, len] + when :FLOAT, :DOUBLE + # Two-argument: length and precision. + "%s(%i,%i)" % [external_type, len, prec] + when :CHAR + if COLUMN_MTYPE_BY_VALUE[mtype] == :MYSQL + # When the mtype is :MYSQL, the column is actually + # stored as VARCHAR despite being a CHAR. This is + # done for CHAR columns having multi-byte character + # sets in order to limit size. Note that such data + # are still space-padded to at least len. + "VARCHAR(%i)" % [len] + else + "CHAR(%i)" % [len] + end + when :DECIMAL + # The DECIMAL type is designated as DECIMAL(M,D) + # however the M and D definitions are not stored + # in the InnoDB data dictionary. We need to define + # the column as something which will extract the + # raw bytes in order to read the column, but we + # can't figure out the right decimal type. The + # len stored here is actually the on-disk storage + # size. + "CHAR(%i)" % [len] + else + external_type + end + end + + # Return a full data type given an mtype and prtype, such + # as ['VARCHAR(10)', :NOT_NULL] or [:INT, :UNSIGNED]. + def self.mtype_prtype_to_data_type(mtype, prtype, len, prec) + type = mtype_prtype_to_type_string(mtype, prtype, len, prec) + raise "Unsupported type (mtype #{mtype}, prtype #{prtype})" unless type + + data_type = [type] + data_type << :NOT_NULL if prtype & COLUMN_PRTYPE_FLAG[:NOT_NULL] != 0 + data_type << :UNSIGNED if prtype & COLUMN_PRTYPE_FLAG[:UNSIGNED] != 0 + + data_type + end + + extend Forwardable + + attr_reader :innodb_system + + def_delegator :innodb_system, :data_dictionary + + def initialize(innodb_system) + @innodb_system = innodb_system + end + + private + + # A helper method to reach inside the system space and retrieve + # the data dictionary index locations from the data dictionary + # header. + def _data_dictionary_indexes + innodb_system.system_space.data_dictionary_page.data_dictionary_header[:indexes] + end + + def _populate_index_with_system_and_non_key_columns(new_table, new_index) + if new_index.type == :clustered + db_trx_id = Innodb::DataDictionary::Column.new(name: "DB_TRX_ID", description: %i[DB_TRX_ID], table: new_table) + new_index.column_references.make(column: db_trx_id, usage: :sys, index: new_index) + db_roll_ptr = Innodb::DataDictionary::Column.new(name: "DB_ROLL_PTR", description: %i[DB_ROLL_PTR], + table: new_table) + new_index.column_references.make(column: db_roll_ptr, usage: :sys, index: new_index) + + new_table.columns.each do |column| + unless new_index.column_references.find(name: column.name) + new_index.column_references.make(column: column, usage: :row, + index: new_index) + end + end + else + clustered_index = new_table.indexes.find(type: :clustered) + clustered_index.column_references.each do |column| + new_index.column_references.make(column: column, usage: :row, index: new_index) unless column.usage == :sys + end + end + end + + public + + def populate_data_dictionary_with_system_table_definitions + system_tablespace = data_dictionary.tablespaces.make(name: "innodb_system", innodb_space_id: 0) + + SYSTEM_TABLES.each do |table| + new_table = data_dictionary.tables.make(name: table[:name], tablespace: system_tablespace) + + table[:columns].each do |column| + new_table.columns.make(name: column[:name], description: column[:description], table: new_table) + end + + table[:indexes].each do |index| + new_index = new_table.indexes.make( + name: index[:name], + type: index[:type], + table: new_table, + tablespace: system_tablespace, + root_page_number: _data_dictionary_indexes[table[:name]][index[:name]] + ) + index[:column_names].each do |column_name| + new_index.column_references.make( + column: new_table.columns.find(name: column_name), + usage: :key, + index: new_index + ) + end + _populate_index_with_system_and_non_key_columns(new_table, new_index) + end + end + + nil + end + + def populate_data_dictionary_from_system_tables + # Read the entire contents of all tables for efficiency sake, since we'll need to do many sub-iterations + # below and don't want to re-parse the records every time. + sys_tables = innodb_system.index_by_name("SYS_TABLES", "PRIMARY").each_record.map(&:fields) + sys_columns = innodb_system.index_by_name("SYS_COLUMNS", "PRIMARY").each_record.map(&:fields) + sys_indexes = innodb_system.index_by_name("SYS_INDEXES", "PRIMARY").each_record.map(&:fields) + sys_fields = innodb_system.index_by_name("SYS_FIELDS", "PRIMARY").each_record.map(&:fields) + + sys_tables.each do |table_record| + tablespace = data_dictionary.tablespaces.find(innodb_space_id: table_record["SPACE"]) + tablespace ||= data_dictionary.tablespaces.make(name: table_record["NAME"], + innodb_space_id: table_record["SPACE"]) + + new_table = data_dictionary.tables.make(name: table_record["NAME"], tablespace: tablespace, + innodb_table_id: table_record["ID"]) + + sys_columns.select { |r| r["TABLE_ID"] == table_record["ID"] }.each do |column_record| + description = self.class.mtype_prtype_to_data_type( + column_record["MTYPE"], + column_record["PRTYPE"], + column_record["LEN"], + column_record["PREC"] + ) + new_table.columns.make(name: column_record["NAME"], description: description, table: new_table) + end + + sys_indexes.select { |r| r["TABLE_ID"] == table_record["ID"] }.each do |index_record| + raise "Different tablespace between table and index" unless table_record["SPACE"] == index_record["SPACE"] + + type = index_record["TYPE"] & INDEX_TYPE_FLAG[:CLUSTERED] ? :clustered : :secondary + new_index = new_table.indexes.make( + name: index_record["NAME"], + type: type, + table: new_table, + tablespace: tablespace, + root_page_number: index_record["PAGE_NO"], + innodb_index_id: index_record["ID"] + ) + sys_fields.select { |r| r["INDEX_ID"] == index_record["ID"] }.each do |field_record| + new_index.column_references.make(column: new_table.columns.find(name: field_record["COL_NAME"]), + usage: :key, index: new_index) + end + + _populate_index_with_system_and_non_key_columns(new_table, new_index) + end + end + + nil + end + + def populate_data_dictionary + populate_data_dictionary_with_system_table_definitions + populate_data_dictionary_from_system_tables + + nil + end + end +end diff --git a/lib/innodb/system.rb b/lib/innodb/system.rb index 24137457..6039da85 100644 --- a/lib/innodb/system.rb +++ b/lib/innodb/system.rb @@ -1,230 +1,187 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true # A class representing an entire InnoDB system, having a system tablespace # and any number of attached single-table tablespaces. -class Innodb::System - # A hash of configuration options by configuration key. - attr_reader :config - - # A hash of spaces by space ID. - attr_reader :spaces - - # Array of space names for which a space file was not found. - attr_reader :orphans - - # The Innodb::DataDictionary for this system. - attr_reader :data_dictionary - - # The space ID of the system space, always 0. - SYSTEM_SPACE_ID = 0 - - def initialize(arg) - if arg.is_a?(Array) && arg.size > 1 - data_filenames = arg - else - arg = arg.first if arg.is_a?(Array) - if File.directory?(arg) - data_filenames = Dir.glob(arg + "/ibdata?").sort - if data_filenames.empty? - raise "Couldn't find any ibdata files in #{arg}" - end - else - data_filenames = [arg] - end - end +module Innodb + class System + # A hash of configuration options by configuration key. + attr_reader :config - @spaces = {} - @orphans = [] - @config = { - :datadir => File.dirname(data_filenames.first), - } + # A hash of spaces by space ID. + attr_reader :spaces - add_space_file(data_filenames) + # Array of space names for which a space file was not found. + attr_reader :orphans - @data_dictionary = Innodb::DataDictionary.new(system_space) - end + # The Innodb::DataDictionary for this system. + attr_reader :data_dictionary - # A helper to get the system space. - def system_space - spaces[SYSTEM_SPACE_ID] - end + # The space ID of the system space, always 0. + SYSTEM_SPACE_ID = 0 - # Add an already-constructed Innodb::Space object. - def add_space(space) - unless space.is_a?(Innodb::Space) - raise "Object was not an Innodb::Space" - end + # The space ID of the mysql.ibd space, always 4294967294 (2**32-2). + MYSQL_SPACE_ID = 4_294_967_294 - spaces[space.space_id.to_i] = space - end + def initialize(arg, data_directory: nil) + @data_dictionary = Innodb::DataDictionary.new - # Add a space by filename. - def add_space_file(space_filenames) - space = Innodb::Space.new(space_filenames) - space.innodb_system = self - add_space(space) - end + if arg.is_a?(Array) && arg.size > 1 + data_filenames = arg + else + arg = arg.first if arg.is_a?(Array) + if File.directory?(arg) + data_filenames = Dir.glob("#{arg}/ibdata?").sort + raise "Couldn't find any ibdata files in #{arg}" if data_filenames.empty? + else + data_filenames = [arg] + end + end - # Add an orphaned space. - def add_space_orphan(space_file) - orphans << space_file - end + @spaces = {} + @orphans = [] + @config = { + data_directory: data_directory || File.dirname(data_filenames.first), + } - # Add a space by table name, constructing an appropriate filename - # from the provided table name. - def add_table(table_name) - space_file = "%s/%s.ibd" % [config[:datadir], table_name] - if File.exist?(space_file) - add_space_file(space_file) - else - add_space_orphan(table_name) - end - end + add_space_file(data_filenames) - # Return an Innodb::Space object for a given space ID, looking up - # and adding the single-table space if necessary. - def space(space_id) - return spaces[space_id] if spaces[space_id] + add_mysql_space_file + add_all_ibd_files - unless table_record = data_dictionary.table_by_space_id(space_id) - raise "Table with space ID #{space_id} not found" - end + @internal_data_dictionary = if system_space.page(0).prev > 80_000 # ugh + Innodb::SdiDataDictionary.new(self) + else + Innodb::SysDataDictionary.new(self) + end + @internal_data_dictionary.populate_data_dictionary + data_dictionary.refresh - add_table(table_record["NAME"]) + data_dictionary.tables.each do |table| + add_table(table.name) unless spaces[table.tablespace.innodb_space_id] + end + end - spaces[space_id] - end + def data_directory + config[:data_directory] + end - # Return an Innodb::Space object by table name. - def space_by_table_name(table_name) - unless table_record = data_dictionary.table_by_name(table_name) - raise "Table #{table_name} not found" + # A helper to get the system space. + def system_space + spaces[SYSTEM_SPACE_ID] end - if table_record["SPACE"] == 0 - return nil + def mysql_space + spaces[MYSQL_SPACE_ID] end - space(table_record["SPACE"]) - end + # Add an already-constructed Innodb::Space object. + def add_space(space) + raise "Object was not an Innodb::Space" unless space.is_a?(Innodb::Space) - # Iterate through all table names. - def each_table_name - unless block_given? - return enum_for(:each_table_name) + spaces[space.space_id] = space end - data_dictionary.each_table do |record| - yield record["NAME"] + # Add a space by filename. + def add_space_file(space_filenames) + space = Innodb::Space.new(space_filenames, innodb_system: self) + add_space(space) unless spaces[space.space_id] end - nil - end - - # Iterate throught all orphaned spaces. - def each_orphan - unless block_given? - return enum_for(:each_orphan) + # Add an orphaned space. + def add_space_orphan(space_file) + orphans << space_file end - orphans.each do |space_name| - yield space_name + # Add a space by table name, constructing an appropriate filename + # from the provided table name. + def add_table(table_name) + space_file = File.join(config[:data_directory], format("%s.ibd", table_name)) + + if File.exist?(space_file) + add_space_file(space_file) + else + add_space_orphan(table_name) + end end - nil - end + # Return an Innodb::Space object for a given space ID, looking up + # and adding the single-table space if necessary. + def space(space_id) + return spaces[space_id] if spaces[space_id] - # Iterate through all column names by table name. - def each_column_name_by_table_name(table_name) - unless block_given? - return enum_for(:each_column_name_by_table_name, table_name) - end + unless (table = data_dictionary.tables.find(innodb_space_id: space_id)) + raise "Table with space ID #{space_id} not found" + end - data_dictionary.each_column_by_table_name(table_name) do |record| - yield record["NAME"] + add_table(table.name) + + spaces[space_id] end - nil - end + def space_by_table_name(table_name) + space_id = data_dictionary.tables.find(name: table_name)&.tablespace&.innodb_space_id - # Iterate through all index names by table name. - def each_index_name_by_table_name(table_name) - unless block_given? - return enum_for(:each_index_name_by_table_name, table_name) + spaces[space_id] if space_id end - data_dictionary.each_index_by_table_name(table_name) do |record| - yield record["NAME"] + def add_mysql_space_file + mysql_ibd = File.join(data_directory, "mysql.ibd") + add_space_file(mysql_ibd) if File.exist?(mysql_ibd) end - nil - end + # Iterate through all table names. + def each_ibd_file_name(&block) + return enum_for(:each_ibd_file_name) unless block_given? - # Iterate through all field names in a given index by table name - # and index name. - def each_index_field_name_by_index_name(table_name, index_name) - unless block_given? - return enum_for(:each_index_field_name_by_index_name, - table_name, index_name) - end + Dir.glob(File.join(data_directory, "**/*.ibd")) + .map { |f| f.sub(File.join(data_directory, "/"), "") }.each(&block) - data_dictionary.each_field_by_index_name(table_name, index_name) do |record| - yield record["COL_NAME"] + nil end - nil - end + def add_all_ibd_files + each_ibd_file_name do |file_name| + add_space_file(File.join(data_directory, file_name)) + end - # Return the table name given a table ID. - def table_name_by_id(table_id) - if table_record = data_dictionary.table_by_id(table_id) - table_record["NAME"] + nil end - end - # Return the index name given an index ID. - def index_name_by_id(index_id) - if index_record = data_dictionary.index_by_id(index_id) - index_record["NAME"] + def each_space(&block) + return enum_for(:each_space) unless block_given? + + spaces.each_value(&block) + + nil end - end - # Return the clustered index name given a table name. - def clustered_index_by_table_name(table_name) - data_dictionary.clustered_index_name_by_table_name(table_name) - end + # Iterate throught all orphaned spaces. + def each_orphan(&block) + return enum_for(:each_orphan) unless block_given? + + orphans.each(&block) - # Return an array of the table name and index name given an index ID. - def table_and_index_name_by_id(index_id) - if dd_index = data_dictionary.data_dictionary_index_ids[index_id] - # This is a data dictionary index, which won't be found in the data - # dictionary itself. - [dd_index[:table], dd_index[:index]] - elsif index_record = data_dictionary.index_by_id(index_id) - # This is a system or user index. - [table_name_by_id(index_record["TABLE_ID"]), index_record["NAME"]] + nil end - end - # Return an Innodb::Index object given a table name and index name. - def index_by_name(table_name, index_name) - index_record = data_dictionary.index_by_name(table_name, index_name) + # Return an Innodb::Index object given a table name and index name. + def index_by_name(table_name, index_name) + table = data_dictionary.tables.find(name: table_name) + index = table.indexes.find(name: index_name) - index_space = space(index_record["SPACE"]) - describer = data_dictionary.record_describer_by_index_name(table_name, index_name) - index = index_space.index(index_record["PAGE_NO"], describer) + space(index.tablespace.innodb_space_id).index(index.root_page_number, index.record_describer) + end - index - end + # Return the clustered index given a table ID. + def clustered_index_by_table_id(table_id) + table = data_dictionary.tables.find(innodb_table_id: table_id) + return unless table - # Return the clustered index given a table ID. - def clustered_index_by_table_id(table_id) - if table_name = table_name_by_id(table_id) - index_by_name(table_name, clustered_index_by_table_name(table_name)) + index_by_name(table.name, table.clustered_index.name) end - end - def history - Innodb::History.new(self) + def history + Innodb::History.new(self) + end end end diff --git a/lib/innodb/undo_log.rb b/lib/innodb/undo_log.rb index 4c8222d7..f6a95351 100644 --- a/lib/innodb/undo_log.rb +++ b/lib/innodb/undo_log.rb @@ -1,134 +1,156 @@ -# -*- encoding : utf-8 -*- - -class Innodb::UndoLog - attr_reader :page - attr_reader :position - def initialize(page, position) - @page = page - @position = position - end - - def size_xa_header - 4 + 4 + 4 + 128 - end +# frozen_string_literal: true + +module Innodb + class UndoLog + Header = Struct.new( + :trx_id, + :trx_no, + :delete_mark_flag, + :log_start_offset, + :xid_flag, + :ddl_flag, + :ddl_table_id, + :next_log_offset, + :prev_log_offset, + :history_list_node, + :xid, + keyword_init: true + ) + + HeaderXid = Struct.new( + :format, + :trid_len, + :bqual_len, + :data, + keyword_init: true + ) + + attr_reader :page + attr_reader :position + + def initialize(page, position) + @page = page + @position = position + end - def size_header - 8 + 8 + 2 + 2 + 1 + 1 + 8 + 2 + 2 + Innodb::List::NODE_SIZE + size_xa_header - end + def size_xa_header + 4 + 4 + 4 + 128 + end - def header - @header ||= page.cursor(@position).name("header") do |c| - xid_flag = nil - { - :trx_id => c.name("trx_id") { c.get_uint64 }, - :trx_no => c.name("trx_no") { c.get_uint64 }, - :delete_mark_flag => c.name("delete_mark_flag") { (c.get_uint16 != 0) }, - :log_start_offset => c.name("log_start_offset") { c.get_uint16 }, - :xid_flag => c.name("xid_flag") { xid_flag = (c.get_uint8 != 0) }, - :ddl_flag => c.name("ddl_flag") { (c.get_uint8 != 0) }, - :ddl_table_id => c.name("ddl_table_id") { c.get_uint64 }, - :next_log_offset => c.name("next_log_offset") { c.get_uint16 }, - :prev_log_offset => c.name("prev_log_offset") { c.get_uint16 }, - :history_list_node => c.name("history_list_node") { - Innodb::List.get_node(c) - }, - :xid => c.name("xid") { - if xid_flag - { - :format => c.name("format") { c.get_uint32 }, - :trid_len => c.name("trid_len") { c.get_uint32 }, - :bqual_len => c.name("bqual_len") { c.get_uint32 }, - :data => c.name("data") { c.get_bytes(128) }, - } - end - }, - } + def size_header + 8 + 8 + 2 + 2 + 1 + 1 + 8 + 2 + 2 + Innodb::List::NODE_SIZE + size_xa_header end - end - def prev_address - header[:history_list_node][:prev] - end + def header + @header ||= page.cursor(@position).name("header") do |c| + header = Header.new( + trx_id: c.name("trx_id") { c.read_uint64 }, + trx_no: c.name("trx_no") { c.read_uint64 }, + delete_mark_flag: c.name("delete_mark_flag") { (c.read_uint16 != 0) }, + log_start_offset: c.name("log_start_offset") { c.read_uint16 }, + xid_flag: c.name("xid_flag") { (c.read_uint8 != 0) }, + ddl_flag: c.name("ddl_flag") { (c.read_uint8 != 0) }, + ddl_table_id: c.name("ddl_table_id") { c.read_uint64 }, + next_log_offset: c.name("next_log_offset") { c.read_uint16 }, + prev_log_offset: c.name("prev_log_offset") { c.read_uint16 }, + history_list_node: c.name("history_list_node") { Innodb::List.get_node(c) } + ) + + if header.xid_flag + header.xid = c.name("xid") do + HeaderXid.new( + format: c.name("format") { c.read_uint32 }, + trid_len: c.name("trid_len") { c.read_uint32 }, + bqual_len: c.name("bqual_len") { c.read_uint32 }, + data: c.name("data") { c.read_bytes(128) } + ) + end + end - def next_address - header[:history_list_node][:next] - end + header + end + end - def undo_record(offset) - new_undo_record = Innodb::UndoRecord.new(page, offset) - new_undo_record.undo_log = self - new_undo_record - end + def prev_address + header.history_list_node.prev + end - def min_undo_record - undo_record(header[:log_start_offset]) - end + def next_address + header.history_list_node.next + end - class UndoRecordCursor - def initialize(undo_log, offset, direction=:forward) - @initial = true - @undo_log = undo_log - @offset = offset - @direction = direction - - case offset - when :min - @undo_record = @undo_log.min_undo_record - when :max - raise "Not implemented" - else - @undo_record = @undo_log.undo_record(offset) - end + def undo_record(offset) + new_undo_record = Innodb::UndoRecord.new(page, offset) + new_undo_record.undo_log = self + new_undo_record end - def next_undo_record - if rec = @undo_record.next - @undo_record = rec - end + def min_undo_record + undo_record(header.log_start_offset) end - def prev_undo_record - if rec = @undo_record.prev - @undo_record = rec + class UndoRecordCursor + def initialize(undo_log, offset, direction = :forward) + @initial = true + @undo_log = undo_log + @offset = offset + @direction = direction + + case offset + when :min + @undo_record = @undo_log.min_undo_record + when :max + raise "Not implemented" + else + @undo_record = @undo_log.undo_record(offset) + end end - end - def undo_record - if @initial - @initial = false - return @undo_record + def next_undo_record + rec = @undo_record.next + @undo_record = rec if rec end - case @direction - when :forward - next_undo_record - when :backward - prev_undo_record + def prev_undo_record + rec = @undo_record.prev + @undo_record = rec if rec end - end - def each_undo_record - unless block_given? - return enum_for(:each_undo_record) + def undo_record + if @initial + @initial = false + return @undo_record + end + + case @direction + when :forward + next_undo_record + when :backward + prev_undo_record + end end - while rec = undo_record - yield rec + def each_undo_record + return enum_for(:each_undo_record) unless block_given? + + while (rec = undo_record) + yield rec + end end end - end - def undo_record_cursor(offset, direction=:forward) - UndoRecordCursor.new(self, offset, direction) - end + def undo_record_cursor(offset, direction = :forward) + UndoRecordCursor.new(self, offset, direction) + end - def first_undo_record_cursor - undo_record_cursor(header[:log_start_offset]) - end + def first_undo_record_cursor + undo_record_cursor(header.log_start_offset) + end - def dump - puts "header:" - pp header - puts + def dump + puts "header:" + pp header + puts + end end end diff --git a/lib/innodb/undo_record.rb b/lib/innodb/undo_record.rb index 094c70dd..29f389d9 100644 --- a/lib/innodb/undo_record.rb +++ b/lib/innodb/undo_record.rb @@ -1,165 +1,204 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true + +require "forwardable" # A single undo log record. -class Innodb::UndoRecord - attr_reader :undo_page - attr_reader :position +module Innodb + class UndoRecord + extend Forwardable + + Header = Struct.new( + :prev, + :next, + :type, + :extern_flag, + :info, + keyword_init: true + ) + + HeaderInfo = Struct.new( + :order_may_change, + :size_may_change, + keyword_init: true + ) + + Record = Struct.new( + :page, + :offset, + :header, + :undo_no, + :table_id, + :info_bits, + :trx_id, + :roll_ptr, + :data, + :key, + :row, + keyword_init: true + ) + + Field = Struct.new( + :name, + :type, + :value, + keyword_init: true + ) + + attr_reader :undo_page + attr_reader :position + + attr_accessor :undo_log + attr_accessor :index_page + + def initialize(undo_page, position) + @undo_page = undo_page + @position = position + + @undo_log = nil + @index_page = nil + end - attr_accessor :undo_log - attr_accessor :index_page + def new_subordinate(undo_page, position) + new_undo_record = self.class.new(undo_page, position) + new_undo_record.undo_log = undo_log + new_undo_record.index_page = index_page - def initialize(undo_page, position) - @undo_page = undo_page - @position = position + new_undo_record + end - @undo_log = nil - @index_page = nil - end + # The header really starts 2 bytes before the undo record position, as the + # pointer to the previous record is written there. + def pos_header + @position - 2 + end - def new_subordinate(undo_page, position) - new_undo_record = self.class.new(undo_page, position) - new_undo_record.undo_log = undo_log - new_undo_record.index_page = index_page + # The size of the header. + def size_header + 2 + 2 + 1 + end - new_undo_record - end + def pos_record + pos_header + size_header + end - # The header really starts 2 bytes before the undo record position, as the - # pointer to the previous record is written there. - def pos_header - @position - 2 - end + # Return a BufferCursor starting before the header. + def cursor(position) + new_cursor = @undo_page.cursor(position) + new_cursor.push_name("undo_log[#{@undo_log.position}]") if @undo_log + new_cursor.push_name("undo_record[#{@position}]") + new_cursor + end - # The size of the header. - def size_header - 2 + 2 + 1 - end + # Possible undo record types. + TYPE = { + 11 => :insert, + 12 => :update_existing, + 13 => :update_deleted, + 14 => :delete, + }.freeze + + TYPES_WITH_PREVIOUS_VERSIONS = %i[ + update_existing + update_deleted + delete + ].freeze + + TYPE_MASK = 0x0f + COMPILATION_INFO_MASK = 0x70 + COMPILATION_INFO_SHIFT = 4 + COMPILATION_INFO_NO_ORDER_CHANGE_BV = 1 + COMPILATION_INFO_NO_SIZE_CHANGE_BV = 2 + EXTERN_FLAG = 0x80 + + def header + @header ||= cursor(pos_header).name("header") do |c| + header = Header.new( + prev: c.name("prev") { c.read_uint16 }, + next: c.name("next") { c.read_uint16 } + ) + + info = c.name("info") { c.read_uint8 } + cmpl = (info & COMPILATION_INFO_MASK) >> COMPILATION_INFO_SHIFT + header.type = TYPE[info & TYPE_MASK] + header.extern_flag = (info & EXTERN_FLAG) != 0 + header.info = HeaderInfo.new( + order_may_change: (cmpl & COMPILATION_INFO_NO_ORDER_CHANGE_BV).zero?, + size_may_change: (cmpl & COMPILATION_INFO_NO_SIZE_CHANGE_BV).zero? + ) + + header + end + end - def pos_record - pos_header + size_header - end + def_delegator :header, :type - # Return a BufferCursor starting before the header. - def cursor(position) - new_cursor = @undo_page.cursor(position) - if @undo_log - new_cursor.push_name("undo_log[#{@undo_log.position}]") + def previous_version? + TYPES_WITH_PREVIOUS_VERSIONS.include?(type) end - new_cursor.push_name("undo_record[#{@position}]") - new_cursor - end - # Possible undo record types. - TYPE = { - 11 => :insert, - 12 => :update_existing, - 13 => :update_deleted, - 14 => :delete, - } - - TYPE_MASK = 0x0f - COMPILATION_INFO_MASK = 0x70 - COMPILATION_INFO_SHIFT = 4 - COMPILATION_INFO_NO_ORDER_CHANGE_BV = 1 - COMPILATION_INFO_NO_SIZE_CHANGE_BV = 2 - EXTERN_FLAG = 0x80 - - def header - @header ||= cursor(pos_header).name("header") do |c| - header = { - :prev => c.name("prev") { c.get_uint16 }, - :next => c.name("next") { c.get_uint16 }, - } - - info = c.name("info") { c.get_uint8 } - cmpl = (info & COMPILATION_INFO_MASK) >> COMPILATION_INFO_SHIFT - header[:type] = TYPE[info & TYPE_MASK] - header[:extern_flag] = (info & EXTERN_FLAG) != 0 - header[:info] = { - :order_may_change => (cmpl & COMPILATION_INFO_NO_ORDER_CHANGE_BV) == 0, - :size_may_change => (cmpl & COMPILATION_INFO_NO_SIZE_CHANGE_BV) == 0, - } - - header - end - end + def get(prev_or_next) + return if header[prev_or_next].zero? - def type - header[:type] - end + new_undo_record = new_subordinate(@undo_page, header[prev_or_next]) + new_undo_record if new_undo_record.type + end - def has_previous_version? - [:update_existing, :update_deleted, :delete].include?(type) - end + def prev + get(:prev) + end - def get(prev_or_next) - if header[prev_or_next] != 0 - new_undo_record = new_subordinate(@undo_page, header[prev_or_next]) - if new_undo_record.type - new_undo_record - end + def next + get(:next) end - end - def prev - get(:prev) - end + def record_size + header[:next] - @position - size_header + end - def next - get(:next) - end + def read_record + cursor(pos_record).name("record") do |c| + this_record = Record.new( + page: undo_page.offset, + offset: position, + header: header, + undo_no: c.name("undo_no") { c.read_imc_uint64 }, + table_id: c.name("table_id") { c.read_imc_uint64 } + ) + + if previous_version? + this_record.info_bits = c.name("info_bits") { c.read_uint8 } + this_record.trx_id = c.name("trx_id") { c.read_ic_uint64 } + this_record.roll_ptr = c.name("roll_ptr") do + Innodb::DataType::RollPointerType.parse_roll_pointer(c.read_ic_uint64) + end + end - def record_size - header[:next] - @position - size_header - end + if index_page + read_record_fields(this_record, c) + else + # Slurp up the remaining data as a string. + this_record.data = c.read_bytes(header[:next] - c.position - 2) + end - def read_record - cursor(pos_record).name("record") do |c| - this_record = { - :page => undo_page.offset, - :offset => position, - :header => header, - :undo_no => c.name("undo_no") { c.get_imc_uint64 }, - :table_id => c.name("table_id") { c.get_imc_uint64 }, - } - - if has_previous_version? - this_record[:info_bits] = c.name("info_bits") { c.get_uint8 } - this_record[:trx_id] = c.name("trx_id") { c.get_ic_uint64 } - this_record[:roll_ptr] = c.name("roll_ptr") { - Innodb::DataType::RollPointerType.parse_roll_pointer(c.get_ic_uint64) - } + this_record end + end - if index_page - read_record_fields(this_record, c) - else - # Slurp up the remaining data as a string. - this_record[:data] = c.get_bytes(header[:next] - c.position - 2) - end + def read_record_fields(this_record, cursor) + this_record.key = [] + index_page.record_format[:key].each do |field| + length = cursor.name("field_length") { cursor.read_ic_uint32 } + value = cursor.name(field.name) { field.value_by_length(cursor, length) } - this_record - end - end + this_record.key[field.position] = Field.new(name: field.name, type: field.data_type.name, value: value) + end - def read_record_fields(this_record, c) - this_record[:key] = [] - index_page.record_format[:key].each do |field| - this_record[:key][field.position] = { - :name => field.name, - :type => field.data_type.name, - :value => c.name(field.name) { - field_length = c.name("field_length") { c.get_ic_uint32 } - field.value_by_length(c, field_length) - } - } - end + return unless previous_version? - if has_previous_version? - field_count = c.name("field_count") { c.get_ic_uint32 } - this_record[:row] = Array.new(index_page.record_format[:row].size) + field_count = cursor.name("field_count") { cursor.read_ic_uint32 } + this_record.row = Array.new(index_page.record_format[:row].size) field_count.times do - field_number = c.name("field_number[#{field_count}]") { c.get_ic_uint32 } + field_number = cursor.name("field_number[#{field_count}]") { cursor.read_ic_uint32 } field = nil field_index = nil index_page.record_format[:row].each_with_index do |candidate_field, index| @@ -168,140 +207,109 @@ def read_record_fields(this_record, c) field_index = index end end - raise "Unknown field #{field_number}" unless field - this_record[:row][field_index] = { - :name => field.name, - :type => field.data_type.name, - :value => c.name(field.name) { - field_length = c.name("field_length") { c.get_ic_uint32 } - field.value_by_length(c, field_length) - } - } - end - end - end - - def undo_record - @undo_record ||= read_record - end - def undo_no - undo_record[:undo_no] - end - - def table_id - undo_record[:table_id] - end - - def trx_id - undo_record[:trx_id] - end + raise "Unknown field #{field_number}" unless field - def roll_ptr - undo_record[:roll_ptr] - end + length = cursor.name("field_length") { cursor.read_ic_uint32 } + value = cursor.name(field.name) { field.value_by_length(cursor, length) } - def key - undo_record[:key] - end + this_record.row[field_index] = Field.new(name: field.name, type: field.data_type.name, value: value) + end + end - def page - undo_record[:page] - end + def undo_record + @undo_record ||= read_record + end - def offset - undo_record[:offset] - end + def_delegator :undo_record, :undo_no + def_delegator :undo_record, :table_id + def_delegator :undo_record, :trx_id + def_delegator :undo_record, :roll_ptr + def_delegator :undo_record, :key + def_delegator :undo_record, :page + def_delegator :undo_record, :offset - def key_string - key && key.map { |r| "%s=%s" % [r[:name], r[:value].inspect] }.join(", ") - end + def key_string + key&.map { |r| "%s=%s" % [r[:name], r[:value].inspect] }&.join(", ") + end - def row - undo_record[:row] - end + def row + undo_record[:row] + end - def row_string - row && row.select { |r| !r.nil? }.map { |r| r && "%s=%s" % [r[:name], r[:value].inspect] }.join(", ") - end + def row_string + row&.compact&.map { |r| r && format("%s=%s", r[:name], r[:value].inspect) }&.join(", ") + end - def string - "(%s) → (%s)" % [key_string, row_string] - end + def string + "(%s) → (%s)" % [key_string, row_string] + end - # Find the previous row version by following the roll_ptr from one undo - # record to the next (backwards through the record version history). Since - # we are operating without the benefit of knowing about active transactions - # and without protection from purge, check that everything looks sane before - # returning it. - def prev_by_history - unless has_previous_version? + # Find the previous row version by following the roll_ptr from one undo + # record to the next (backwards through the record version history). Since + # we are operating without the benefit of knowing about active transactions + # and without protection from purge, check that everything looks sane before + # returning it. + def prev_by_history # This undo record type has no previous version information. - return nil - end + return unless previous_version? - undo_log = roll_ptr[:undo_log] - older_undo_page = @undo_page.space.page(undo_log[:page]) + undo_log = roll_ptr[:undo_log] + older_undo_page = @undo_page.space.page(undo_log[:page]) - unless older_undo_page and older_undo_page.is_a?(Innodb::Page::UndoLog) # The page was probably re-used for something else. - return nil - end + return unless older_undo_page.is_a?(Innodb::Page::UndoLog) - older_undo_record = new_subordinate(older_undo_page, - undo_log[:offset]) + older_undo_record = new_subordinate(older_undo_page, undo_log[:offset]) - unless older_undo_record and table_id == older_undo_record.table_id # The record space was probably re-used for something else. - return nil - end + return unless older_undo_record && table_id == older_undo_record.table_id - unless older_undo_record.trx_id.nil? or trx_id >= older_undo_record.trx_id # The trx_id should not be newer; but may be absent (for insert). - return nil - end + return unless older_undo_record.trx_id.nil? || trx_id >= older_undo_record.trx_id - older_undo_record - end - - def dump - puts "Undo record at offset %i" % offset - puts - - puts "Header:" - puts " %-25s: %i" % ["Previous record offset", header[:prev]] - puts " %-25s: %i" % ["Next record offset", header[:next]] - puts " %-25s: %s" % ["Type", header[:type]] - puts - - puts "System fields:" - puts " Transaction ID: %s" % trx_id - puts " Roll Pointer:" - puts " Undo Log: page %i, offset %i" % [ - roll_ptr[:undo_log][:page], - roll_ptr[:undo_log][:offset], - ] - puts " Rollback Segment ID: %i" % roll_ptr[:rseg_id] - puts - - puts "Key fields:" - key.each do |field| - puts " %s: %s" % [ - field[:name], - field[:value].inspect, - ] + older_undo_record end - puts - - puts "Non-key fields:" - row.each do |field| - next if !field - puts " %s: %s" % [ - field[:name], - field[:value].inspect, + + def dump + puts "Undo record at offset %i" % offset + puts + + puts "Header:" + puts " %-25s: %i" % ["Previous record offset", header[:prev]] + puts " %-25s: %i" % ["Next record offset", header[:next]] + puts " %-25s: %s" % ["Type", header[:type]] + puts + + puts "System fields:" + puts " Transaction ID: %s" % trx_id + puts " Roll Pointer:" + puts " Undo Log: page %i, offset %i" % [ + roll_ptr[:undo_log][:page], + roll_ptr[:undo_log][:offset], ] + puts " Rollback Segment ID: %i" % roll_ptr[:rseg_id] + puts + + puts "Key fields:" + key.each do |field| + puts " %s: %s" % [ + field[:name], + field[:value].inspect, + ] + end + puts + + puts "Non-key fields:" + row.each do |field| + next unless field + + puts " %s: %s" % [ + field[:name], + field[:value].inspect, + ] + end + puts end - puts end - end diff --git a/lib/innodb/util/buffer_cursor.rb b/lib/innodb/util/buffer_cursor.rb index 4bbd976f..29ef30a4 100644 --- a/lib/innodb/util/buffer_cursor.rb +++ b/lib/innodb/util/buffer_cursor.rb @@ -1,4 +1,4 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true require "bindata" @@ -17,7 +17,7 @@ class StackEntry attr_accessor :direction attr_accessor :name - def initialize(cursor, position=0, direction=:forward, name=nil) + def initialize(cursor, position = 0, direction = :forward, name = nil) @cursor = cursor @position = position @direction = direction @@ -37,21 +37,25 @@ def dup end end - @@global_tracing = false + @global_tracing = false # Enable tracing for all BufferCursor objects globally. - def self.trace!(arg=true) - @@global_tracing = arg + def self.trace!(arg = true) # rubocop:disable Style/OptionalBooleanParameter + @global_tracing = arg + end + + def self.global_tracing? + @global_tracing end # Initialize a cursor within a buffer at the given position. def initialize(buffer, position) @buffer = buffer - @stack = [ StackEntry.new(self, position) ] + @stack = [StackEntry.new(self, position)] trace false trace_with :print_trace - trace_to STDOUT + trace_to $stdout end def inspect @@ -62,46 +66,49 @@ def inspect ] end - def trace(arg=true) + def trace(arg = true) # rubocop:disable Style/OptionalBooleanParameter @instance_tracing = arg + self end # Print a trace output for this cursor. The method is passed a cursor object, # position, raw byte buffer, and array of names. - def print_trace(cursor, position, bytes, name) + def print_trace(_cursor, position, bytes, name) slice_size = 16 - bytes.each_slice(slice_size).each_with_index do |slice_bytes, slice_count| + bytes.each_slice(slice_size).with_index do |slice_bytes, slice_count| @trace_io.puts "%06i %s %-32s %s" % [ position + (slice_count * slice_size), direction == :backward ? "←" : "→", slice_bytes.map { |n| "%02x" % n }.join, - slice_count == 0 ? name.join(".") : "↵", + slice_count.zero? ? name.join(".") : "↵", ] end end def trace_to(file) @trace_io = file + self end # Set a Proc or method on self to trace with. - def trace_with(arg=nil) + def trace_with(arg = nil) if arg.nil? @trace_proc = nil - elsif arg.class == Proc + elsif arg.instance_of?(Proc) @trace_proc = arg - elsif arg.class == Symbol - @trace_proc = lambda { |cursor, position, bytes, name| self.send(arg, cursor, position, bytes, name) } + elsif arg.instance_of?(Symbol) + @trace_proc = ->(cursor, position, bytes, name) { send(arg, cursor, position, bytes, name) } else raise "Don't know how to trace with #{arg}" end + self end def tracing_enabled? - (@@global_tracing or @instance_tracing) && @trace_proc + (self.class.global_tracing? || @instance_tracing) && @trace_proc end # Generate a trace record from the current cursor. @@ -123,14 +130,10 @@ def pop_name end # Set the field name. - def name(name_arg=nil) - if name_arg.nil? - return current.name - end + def name(name_arg = nil) + return current.name if name_arg.nil? - unless block_given? - raise "No block given" - end + raise "No block given" unless block_given? current.name.push name_arg ret = yield(self) @@ -139,12 +142,11 @@ def name(name_arg=nil) end # Return the direction of the current cursor. - def direction(direction_arg=nil) - if direction_arg.nil? - return current.direction - end + def direction(direction_arg = nil) + return current.direction if direction_arg.nil? current.direction = direction_arg + self end @@ -166,34 +168,40 @@ def position # Move the current cursor to a new absolute position. def seek(position) current.position = position if position + self end # Adjust the current cursor to a new relative position. def adjust(relative_position) current.position += relative_position + self end # Save the current cursor position and start a new (nested, stacked) cursor. - def push(position=nil) + def push(position = nil) @stack.push current.dup seek(position) + self end # Restore the last cursor position. def pop raise "No cursors to pop" unless @stack.size > 1 + @stack.pop + self end # Execute a block and restore the cursor to the previous position after # the block returns. Return the block's return value after restoring the # cursor. Optionally seek to provided position before executing block. - def peek(position=nil) + def peek(position = nil) raise "No block given" unless block_given? + push(position) result = yield(self) pop @@ -219,105 +227,101 @@ def read_and_advance(length) end # Return raw bytes. - def get_bytes(length) + def read_bytes(length) read_and_advance(length) end # Return a null-terminated string. - def get_string(length) + def read_string(length) BinData::Stringz.read(read_and_advance(length)) end # Iterate through length bytes returning each as an unsigned 8-bit integer. - def each_byte_as_uint8(length) - unless block_given? - return enum_for(:each_byte_as_uint8, length) - end + def each_byte_as_uint8(length, &block) + return enum_for(:each_byte_as_uint8, length) unless block_given? - read_and_advance(length).bytes.each do |byte| - yield byte - end + read_and_advance(length).bytes.each(&block) nil end # Return raw bytes as hex. - def get_hex(length) + def read_hex(length) read_and_advance(length).bytes.map { |c| "%02x" % c }.join end # Read an unsigned 8-bit integer. - def get_uint8(position=nil) + def read_uint8(position = nil) seek(position) data = read_and_advance(1) BinData::Uint8.read(data).to_i end # Read a big-endian unsigned 16-bit integer. - def get_uint16(position=nil) + def read_uint16(position = nil) seek(position) data = read_and_advance(2) BinData::Uint16be.read(data).to_i end # Read a big-endian signed 16-bit integer. - def get_sint16(position=nil) + def read_sint16(position = nil) seek(position) data = read_and_advance(2) BinData::Int16be.read(data).to_i end # Read a big-endian unsigned 24-bit integer. - def get_uint24(position=nil) + def read_uint24(position = nil) seek(position) data = read_and_advance(3) BinData::Uint24be.read(data).to_i end # Read a big-endian unsigned 32-bit integer. - def get_uint32(position=nil) + def read_uint32(position = nil) seek(position) data = read_and_advance(4) BinData::Uint32be.read(data).to_i end # Read a big-endian unsigned 48-bit integer. - def get_uint48(position=nil) + def read_uint48(position = nil) seek(position) data = read_and_advance(6) BinData::Uint48be.read(data).to_i end # Read a big-endian unsigned 64-bit integer. - def get_uint64(position=nil) + def read_uint64(position = nil) seek(position) data = read_and_advance(8) BinData::Uint64be.read(data).to_i end # Read a big-endian unsigned integer given its size in bytes. - def get_uint_by_size(size) + def read_uint_by_size(size) case size when 1 - get_uint8 + read_uint8 when 2 - get_uint16 + read_uint16 when 3 - get_uint24 + read_uint24 when 4 - get_uint32 + read_uint32 when 6 - get_uint48 + read_uint48 when 8 - get_uint64 + read_uint64 else raise "Integer size #{size} not implemented" end end # Read an array of count unsigned integers given their size in bytes. - def get_uint_array_by_size(size, count) - (0...count).to_a.inject([]) { |a, n| a << get_uint_by_size(size); a } + def read_uint_array_by_size(size, count) + count.times.map { read_uint_by_size(size) } end # Read an InnoDB-compressed unsigned 32-bit integer (1-5 bytes). @@ -327,30 +331,28 @@ def get_uint_array_by_size(size, count) # flag for integers >= 0xf0000000. # # Optionally accept a flag (first byte) if it has already been read (as is - # the case in get_imc_uint64). - def get_ic_uint32(flag=nil) - name("ic_uint32") { - if !flag - flag = peek { name("uint8_or_flag") { get_uint8 } } - end + # the case in read_imc_uint64). + def read_ic_uint32(flag = nil) + name("ic_uint32") do + flag ||= peek { name("uint8_or_flag") { read_uint8 } } case when flag < 0x80 adjust(+1) flag when flag < 0xc0 - name("uint16") { get_uint16 } & 0x7fff + name("uint16") { read_uint16 } & 0x7fff when flag < 0xe0 - name("uint24") { get_uint24 } & 0x3fffff + name("uint24") { read_uint24 } & 0x3fffff when flag < 0xf0 - name("uint32") { get_uint32 } & 0x1fffffff + name("uint32") { read_uint32 } & 0x1fffffff when flag == 0xf0 adjust(+1) # Skip the flag byte. - name("uint32+1") { get_uint32 } + name("uint32+1") { read_uint32 } else - raise "Invalid flag #{flag.to_s} seen" + raise "Invalid flag #{flag} seen" end - } + end end # Read an InnoDB-compressed unsigned 64-bit integer (5-9 bytes). @@ -359,13 +361,13 @@ def get_ic_uint32(flag=nil) # integer (1-5 bytes) while the low 32 bits are stored as a standard # big-endian 32-bit integer (4 bytes). This makes a combined size of # between 5 and 9 bytes. - def get_ic_uint64 - name("ic_uint64") { - high = name("high") { get_ic_uint32 } - low = name("low") { name("uint32") { get_uint32 } } + def read_ic_uint64 + name("ic_uint64") do + high = name("high") { read_ic_uint32 } + low = name("low") { name("uint32") { read_uint32 } } (high << 32) | low - } + end end # Read an InnoDB-"much compressed" unsigned 64-bit integer (1-11 bytes). @@ -376,31 +378,31 @@ def get_ic_uint64 # is also a flag) of the low 32 bits of the value, also as an InnoDB- # compressed 32-bit unsigned integer. This makes for a combined size # of between 1 and 11 bytes. - def get_imc_uint64 - name("imc_uint64") { + def read_imc_uint64 + name("imc_uint64") do high = 0 - flag = peek { name("uint8_or_flag") { get_uint8 } } + flag = peek { name("uint8_or_flag") { read_uint8 } } if flag == 0xff # The high 32-bits are stored first as an ic_uint32. adjust(+1) # Skip the flag byte. - high = name("high") { get_ic_uint32 } + high = name("high") { read_ic_uint32 } flag = nil end # The low 32-bits are stored as an ic_uint32; pass the flag we already # read, so we don't have to read it again. - low = name("low") { get_ic_uint32(flag) } + low = name("low") { read_ic_uint32(flag) } (high << 32) | low - } + end end # Read an array of 1-bit integers. - def get_bit_array(num_bits) + def read_bit_array(num_bits) size = (num_bits + 7) / 8 data = read_and_advance(size) - bit_array = BinData::Array.new(:type => :bit1, :initial_length => size * 8) + bit_array = BinData::Array.new(type: :bit1, initial_length: size * 8) bit_array.read(data).to_ary end end diff --git a/lib/innodb/util/hex_format.rb b/lib/innodb/util/hex_format.rb new file mode 100644 index 00000000..db162958 --- /dev/null +++ b/lib/innodb/util/hex_format.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module HexFormat + LINE_SIZE = 16 + GROUP_SIZE = 8 + GROUP_FORMAT_LENGTH = ((LINE_SIZE.to_f / GROUP_SIZE).ceil * (GROUP_SIZE * 3)) + + def self.format_group(data) + data.map { |n| "%02x" % n.ord }.join(" ") + end + + def self.format_groups(data, size) + data.each_slice(size).map { |g| format_group(g) }.join(" ") + end + + def self.format_printable(data) + data.join.gsub(/[^[:print:]]/, ".") + end + + def self.format_hex(data) + data.chars.each_slice(LINE_SIZE).with_index do |bytes, i| + yield format("%08i %-#{GROUP_FORMAT_LENGTH}s |%-#{LINE_SIZE}s|", + (i * LINE_SIZE), format_groups(bytes, GROUP_SIZE), format_printable(bytes)) + end + + nil + end + + def self.puts(data, io: $stdout) + format_hex(data) { |line| io.puts(line) } + end +end diff --git a/lib/innodb/util/read_bits_at_offset.rb b/lib/innodb/util/read_bits_at_offset.rb index 76f83f4d..3e6f7eef 100644 --- a/lib/innodb/util/read_bits_at_offset.rb +++ b/lib/innodb/util/read_bits_at_offset.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ReadBitsAtOffset # Read a given number of bits from an integer at a specific bit offset. The # value returned is 0-based so does not need further shifting or adjustment. @@ -5,4 +7,3 @@ def read_bits_at_offset(data, bits, offset) ((data & (((1 << bits) - 1) << offset)) >> offset) end end - diff --git a/lib/innodb/version.rb b/lib/innodb/version.rb index ae156eb0..492ee8f6 100644 --- a/lib/innodb/version.rb +++ b/lib/innodb/version.rb @@ -1,5 +1,5 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true module Innodb - VERSION = "0.9.16" + VERSION = "0.14.0" end diff --git a/lib/innodb/xdes.rb b/lib/innodb/xdes.rb index e55c2159..07896131 100644 --- a/lib/innodb/xdes.rb +++ b/lib/innodb/xdes.rb @@ -1,166 +1,168 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true + +require "forwardable" # An InnoDB "extent descriptor entry" or "+XDES+". These structures are used # in the +XDES+ entry array contained in +FSP_HDR+ and +XDES+ pages. # # Note the distinction between +XDES+ _entries_ and +XDES+ _pages_. -class Innodb::Xdes - # Number of bits per page in the +XDES+ entry bitmap field. Currently - # +XDES+ entries store two bits per page, with the following meanings: - # - # * 1 = free (the page is free, or not in use) - # * 2 = clean (currently unused, always 1 when initialized) - BITS_PER_PAGE = 2 - - # The bit value for a free page. - BITMAP_BV_FREE = 1 - - # The bit value for a clean page (currently unused in InnoDB). - BITMAP_BV_CLEAN = 2 - - # The bitwise-OR of all bitmap bit values. - BITMAP_BV_ALL = (BITMAP_BV_FREE | BITMAP_BV_CLEAN) - - # The values used in the +:state+ field indicating what the extent is - # used for (or what list it is on). - STATES = { - 1 => :free, # The extent is completely empty and unused, and should - # be present on the filespace's FREE list. - - 2 => :free_frag, # Some pages of the extent are used individually, and - # the extent should be present on the filespace's - # FREE_FRAG list. - - 3 => :full_frag, # All pages of the extent are used individually, and - # the extent should be present on the filespace's - # FULL_FRAG list. - - 4 => :fseg, # The extent is wholly allocated to a file segment. - # Additional information about the state of this extent - # can be derived from the its presence on particular - # file segment lists (FULL, NOT_FULL, or FREE). - } - - def initialize(page, cursor) - @page = page - @xdes = read_xdes_entry(page, cursor) - end +module Innodb + class Xdes + # An XDES entry structure. + Entry = Struct.new(:offset, :start_page, :end_page, :fseg_id, :this, :list, :state, :bitmap, keyword_init: true) + + PageStatus = Struct.new(:free, :clean, keyword_init: true) + + # Number of bits per page in the +XDES+ entry bitmap field. Currently + # +XDES+ entries store two bits per page, with the following meanings: + # + # * 1 = free (the page is free, or not in use) + # * 2 = clean (currently unused, always 1 when initialized) + BITS_PER_PAGE = 2 + + # The bit value for a free page. + BITMAP_BV_FREE = 1 + + # The bit value for a clean page (currently unused in InnoDB). + BITMAP_BV_CLEAN = 2 + + # The bitwise-OR of all bitmap bit values. + BITMAP_BV_ALL = (BITMAP_BV_FREE | BITMAP_BV_CLEAN) + + # The values used in the +:state+ field indicating what the extent is + # used for (or what list it is on). + STATES = { + # The extent is completely empty and unused, and should be present on the filespace's FREE list. + 1 => :free, + + # Some pages of the extent are used individually, and the extent should be present on the filespace's + # FREE_FRAG list. + 2 => :free_frag, + + # All pages of the extent are used individually, and the extent should be present on the filespace's + # FULL_FRAG list. + 3 => :full_frag, + + # The extent is wholly allocated to a file segment. Additional information about the state of this extent can + # be derived from the its presence on particular file segment lists (FULL, NOT_FULL, or FREE). + 4 => :fseg, + }.freeze + + attr_reader :xdes + + extend Forwardable + + def_delegator :xdes, :offset + def_delegator :xdes, :start_page + def_delegator :xdes, :end_page + def_delegator :xdes, :fseg_id + def_delegator :xdes, :this + def_delegator :xdes, :list + def_delegator :xdes, :state + def_delegator :xdes, :bitmap + + def initialize(page, cursor) + @page = page + @xdes = read_xdes_entry(page, cursor) + end - # Size (in bytes) of the bitmap field in the +XDES+ entry. - def size_bitmap - (@page.space.pages_per_extent * BITS_PER_PAGE) / 8 - end + # Size (in bytes) of the bitmap field in the +XDES+ entry. + def size_bitmap + (@page.space.pages_per_extent * BITS_PER_PAGE) / 8 + end - # Size (in bytes) of the an +XDES+ entry. - def size_entry - 8 + Innodb::List::NODE_SIZE + 4 + size_bitmap - end + # Size (in bytes) of the an +XDES+ entry. + def size_entry + 8 + Innodb::List::NODE_SIZE + 4 + size_bitmap + end - # Read an XDES entry from a cursor. - def read_xdes_entry(page, cursor) - extent_number = (cursor.position - page.pos_xdes_array) / size_entry - start_page = page.offset + (extent_number * page.space.pages_per_extent) - cursor.name("xdes[#{extent_number}]") do |c| - { - :offset => c.position, - :start_page => start_page, - :end_page => start_page + page.space.pages_per_extent - 1, - :fseg_id => c.name("fseg_id") { c.get_uint64 }, - :this => {:page => page.offset, :offset => c.position}, - :list => c.name("list") { Innodb::List.get_node(c) }, - :state => c.name("state") { STATES[c.get_uint32] }, - :bitmap => c.name("bitmap") { c.get_bytes(size_bitmap) }, - } + # Read an XDES entry from a cursor. + def read_xdes_entry(page, cursor) + extent_number = (cursor.position - page.pos_xdes_array) / size_entry + start_page = page.offset + (extent_number * page.space.pages_per_extent) + cursor.name("xdes[#{extent_number}]") do |c| + Entry.new( + offset: c.position, + start_page: start_page, + end_page: start_page + page.space.pages_per_extent - 1, + fseg_id: c.name("fseg_id") { c.read_uint64 }, + this: Innodb::Page::Address.new(page: page.offset, offset: c.position), + list: c.name("list") { Innodb::List.get_node(c) }, + state: c.name("state") { STATES[c.read_uint32] }, + bitmap: c.name("bitmap") { c.read_bytes(size_bitmap) } + ) + end end - end - # Return the stored extent descriptor entry. - def xdes - @xdes - end + # Return whether this XDES entry is allocated to an fseg (the whole extent + # then belongs to the fseg). + def allocated_to_fseg? + fseg_id != 0 + end - def offset; @xdes[:offset]; end - def start_page; @xdes[:start_page]; end - def end_page; @xdes[:end_page]; end - def fseg_id; @xdes[:fseg_id]; end - def this; @xdes[:this]; end - def list; @xdes[:list]; end - def state; @xdes[:state]; end - def bitmap; @xdes[:bitmap]; end - - # Return whether this XDES entry is allocated to an fseg (the whole extent - # then belongs to the fseg). - def allocated_to_fseg? - fseg_id != 0 - end + # Return the status for a given page. This is relatively inefficient as + # implemented and could be done better. + def page_status(page_number) + page_status_array = each_page_status.to_a + page_status_array[page_number - start_page][1] + end - # Return the status for a given page. This is relatively inefficient as - # implemented and could be done better. - def page_status(page_number) - page_status_array = each_page_status.to_a - page_status_array[page_number - xdes[:start_page]][1] - end + # Iterate through all pages represented by this extent descriptor, + # yielding a page status hash for each page, containing the following + # fields: + # + # :page The page number. + # :free Boolean indicating whether the page is free. + # :clean Boolean indicating whether the page is clean (currently + # this bit is unused by InnoDB, and always set true). + def each_page_status + return enum_for(:each_page_status) unless block_given? + + bitmap.each_byte.with_index do |byte, byte_index| + (0..3).each do |page_offset| + page_number = start_page + (byte_index * 4) + page_offset + page_bits = ((byte >> (page_offset * BITS_PER_PAGE)) & BITMAP_BV_ALL) + page_status = PageStatus.new( + free: (page_bits & BITMAP_BV_FREE != 0), + clean: (page_bits & BITMAP_BV_CLEAN != 0) + ) + yield page_number, page_status + end + end - # Iterate through all pages represented by this extent descriptor, - # yielding a page status hash for each page, containing the following - # fields: - # - # :page The page number. - # :free Boolean indicating whether the page is free. - # :clean Boolean indicating whether the page is clean (currently - # this bit is unused by InnoDB, and always set true). - def each_page_status - unless block_given? - return enum_for(:each_page_status) + nil end - bitmap = xdes[:bitmap].enum_for(:each_byte) - - bitmap.each_with_index do |byte, byte_index| - (0..3).each do |page_offset| - page_number = xdes[:start_page] + (byte_index * 4) + page_offset - page_bits = ((byte >> (page_offset * BITS_PER_PAGE)) & BITMAP_BV_ALL) - page_status = { - :free => (page_bits & BITMAP_BV_FREE != 0), - :clean => (page_bits & BITMAP_BV_CLEAN != 0), - } - yield page_number, page_status + # Return the count of free pages (free bit is true) on this extent. + def free_pages + each_page_status.inject(0) do |sum, (_page_number, page_status)| + sum += 1 if page_status.free + sum end end - nil - end - - # Return the count of free pages (free bit is true) on this extent. - def free_pages - each_page_status.inject(0) do |sum, (page_number, page_status)| - sum += 1 if page_status[:free] - sum + # Return the count of used pages (free bit is false) on this extent. + def used_pages + @page.space.pages_per_extent - free_pages end - end - - # Return the count of used pages (free bit is false) on this extent. - def used_pages - @page.space.pages_per_extent - free_pages - end - # Return the address of the previous list pointer from the list node - # contained within the XDES entry. This is used by +Innodb::List::Xdes+ - # to iterate through XDES entries in a list. - def prev_address - xdes[:list][:prev] - end + # Return the address of the previous list pointer from the list node + # contained within the XDES entry. This is used by +Innodb::List::Xdes+ + # to iterate through XDES entries in a list. + def prev_address + list.prev + end - # Return the address of the next list pointer from the list node - # contained within the XDES entry. This is used by +Innodb::List::Xdes+ - # to iterate through XDES entries in a list. - def next_address - xdes[:list][:next] - end + # Return the address of the next list pointer from the list node + # contained within the XDES entry. This is used by +Innodb::List::Xdes+ + # to iterate through XDES entries in a list. + def next_address + list.next + end - # Compare one Innodb::Xdes to another. - def ==(other) - xdes[:this][:page] == other.xdes[:this][:page] && - xdes[:this][:offset] == other.xdes[:this][:offset] + # Compare one Innodb::Xdes to another. + def ==(other) + this.page == other.this.page && this.offset == other.this.offset + end end end diff --git a/spec/data/ib_logfile0 b/spec/data/orphan/ib_logfile0 similarity index 100% rename from spec/data/ib_logfile0 rename to spec/data/orphan/ib_logfile0 diff --git a/spec/data/ib_logfile1 b/spec/data/orphan/ib_logfile1 similarity index 100% rename from spec/data/ib_logfile1 rename to spec/data/orphan/ib_logfile1 diff --git a/spec/data/ibdata1 b/spec/data/orphan/ibdata1 similarity index 100% rename from spec/data/ibdata1 rename to spec/data/orphan/ibdata1 diff --git a/spec/data/sakila/5.7/ib_logfile0 b/spec/data/sakila/5.7/ib_logfile0 new file mode 100644 index 00000000..248b2f64 Binary files /dev/null and b/spec/data/sakila/5.7/ib_logfile0 differ diff --git a/spec/data/sakila/5.7/ib_logfile1 b/spec/data/sakila/5.7/ib_logfile1 new file mode 100644 index 00000000..274bba07 Binary files /dev/null and b/spec/data/sakila/5.7/ib_logfile1 differ diff --git a/spec/data/sakila/5.7/ibdata1 b/spec/data/sakila/5.7/ibdata1 new file mode 100644 index 00000000..de1c3a88 Binary files /dev/null and b/spec/data/sakila/5.7/ibdata1 differ diff --git a/spec/data/sakila/5.7/sakila/FTS_000000000000002d_0000000000000045_INDEX_1.ibd b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_0000000000000045_INDEX_1.ibd new file mode 100644 index 00000000..ba78956f Binary files /dev/null and b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_0000000000000045_INDEX_1.ibd differ diff --git a/spec/data/sakila/5.7/sakila/FTS_000000000000002d_0000000000000045_INDEX_2.ibd b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_0000000000000045_INDEX_2.ibd new file mode 100644 index 00000000..f90ca050 Binary files /dev/null and b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_0000000000000045_INDEX_2.ibd differ diff --git a/spec/data/sakila/5.7/sakila/FTS_000000000000002d_0000000000000045_INDEX_3.ibd b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_0000000000000045_INDEX_3.ibd new file mode 100644 index 00000000..a9cedc2c Binary files /dev/null and b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_0000000000000045_INDEX_3.ibd differ diff --git a/spec/data/sakila/5.7/sakila/FTS_000000000000002d_0000000000000045_INDEX_4.ibd b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_0000000000000045_INDEX_4.ibd new file mode 100644 index 00000000..b73a8a1b Binary files /dev/null and b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_0000000000000045_INDEX_4.ibd differ diff --git a/spec/data/sakila/5.7/sakila/FTS_000000000000002d_0000000000000045_INDEX_5.ibd b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_0000000000000045_INDEX_5.ibd new file mode 100644 index 00000000..a5903bbe Binary files /dev/null and b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_0000000000000045_INDEX_5.ibd differ diff --git a/spec/data/sakila/5.7/sakila/FTS_000000000000002d_0000000000000045_INDEX_6.ibd b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_0000000000000045_INDEX_6.ibd new file mode 100644 index 00000000..87586f40 Binary files /dev/null and b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_0000000000000045_INDEX_6.ibd differ diff --git a/spec/data/sakila/5.7/sakila/FTS_000000000000002d_BEING_DELETED.ibd b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_BEING_DELETED.ibd new file mode 100644 index 00000000..dda81f94 Binary files /dev/null and b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_BEING_DELETED.ibd differ diff --git a/spec/data/sakila/5.7/sakila/FTS_000000000000002d_BEING_DELETED_CACHE.ibd b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_BEING_DELETED_CACHE.ibd new file mode 100644 index 00000000..de6aed81 Binary files /dev/null and b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_BEING_DELETED_CACHE.ibd differ diff --git a/spec/data/sakila/5.7/sakila/FTS_000000000000002d_CONFIG.ibd b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_CONFIG.ibd new file mode 100644 index 00000000..582c794f Binary files /dev/null and b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_CONFIG.ibd differ diff --git a/spec/data/sakila/5.7/sakila/FTS_000000000000002d_DELETED.ibd b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_DELETED.ibd new file mode 100644 index 00000000..f1cd7994 Binary files /dev/null and b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_DELETED.ibd differ diff --git a/spec/data/sakila/5.7/sakila/FTS_000000000000002d_DELETED_CACHE.ibd b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_DELETED_CACHE.ibd new file mode 100644 index 00000000..e1ffc2a9 Binary files /dev/null and b/spec/data/sakila/5.7/sakila/FTS_000000000000002d_DELETED_CACHE.ibd differ diff --git a/spec/data/sakila/5.7/sakila/actor.ibd b/spec/data/sakila/5.7/sakila/actor.ibd new file mode 100644 index 00000000..408ddc7e Binary files /dev/null and b/spec/data/sakila/5.7/sakila/actor.ibd differ diff --git a/spec/data/sakila/5.7/sakila/address.ibd b/spec/data/sakila/5.7/sakila/address.ibd new file mode 100644 index 00000000..1555789e Binary files /dev/null and b/spec/data/sakila/5.7/sakila/address.ibd differ diff --git a/spec/data/sakila/5.7/sakila/category.ibd b/spec/data/sakila/5.7/sakila/category.ibd new file mode 100644 index 00000000..f3442386 Binary files /dev/null and b/spec/data/sakila/5.7/sakila/category.ibd differ diff --git a/spec/data/sakila/5.7/sakila/city.ibd b/spec/data/sakila/5.7/sakila/city.ibd new file mode 100644 index 00000000..ba098fa8 Binary files /dev/null and b/spec/data/sakila/5.7/sakila/city.ibd differ diff --git a/spec/data/sakila/5.7/sakila/country.ibd b/spec/data/sakila/5.7/sakila/country.ibd new file mode 100644 index 00000000..523bad1f Binary files /dev/null and b/spec/data/sakila/5.7/sakila/country.ibd differ diff --git a/spec/data/sakila/5.7/sakila/customer.ibd b/spec/data/sakila/5.7/sakila/customer.ibd new file mode 100644 index 00000000..453b93b8 Binary files /dev/null and b/spec/data/sakila/5.7/sakila/customer.ibd differ diff --git a/spec/data/sakila/5.7/sakila/film.ibd b/spec/data/sakila/5.7/sakila/film.ibd new file mode 100644 index 00000000..981bc399 Binary files /dev/null and b/spec/data/sakila/5.7/sakila/film.ibd differ diff --git a/spec/data/sakila/5.7/sakila/film_actor.ibd b/spec/data/sakila/5.7/sakila/film_actor.ibd new file mode 100644 index 00000000..865e0b9b Binary files /dev/null and b/spec/data/sakila/5.7/sakila/film_actor.ibd differ diff --git a/spec/data/sakila/5.7/sakila/film_category.ibd b/spec/data/sakila/5.7/sakila/film_category.ibd new file mode 100644 index 00000000..b62ad384 Binary files /dev/null and b/spec/data/sakila/5.7/sakila/film_category.ibd differ diff --git a/spec/data/sakila/5.7/sakila/film_text.ibd b/spec/data/sakila/5.7/sakila/film_text.ibd new file mode 100644 index 00000000..628eea03 Binary files /dev/null and b/spec/data/sakila/5.7/sakila/film_text.ibd differ diff --git a/spec/data/sakila/5.7/sakila/inventory.ibd b/spec/data/sakila/5.7/sakila/inventory.ibd new file mode 100644 index 00000000..6ae30713 Binary files /dev/null and b/spec/data/sakila/5.7/sakila/inventory.ibd differ diff --git a/spec/data/sakila/5.7/sakila/language.ibd b/spec/data/sakila/5.7/sakila/language.ibd new file mode 100644 index 00000000..bd6b51c8 Binary files /dev/null and b/spec/data/sakila/5.7/sakila/language.ibd differ diff --git a/spec/data/sakila/5.7/sakila/payment.ibd b/spec/data/sakila/5.7/sakila/payment.ibd new file mode 100644 index 00000000..7f23ad49 Binary files /dev/null and b/spec/data/sakila/5.7/sakila/payment.ibd differ diff --git a/spec/data/sakila/5.7/sakila/rental.ibd b/spec/data/sakila/5.7/sakila/rental.ibd new file mode 100644 index 00000000..34dbb0c2 Binary files /dev/null and b/spec/data/sakila/5.7/sakila/rental.ibd differ diff --git a/spec/data/sakila/5.7/sakila/staff.ibd b/spec/data/sakila/5.7/sakila/staff.ibd new file mode 100644 index 00000000..3531c57a Binary files /dev/null and b/spec/data/sakila/5.7/sakila/staff.ibd differ diff --git a/spec/data/sakila/5.7/sakila/store.ibd b/spec/data/sakila/5.7/sakila/store.ibd new file mode 100644 index 00000000..ed9d9f6e Binary files /dev/null and b/spec/data/sakila/5.7/sakila/store.ibd differ diff --git a/spec/data/sakila/8.0/ibdata1 b/spec/data/sakila/8.0/ibdata1 new file mode 100644 index 00000000..de0ac7bf Binary files /dev/null and b/spec/data/sakila/8.0/ibdata1 differ diff --git a/spec/data/sakila/8.0/mysql.ibd b/spec/data/sakila/8.0/mysql.ibd new file mode 100644 index 00000000..c8fffdbd Binary files /dev/null and b/spec/data/sakila/8.0/mysql.ibd differ diff --git a/spec/data/sakila/8.0/sakila/actor.ibd b/spec/data/sakila/8.0/sakila/actor.ibd new file mode 100644 index 00000000..c327fca5 Binary files /dev/null and b/spec/data/sakila/8.0/sakila/actor.ibd differ diff --git a/spec/data/sakila/8.0/sakila/address.ibd b/spec/data/sakila/8.0/sakila/address.ibd new file mode 100644 index 00000000..d0d77417 Binary files /dev/null and b/spec/data/sakila/8.0/sakila/address.ibd differ diff --git a/spec/data/sakila/8.0/sakila/category.ibd b/spec/data/sakila/8.0/sakila/category.ibd new file mode 100644 index 00000000..9f25f016 Binary files /dev/null and b/spec/data/sakila/8.0/sakila/category.ibd differ diff --git a/spec/data/sakila/8.0/sakila/city.ibd b/spec/data/sakila/8.0/sakila/city.ibd new file mode 100644 index 00000000..ac850651 Binary files /dev/null and b/spec/data/sakila/8.0/sakila/city.ibd differ diff --git a/spec/data/sakila/8.0/sakila/country.ibd b/spec/data/sakila/8.0/sakila/country.ibd new file mode 100644 index 00000000..a0872366 Binary files /dev/null and b/spec/data/sakila/8.0/sakila/country.ibd differ diff --git a/spec/data/sakila/8.0/sakila/customer.ibd b/spec/data/sakila/8.0/sakila/customer.ibd new file mode 100644 index 00000000..31c35fb0 Binary files /dev/null and b/spec/data/sakila/8.0/sakila/customer.ibd differ diff --git a/spec/data/sakila/8.0/sakila/film.ibd b/spec/data/sakila/8.0/sakila/film.ibd new file mode 100644 index 00000000..dff40599 Binary files /dev/null and b/spec/data/sakila/8.0/sakila/film.ibd differ diff --git a/spec/data/sakila/8.0/sakila/film_actor.ibd b/spec/data/sakila/8.0/sakila/film_actor.ibd new file mode 100644 index 00000000..32ec10ac Binary files /dev/null and b/spec/data/sakila/8.0/sakila/film_actor.ibd differ diff --git a/spec/data/sakila/8.0/sakila/film_category.ibd b/spec/data/sakila/8.0/sakila/film_category.ibd new file mode 100644 index 00000000..88b5f746 Binary files /dev/null and b/spec/data/sakila/8.0/sakila/film_category.ibd differ diff --git a/spec/data/sakila/8.0/sakila/film_text.ibd b/spec/data/sakila/8.0/sakila/film_text.ibd new file mode 100644 index 00000000..37f632a7 Binary files /dev/null and b/spec/data/sakila/8.0/sakila/film_text.ibd differ diff --git a/spec/data/sakila/8.0/sakila/fts_0000000000000451_00000000000000f1_index_1.ibd b/spec/data/sakila/8.0/sakila/fts_0000000000000451_00000000000000f1_index_1.ibd new file mode 100644 index 00000000..11333d38 Binary files /dev/null and b/spec/data/sakila/8.0/sakila/fts_0000000000000451_00000000000000f1_index_1.ibd differ diff --git a/spec/data/sakila/8.0/sakila/fts_0000000000000451_00000000000000f1_index_2.ibd b/spec/data/sakila/8.0/sakila/fts_0000000000000451_00000000000000f1_index_2.ibd new file mode 100644 index 00000000..9e9223f6 Binary files /dev/null and b/spec/data/sakila/8.0/sakila/fts_0000000000000451_00000000000000f1_index_2.ibd differ diff --git a/spec/data/sakila/8.0/sakila/fts_0000000000000451_00000000000000f1_index_3.ibd b/spec/data/sakila/8.0/sakila/fts_0000000000000451_00000000000000f1_index_3.ibd new file mode 100644 index 00000000..6d2c6adb Binary files /dev/null and b/spec/data/sakila/8.0/sakila/fts_0000000000000451_00000000000000f1_index_3.ibd differ diff --git a/spec/data/sakila/8.0/sakila/fts_0000000000000451_00000000000000f1_index_4.ibd b/spec/data/sakila/8.0/sakila/fts_0000000000000451_00000000000000f1_index_4.ibd new file mode 100644 index 00000000..b0cd94e0 Binary files /dev/null and b/spec/data/sakila/8.0/sakila/fts_0000000000000451_00000000000000f1_index_4.ibd differ diff --git a/spec/data/sakila/8.0/sakila/fts_0000000000000451_00000000000000f1_index_5.ibd b/spec/data/sakila/8.0/sakila/fts_0000000000000451_00000000000000f1_index_5.ibd new file mode 100644 index 00000000..a7b6a682 Binary files /dev/null and b/spec/data/sakila/8.0/sakila/fts_0000000000000451_00000000000000f1_index_5.ibd differ diff --git a/spec/data/sakila/8.0/sakila/fts_0000000000000451_00000000000000f1_index_6.ibd b/spec/data/sakila/8.0/sakila/fts_0000000000000451_00000000000000f1_index_6.ibd new file mode 100644 index 00000000..29157a0e Binary files /dev/null and b/spec/data/sakila/8.0/sakila/fts_0000000000000451_00000000000000f1_index_6.ibd differ diff --git a/spec/data/sakila/8.0/sakila/fts_0000000000000451_being_deleted.ibd b/spec/data/sakila/8.0/sakila/fts_0000000000000451_being_deleted.ibd new file mode 100644 index 00000000..7164f685 Binary files /dev/null and b/spec/data/sakila/8.0/sakila/fts_0000000000000451_being_deleted.ibd differ diff --git a/spec/data/sakila/8.0/sakila/fts_0000000000000451_being_deleted_cache.ibd b/spec/data/sakila/8.0/sakila/fts_0000000000000451_being_deleted_cache.ibd new file mode 100644 index 00000000..19caf294 Binary files /dev/null and b/spec/data/sakila/8.0/sakila/fts_0000000000000451_being_deleted_cache.ibd differ diff --git a/spec/data/sakila/8.0/sakila/fts_0000000000000451_config.ibd b/spec/data/sakila/8.0/sakila/fts_0000000000000451_config.ibd new file mode 100644 index 00000000..fc4f2b41 Binary files /dev/null and b/spec/data/sakila/8.0/sakila/fts_0000000000000451_config.ibd differ diff --git a/spec/data/sakila/8.0/sakila/fts_0000000000000451_deleted.ibd b/spec/data/sakila/8.0/sakila/fts_0000000000000451_deleted.ibd new file mode 100644 index 00000000..67315cc0 Binary files /dev/null and b/spec/data/sakila/8.0/sakila/fts_0000000000000451_deleted.ibd differ diff --git a/spec/data/sakila/8.0/sakila/fts_0000000000000451_deleted_cache.ibd b/spec/data/sakila/8.0/sakila/fts_0000000000000451_deleted_cache.ibd new file mode 100644 index 00000000..94f8374a Binary files /dev/null and b/spec/data/sakila/8.0/sakila/fts_0000000000000451_deleted_cache.ibd differ diff --git a/spec/data/sakila/8.0/sakila/inventory.ibd b/spec/data/sakila/8.0/sakila/inventory.ibd new file mode 100644 index 00000000..003c2f0e Binary files /dev/null and b/spec/data/sakila/8.0/sakila/inventory.ibd differ diff --git a/spec/data/sakila/8.0/sakila/language.ibd b/spec/data/sakila/8.0/sakila/language.ibd new file mode 100644 index 00000000..0a83cb48 Binary files /dev/null and b/spec/data/sakila/8.0/sakila/language.ibd differ diff --git a/spec/data/sakila/8.0/sakila/payment.ibd b/spec/data/sakila/8.0/sakila/payment.ibd new file mode 100644 index 00000000..0da2714e Binary files /dev/null and b/spec/data/sakila/8.0/sakila/payment.ibd differ diff --git a/spec/data/sakila/8.0/sakila/rental.ibd b/spec/data/sakila/8.0/sakila/rental.ibd new file mode 100644 index 00000000..e930f2d3 Binary files /dev/null and b/spec/data/sakila/8.0/sakila/rental.ibd differ diff --git a/spec/data/sakila/8.0/sakila/staff.ibd b/spec/data/sakila/8.0/sakila/staff.ibd new file mode 100644 index 00000000..08e2c953 Binary files /dev/null and b/spec/data/sakila/8.0/sakila/staff.ibd differ diff --git a/spec/data/sakila/8.0/sakila/store.ibd b/spec/data/sakila/8.0/sakila/store.ibd new file mode 100644 index 00000000..5bfe38ae Binary files /dev/null and b/spec/data/sakila/8.0/sakila/store.ibd differ diff --git a/spec/data/sakila/8.0/sys/sys_config.ibd b/spec/data/sakila/8.0/sys/sys_config.ibd new file mode 100644 index 00000000..8aad6b39 Binary files /dev/null and b/spec/data/sakila/8.0/sys/sys_config.ibd differ diff --git a/spec/data/sakila/8.0/undo_001 b/spec/data/sakila/8.0/undo_001 new file mode 100644 index 00000000..0dfcbc88 Binary files /dev/null and b/spec/data/sakila/8.0/undo_001 differ diff --git a/spec/data/sakila/8.0/undo_002 b/spec/data/sakila/8.0/undo_002 new file mode 100644 index 00000000..d16e7ded Binary files /dev/null and b/spec/data/sakila/8.0/undo_002 differ diff --git a/spec/data/sakila/8.4/ibdata1 b/spec/data/sakila/8.4/ibdata1 new file mode 100644 index 00000000..97e05284 Binary files /dev/null and b/spec/data/sakila/8.4/ibdata1 differ diff --git a/spec/data/sakila/8.4/mysql.ibd b/spec/data/sakila/8.4/mysql.ibd new file mode 100644 index 00000000..6c759f37 Binary files /dev/null and b/spec/data/sakila/8.4/mysql.ibd differ diff --git a/spec/data/sakila/8.4/sakila/actor.ibd b/spec/data/sakila/8.4/sakila/actor.ibd new file mode 100644 index 00000000..76a6f76a Binary files /dev/null and b/spec/data/sakila/8.4/sakila/actor.ibd differ diff --git a/spec/data/sakila/8.4/sakila/address.ibd b/spec/data/sakila/8.4/sakila/address.ibd new file mode 100644 index 00000000..521b68d3 Binary files /dev/null and b/spec/data/sakila/8.4/sakila/address.ibd differ diff --git a/spec/data/sakila/8.4/sakila/category.ibd b/spec/data/sakila/8.4/sakila/category.ibd new file mode 100644 index 00000000..2e5f492f Binary files /dev/null and b/spec/data/sakila/8.4/sakila/category.ibd differ diff --git a/spec/data/sakila/8.4/sakila/city.ibd b/spec/data/sakila/8.4/sakila/city.ibd new file mode 100644 index 00000000..4f84a723 Binary files /dev/null and b/spec/data/sakila/8.4/sakila/city.ibd differ diff --git a/spec/data/sakila/8.4/sakila/country.ibd b/spec/data/sakila/8.4/sakila/country.ibd new file mode 100644 index 00000000..547b1244 Binary files /dev/null and b/spec/data/sakila/8.4/sakila/country.ibd differ diff --git a/spec/data/sakila/8.4/sakila/customer.ibd b/spec/data/sakila/8.4/sakila/customer.ibd new file mode 100644 index 00000000..7f53e058 Binary files /dev/null and b/spec/data/sakila/8.4/sakila/customer.ibd differ diff --git a/spec/data/sakila/8.4/sakila/film.ibd b/spec/data/sakila/8.4/sakila/film.ibd new file mode 100644 index 00000000..00818a46 Binary files /dev/null and b/spec/data/sakila/8.4/sakila/film.ibd differ diff --git a/spec/data/sakila/8.4/sakila/film_actor.ibd b/spec/data/sakila/8.4/sakila/film_actor.ibd new file mode 100644 index 00000000..0aa3340d Binary files /dev/null and b/spec/data/sakila/8.4/sakila/film_actor.ibd differ diff --git a/spec/data/sakila/8.4/sakila/film_category.ibd b/spec/data/sakila/8.4/sakila/film_category.ibd new file mode 100644 index 00000000..b1018a29 Binary files /dev/null and b/spec/data/sakila/8.4/sakila/film_category.ibd differ diff --git a/spec/data/sakila/8.4/sakila/film_text.ibd b/spec/data/sakila/8.4/sakila/film_text.ibd new file mode 100644 index 00000000..5f1f75c8 Binary files /dev/null and b/spec/data/sakila/8.4/sakila/film_text.ibd differ diff --git a/spec/data/sakila/8.4/sakila/fts_0000000000000431_00000000000000b6_index_1.ibd b/spec/data/sakila/8.4/sakila/fts_0000000000000431_00000000000000b6_index_1.ibd new file mode 100644 index 00000000..9c0c314c Binary files /dev/null and b/spec/data/sakila/8.4/sakila/fts_0000000000000431_00000000000000b6_index_1.ibd differ diff --git a/spec/data/sakila/8.4/sakila/fts_0000000000000431_00000000000000b6_index_2.ibd b/spec/data/sakila/8.4/sakila/fts_0000000000000431_00000000000000b6_index_2.ibd new file mode 100644 index 00000000..e234b45c Binary files /dev/null and b/spec/data/sakila/8.4/sakila/fts_0000000000000431_00000000000000b6_index_2.ibd differ diff --git a/spec/data/sakila/8.4/sakila/fts_0000000000000431_00000000000000b6_index_3.ibd b/spec/data/sakila/8.4/sakila/fts_0000000000000431_00000000000000b6_index_3.ibd new file mode 100644 index 00000000..4a2caab8 Binary files /dev/null and b/spec/data/sakila/8.4/sakila/fts_0000000000000431_00000000000000b6_index_3.ibd differ diff --git a/spec/data/sakila/8.4/sakila/fts_0000000000000431_00000000000000b6_index_4.ibd b/spec/data/sakila/8.4/sakila/fts_0000000000000431_00000000000000b6_index_4.ibd new file mode 100644 index 00000000..b15db73f Binary files /dev/null and b/spec/data/sakila/8.4/sakila/fts_0000000000000431_00000000000000b6_index_4.ibd differ diff --git a/spec/data/sakila/8.4/sakila/fts_0000000000000431_00000000000000b6_index_5.ibd b/spec/data/sakila/8.4/sakila/fts_0000000000000431_00000000000000b6_index_5.ibd new file mode 100644 index 00000000..113e4c7c Binary files /dev/null and b/spec/data/sakila/8.4/sakila/fts_0000000000000431_00000000000000b6_index_5.ibd differ diff --git a/spec/data/sakila/8.4/sakila/fts_0000000000000431_00000000000000b6_index_6.ibd b/spec/data/sakila/8.4/sakila/fts_0000000000000431_00000000000000b6_index_6.ibd new file mode 100644 index 00000000..daae505a Binary files /dev/null and b/spec/data/sakila/8.4/sakila/fts_0000000000000431_00000000000000b6_index_6.ibd differ diff --git a/spec/data/sakila/8.4/sakila/fts_0000000000000431_being_deleted.ibd b/spec/data/sakila/8.4/sakila/fts_0000000000000431_being_deleted.ibd new file mode 100644 index 00000000..1dc37ca2 Binary files /dev/null and b/spec/data/sakila/8.4/sakila/fts_0000000000000431_being_deleted.ibd differ diff --git a/spec/data/sakila/8.4/sakila/fts_0000000000000431_being_deleted_cache.ibd b/spec/data/sakila/8.4/sakila/fts_0000000000000431_being_deleted_cache.ibd new file mode 100644 index 00000000..8a159fda Binary files /dev/null and b/spec/data/sakila/8.4/sakila/fts_0000000000000431_being_deleted_cache.ibd differ diff --git a/spec/data/sakila/8.4/sakila/fts_0000000000000431_config.ibd b/spec/data/sakila/8.4/sakila/fts_0000000000000431_config.ibd new file mode 100644 index 00000000..2bf2c002 Binary files /dev/null and b/spec/data/sakila/8.4/sakila/fts_0000000000000431_config.ibd differ diff --git a/spec/data/sakila/8.4/sakila/fts_0000000000000431_deleted.ibd b/spec/data/sakila/8.4/sakila/fts_0000000000000431_deleted.ibd new file mode 100644 index 00000000..d0debe6e Binary files /dev/null and b/spec/data/sakila/8.4/sakila/fts_0000000000000431_deleted.ibd differ diff --git a/spec/data/sakila/8.4/sakila/fts_0000000000000431_deleted_cache.ibd b/spec/data/sakila/8.4/sakila/fts_0000000000000431_deleted_cache.ibd new file mode 100644 index 00000000..5b595a30 Binary files /dev/null and b/spec/data/sakila/8.4/sakila/fts_0000000000000431_deleted_cache.ibd differ diff --git a/spec/data/sakila/8.4/sakila/inventory.ibd b/spec/data/sakila/8.4/sakila/inventory.ibd new file mode 100644 index 00000000..b3f51fc7 Binary files /dev/null and b/spec/data/sakila/8.4/sakila/inventory.ibd differ diff --git a/spec/data/sakila/8.4/sakila/language.ibd b/spec/data/sakila/8.4/sakila/language.ibd new file mode 100644 index 00000000..2f8ad499 Binary files /dev/null and b/spec/data/sakila/8.4/sakila/language.ibd differ diff --git a/spec/data/sakila/8.4/sakila/payment.ibd b/spec/data/sakila/8.4/sakila/payment.ibd new file mode 100644 index 00000000..cba253df Binary files /dev/null and b/spec/data/sakila/8.4/sakila/payment.ibd differ diff --git a/spec/data/sakila/8.4/sakila/rental.ibd b/spec/data/sakila/8.4/sakila/rental.ibd new file mode 100644 index 00000000..2e66b1f5 Binary files /dev/null and b/spec/data/sakila/8.4/sakila/rental.ibd differ diff --git a/spec/data/sakila/8.4/sakila/staff.ibd b/spec/data/sakila/8.4/sakila/staff.ibd new file mode 100644 index 00000000..5df89bdf Binary files /dev/null and b/spec/data/sakila/8.4/sakila/staff.ibd differ diff --git a/spec/data/sakila/8.4/sakila/store.ibd b/spec/data/sakila/8.4/sakila/store.ibd new file mode 100644 index 00000000..06b88110 Binary files /dev/null and b/spec/data/sakila/8.4/sakila/store.ibd differ diff --git a/spec/data/sakila/8.4/undo_001 b/spec/data/sakila/8.4/undo_001 new file mode 100644 index 00000000..19f695d5 Binary files /dev/null and b/spec/data/sakila/8.4/undo_001 differ diff --git a/spec/data/sakila/8.4/undo_002 b/spec/data/sakila/8.4/undo_002 new file mode 100644 index 00000000..1e1ba26e Binary files /dev/null and b/spec/data/sakila/8.4/undo_002 differ diff --git a/spec/data/sakila/single_space/5.5/ib_logfile0 b/spec/data/sakila/single_space/5.5/ib_logfile0 new file mode 100644 index 00000000..ab87bfe0 Binary files /dev/null and b/spec/data/sakila/single_space/5.5/ib_logfile0 differ diff --git a/spec/data/sakila/single_space/5.5/ib_logfile1 b/spec/data/sakila/single_space/5.5/ib_logfile1 new file mode 100644 index 00000000..b58c5911 Binary files /dev/null and b/spec/data/sakila/single_space/5.5/ib_logfile1 differ diff --git a/spec/data/sakila/single_space/5.5/ibdata1 b/spec/data/sakila/single_space/5.5/ibdata1 new file mode 100644 index 00000000..f02ff712 Binary files /dev/null and b/spec/data/sakila/single_space/5.5/ibdata1 differ diff --git a/spec/data/sakila/single_space/5.7/ib_logfile0 b/spec/data/sakila/single_space/5.7/ib_logfile0 new file mode 100644 index 00000000..2ab5e2d9 Binary files /dev/null and b/spec/data/sakila/single_space/5.7/ib_logfile0 differ diff --git a/spec/data/sakila/single_space/5.7/ib_logfile1 b/spec/data/sakila/single_space/5.7/ib_logfile1 new file mode 100644 index 00000000..274bba07 Binary files /dev/null and b/spec/data/sakila/single_space/5.7/ib_logfile1 differ diff --git a/spec/data/sakila/single_space/5.7/ibdata1 b/spec/data/sakila/single_space/5.7/ibdata1 new file mode 100644 index 00000000..c7a55ddb Binary files /dev/null and b/spec/data/sakila/single_space/5.7/ibdata1 differ diff --git a/spec/innodb/checksum_spec.rb b/spec/innodb/checksum_spec.rb index ccf579e7..467fb6bf 100644 --- a/spec/innodb/checksum_spec.rb +++ b/spec/innodb/checksum_spec.rb @@ -1,4 +1,4 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true require "spec_helper" @@ -9,10 +9,10 @@ end it "calculates correct values" do - Innodb::Checksum.fold_pair(0x00, 0x00).should eql 3277101703 - Innodb::Checksum.fold_pair(0x00, 0xff).should eql 3277088390 - Innodb::Checksum.fold_pair(0xff, 0x00).should eql 3277088120 - Innodb::Checksum.fold_pair(0xff, 0xff).should eql 3277101943 + Innodb::Checksum.fold_pair(0x00, 0x00).should eql 3_277_101_703 + Innodb::Checksum.fold_pair(0x00, 0xff).should eql 3_277_088_390 + Innodb::Checksum.fold_pair(0xff, 0x00).should eql 3_277_088_120 + Innodb::Checksum.fold_pair(0xff, 0xff).should eql 3_277_101_943 end end @@ -22,7 +22,7 @@ end it "calculates correct values" do - Innodb::Checksum.fold_enumerator(0..255).should eql 1406444672 + Innodb::Checksum.fold_enumerator(0..255).should eql 1_406_444_672 end end @@ -32,7 +32,7 @@ end it "calculates correct values" do - Innodb::Checksum.fold_string("hello world").should eql 2249882843 + Innodb::Checksum.fold_string("hello world").should eql 2_249_882_843 end end end diff --git a/spec/innodb/data_dictionary_spec.rb b/spec/innodb/data_dictionary_spec.rb deleted file mode 100644 index d3f2fa17..00000000 --- a/spec/innodb/data_dictionary_spec.rb +++ /dev/null @@ -1,210 +0,0 @@ -# -*- encoding : utf-8 -*- - -require 'spec_helper' - -describe Innodb::DataDictionary do - before :all do - @system = Innodb::System.new("spec/data/sakila/compact/ibdata1") - @dict = @system.data_dictionary - end - - describe "#mtype_prtype_to_type_string" do - it "produces the correct type string or symbol" do - type = Innodb::DataDictionary.mtype_prtype_to_type_string(6, 1794, 2, 0) - type.should eql :SMALLINT - end - end - - describe "#mtype_prtype_to_data_type" do - it "produces the correct type array" do - type = Innodb::DataDictionary.mtype_prtype_to_data_type(6, 1794, 2, 0) - type.should be_an_instance_of Array - type.should eql [:SMALLINT, :NOT_NULL, :UNSIGNED] - end - end - - describe "#data_dictionary_indexes" do - it "is a Hash" do - @dict.data_dictionary_indexes.should be_an_instance_of Hash - end - - it "contains Hashes" do - key = @dict.data_dictionary_indexes.keys.first - @dict.data_dictionary_indexes[key].should be_an_instance_of Hash - end - end - - describe "#data_dictionary_index" do - it "returns Innodb::Index objects" do - index = @dict.data_dictionary_index(:SYS_TABLES, :PRIMARY) - index.should be_an_instance_of Innodb::Index - end - end - - describe "#each_data_dictionary_index_root_page_number" do - it "is an enumerator" do - is_enumerator?(@dict.each_data_dictionary_index_root_page_number).should be_truthy - end - end - - describe "#each_data_dictionary_index" do - it "is an enumerator" do - is_enumerator?(@dict.each_data_dictionary_index).should be_truthy - end - end - - describe "#each_record_from_data_dictionary_index" do - it "is an enumerator" do - is_enumerator?(@dict.each_record_from_data_dictionary_index(:SYS_TABLES, :PRIMARY)).should be_truthy - end - end - - describe "#each_table" do - it "is an enumerator" do - is_enumerator?(@dict.each_table).should be_truthy - end - end - - describe "#each_column" do - it "is an enumerator" do - is_enumerator?(@dict.each_column).should be_truthy - end - end - - describe "#each_index" do - it "is an enumerator" do - is_enumerator?(@dict.each_index).should be_truthy - end - end - - describe "#each_field" do - it "is an enumerator" do - is_enumerator?(@dict.each_field).should be_truthy - end - end - - describe "#table_by_id" do - it "finds the correct table" do - table = @dict.table_by_id(19) - table.should be_an_instance_of Hash - table["NAME"].should eql "sakila/film" - end - end - - describe "#table_by_name" do - it "finds the correct table" do - table = @dict.table_by_name("sakila/film") - table.should be_an_instance_of Hash - table["NAME"].should eql "sakila/film" - end - end - - describe "#table_by_space_id" do - it "finds the correct table" do - table = @dict.table_by_space_id(7) - table.should be_an_instance_of Hash - table["NAME"].should eql "sakila/film" - end - end - - describe "#column_by_name" do - it "finds the correct column" do - column = @dict.column_by_name("sakila/film", "film_id") - column.should be_an_instance_of Hash - column["NAME"].should eql "film_id" - end - end - - describe "#index_by_id" do - it "finds the correct index" do - index = @dict.index_by_id(27) - index.should be_an_instance_of Hash - index["NAME"].should eql "PRIMARY" - end - end - - describe "#index_by_name" do - it "finds the correct index" do - index = @dict.index_by_name("sakila/film", "PRIMARY") - index.should be_an_instance_of Hash - index["NAME"].should eql "PRIMARY" - end - end - - describe "#each_index_by_space_id" do - it "is an enumerator" do - is_enumerator?(@dict.each_index_by_space_id(7)).should be_truthy - end - end - - describe "#each_index_by_table_id" do - it "is an enumerator" do - is_enumerator?(@dict.each_index_by_table_id(19)).should be_truthy - end - end - - describe "#each_index_by_table_name" do - it "is an enumerator" do - is_enumerator?(@dict.each_index_by_table_name("sakila/film")).should be_truthy - end - end - - describe "#each_field_by_index_id" do - it "is an enumerator" do - is_enumerator?(@dict.each_field_by_index_id(27)).should be_truthy - end - end - - describe "#each_field_by_index_name" do - it "is an enumerator" do - is_enumerator?(@dict.each_field_by_index_name("sakila/film", "PRIMARY")).should be_truthy - end - end - - describe "#each_column_by_table_id" do - it "is an enumerator" do - is_enumerator?(@dict.each_column_by_table_id(19)).should be_truthy - end - end - - describe "#each_column_by_table_name" do - it "is an enumerator" do - is_enumerator?(@dict.each_column_by_table_name("sakila/film")).should be_truthy - end - end - - describe "#each_column_in_index_by_name" do - it "is an enumerator" do - is_enumerator?(@dict.each_column_in_index_by_name("sakila/film", "PRIMARY")).should be_truthy - end - end - - describe "#each_column_not_in_index_by_name" do - it "is an enumerator" do - is_enumerator?(@dict.each_column_not_in_index_by_name("sakila/film", "PRIMARY")).should be_truthy - end - end - - describe "#clustered_index_name_by_table_name" do - end - - describe "#each_column_description_by_index_name" do - it "is an enumerator" do - is_enumerator?(@dict.each_column_description_by_index_name("sakila/film", "PRIMARY")).should be_truthy - end - end - - describe "#record_describer_by_index_name" do - it "returns an Innodb::RecordDescriber" do - desc = @dict.record_describer_by_index_name("sakila/film", "PRIMARY") - desc.should be_an_instance_of Innodb::RecordDescriber - end - end - - describe "#record_describer_by_index_id" do - it "returns an Innodb::RecordDescriber" do - desc = @dict.record_describer_by_index_id(27) - desc.should be_an_instance_of Innodb::RecordDescriber - end - end -end diff --git a/spec/innodb/data_type_spec.rb b/spec/innodb/data_type_spec.rb index b4bbdf1a..309625fb 100644 --- a/spec/innodb/data_type_spec.rb +++ b/spec/innodb/data_type_spec.rb @@ -1,64 +1,41 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true require "spec_helper" require "stringio" describe Innodb::DataType do - - it "makes proper data type names" do - Innodb::DataType.make_name("BIGINT", [], [:UNSIGNED]).should eql "BIGINT UNSIGNED" - Innodb::DataType.make_name("SMALLINT", [], []).should eql "SMALLINT" - Innodb::DataType.make_name("VARCHAR", [32], []).should eql "VARCHAR(32)" - Innodb::DataType.make_name("CHAR", [16], []).should eql "CHAR(16)" - Innodb::DataType.make_name("CHAR", [], []).should eql "CHAR" - Innodb::DataType.make_name("VARBINARY", [48], []).should eql "VARBINARY(48)" - Innodb::DataType.make_name("BINARY", [64], []).should eql "BINARY(64)" - Innodb::DataType.make_name("BINARY", [], []).should eql "BINARY" - end - - describe Innodb::DataType::CharacterType do - it "handles optional width" do - Innodb::DataType.new(:CHAR, [], []).width.should eql 1 - Innodb::DataType.new(:CHAR, [16], []).width.should eql 16 + describe Innodb::DataType::Character do + it "handles optional length" do + Innodb::DataType.parse("CHAR", []).length.should eql 1 + Innodb::DataType.parse("CHAR(16)", []).length.should eql 16 end - end - describe Innodb::DataType::VariableCharacterType do it "throws an error on invalid modifiers" do - expect { Innodb::DataType.new(:VARCHAR, [], []) }. - to raise_error "Invalid width specification" - expect { Innodb::DataType.new(:VARCHAR, [1,1], []) }. - to raise_error "Invalid width specification" + expect { Innodb::DataType.parse("VARCHAR", []) }.to raise_error Innodb::DataType::InvalidSpecificationError + expect { Innodb::DataType.parse("VARCHAR(1,1)", []) }.to raise_error Innodb::DataType::InvalidSpecificationError end - end - describe Innodb::DataType::BinaryType do - it "handles optional width" do - Innodb::DataType.new(:BINARY, [], []).width.should eql 1 - Innodb::DataType.new(:BINARY, [16], []).width.should eql 16 + it "handles optional length" do + Innodb::DataType.parse("BINARY", []).length.should eql 1 + Innodb::DataType.parse("BINARY(16)", []).length.should eql 16 end - end - describe Innodb::DataType::VariableBinaryType do it "throws an error on invalid modifiers" do - expect { Innodb::DataType.new(:VARBINARY, [], []) }. - to raise_error "Invalid width specification" - expect { Innodb::DataType.new(:VARBINARY, [1,1], []) }. - to raise_error "Invalid width specification" + expect { Innodb::DataType.parse("VARBINARY", []) }.to raise_error Innodb::DataType::InvalidSpecificationError + expect { Innodb::DataType.parse("VARBINARY(1,1)", []) }.to raise_error Innodb::DataType::InvalidSpecificationError end end - describe Innodb::DataType::IntegerType do + describe Innodb::DataType::Integer do before :all do @data = { - :offset => {}, - :buffer => "", + offset: {}, + buffer: "".dup, } # Bytes 0x00 through 0x0f at offset 0. @data[:offset][:bytes_00_0f] = @data[:buffer].size - @data[:buffer] << - "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + @data[:buffer] << "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" # Maximum-sized integers for each type. @data[:offset][:max_uint] = @data[:buffer].size @@ -66,13 +43,11 @@ # InnoDB-munged signed positive integers. @data[:offset][:innodb_sint_pos] = @data[:buffer].size - @data[:buffer] << - "\x80\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + @data[:buffer] << "\x80\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" # InnoDB-munged signed negative integers. @data[:offset][:innodb_sint_neg] = @data[:buffer].size - @data[:buffer] << - "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + @data[:buffer] << "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" @buffer = StringIO.new(@data[:buffer]) end @@ -82,8 +57,8 @@ end it "returns a TINYINT value correctly" do - data_type = Innodb::DataType.new(:TINYINT, [], []) - data_type.should be_an_instance_of Innodb::DataType::IntegerType + data_type = Innodb::DataType.parse("TINYINT", []) + data_type.should be_an_instance_of Innodb::DataType::Integer @buffer.seek(@data[:offset][:innodb_sint_pos]) data_type.value(@buffer.read(1)).should eql 0x00 @buffer.seek(@data[:offset][:innodb_sint_neg]) @@ -91,8 +66,8 @@ end it "returns a TINYINT UNSIGNED value correctly" do - data_type = Innodb::DataType.new(:TINYINT, [], [:UNSIGNED]) - data_type.should be_an_instance_of Innodb::DataType::IntegerType + data_type = Innodb::DataType.parse("TINYINT", %i[UNSIGNED]) + data_type.should be_an_instance_of Innodb::DataType::Integer data_type.value(@buffer.read(1)).should eql 0x00 data_type.value(@buffer.read(1)).should eql 0x01 data_type.value(@buffer.read(1)).should eql 0x02 @@ -102,17 +77,17 @@ end it "returns a SMALLINT value correctly" do - data_type = Innodb::DataType.new(:SMALLINT, [], []) - data_type.should be_an_instance_of Innodb::DataType::IntegerType + data_type = Innodb::DataType.parse("SMALLINT", []) + data_type.should be_an_instance_of Innodb::DataType::Integer @buffer.seek(@data[:offset][:innodb_sint_pos]) data_type.value(@buffer.read(2)).should eql 0x0001 @buffer.seek(@data[:offset][:innodb_sint_neg]) - data_type.value(@buffer.read(2)).should eql -32767 + data_type.value(@buffer.read(2)).should eql -32_767 end it "returns a SMALLINT UNSIGNED value correctly" do - data_type = Innodb::DataType.new(:SMALLINT, [], [:UNSIGNED]) - data_type.should be_an_instance_of Innodb::DataType::IntegerType + data_type = Innodb::DataType.parse("SMALLINT", %i[UNSIGNED]) + data_type.should be_an_instance_of Innodb::DataType::Integer data_type.value(@buffer.read(2)).should eql 0x0001 data_type.value(@buffer.read(2)).should eql 0x0203 data_type.value(@buffer.read(2)).should eql 0x0405 @@ -122,17 +97,17 @@ end it "returns a MEDIUMINT value correctly" do - data_type = Innodb::DataType.new(:MEDIUMINT, [], []) - data_type.should be_an_instance_of Innodb::DataType::IntegerType + data_type = Innodb::DataType.parse("MEDIUMINT", []) + data_type.should be_an_instance_of Innodb::DataType::Integer @buffer.seek(@data[:offset][:innodb_sint_pos]) data_type.value(@buffer.read(3)).should eql 0x000102 @buffer.seek(@data[:offset][:innodb_sint_neg]) - data_type.value(@buffer.read(3)).should eql -8388350 + data_type.value(@buffer.read(3)).should eql -8_388_350 end it "returns a MEDIUMINT UNSIGNED value correctly" do - data_type = Innodb::DataType.new(:MEDIUMINT, [], [:UNSIGNED]) - data_type.should be_an_instance_of Innodb::DataType::IntegerType + data_type = Innodb::DataType.parse("MEDIUMINT", %i[UNSIGNED]) + data_type.should be_an_instance_of Innodb::DataType::Integer data_type.value(@buffer.read(3)).should eql 0x000102 data_type.value(@buffer.read(3)).should eql 0x030405 data_type.value(@buffer.read(3)).should eql 0x060708 @@ -142,17 +117,17 @@ end it "returns an INT value correctly" do - data_type = Innodb::DataType.new(:INT, [], []) - data_type.should be_an_instance_of Innodb::DataType::IntegerType + data_type = Innodb::DataType.parse("INT", []) + data_type.should be_an_instance_of Innodb::DataType::Integer @buffer.seek(@data[:offset][:innodb_sint_pos]) data_type.value(@buffer.read(4)).should eql 0x00010203 @buffer.seek(@data[:offset][:innodb_sint_neg]) - data_type.value(@buffer.read(4)).should eql -2147417597 + data_type.value(@buffer.read(4)).should eql -2_147_417_597 end it "returns an INT UNSIGNED value correctly" do - data_type = Innodb::DataType.new(:INT, [], [:UNSIGNED]) - data_type.should be_an_instance_of Innodb::DataType::IntegerType + data_type = Innodb::DataType.parse("INT", %i[UNSIGNED]) + data_type.should be_an_instance_of Innodb::DataType::Integer data_type.value(@buffer.read(4)).should eql 0x00010203 data_type.value(@buffer.read(4)).should eql 0x04050607 data_type.value(@buffer.read(4)).should eql 0x08090a0b @@ -162,15 +137,15 @@ end it "returns a BIGINT value correctly" do - data_type = Innodb::DataType.new(:BIGINT, [], []) + data_type = Innodb::DataType.parse("BIGINT", []) @buffer.seek(@data[:offset][:innodb_sint_pos]) data_type.value(@buffer.read(8)).should eql 0x0001020304050607 @buffer.seek(@data[:offset][:innodb_sint_neg]) - data_type.value(@buffer.read(8)).should eql -9223088349902469625 + data_type.value(@buffer.read(8)).should eql -9_223_088_349_902_469_625 end it "returns a BIGINT UNSIGNED value correctly" do - data_type = Innodb::DataType.new(:BIGINT, [], [:UNSIGNED]) + data_type = Innodb::DataType.parse("BIGINT", %i[UNSIGNED]) data_type.value(@buffer.read(8)).should eql 0x0001020304050607 data_type.value(@buffer.read(8)).should eql 0x08090a0b0c0d0e0f @buffer.seek(@data[:offset][:max_uint]) diff --git a/spec/innodb/date_and_time_types_spec.rb b/spec/innodb/date_and_time_types_spec.rb index 3ef449ed..d6b7bd9a 100644 --- a/spec/innodb/date_and_time_types_spec.rb +++ b/spec/innodb/date_and_time_types_spec.rb @@ -1,8 +1,8 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true -require 'spec_helper' +require "spec_helper" -class Innodb::RecordDescriber::DateTimeTypes < Innodb::RecordDescriber +class DateTimeTypes < Innodb::RecordDescriber type :clustered key "c01", "INT", :NOT_NULL row "c02", "YEAR" @@ -27,7 +27,7 @@ class Innodb::RecordDescriber::DateTimeTypes < Innodb::RecordDescriber describe Innodb::RecordDescriber do before :all do @space = Innodb::Space.new("spec/data/t_date_and_time_types.ibd") - @space.record_describer = Innodb::RecordDescriber::DateTimeTypes.new + @space.record_describer = DateTimeTypes.new end describe "#index" do diff --git a/spec/innodb/index_spec.rb b/spec/innodb/index_spec.rb index fc7b8faa..3b74c065 100644 --- a/spec/innodb/index_spec.rb +++ b/spec/innodb/index_spec.rb @@ -1,4 +1,4 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true require "spec_helper" @@ -21,7 +21,7 @@ class TTenKRowsDescriber < Innodb::RecordDescriber end it "handles failed searches" do - rec = @index.linear_search([999999]) + rec = @index.linear_search([999_999]) rec.should be_nil end @@ -29,13 +29,13 @@ class TTenKRowsDescriber < Innodb::RecordDescriber rec = @index.linear_search([1]) rec.key[0][:value].should eql 1 - rec = @index.linear_search([10000]) - rec.key[0][:value].should eql 10000 + rec = @index.linear_search([10_000]) + rec.key[0][:value].should eql 10_000 rec = @index.linear_search([0]) rec.should be_nil - rec = @index.linear_search([10001]) + rec = @index.linear_search([10_001]) rec.should be_nil end end @@ -47,7 +47,7 @@ class TTenKRowsDescriber < Innodb::RecordDescriber end it "handles failed searches" do - rec = @index.binary_search([999999]) + rec = @index.binary_search([999_999]) rec.should be_nil end @@ -55,31 +55,31 @@ class TTenKRowsDescriber < Innodb::RecordDescriber rec = @index.binary_search([1]) rec.key[0][:value].should eql 1 - rec = @index.binary_search([10000]) - rec.key[0][:value].should eql 10000 + rec = @index.binary_search([10_000]) + rec.key[0][:value].should eql 10_000 rec = @index.binary_search([0]) rec.should be_nil - rec = @index.binary_search([10001]) + rec = @index.binary_search([10_001]) rec.should be_nil end it "is much more efficient than linear_search" do Innodb::Stats.reset - rec = @index.linear_search([5000]) + @index.linear_search([5_000]) linear_compares = Innodb::Stats.get(:compare_key) Innodb::Stats.reset - rec = @index.binary_search([5000]) + @index.binary_search([5_000]) binary_compares = Innodb::Stats.get(:compare_key) - ((linear_compares.to_f / binary_compares.to_f) > 10).should be_truthy + ((linear_compares.to_f / binary_compares) > 10).should be_truthy end it "can find 200 random rows" do missing_keys = {} - (200.times.map { (rand() * 10000 + 1).floor }).map do |i| + (200.times.map { ((rand * 10_000) + 1).floor }).map do |i| rec = @index.binary_search([i]) if rec.nil? missing_keys[i] = :missing_key @@ -112,14 +112,14 @@ class TTenKRowsDescriber < Innodb::RecordDescriber page = @index.max_page_at_level(0) page.level.should eql 0 rec = page.max_record - rec.key[0][:value].should eql 10000 + rec.key[0][:value].should eql 10_000 end end describe "#max_record" do it "returns the max record" do rec = @index.max_record - rec.key[0][:value].should eql 10000 + rec.key[0][:value].should eql 10_000 end end @@ -190,7 +190,7 @@ class TTenKRowsDescriber < Innodb::RecordDescriber cursor.record.should be_nil cursor = @index.cursor(:max, :forward) - cursor.record.key[0][:value].to_i.should eql 10000 + cursor.record.key[0][:value].to_i.should eql 10_000 cursor.record.should be_nil end end @@ -202,4 +202,3 @@ class TTenKRowsDescriber < Innodb::RecordDescriber end end end - diff --git a/spec/innodb/list/history_spec.rb b/spec/innodb/list/history_spec.rb new file mode 100644 index 00000000..0122d8a3 --- /dev/null +++ b/spec/innodb/list/history_spec.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require "spec_helper" + +describe Innodb::List::History do + before :all do + @system = Innodb::System.new("spec/data/sakila/compact/ibdata1") + @empty_list = @system.history.each_history_list.first&.list + end + + it "can read an empty list" do + @empty_list.empty?.should(be_truthy) + end +end diff --git a/spec/innodb/list/inode_spec.rb b/spec/innodb/list/inode_spec.rb new file mode 100644 index 00000000..66bc888e --- /dev/null +++ b/spec/innodb/list/inode_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require "spec_helper" + +describe Innodb::List::Inode do + before :all do + @system = Innodb::System.new("spec/data/sakila/compact/ibdata1") + @empty_list = @system.space_by_table_name("sakila/film").list(:full_inodes) + end + + it "can read an empty list" do + @empty_list.empty?.should(be_truthy) + end + + it "only iterates through INODE pages" do + @system.system_space.list(:full_inodes).each.map(&:type).should(eql(%i[INODE INODE])) + end +end diff --git a/spec/innodb/list/undo_page_spec.rb b/spec/innodb/list/undo_page_spec.rb new file mode 100644 index 00000000..935febf2 --- /dev/null +++ b/spec/innodb/list/undo_page_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require "spec_helper" + +describe Innodb::List::UndoPage do + # TODO: Add some kind of test here. +end diff --git a/spec/innodb/list/xdes_spec.rb b/spec/innodb/list/xdes_spec.rb new file mode 100644 index 00000000..0c4c6a29 --- /dev/null +++ b/spec/innodb/list/xdes_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require "spec_helper" + +describe Innodb::List::Xdes do + # TODO: Add some kind of test here. +end diff --git a/spec/innodb/log_block_spec.rb b/spec/innodb/log_block_spec.rb index e040cf03..79830ac7 100644 --- a/spec/innodb/log_block_spec.rb +++ b/spec/innodb/log_block_spec.rb @@ -1,15 +1,16 @@ -# -*- encoding : utf-8 -*- -require 'spec_helper' +# frozen_string_literal: true + +require "spec_helper" describe Innodb::LogBlock do before :all do - @log = Innodb::Log.new("spec/data/ib_logfile0") + @log = Innodb::Log.new("spec/data/sakila/compact/ib_logfile0") @block = @log.block(0) end describe "#block" do it "has a correct checksum" do - @block.checksum.should eql 1706444976 + @block.checksum.should eql 1_706_444_976 end it "is not corrupt" do @@ -17,13 +18,12 @@ end it "returns a valid header" do - @block.header.should eql({ - :flush => true, - :block_number => 17, - :data_length => 512, - :first_rec_group => 12, - :checkpoint_no => 5, - }) + h = @block.header + h.flush.should eql true + h.block_number.should eql 17 + h.data_length.should eql 512 + h.first_rec_group.should eql 12 + h.checkpoint_no.should eql 5 end end end diff --git a/spec/innodb/log_group_spec.rb b/spec/innodb/log_group_spec.rb index 8d94e6fb..9e0cba1a 100644 --- a/spec/innodb/log_group_spec.rb +++ b/spec/innodb/log_group_spec.rb @@ -1,10 +1,14 @@ -# -*- encoding : utf-8 -*- -require 'spec_helper' +# frozen_string_literal: true + +require "spec_helper" describe Innodb::LogGroup do before :all do - @log_files = ["spec/data/ib_logfile0", "spec/data/ib_logfile1"] - @log_file_size = 5242880 + @log_files = %w[ + spec/data/sakila/compact/ib_logfile0 + spec/data/sakila/compact/ib_logfile1 + ] + @log_file_size = 5_242_880 @log_group = Innodb::LogGroup.new(@log_files) end @@ -72,7 +76,7 @@ describe "#record" do it "returns an instance of Innodb::LogRecord" do - @log_group.record(8204).should be_a Innodb::LogRecord + @log_group.record(8_204).should be_a Innodb::LogRecord end end end diff --git a/spec/innodb/log_reader_spec.rb b/spec/innodb/log_reader_spec.rb index adb6cca8..40b044f7 100644 --- a/spec/innodb/log_reader_spec.rb +++ b/spec/innodb/log_reader_spec.rb @@ -1,38 +1,39 @@ -# -*- encoding : utf-8 -*- -require 'spec_helper' +# frozen_string_literal: true + +require "spec_helper" describe Innodb::LogReader do before :all do - log_files = ["spec/data/ib_logfile0", "spec/data/ib_logfile1"] + log_files = ["spec/data/sakila/compact/ib_logfile0", "spec/data/sakila/compact/ib_logfile1"] @group = Innodb::LogGroup.new(log_files) @reader = @group.reader end describe "#seek" do it "repositions the reader" do - @reader.seek(8204).tell.should eql 8204 + @reader.seek(8_204).tell.should eql 8_204 end it "detects out of bounds seeks" do - expect { @reader.seek(8192) }.to raise_error "LSN 8192 is out of bounds" + expect { @reader.seek(8_192) }.to raise_error "LSN 8192 is out of bounds" end end describe "#tell" do it "returns the current LSN position" do - @reader.seek(8205).tell.should eql 8205 - @reader.seek(8204).tell.should eql 8204 + @reader.seek(8_205).tell.should eql 8_205 + @reader.seek(8_204).tell.should eql 8_204 end end describe "#record" do - it "returns an instance of Innodb::LogRecord" do + it "returns an instance of Innodb::LogRecord" do record = @reader.record record.should be_an_instance_of Innodb::LogRecord end it "repositions the reader after reading a record" do - @reader.tell.should eql 8207 + @reader.tell.should eql 8_207 end it "reads records across blocks" do diff --git a/spec/innodb/log_record_spec.rb b/spec/innodb/log_record_spec.rb index a01d7d90..1e4475cc 100644 --- a/spec/innodb/log_record_spec.rb +++ b/spec/innodb/log_record_spec.rb @@ -1,28 +1,32 @@ -# -*- encoding : utf-8 -*- -require 'spec_helper' +# frozen_string_literal: true + +require "spec_helper" describe Innodb::LogRecord do before :all do - log_files = ["spec/data/ib_logfile0", "spec/data/ib_logfile1"] + log_files = %w[ + spec/data/sakila/compact/ib_logfile0 + spec/data/sakila/compact/ib_logfile1 + ] @group = Innodb::LogGroup.new(log_files) end describe :INIT_FILE_PAGE do before(:all) do - @rec = @group.reader.seek(8204).record + @rec = @group.reader.seek(8_204).record end it "has the correct size" do @rec.size.should eql 3 end it "has the correct LSN" do - @rec.lsn.should eql [8204, 8207] + @rec.lsn.should eql [8_204, 8_207] end it "has the correct preamble" do - @rec.preamble.should eql( - :type => :INIT_FILE_PAGE, - :page_number => 1, - :space => 0, - :single_record => false) + p = @rec.preamble + p.type.should eql :INIT_FILE_PAGE + p.page_number.should eql 1 + p.space.should eql 0 + p.single_record.should eql false end it "has an empty payload" do @rec.payload.should eql({}) @@ -31,20 +35,20 @@ describe :IBUF_BITMAP_INIT do before(:all) do - @rec = @group.reader.seek(8207).record + @rec = @group.reader.seek(8_207).record end it "has the correct size" do @rec.size.should eql 3 end it "has the correct LSN" do - @rec.lsn.should eql [8207, 8210] + @rec.lsn.should eql [8_207, 8_210] end it "has the correct preamble" do - @rec.preamble.should eql( - :type => :IBUF_BITMAP_INIT, - :page_number => 1, - :space => 0, - :single_record => false) + p = @rec.preamble + p.type.should eql :IBUF_BITMAP_INIT + p.page_number.should eql 1 + p.space.should eql 0 + p.single_record.should eql false end it "has an empty payload" do @rec.payload.should eql({}) @@ -53,28 +57,29 @@ describe :REC_INSERT do before(:all) do - @rec = @group.reader.seek(1589112).record + @rec = @group.reader.seek(1_589_112).record end it "has the correct size" do @rec.size.should eql 36 end it "has the correct LSN" do - @rec.lsn.should eql [1589112, 1589148] + @rec.lsn.should eql [1_589_112, 1_589_148] end it "has the correct preamble" do - @rec.preamble.should eql( - :type => :REC_INSERT, - :page_number => 9, - :space => 0, - :single_record => false) + p = @rec.preamble + p.type.should eql :REC_INSERT + p.page_number.should eql 9 + p.space.should eql 0 + p.single_record.should eql false end it "has the correct payload" do @rec.payload.values.first.should include( - :mismatch_index => 0, - :page_offset => 101, - :end_seg_len => 27, - :info_and_status_bits => 0, - :origin_offset => 8) + mismatch_index: 0, + page_offset: 101, + end_seg_len: 27, + info_and_status_bits: 0, + origin_offset: 8 + ) end end end diff --git a/spec/innodb/log_spec.rb b/spec/innodb/log_spec.rb index 94ee7336..62cb3796 100644 --- a/spec/innodb/log_spec.rb +++ b/spec/innodb/log_spec.rb @@ -1,11 +1,10 @@ -# -*- encoding : utf-8 -*- -require 'spec_helper' +# frozen_string_literal: true -describe Innodb::Log do - LOG_CHECKPOINT_FSP_MAGIC_N_VAL = 1441231243 +require "spec_helper" +describe Innodb::Log do before :all do - @log = Innodb::Log.new("spec/data/ib_logfile0") + @log = Innodb::Log.new("spec/data/sakila/compact/ib_logfile0") end describe "#new" do @@ -19,14 +18,14 @@ end describe "#size" do - it "returns 5242880 bytes" do - @log.size.should eql 5242880 + it "returns 5,242,880 bytes" do + @log.size.should eql 5_242_880 end end describe "#blocks" do - it "returns 10236 blocks" do - @log.blocks.should eql 10236 + it "returns 10,236 blocks" do + @log.blocks.should eql 10_236 end end @@ -37,7 +36,7 @@ it "does not return an invalid block" do @log.block(-1).should be_nil - @log.block(10236).should be_nil + @log.block(10_236).should be_nil end end @@ -50,62 +49,49 @@ end describe "#header" do - it "returns a Hash" do - @log.header.should be_an_instance_of Hash - end - - it "has only Symbol keys" do - classes = @log.header.keys.map { |k| k.class }.uniq - classes.should eql [Symbol] + it "returns a Innodb::Log::Header" do + @log.header.should be_an_instance_of Innodb::Log::Header end it "has the right keys and values" do @log.header.size.should eql 4 - @log.header.should include( - :group_id => 0, - :start_lsn => 8192, - :file_no => 0, - :created_by => " ") + @log.header.group_id.should eql 0 + @log.header.start_lsn.should eql 8_192 + @log.header.file_no.should eql 0 + @log.header.created_by.should eql " " end end describe "#checkpoint" do - it "returns a Hash" do - @log.checkpoint.should be_an_instance_of Hash + it "returns a Innodb::Log::CheckpointSet" do + @log.checkpoint.should be_an_instance_of Innodb::Log::CheckpointSet @log.checkpoint.size.should eql 2 end - it "has only Symbol keys" do - classes = @log.checkpoint.keys.map { |k| k.class }.uniq - classes.should eql [Symbol] - end - it "has a correct checkpoint_1" do - @log.checkpoint[:checkpoint_1].should include( - :number => 10, - :lsn => 1603732, - :lsn_offset => 1597588, - :buffer_size => 1048576, - :archived_lsn => 18446744073709551615, - # :group_array - :checksum_1 => 654771786, - :checksum_2 => 1113429956, - :fsp_free_limit => 5, - :fsp_magic => LOG_CHECKPOINT_FSP_MAGIC_N_VAL) + c = @log.checkpoint.checkpoint_1 + c.number.should eql 14 + c.lsn.should eql 8_400_260 + c.lsn_offset.should eql 8_396_164 + c.buffer_size.should eql 1_048_576 + c.archived_lsn.should eql 18_446_744_073_709_551_615 + c.checksum_1.should eql 2_424_759_900 + c.checksum_2.should eql 3_026_016_186 + c.fsp_free_limit.should eql 9 + c.fsp_magic.should eql 1_441_231_243 # LOG_CHECKPOINT_FSP_MAGIC_N_VAL end it "has a correct checkpoint_2" do - @log.checkpoint[:checkpoint_2].should include( - :number => 11, - :lsn => 1603732, - :lsn_offset => 1597588, - :buffer_size => 1048576, - :archived_lsn => 18446744073709551615, - # :group_array - :checksum_1 => 843938123, - :checksum_2 => 674570893, - :fsp_free_limit => 5, - :fsp_magic => LOG_CHECKPOINT_FSP_MAGIC_N_VAL) + c = @log.checkpoint.checkpoint_2 + c.number.should eql 15 + c.lsn.should eql 8_400_260 + c.lsn_offset.should eql 8_396_164 + c.buffer_size.should eql 1_048_576 + c.archived_lsn.should eql 18_446_744_073_709_551_615 + c.checksum_1.should eql 125_194_589 + c.checksum_2.should eql 1_139_538_825 + c.fsp_free_limit.should eql 9 + c.fsp_magic.should eql 1_441_231_243 # LOG_CHECKPOINT_FSP_MAGIC_N_VAL end end end diff --git a/spec/innodb/mysql_collation_spec.rb b/spec/innodb/mysql_collation_spec.rb new file mode 100644 index 00000000..9ca5fad1 --- /dev/null +++ b/spec/innodb/mysql_collation_spec.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require "spec_helper" + +describe Innodb::MysqlCollation do + describe ".collations" do + it "is an Array" do + Innodb::MysqlCollation.collations.should be_an_instance_of Array + end + + # The count/minmax here can be updated when lib/innodb/mysql_collations.rb is updated, but it should probably never + # decrease, only increase. Proceed with caution. + + it "has 287 entries" do + Innodb::MysqlCollation.collations.count.should eql 287 + end + + it "has entries from id 1 to 323" do + Innodb::MysqlCollation.collations.map(&:id).minmax.should eql [1, 323] + end + + it "has only Innodb::MysqlCollation entries" do + Innodb::MysqlCollation.collations.map(&:class).uniq.should eql [Innodb::MysqlCollation] + end + + it "has mbminlen values of 1, 2, 4" do + Innodb::MysqlCollation.collations.map(&:mbminlen).sort.uniq.should eql [1, 2, 4] + end + + it "has mbmaxlen values of 1, 2, 3, 4, 5" do + Innodb::MysqlCollation.collations.map(&:mbmaxlen).sort.uniq.should eql [1, 2, 3, 4, 5] + end + end + + describe ".by_id" do + it "can look up utf8mb4_general_ci by id" do + Innodb::MysqlCollation.by_id(45).name.should eql "utf8mb4_general_ci" + end + end + + describe ".by_name" do + it "can look up utf8mb4_general_ci by name" do + Innodb::MysqlCollation.by_name("utf8mb4_general_ci").id.should eql 45 + end + end + + describe "#fixed?" do + it "works properly for two example collations" do + Innodb::MysqlCollation.by_name("ascii_general_ci").fixed?.should be true + Innodb::MysqlCollation.by_name("utf8mb4_general_ci").fixed?.should be false + end + end + + describe "#variable?" do + it "works properly for two example collations" do + Innodb::MysqlCollation.by_name("ascii_general_ci").variable?.should be false + Innodb::MysqlCollation.by_name("utf8mb4_general_ci").variable?.should be true + end + end +end diff --git a/spec/innodb/numeric_types_spec.rb b/spec/innodb/numeric_types_spec.rb index ae13ba56..2cc051d3 100644 --- a/spec/innodb/numeric_types_spec.rb +++ b/spec/innodb/numeric_types_spec.rb @@ -1,20 +1,21 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true -require 'spec_helper' +require "spec_helper" -class Innodb::RecordDescriber::NumericTypes < Innodb::RecordDescriber +# rubocop:disable Style/NumericLiterals +class NumericTypes < Innodb::RecordDescriber type :clustered - key "c01", "INT", :UNSIGNED, :NOT_NULL + key "c01", "INT", :UNSIGNED, :NOT_NULL row "c02", "TINYINT" - row "c03", "TINYINT", :UNSIGNED + row "c03", "TINYINT", :UNSIGNED row "c04", "SMALLINT" - row "c05", "SMALLINT", :UNSIGNED + row "c05", "SMALLINT", :UNSIGNED row "c06", "MEDIUMINT" row "c07", "MEDIUMINT", :UNSIGNED row "c08", "INT" - row "c09", "INT", :UNSIGNED + row "c09", "INT", :UNSIGNED row "c10", "BIGINT" - row "c11", "BIGINT", :UNSIGNED + row "c11", "BIGINT", :UNSIGNED row "c12", "FLOAT" row "c13", "FLOAT" row "c14", "DOUBLE" @@ -30,152 +31,158 @@ class Innodb::RecordDescriber::NumericTypes < Innodb::RecordDescriber # Zero. row0 = [ - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0.0, - 0.0, - 0.0, - 0.0, - "0.0", - "0.0", - "0.0", - "0.0", - "0b0", - "0b0", - "0b0"] + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0.0, + 0.0, + 0.0, + 0.0, + "0.0", + "0.0", + "0.0", + "0.0", + "0b0", + "0b0", + "0b0", +] # Minus one. row1 = [ - -1, - 0, - -1, - 0, - -1, - 0, - -1, - 0, - -1, - 0, - -1.0, - 0.0, - -1.0, - 0.0, - "-1.0", - "0.0", - "-1.0", - "-1.0", - "0b1", - "0b11111111111111111111111111111111", - "0b1111111111111111111111111111111111111111111111111111111111111111"] + -1, + 0, + -1, + 0, + -1, + 0, + -1, + 0, + -1, + 0, + -1.0, + 0.0, + -1.0, + 0.0, + "-1.0", + "0.0", + "-1.0", + "-1.0", + "0b1", + "0b11111111111111111111111111111111", + "0b1111111111111111111111111111111111111111111111111111111111111111", +] # One. row2 = [ - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1.0, - 1.0, - 1.0, - 1.0, - "1.0", - "1.0", - "1.0", - "1.0", - "0b1", - "0b1", - "0b1"] + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1.0, + 1.0, + 1.0, + 1.0, + "1.0", + "1.0", + "1.0", + "1.0", + "0b1", + "0b1", + "0b1", +] # Minimum values. row3 = [ - -128, - 0, - -32768, - 0, - -8388608, - 0, - -2147483648, - 0, - -9223372036854775808, - 0, - -1.1754943508222875e-38, - 0.0, - -2.2250738585072014e-208, - 0.0, - "-9999999999.0", - "0.0", - "-99999999999999999999999999999999999999999999999999999999999999999.0", - "-99999.999999999999999999999999999999", - "0b0", - "0b0", - "0b0"] + -128, + 0, + -32768, + 0, + -8388608, + 0, + -2147483648, + 0, + -9223372036854775808, + 0, + -1.1754943508222875e-38, + 0.0, + -2.2250738585072014e-208, + 0.0, + "-9999999999.0", + "0.0", + "-99999999999999999999999999999999999999999999999999999999999999999.0", + "-99999.999999999999999999999999999999", + "0b0", + "0b0", + "0b0", +] # Maximum values. row4 = [ - 127, - 255, - 32767, - 65535, - 8388607, - 16777215, - 2147483647, - 4294967295, - 9223372036854775807, - 18446744073709551615, - 3.4028234663852886e+38, - 3.4028234663852886e+38, - 1.7976931348623157e+308, - 1.7976931348623157e+308, - "9999999999.0", - "9999999999.0", - "99999999999999999999999999999999999999999999999999999999999999999.0", - "99999.999999999999999999999999999999", - "0b1", - "0b11111111111111111111111111111111", - "0b1111111111111111111111111111111111111111111111111111111111111111"] + 127, + 255, + 32767, + 65535, + 8388607, + 16777215, + 2147483647, + 4294967295, + 9223372036854775807, + 18446744073709551615, + 3.4028234663852886e+38, + 3.4028234663852886e+38, + 1.7976931348623157e+308, + 1.7976931348623157e+308, + "9999999999.0", + "9999999999.0", + "99999999999999999999999999999999999999999999999999999999999999999.0", + "99999.999999999999999999999999999999", + "0b1", + "0b11111111111111111111111111111111", + "0b1111111111111111111111111111111111111111111111111111111111111111", +] # Random values. row5 = [ - -92, - 216, - -21244, - 37375, - -2029076, - 13161062, - -561256167, - 2859565307, - -2989164089322500559, - 4909805763357741578, - 8.007314291015967e+37, - 2.3826953364781035e+38, - -1.0024988592301854e+308, - 3.8077578553713446e+307, - "-2118290683.0", - "7554694345.0", - "36896958284301606307227443682014665342058559023876912710455539626.0", - "59908.987290718443144993967601373349", - "0b0", - "0b1110000001101111100011001110100", - "0b1001001010001001000111001010000011000011110110011100000101000010"] + -92, + 216, + -21244, + 37375, + -2029076, + 13161062, + -561256167, + 2859565307, + -2989164089322500559, + 4909805763357741578, + 8.007314291015967e+37, + 2.3826953364781035e+38, + -1.0024988592301854e+308, + 3.8077578553713446e+307, + "-2118290683.0", + "7554694345.0", + "36896958284301606307227443682014665342058559023876912710455539626.0", + "59908.987290718443144993967601373349", + "0b0", + "0b1110000001101111100011001110100", + "0b1001001010001001000111001010000011000011110110011100000101000010", +] describe Innodb::RecordDescriber do before :all do @space = Innodb::Space.new("spec/data/t_numeric_types.ibd") - @space.record_describer = Innodb::RecordDescriber::NumericTypes.new + @space.record_describer = NumericTypes.new end describe "#index" do @@ -196,3 +203,4 @@ class Innodb::RecordDescriber::NumericTypes < Innodb::RecordDescriber end end end +# rubocop:enable Style/NumericLiterals diff --git a/spec/innodb/page/fsp_hdr_xdes_spec.rb b/spec/innodb/page/fsp_hdr_xdes_spec.rb index 66074e19..84355173 100644 --- a/spec/innodb/page/fsp_hdr_xdes_spec.rb +++ b/spec/innodb/page/fsp_hdr_xdes_spec.rb @@ -1,4 +1,4 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true require "spec_helper" @@ -9,9 +9,9 @@ end describe "class" do - it "registers itself in Innodb::Page::SPECIALIZED_CLASSES" do - Innodb::Page::SPECIALIZED_CLASSES[:FSP_HDR].should eql Innodb::Page::FspHdrXdes - Innodb::Page::SPECIALIZED_CLASSES[:XDES].should eql Innodb::Page::FspHdrXdes + it "registers itself as a specialized page type" do + Innodb::Page.specialization_for?(:FSP_HDR).should be_truthy + Innodb::Page.specialization_for?(:XDES).should be_truthy end end @@ -24,4 +24,16 @@ @page.should be_a Innodb::Page end end + + describe "#each_list" do + it "returns an appropriate set of lists" do + @page.each_list.map { |name, _| name }.should include( + :free, + :free_frag, + :full_frag, + :full_inodes, + :free_inodes + ) + end + end end diff --git a/spec/innodb/page/index_spec.rb b/spec/innodb/page/index_spec.rb index 38002ed6..111f3855 100644 --- a/spec/innodb/page/index_spec.rb +++ b/spec/innodb/page/index_spec.rb @@ -1,4 +1,4 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true require "spec_helper" @@ -9,8 +9,8 @@ end describe "class" do - it "registers itself in Innodb::Page::SPECIALIZED_CLASSES" do - Innodb::Page::SPECIALIZED_CLASSES[:INDEX].should eql Innodb::Page::Index + it "registers itself as a specialized page type" do + Innodb::Page.specialization_for?(:INDEX).should be_truthy end end @@ -25,27 +25,26 @@ end describe "#page_header" do - it "is a Hash" do - @page.page_header.should be_an_instance_of Hash + it "is a Innodb::Page::Index::PageHeader" do + @page.page_header.should be_an_instance_of Innodb::Page::Index::PageHeader end it "has the right keys and values" do - @page.page_header.keys.size.should eql 13 - @page.page_header[:n_dir_slots].should eql 2 - @page.page_header[:heap_top].should eql 120 - @page.page_header[:garbage_offset].should eql 0 - @page.page_header[:garbage_size].should eql 0 - @page.page_header[:last_insert_offset].should eql 0 - @page.page_header[:direction].should eql :no_direction - @page.page_header[:n_direction].should eql 0 - @page.page_header[:n_recs].should eql 0 - @page.page_header[:max_trx_id].should eql 0 - @page.page_header[:level].should eql 0 - @page.page_header[:index_id].should eql 16 - @page.page_header[:n_heap].should eql 2 - @page.page_header[:format].should eql :compact - end - + @page.page_header.n_dir_slots.should eql 2 + @page.page_header.heap_top.should eql 120 + @page.page_header.garbage_offset.should eql 0 + @page.page_header.garbage_size.should eql 0 + @page.page_header.last_insert_offset.should eql 0 + @page.page_header.direction.should eql :no_direction + @page.page_header.n_direction.should eql 0 + @page.page_header.n_recs.should eql 0 + @page.page_header.max_trx_id.should eql 0 + @page.page_header.level.should eql 0 + @page.page_header.index_id.should eql 16 + @page.page_header.n_heap.should eql 2 + @page.page_header.format.should eql :compact + end + it "has helper functions" do @page.level.should eql @page.page_header[:level] @page.records.should eql @page.page_header[:n_recs] @@ -56,7 +55,7 @@ describe "#free_space" do it "returns the amount of free space" do - @page.free_space.should eql 16252 + @page.free_space.should eql 16_252 end end @@ -73,12 +72,11 @@ end describe "#fseg_header" do - it "is a Hash" do - @page.fseg_header.should be_an_instance_of Hash + it "is a Innodb::Page::Index::FsegHeader" do + @page.fseg_header.should be_an_instance_of Innodb::Page::Index::FsegHeader end - + it "has the right keys and values" do - @page.fseg_header.keys.size.should eql 2 @page.fseg_header[:leaf].should be_an_instance_of Innodb::Inode @page.fseg_header[:internal].should be_an_instance_of Innodb::Inode end @@ -89,18 +87,17 @@ @header = @page.record_header(@page.cursor(@page.pos_infimum)) end - it "is a Hash" do - @header.should be_an_instance_of Hash + it "is a Innodb::Page::Index::RecordHeader" do + @header.should be_an_instance_of Innodb::Page::Index::RecordHeader end - + it "has the right keys and values" do - @header.size.should eql 7 - @header[:type].should eql :infimum - @header[:next].should eql 112 - @header[:heap_number].should eql 0 - @header[:n_owned].should eql 1 - @header[:min_rec].should eql false - @header[:deleted].should eql false + @header.type.should eql :infimum + @header.next.should eql 112 + @header.heap_number.should eql 0 + @header.n_owned.should eql 1 + @header.min_rec?.should eql false + @header.deleted?.should eql false end end @@ -109,7 +106,7 @@ rec = @page.infimum rec.should be_an_instance_of Innodb::Record rec.record[:data].should eql "infimum\x00" - rec.header.should be_an_instance_of Hash + rec.header.should be_an_instance_of Innodb::Page::Index::RecordHeader rec.header[:type].should eql :infimum end @@ -117,7 +114,7 @@ rec = @page.supremum rec.should be_an_instance_of Innodb::Record rec.record[:data].should eql "supremum" - rec.header.should be_an_instance_of Hash + rec.header.should be_an_instance_of Innodb::Page::Index::RecordHeader rec.header[:type].should eql :supremum end end diff --git a/spec/innodb/page/inode_spec.rb b/spec/innodb/page/inode_spec.rb index 3739eac3..35f26455 100644 --- a/spec/innodb/page/inode_spec.rb +++ b/spec/innodb/page/inode_spec.rb @@ -1,16 +1,16 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true require "spec_helper" describe Innodb::Page::Inode do before :all do - @space = Innodb::Space.new("spec/data/ibdata1") + @space = Innodb::Space.new("spec/data/sakila/compact/ibdata1") @page = @space.page(2) end describe "class" do - it "registers itself in Innodb::Page::SPECIALIZED_CLASSES" do - Innodb::Page::SPECIALIZED_CLASSES[:INODE].should eql Innodb::Page::Inode + it "registers itself as a specialized page type" do + Innodb::Page.specialization_for?(:INODE).should be_truthy end end @@ -25,14 +25,14 @@ end describe "#list_entry" do - it "is a Hash" do - @page.list_entry.should be_an_instance_of Hash + it "is a Innodb::List::Node" do + @page.list_entry.should be_an_instance_of Innodb::List::Node end it "has the right keys and values" do @page.list_entry.size.should eql 2 @page.list_entry[:prev].should eql nil - @page.list_entry[:next].should eql nil + @page.list_entry[:next].should eql Innodb::Page::Address.new(page: 243, offset: 38) end it "has helper functions" do @@ -43,10 +43,10 @@ describe "#each_inode" do it "yields Innodb::Inode objects" do - @page.each_inode.to_a.map { |v| v.class }.uniq.should eql [Innodb::Inode] + @page.each_inode.to_a.map(&:class).uniq.should eql [Innodb::Inode] end - it "yields Hashes with the right keys and values" do + it "yields objects with the right keys and values" do inode = @page.each_inode.to_a.first inode.fseg_id.should eql 1 inode.not_full_n_used.should eql 0 @@ -60,7 +60,7 @@ describe "#each_allocated_inode" do it "yields Innodb::Inode objects" do - @page.each_allocated_inode.to_a.map { |v| v.class }.uniq.should eql [Innodb::Inode] + @page.each_allocated_inode.to_a.map(&:class).uniq.should eql [Innodb::Inode] end it "yields only allocated inodes" do diff --git a/spec/innodb/page/trx_sys_spec.rb b/spec/innodb/page/trx_sys_spec.rb index 39cebc51..46a142a7 100644 --- a/spec/innodb/page/trx_sys_spec.rb +++ b/spec/innodb/page/trx_sys_spec.rb @@ -1,16 +1,16 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true require "spec_helper" describe Innodb::Page::TrxSys do before :all do - @space = Innodb::Space.new("spec/data/ibdata1") + @space = Innodb::Space.new("spec/data/sakila/compact/ibdata1") @page = @space.page(5) end describe "class" do - it "registers itself in Innodb::Page::SPECIALIZED_CLASSES" do - Innodb::Page::SPECIALIZED_CLASSES[:TRX_SYS].should eql Innodb::Page::TrxSys + it "registers itself as a specialized page type" do + Innodb::Page.specialization_for?(:TRX_SYS).should be_truthy end end @@ -25,17 +25,16 @@ end describe "#trx_sys" do - it "is a Hash" do - @page.trx_sys.should be_an_instance_of Hash + it "is a Innodb::Page::TrxSys::Header" do + @page.trx_sys.should be_an_instance_of Innodb::Page::TrxSys::Header end it "has the right keys and values" do - @page.trx_sys.size.should eql 6 - @page.trx_sys[:trx_id].should eql 1280 - @page.trx_sys[:rsegs].should be_an_instance_of Array - @page.trx_sys[:binary_log].should eql nil - @page.trx_sys[:master_log].should eql nil - @page.trx_sys[:doublewrite].should be_an_instance_of Hash + @page.trx_sys.trx_id.should eql 1280 + @page.trx_sys.rsegs.should be_an_instance_of Array + @page.trx_sys.binary_log.should eql nil + @page.trx_sys.master_log.should eql nil + @page.trx_sys.doublewrite.should be_an_instance_of Innodb::Page::TrxSys::DoublewriteInfo end end end diff --git a/spec/innodb/page_spec.rb b/spec/innodb/page_spec.rb index fc7e3225..6b03a46f 100644 --- a/spec/innodb/page_spec.rb +++ b/spec/innodb/page_spec.rb @@ -1,10 +1,10 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true require "spec_helper" describe Innodb::Page do before :all do - @space = Innodb::Space.new("spec/data/ibdata1") + @space = Innodb::Space.new("spec/data/sakila/compact/ibdata1") @page_data = @space.page_data(0) @page = @space.page(0) end @@ -15,12 +15,12 @@ end it "has only Symbol keys" do - classes = Innodb::Page::PAGE_TYPE.keys.map { |k| k.class }.uniq + classes = Innodb::Page::PAGE_TYPE.keys.map(&:class).uniq classes.should eql [Symbol] end it "has only Hash values" do - classes = Innodb::Page::PAGE_TYPE.values.map { |v| v.class }.uniq + classes = Innodb::Page::PAGE_TYPE.values.map(&:class).uniq classes.should eql [Hash] end end @@ -31,53 +31,47 @@ end it "has only Integer keys" do - classes = Innodb::Page::PAGE_TYPE_BY_VALUE.keys.map { |k| k.class }.uniq + classes = Innodb::Page::PAGE_TYPE_BY_VALUE.keys.map(&:class).uniq classes.should eql [Integer] end it "has only Symbol values" do - classes = Innodb::Page::PAGE_TYPE_BY_VALUE.values.map { |v| v.class }.uniq + classes = Innodb::Page::PAGE_TYPE_BY_VALUE.values.map(&:class).uniq classes.should eql [Symbol] end end - describe "::SPECIALIZED_CLASSES" do + describe "specialized_classes" do it "is a Hash" do - Innodb::Page::SPECIALIZED_CLASSES.should be_an_instance_of Hash + Innodb::Page.specialized_classes.should be_an_instance_of Hash end it "has only Symbol keys" do - classes = Innodb::Page::SPECIALIZED_CLASSES.keys.map { |k| k.class }.uniq - classes.should eql [Symbol] + Innodb::Page.specialized_classes.keys.map(&:class).uniq.should eql [Symbol] end it "has only keys that are keys in ::PAGE_TYPE" do - checks = Innodb::Page::SPECIALIZED_CLASSES.keys.map { |k| - Innodb::Page::PAGE_TYPE.include? k - }.uniq - checks.should eql [true] + Innodb::Page.specialized_classes.keys.all? { |k| Innodb::Page::PAGE_TYPE.include?(k) }.should be_truthy end it "has only Class values" do - classes = Innodb::Page::SPECIALIZED_CLASSES.values.map { |v| v.class }.uniq - classes.should eql [Class] + Innodb::Page.specialized_classes.values.map(&:class).uniq.should eql [Class] end it "has only values subclassing Innodb::Page" do - classes = Innodb::Page::SPECIALIZED_CLASSES.values.map { |v| v.superclass }.uniq - classes.should eql [Innodb::Page] + Innodb::Page.specialized_classes.values.all? { |x| x.is_a?(Innodb::Page) } end end describe "#new" do it "returns a class" do - Innodb::Page.new(@space, @page_data).should be_an_instance_of Innodb::Page + Innodb::Page.new(@space, @page_data).should be_an_instance_of Innodb::Page end end describe "#parse" do it "delegates to the right specialized class" do - Innodb::Page.parse(@space, @page_data).should be_an_instance_of Innodb::Page::FspHdrXdes + Innodb::Page.parse(@space, @page_data).should be_an_instance_of Innodb::Page::FspHdrXdes end end @@ -93,7 +87,7 @@ it "is reading reasonable data" do # This will read the page number from page 0, which should be 0. - @page.cursor(4).get_uint32.should eql 0 + @page.cursor(4).read_uint32.should eql 0 end end @@ -101,31 +95,26 @@ it "returns the value when the value is not UINT_MAX" do Innodb::Page.maybe_undefined(5).should eql 5 end - + it "returns nil when the value is UINT_MAX" do - Innodb::Page.maybe_undefined(4294967295).should eql nil + Innodb::Page.maybe_undefined(4_294_967_295).should eql nil end end describe "#fil_header" do - it "returns a Hash" do - @page.fil_header.should be_an_instance_of Hash - end - - it "has only Symbol keys" do - classes = @page.fil_header.keys.map { |k| k.class }.uniq - classes.should eql [Symbol] + it "returns a Innodb::Page::FilHeader" do + @page.fil_header.should be_an_instance_of Innodb::Page::FilHeader end it "has the right keys and values" do @page.fil_header.size.should eql 8 - @page.fil_header[:checksum].should eql 2067631406 + @page.fil_header[:checksum].should eql 3_774_490_636 @page.fil_header[:offset].should eql 0 @page.fil_header[:prev].should eql 0 @page.fil_header[:next].should eql 0 - @page.fil_header[:lsn].should eql 1601269 + @page.fil_header[:lsn].should eql 8_400_049 @page.fil_header[:type].should eql :FSP_HDR - @page.fil_header[:flush_lsn].should eql 1603732 + @page.fil_header[:flush_lsn].should eql 8_400_260 @page.fil_header[:space_id].should eql 0 end @@ -141,7 +130,7 @@ describe "#checksum_innodb" do it "calculates the right checksum" do - @page.checksum_innodb.should eql 2067631406 + @page.checksum_innodb.should eql 3_774_490_636 @page.corrupt?.should eql false end end diff --git a/spec/innodb/record_describer_spec.rb b/spec/innodb/record_describer_spec.rb index aa3963df..9d12f4f1 100644 --- a/spec/innodb/record_describer_spec.rb +++ b/spec/innodb/record_describer_spec.rb @@ -1,26 +1,26 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true -require 'spec_helper' +require "spec_helper" class PkRecordDescriber < Innodb::RecordDescriber type :clustered - key "c1", :BIGINT, :NOT_NULL, :UNSIGNED - row "c2", "INT" + key "c1", :BIGINT, :NOT_NULL, :UNSIGNED + row "c2", :INT row "c3", "VARCHAR(64)" - key "c4", "INT", :NOT_NULL - row "c5", "VARCHAR(128)", :NOT_NULL - row "c6", "MEDIUMINT", :UNSIGNED + key "c4", :INT, :NOT_NULL + row "c5", "VARCHAR(128)", :NOT_NULL + row "c6", :MEDIUMINT, :UNSIGNED row "c7", "VARBINARY(512)" - row "c8", "BIGINT", :UNSIGNED - row "c9", "BLOB" + row "c8", :BIGINT, :UNSIGNED + row "c9", :BLOB end class SkRecordDescriber < Innodb::RecordDescriber type :secondary - row "c1", :BIGINT, :NOT_NULL, :UNSIGNED - row "c4", "INT", :NOT_NULL - key "c6", "MEDIUMINT", :UNSIGNED - key "c8", "BIGINT", :UNSIGNED + row "c1", :BIGINT, :NOT_NULL, :UNSIGNED + row "c4", :INT, :NOT_NULL + key "c6", :MEDIUMINT, :UNSIGNED + key "c8", :BIGINT, :UNSIGNED end describe Innodb::RecordDescriber do @@ -61,12 +61,10 @@ class SkRecordDescriber < Innodb::RecordDescriber end it "#roll_pointer" do - @rec.roll_pointer.should eql( - :is_insert => true, - :undo_log => { - :offset => 272, - :page => 435}, - :rseg_id => 2) + @rec.roll_pointer.is_insert.should eql true + @rec.roll_pointer.undo_log.offset.should eql 272 + @rec.roll_pointer.undo_log.page.should eql 435 + @rec.roll_pointer.rseg_id.should eql 2 end it "key is (1, 1)" do @@ -75,23 +73,23 @@ class SkRecordDescriber < Innodb::RecordDescriber fields.next[:value].should eql 1 end - it "row is (-1, '1' * 64, '1' * 128, 1, NULL, 1, '1' * 16384)" do + it "row is as expected" do fields = @rec.row.each fields.next[:value].should eql -1 - fields.next[:value].should eql '1' * 64 - fields.next[:value].should eql '1' * 128 + fields.next[:value].should eql "1" * 64 + fields.next[:value].should eql "1" * 128 fields.next[:value].should eql 1 fields.next[:value].should eql :NULL fields.next[:value].should eql 1 - fields.next[:value].should eql '1' * 768 # max prefix + fields.next[:value].should eql "1" * 768 # max prefix end - it "external reference field is [6, 5, 38, 15616]" do + it "external reference field is [6, 5, 38, 15_616]" do extern = @rec.row.last[:extern] extern[:space_id].should eql 6 extern[:page_number].should eql 5 extern[:offset].should eql 38 - extern[:length].should eql 15616 # 16384 - 768 + extern[:length].should eql 15_616 # 16384 - 768 end end @@ -101,14 +99,15 @@ class SkRecordDescriber < Innodb::RecordDescriber end it "#header" do - @rec.header.should include( - :type => :node_pointer, - :length => 5, - :min_rec => true, - :heap_number => 2, - :deleted => false) - @rec.header[:nulls].size.should eql 0 - @rec.header[:lengths].size.should eql 0 + @rec.header.type.should eql :node_pointer + @rec.header.length.should eql 5 + @rec.header.heap_number.should eql 2 + + @rec.header.min_rec?.should eql true + @rec.header.deleted?.should eql false + + @rec.header.nulls.size.should eql 0 + @rec.header.lengths.size.should eql 0 end it "#child_page_number" do diff --git a/spec/innodb/space_spec.rb b/spec/innodb/space_spec.rb index 3b75ac65..479b6465 100644 --- a/spec/innodb/space_spec.rb +++ b/spec/innodb/space_spec.rb @@ -1,11 +1,11 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true -require 'spec_helper' +require "spec_helper" describe Innodb::Space do before :all do - @space = Innodb::Space.new("spec/data/ibdata1") - @space_ibd = Innodb::Space.new("spec/data/t_empty.ibd") + @space = Innodb::Space.new("spec/data/sakila/compact/ibdata1") + @space_ibd = Innodb::Space.new("spec/data/sakila/compact/sakila/film.ibd") end describe "DEFAULT_PAGE_SIZE" do @@ -35,25 +35,25 @@ # This will read the page number from page 0, which should be 0. @space.read_at_offset(4, 4).should eql "\x00\x00\x00\x00" # This will read the page number from page 1, which should be 1. - @space.read_at_offset(16384+4, 4).should eql "\x00\x00\x00\x01" + @space.read_at_offset(16_384 + 4, 4).should eql "\x00\x00\x00\x01" end end describe "#page_size" do it "finds a 16 KiB page size" do - @space.page_size.should eql 16384 + @space.page_size.should eql 16_384 end end describe "#pages" do - it "returns 1152 pages" do - @space.pages.should eql 1152 + it "returns 1,152 pages" do + @space.pages.should eql 1_152 end end describe "#size" do - it "returns 18874368 bytes" do - @space.size.should eql 18874368 + it "returns 18,874,368 bytes" do + @space.size.should eql 18_874_368 end end @@ -65,7 +65,7 @@ describe "#page_data" do it "returns 16 KiB of data" do - @space.page_data(0).size.should eql 16384 + @space.page_data(0).size.should eql 16_384 end end @@ -82,13 +82,13 @@ end it "should return nil for a page that does not exist" do - @space.page(2000).should eql nil + @space.page(2_000).should eql nil end end describe "#fsp" do - it "is a Hash" do - @space.fsp.should be_an_instance_of Hash + it "is a Innodb::Page::FspHdrXdes::Header" do + @space.fsp.should be_an_instance_of Innodb::Page::FspHdrXdes::Header end end @@ -108,7 +108,7 @@ end it "should return nil for a non-system space" do - @space_ibd.trx_sys.should eql nil + expect { @space_ibd.trx_sys }.to raise_error(RuntimeError) end end @@ -118,7 +118,7 @@ end it "should return nil for a non-system space" do - @space_ibd.data_dictionary_page.should eql nil + expect { @space_ibd.data_dictionary_page }.to raise_error(RuntimeError) end end @@ -134,7 +134,7 @@ end it "iterates through indexes" do - @space_ibd.each_index.to_a.size.should eql 1 + @space_ibd.each_index.to_a.size.should eql 4 end it "yields an Innodb::Index" do @@ -148,7 +148,7 @@ end it "iterates through pages" do - @space_ibd.each_page.to_a.size.should eql 6 + @space_ibd.each_page.to_a.size.should eql 21 end it "yields an Array of [page_number, page]" do @@ -175,10 +175,10 @@ @space.xdes_page_for_page(1).should eql 0 @space.xdes_page_for_page(63).should eql 0 @space.xdes_page_for_page(64).should eql 0 - @space.xdes_page_for_page(16383).should eql 0 - @space.xdes_page_for_page(16384).should eql 16384 - @space.xdes_page_for_page(32767).should eql 16384 - @space.xdes_page_for_page(32768).should eql 32768 + @space.xdes_page_for_page(16_383).should eql 0 + @space.xdes_page_for_page(16_384).should eql 16_384 + @space.xdes_page_for_page(32_767).should eql 16_384 + @space.xdes_page_for_page(32_768).should eql 32_768 end end @@ -195,14 +195,14 @@ @space.xdes_entry_for_page(65).should eql 1 @space.xdes_entry_for_page(127).should eql 1 @space.xdes_entry_for_page(128).should eql 2 - @space.xdes_entry_for_page(16383).should eql 255 - @space.xdes_entry_for_page(16384).should eql 0 - @space.xdes_entry_for_page(16385).should eql 0 - @space.xdes_entry_for_page(16448).should eql 1 - @space.xdes_entry_for_page(16511).should eql 1 - @space.xdes_entry_for_page(16512).should eql 2 - @space.xdes_entry_for_page(16576).should eql 3 - @space.xdes_entry_for_page(32767).should eql 255 + @space.xdes_entry_for_page(16_383).should eql 255 + @space.xdes_entry_for_page(16_384).should eql 0 + @space.xdes_entry_for_page(16_385).should eql 0 + @space.xdes_entry_for_page(16_448).should eql 1 + @space.xdes_entry_for_page(16_511).should eql 1 + @space.xdes_entry_for_page(16_512).should eql 2 + @space.xdes_entry_for_page(16_576).should eql 3 + @space.xdes_entry_for_page(32_767).should eql 255 end end @@ -213,8 +213,8 @@ it "returns the correct XDES entry" do xdes = @space.xdes_for_page(0) - (0 >= xdes.start_page).should eql true - (0 <= xdes.end_page).should eql true + (xdes.start_page <= 0).should eql true + (xdes.end_page >= 0).should eql true end end diff --git a/spec/innodb/stats_spec.rb b/spec/innodb/stats_spec.rb index f22f89c0..9e5027ec 100644 --- a/spec/innodb/stats_spec.rb +++ b/spec/innodb/stats_spec.rb @@ -1,4 +1,4 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true require "spec_helper" @@ -41,5 +41,4 @@ Innodb::Stats.data.size.should eql 0 end end - end diff --git a/spec/innodb/sys_data_dictionary_spec.rb b/spec/innodb/sys_data_dictionary_spec.rb new file mode 100644 index 00000000..aa4bd837 --- /dev/null +++ b/spec/innodb/sys_data_dictionary_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require "spec_helper" + +describe Innodb::SysDataDictionary do + before :all do + @system = Innodb::System.new("spec/data/sakila/compact/ibdata1") + @dict = @system.data_dictionary + end + + describe "#mtype_prtype_to_type_string" do + it "produces the correct type string or symbol" do + type = Innodb::SysDataDictionary.mtype_prtype_to_type_string(6, 1794, 2, 0) + type.should eql :SMALLINT + end + end + + describe "#mtype_prtype_to_data_type" do + it "produces the correct type array" do + type = Innodb::SysDataDictionary.mtype_prtype_to_data_type(6, 1794, 2, 0) + type.should be_an_instance_of Array + type.should eql %i[SMALLINT NOT_NULL UNSIGNED] + end + end + + # TODO: Write new tests for SysDataDictionary +end diff --git a/spec/innodb/system_spec.rb b/spec/innodb/system_spec.rb index 6679dfbe..2da5928a 100644 --- a/spec/innodb/system_spec.rb +++ b/spec/innodb/system_spec.rb @@ -1,12 +1,31 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true -require 'spec_helper' +require "spec_helper" describe Innodb::System do before :all do @system = Innodb::System.new("spec/data/sakila/compact/ibdata1") end + describe "data_directory" do + it "cannot find tablespace files when the data directory is wrong" do + broken_system = Innodb::System.new("spec/data/sakila/compact/ibdata1", data_directory: "foo/bar") + space = broken_system.space_by_table_name("sakila/film") + space.should be_nil + end + end + + describe "data_directory" do + it "can find tablespace files using a specified data directory" do + working_system = Innodb::System.new( + "spec/data/sakila/compact/ibdata1", + data_directory: "spec/data/sakila/compact" + ) + space = working_system.space_by_table_name("sakila/film") + space.should be_an_instance_of(Innodb::Space) + end + end + describe "#system_space" do it "returns an Innodb::Space" do @system.system_space.should be_an_instance_of Innodb::Space @@ -68,105 +87,6 @@ end end - describe "#each_table_name" do - it "is an enumerator" do - is_enumerator?(@system.each_table_name).should be_truthy - end - - it "iterates all tables in the system" do - expected = [ - "SYS_FOREIGN", - "SYS_FOREIGN_COLS", - "sakila/actor", - "sakila/address", - "sakila/category", - "sakila/city", - "sakila/country", - "sakila/customer", - "sakila/film", - "sakila/film_actor", - "sakila/film_category", - "sakila/inventory", - "sakila/language", - "sakila/payment", - "sakila/rental", - "sakila/staff", - "sakila/store", - ] - - actual = @system.each_table_name.to_a - result = actual.map { |n| expected.include?(n) }.uniq - result.should eql [true] - end - end - - describe "#each_column_name_by_table_name" do - it "is an enumerator" do - is_enumerator?(@system.each_column_name_by_table_name("sakila/film")).should be_truthy - end - - it "iterates all columns in the table" do - expected = [ - "film_id", - "title", - "description", - "release_year", - "language_id", - "original_language_id", - "rental_duration", - "rental_rate", - "length", - "replacement_cost", - "rating", - "special_features", - "last_update", - ] - - actual = @system.each_column_name_by_table_name("sakila/film").to_a - result = actual.map { |n| expected.include?(n) }.uniq - result.should eql [true] - end - end - - describe "#each_index_name_by_table_name" do - it "is an enumerator" do - is_enumerator?(@system.each_index_name_by_table_name("sakila/film")).should be_truthy - end - - it "iterates all indexes in the table" do - expected = [ - "PRIMARY", - "idx_title", - "idx_fk_language_id", - "idx_fk_original_language_id", - ] - - actual = @system.each_index_name_by_table_name("sakila/film").to_a - result = actual.map { |n| expected.include?(n) }.uniq - result.should eql [true] - end - end - - describe "#table_name_by_id" do - it "returns the correct table name" do - @system.table_name_by_id(19).should eql "sakila/film" - end - end - - describe "#index_name_by_id" do - it "returns the correct index name" do - @system.index_name_by_id(27).should eql "PRIMARY" - end - end - - describe "#table_and_index_name_by_id" do - it "returns the correct table and index name" do - table, index = @system.table_and_index_name_by_id(27) - table.should eql "sakila/film" - index.should eql "PRIMARY" - end - end - describe "#index_by_name" do it "returns an Innodb::Index object" do index = @system.index_by_name("sakila/film", "PRIMARY") @@ -176,7 +96,8 @@ describe "#each_orphan" do before :all do - @system = Innodb::System.new("spec/data/ibdata1") + # Tablespace has a missing tablespace file for test/t_empty. + @system = Innodb::System.new("spec/data/orphan/ibdata1") end it "has an orphan space" do @@ -191,5 +112,4 @@ @system.each_orphan.next.should eql "test/t_empty" end end - end diff --git a/spec/innodb/util/buffer_cursor_spec.rb b/spec/innodb/util/buffer_cursor_spec.rb index 905ddd66..825ac9ac 100644 --- a/spec/innodb/util/buffer_cursor_spec.rb +++ b/spec/innodb/util/buffer_cursor_spec.rb @@ -1,18 +1,17 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true require "spec_helper" describe BufferCursor do before :all do @data = { - :offset => {}, - :buffer => "", + offset: {}, + buffer: "".dup, } # Bytes 0x00 through 0x0f at offset 0. @data[:offset][:bytes_00_0f] = @data[:buffer].size - @data[:buffer] << - "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + @data[:buffer] << "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" # Maximum-sized integers for each type. @data[:offset][:max_uint] = @data[:buffer].size @@ -198,7 +197,7 @@ @cursor.peek { false }.should eql false end - it "doesn't disturb the cursor position or direction on return" do + it "does not disturb the cursor position or direction on return" do @cursor.position.should eql 0 @cursor.direction.should eql :forward @cursor.peek do @@ -209,250 +208,250 @@ @cursor.seek(20).forward @cursor.position.should eql 20 @cursor.direction.should eql :forward - end + end @cursor.position.should eql 10 @cursor.direction.should eql :backward - end + end @cursor.position.should eql 0 @cursor.direction.should eql :forward end end - describe "#get_bytes" do + describe "#read_bytes" do it "returns a raw byte string of the given length" do - @cursor.get_bytes(4).should eql "\x00\x01\x02\x03" + @cursor.read_bytes(4).should eql "\x00\x01\x02\x03" end it "returns a string uncorrupted" do @cursor.seek(@data[:offset][:alphabet]) - @cursor.get_bytes(4).should eql "abcd" + @cursor.read_bytes(4).should eql "abcd" end end - describe "#get_hex" do + describe "#read_hex" do it "returns a hex string of the given length" do - @cursor.get_hex(4).should eql "00010203" - @cursor.get_hex(4).should eql "04050607" - @cursor.get_hex(4).should eql "08090a0b" - @cursor.get_hex(4).should eql "0c0d0e0f" + @cursor.read_hex(4).should eql "00010203" + @cursor.read_hex(4).should eql "04050607" + @cursor.read_hex(4).should eql "08090a0b" + @cursor.read_hex(4).should eql "0c0d0e0f" end end - describe "#get_uint8" do + describe "#read_uint8" do it "reads 1 byte as uint8" do - @cursor.get_uint8.should eql 0x00 - @cursor.get_uint8.should eql 0x01 - @cursor.get_uint8.should eql 0x02 - @cursor.get_uint8.should eql 0x03 + @cursor.read_uint8.should eql 0x00 + @cursor.read_uint8.should eql 0x01 + @cursor.read_uint8.should eql 0x02 + @cursor.read_uint8.should eql 0x03 @cursor.seek(@data[:offset][:max_uint]) - @cursor.get_uint8.should eql 0xff + @cursor.read_uint8.should eql 0xff end end - describe "#get_uint16" do + describe "#read_uint16" do it "returns 2 bytes as uint16" do - @cursor.get_uint16.should eql 0x0001 - @cursor.get_uint16.should eql 0x0203 - @cursor.get_uint16.should eql 0x0405 - @cursor.get_uint16.should eql 0x0607 + @cursor.read_uint16.should eql 0x0001 + @cursor.read_uint16.should eql 0x0203 + @cursor.read_uint16.should eql 0x0405 + @cursor.read_uint16.should eql 0x0607 @cursor.seek(@data[:offset][:max_uint]) - @cursor.get_uint16.should eql 0xffff + @cursor.read_uint16.should eql 0xffff end end - describe "#get_uint24" do + describe "#read_uint24" do it "returns 3 bytes as uint24" do - @cursor.get_uint24.should eql 0x000102 - @cursor.get_uint24.should eql 0x030405 - @cursor.get_uint24.should eql 0x060708 - @cursor.get_uint24.should eql 0x090a0b + @cursor.read_uint24.should eql 0x000102 + @cursor.read_uint24.should eql 0x030405 + @cursor.read_uint24.should eql 0x060708 + @cursor.read_uint24.should eql 0x090a0b @cursor.seek(@data[:offset][:max_uint]) - @cursor.get_uint24.should eql 0xffffff + @cursor.read_uint24.should eql 0xffffff end end - describe "#get_uint32" do + describe "#read_uint32" do it "returns 4 bytes as uint32" do - @cursor.get_uint32.should eql 0x00010203 - @cursor.get_uint32.should eql 0x04050607 - @cursor.get_uint32.should eql 0x08090a0b - @cursor.get_uint32.should eql 0x0c0d0e0f + @cursor.read_uint32.should eql 0x00010203 + @cursor.read_uint32.should eql 0x04050607 + @cursor.read_uint32.should eql 0x08090a0b + @cursor.read_uint32.should eql 0x0c0d0e0f @cursor.seek(@data[:offset][:max_uint]) - @cursor.get_uint32.should eql 0xffffffff + @cursor.read_uint32.should eql 0xffffffff end end - describe "#get_uint64" do + describe "#read_uint64" do it "returns 8 bytes as uint64" do - @cursor.get_uint64.should eql 0x0001020304050607 - @cursor.get_uint64.should eql 0x08090a0b0c0d0e0f + @cursor.read_uint64.should eql 0x0001020304050607 + @cursor.read_uint64.should eql 0x08090a0b0c0d0e0f @cursor.seek(@data[:offset][:max_uint]) - @cursor.get_uint64.should eql 0xffffffffffffffff + @cursor.read_uint64.should eql 0xffffffffffffffff end end - describe "#get_uint_by_size" do + describe "#read_uint_by_size" do it "returns a uint8 for size 1" do - @cursor.get_uint_by_size(1).should eql 0x00 + @cursor.read_uint_by_size(1).should eql 0x00 end it "returns a uint16 for size 2" do - @cursor.get_uint_by_size(2).should eql 0x0001 + @cursor.read_uint_by_size(2).should eql 0x0001 end it "returns a uint24 for size 3" do - @cursor.get_uint_by_size(3).should eql 0x000102 + @cursor.read_uint_by_size(3).should eql 0x000102 end it "returns a uint32 for size 4" do - @cursor.get_uint_by_size(4).should eql 0x00010203 + @cursor.read_uint_by_size(4).should eql 0x00010203 end it "returns a uint64 for size 8" do - @cursor.get_uint_by_size(8).should eql 0x0001020304050607 + @cursor.read_uint_by_size(8).should eql 0x0001020304050607 end end - describe "#get_ic_uint32" do + describe "#read_ic_uint32" do it "reads a 1-byte zero value correctly" do @cursor.seek(@data[:offset][:ic_uint32_00000000]) - @cursor.get_ic_uint32.should eql 0 + @cursor.read_ic_uint32.should eql 0 @cursor.position.should eql @data[:offset][:ic_uint32_00000000] + 1 end it "reads a 1-byte maximal value correctly" do @cursor.seek(@data[:offset][:ic_uint32_0000007f]) - @cursor.get_ic_uint32.should eql 0x7f + @cursor.read_ic_uint32.should eql 0x7f @cursor.position.should eql @data[:offset][:ic_uint32_0000007f] + 1 end it "reads a 2-byte maximal value correctly" do @cursor.seek(@data[:offset][:ic_uint32_00003fff]) - @cursor.get_ic_uint32.should eql 0x3fff + @cursor.read_ic_uint32.should eql 0x3fff @cursor.position.should eql @data[:offset][:ic_uint32_00003fff] + 2 end it "reads a 3-byte maximal value correctly" do @cursor.seek(@data[:offset][:ic_uint32_001fffff]) - @cursor.get_ic_uint32.should eql 0x1fffff + @cursor.read_ic_uint32.should eql 0x1fffff @cursor.position.should eql @data[:offset][:ic_uint32_001fffff] + 3 end it "reads a 4-byte maximal value correctly" do @cursor.seek(@data[:offset][:ic_uint32_0fffffff]) - @cursor.get_ic_uint32.should eql 0x0fffffff + @cursor.read_ic_uint32.should eql 0x0fffffff @cursor.position.should eql @data[:offset][:ic_uint32_0fffffff] + 4 end it "reads a 5-byte maximal value correctly" do @cursor.seek(@data[:offset][:ic_uint32_ffffffff]) - @cursor.get_ic_uint32.should eql 0xffffffff + @cursor.read_ic_uint32.should eql 0xffffffff @cursor.position.should eql @data[:offset][:ic_uint32_ffffffff] + 5 end end - describe "#get_ic_uint64" do + describe "#read_ic_uint64" do it "reads a 5-byte zero value correctly" do @cursor.seek(@data[:offset][:ic_uint64_0000000000000000]) - @cursor.get_ic_uint64.should eql 0 + @cursor.read_ic_uint64.should eql 0 @cursor.position.should eql @data[:offset][:ic_uint64_0000000000000000] + 5 end it "reads a 5-byte interesting value 0x0000000100000001 correctly" do @cursor.seek(@data[:offset][:ic_uint64_0000000100000001]) - @cursor.get_ic_uint64.should eql 0x0000000100000001 + @cursor.read_ic_uint64.should eql 0x0000000100000001 @cursor.position.should eql @data[:offset][:ic_uint64_0000000100000001] + 5 end it "reads a 5-byte interesting value 0x00000000ffffffff correctly" do @cursor.seek(@data[:offset][:ic_uint64_00000000ffffffff]) - @cursor.get_ic_uint64.should eql 0x00000000ffffffff + @cursor.read_ic_uint64.should eql 0x00000000ffffffff @cursor.position.should eql @data[:offset][:ic_uint64_00000000ffffffff] + 5 end it "reads a 9-byte interesting value 0xffffffff00000000 correctly" do @cursor.seek(@data[:offset][:ic_uint64_ffffffff00000000]) - @cursor.get_ic_uint64.should eql 0xffffffff00000000 + @cursor.read_ic_uint64.should eql 0xffffffff00000000 @cursor.position.should eql @data[:offset][:ic_uint64_ffffffff00000000] + 9 end it "reads a 7-byte interesting value 0x0000ffff0000ffff correctly" do @cursor.seek(@data[:offset][:ic_uint64_0000ffff0000ffff]) - @cursor.get_ic_uint64.should eql 0x0000ffff0000ffff + @cursor.read_ic_uint64.should eql 0x0000ffff0000ffff @cursor.position.should eql @data[:offset][:ic_uint64_0000ffff0000ffff] + 7 end it "reads a 9-byte interesting value 0xffff0000ffff0000 correctly" do @cursor.seek(@data[:offset][:ic_uint64_ffff0000ffff0000]) - @cursor.get_ic_uint64.should eql 0xffff0000ffff0000 + @cursor.read_ic_uint64.should eql 0xffff0000ffff0000 @cursor.position.should eql @data[:offset][:ic_uint64_ffff0000ffff0000] + 9 end it "reads a 9-byte maximal value correctly" do @cursor.seek(@data[:offset][:ic_uint64_ffffffffffffffff]) - @cursor.get_ic_uint64.should eql 0xffffffffffffffff + @cursor.read_ic_uint64.should eql 0xffffffffffffffff @cursor.position.should eql @data[:offset][:ic_uint64_ffffffffffffffff] + 9 end end - describe "#get_imc_uint64" do + describe "#read_imc_uint64" do it "reads a 1-byte zero value correctly" do @cursor.seek(@data[:offset][:imc_uint64_0000000000000000]) - @cursor.get_imc_uint64.should eql 0 + @cursor.read_imc_uint64.should eql 0 @cursor.position.should eql @data[:offset][:imc_uint64_0000000000000000] + 1 end it "reads a 3-byte interesting value 0x0000000100000001 correctly" do @cursor.seek(@data[:offset][:imc_uint64_0000000100000001]) - @cursor.get_imc_uint64.should eql 0x0000000100000001 + @cursor.read_imc_uint64.should eql 0x0000000100000001 @cursor.position.should eql @data[:offset][:imc_uint64_0000000100000001] + 3 end it "reads a 5-byte interesting value 0x00000000ffffffff correctly" do @cursor.seek(@data[:offset][:imc_uint64_00000000ffffffff]) - @cursor.get_imc_uint64.should eql 0x00000000ffffffff + @cursor.read_imc_uint64.should eql 0x00000000ffffffff @cursor.position.should eql @data[:offset][:imc_uint64_00000000ffffffff] + 5 end it "reads a 7-byte interesting value 0xffffffff00000000 correctly" do @cursor.seek(@data[:offset][:imc_uint64_ffffffff00000000]) - @cursor.get_imc_uint64.should eql 0xffffffff00000000 + @cursor.read_imc_uint64.should eql 0xffffffff00000000 @cursor.position.should eql @data[:offset][:imc_uint64_ffffffff00000000] + 7 end it "reads a 7-byte interesting value 0x0000ffff0000ffff correctly" do @cursor.seek(@data[:offset][:imc_uint64_0000ffff0000ffff]) - @cursor.get_imc_uint64.should eql 0x0000ffff0000ffff + @cursor.read_imc_uint64.should eql 0x0000ffff0000ffff @cursor.position.should eql @data[:offset][:imc_uint64_0000ffff0000ffff] + 7 end it "reads a 11-byte interesting value 0xffff0000ffff0000 correctly" do @cursor.seek(@data[:offset][:imc_uint64_ffff0000ffff0000]) - @cursor.get_imc_uint64.should eql 0xffff0000ffff0000 + @cursor.read_imc_uint64.should eql 0xffff0000ffff0000 @cursor.position.should eql @data[:offset][:imc_uint64_ffff0000ffff0000] + 11 end it "reads a 11-byte maximal value correctly" do @cursor.seek(@data[:offset][:imc_uint64_ffffffffffffffff]) - @cursor.get_imc_uint64.should eql 0xffffffffffffffff + @cursor.read_imc_uint64.should eql 0xffffffffffffffff @cursor.position.should eql @data[:offset][:imc_uint64_ffffffffffffffff] + 11 end end - describe "#get_bit_array" do + describe "#read_bit_array" do it "returns an array of bits" do - @cursor.get_bit_array(64).uniq.sort.should eql [0, 1] + @cursor.read_bit_array(64).uniq.sort.should eql [0, 1] end it "returns the right bits" do - @cursor.get_bit_array(8).should eql [0, 0, 0, 0, 0, 0, 0, 0] - @cursor.get_bit_array(8).should eql [0, 0, 0, 0, 0, 0, 0, 1] + @cursor.read_bit_array(8).should eql [0, 0, 0, 0, 0, 0, 0, 0] + @cursor.read_bit_array(8).should eql [0, 0, 0, 0, 0, 0, 0, 1] @cursor.seek(@data[:offset][:max_uint]) - @cursor.get_bit_array(8).should eql [1, 1, 1, 1, 1, 1, 1, 1] + @cursor.read_bit_array(8).should eql [1, 1, 1, 1, 1, 1, 1, 1] end it "can handle large bit arrays" do - @cursor.get_bit_array(64).size.should eql 64 + @cursor.read_bit_array(64).size.should eql 64 end end @@ -460,18 +459,18 @@ it "enables tracing globally" do BufferCursor.trace! - trace_string = "" + trace_string = "".dup trace_output = StringIO.new(trace_string, "w") c1 = BufferCursor.new(@buffer, 0) c1.trace_to(trace_output) - c1.get_bytes(4).should eql "\x00\x01\x02\x03" + c1.read_bytes(4).should eql "\x00\x01\x02\x03" trace_string.should match(/000000 → 00010203/) c2 = BufferCursor.new(@buffer, 0) c2.trace_to(trace_output) - c2.seek(4).get_bytes(4).should eql "\x04\x05\x06\x07" + c2.seek(4).read_bytes(4).should eql "\x04\x05\x06\x07" trace_string.should match(/000004 → 04050607/) @@ -481,19 +480,19 @@ describe "#trace" do it "enables tracing per instance" do - trace_string = "" + trace_string = "".dup trace_output = StringIO.new(trace_string, "w") c1 = BufferCursor.new(@buffer, 0) c1.trace c1.trace_to(trace_output) - c1.get_bytes(4).should eql "\x00\x01\x02\x03" + c1.read_bytes(4).should eql "\x00\x01\x02\x03" trace_string.should match(/000000 → 00010203/) c2 = BufferCursor.new(@buffer, 0) c2.trace_to(trace_output) - c2.seek(4).get_bytes(4).should eql "\x04\x05\x06\x07" + c2.seek(4).read_bytes(4).should eql "\x04\x05\x06\x07" trace_string.should_not match(/000004 → 04050607/) end diff --git a/spec/innodb/xdes_spec.rb b/spec/innodb/xdes_spec.rb index 804ccbcf..c0bcb203 100644 --- a/spec/innodb/xdes_spec.rb +++ b/spec/innodb/xdes_spec.rb @@ -1,10 +1,10 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true require "spec_helper" describe Innodb::Xdes do before :all do - @space = Innodb::Space.new("spec/data/ibdata1") + @space = Innodb::Space.new("spec/data/sakila/compact/ibdata1") @page = @space.page(0) @cursor = @page.cursor(@page.pos_xdes_array) @xdes0 = Innodb::Xdes.new(@page, @cursor) @@ -17,12 +17,12 @@ end it "has only Integer keys" do - classes = Innodb::Xdes::STATES.keys.map { |k| k.class }.uniq + classes = Innodb::Xdes::STATES.keys.map(&:class).uniq classes.should eql [Integer] end it "has only Symbol values" do - classes = Innodb::Xdes::STATES.values.map { |v| v.class }.uniq + classes = Innodb::Xdes::STATES.values.map(&:class).uniq classes.should eql [Symbol] end end @@ -47,15 +47,15 @@ it "has the right methods and values" do @xdes0.start_page.should eql 0 @xdes0.fseg_id.should eql 0 - @xdes0.this.should be_an_instance_of Hash - @xdes0.list.should be_an_instance_of Hash + @xdes0.this.should be_an_instance_of Innodb::Page::Address + @xdes0.list.should be_an_instance_of Innodb::List::Node @xdes0.bitmap.size.should eql 16 end end describe "#xdes" do it "is a Hash" do - @xdes0.xdes.should be_an_instance_of Hash + @xdes0.xdes.should be_an_instance_of Innodb::Xdes::Entry end end @@ -69,7 +69,7 @@ describe "#page_status" do it "returns the status of a page" do status = @xdes0.page_status(0) - status.should be_an_instance_of Hash + status.should be_an_instance_of Innodb::Xdes::PageStatus status.size.should eql 2 status[:free].should eql false status[:clean].should eql true @@ -82,7 +82,7 @@ end it "yields Hashes" do - @xdes0.each_page_status.to_a.map { |v| v[1].class }.uniq.should eql [Hash] + @xdes0.each_page_status.to_a.map { |v| v[1].class }.uniq.should eql [Innodb::Xdes::PageStatus] end it "yields Hashes with the right keys and values" do diff --git a/spec/innodb_spec.rb b/spec/innodb_spec.rb index edede250..d20ec81d 100644 --- a/spec/innodb_spec.rb +++ b/spec/innodb_spec.rb @@ -1,4 +1,4 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true require "spec_helper" diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index bf97e2d8..6b6c142b 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,12 +1,12 @@ -# -*- encoding : utf-8 -*- +# frozen_string_literal: true -require File.join(File.dirname(__FILE__), '..', 'lib', 'innodb') +require File.join(File.dirname(__FILE__), "..", "lib", "innodb") RSpec.configure do |config| # Enable the below to allow easier fixing of deprecated RSpec syntax. - #config.raise_errors_for_deprecations! + # config.raise_errors_for_deprecations! config.expect_with :rspec do |c| - c.syntax = [:should, :expect] + c.syntax = %i[should expect] end end