diff --git a/lib/matplotlib/afm.py b/lib/matplotlib/afm.py
index 30d335879..3a06c4618 100644
--- a/lib/matplotlib/afm.py
+++ b/lib/matplotlib/afm.py
@@ -189,23 +189,22 @@ def _parse_char_metrics(fh):
     ascii_d = {}
     name_d = {}
     for line in fh:
-        line = line.rstrip().decode('ascii')  # Convert from byte-literal
-        if line.startswith('EndCharMetrics'):
+        line = line.rstrip()
+        if line.startswith(b'EndCharMetrics'):
             return ascii_d, name_d
         # Split the metric line into a dictionary, keyed by metric identifiers
-        vals = dict(s.strip().split(' ', 1) for s in line.split(';') if s)
+        vals = dict(s.strip().split(b' ', 1) for s in line.split(b';') if s)
         # There may be other metrics present, but only these are needed
-        if not {'C', 'WX', 'N', 'B'}.issubset(vals):
+        if not {b'C', b'WX', b'N', b'B'}.issubset(vals):
             raise RuntimeError('Bad char metrics line: %s' % line)
-        num = _to_int(vals['C'])
-        wx = _to_float(vals['WX'])
-        name = vals['N']
-        bbox = _to_list_of_floats(vals['B'])
-        bbox = list(map(int, bbox))
+        num = _to_int(vals[b'C'])
+        wx = _to_float(vals[b'WX'])
+        name = vals[b'N'].decode('ascii')
+        bbox = _to_list_of_ints(vals[b'B'])
         # Workaround: If the character name is 'Euro', give it the
         # corresponding character code, according to WinAnsiEncoding (see PDF
         # Reference).
-        if name == 'Euro':
+        if name == b'Euro':
             num = 128
         if num != -1:
             ascii_d[num] = (wx, name, bbox)