From 8782bf8cb0837ae1f1b07dfc635d634f26154cd6 Mon Sep 17 00:00:00 2001 From: Andrey Maltsev Date: Tue, 4 Apr 2023 09:23:29 +0000 Subject: [PATCH 1/4] Update test_hashlib.py from Cpython v3.11.2 --- Lib/test/test_hashlib.py | 65 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 5c2663e8ee..da748b2e78 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -10,6 +10,7 @@ from binascii import unhexlify import hashlib import importlib +import io import itertools import os import sys @@ -20,6 +21,7 @@ from test import support from test.support import _4G, bigmemtest from test.support.import_helper import import_fresh_module +from test.support import os_helper from test.support import threading_helper from test.support import warnings_helper from http.client import HTTPException @@ -102,8 +104,7 @@ class HashLibTestCase(unittest.TestCase): 'sha384', 'SHA384', 'sha512', 'SHA512', 'blake2b', 'blake2s', 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', - 'shake_128', 'shake_256' - ) + 'shake_128', 'shake_256') shakes = {'shake_128', 'shake_256'} @@ -384,6 +385,36 @@ def check(self, name, data, hexdigest, shake=False, **kwargs): if not shake: self.assertEqual(len(digest), m.digest_size) + if not shake and kwargs.get("key") is None: + # skip shake and blake2 extended parameter tests + self.check_file_digest(name, data, hexdigest) + + def check_file_digest(self, name, data, hexdigest): + hexdigest = hexdigest.lower() + try: + hashlib.new(name) + except ValueError: + # skip, algorithm is blocked by security policy. + return + digests = [name] + digests.extend(self.constructors_to_test[name]) + + with open(os_helper.TESTFN, "wb") as f: + f.write(data) + + try: + for digest in digests: + buf = io.BytesIO(data) + buf.seek(0) + self.assertEqual( + hashlib.file_digest(buf, digest).hexdigest(), hexdigest + ) + with open(os_helper.TESTFN, "rb") as f: + digestobj = hashlib.file_digest(f, digest) + self.assertEqual(digestobj.hexdigest(), hexdigest) + finally: + os.unlink(os_helper.TESTFN) + def check_no_unicode(self, algorithm_name): # Unicode objects are not allowed as input. constructors = self.constructors_to_test[algorithm_name] @@ -898,6 +929,7 @@ def test_gil(self): ) @threading_helper.reap_threads + @threading_helper.requires_working_threading() def test_threaded_hashing(self): # Updating the same hash object from several threads at once # using data chunk sizes containing the same byte sequences. @@ -1142,6 +1174,35 @@ def test_normalized_name(self): self.assertNotIn("blake2b512", hashlib.algorithms_available) self.assertNotIn("sha3-512", hashlib.algorithms_available) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_file_digest(self): + data = b'a' * 65536 + d1 = hashlib.sha256() + self.addCleanup(os.unlink, os_helper.TESTFN) + with open(os_helper.TESTFN, "wb") as f: + for _ in range(10): + d1.update(data) + f.write(data) + + with open(os_helper.TESTFN, "rb") as f: + d2 = hashlib.file_digest(f, hashlib.sha256) + + self.assertEqual(d1.hexdigest(), d2.hexdigest()) + self.assertEqual(d1.name, d2.name) + self.assertIs(type(d1), type(d2)) + + with self.assertRaises(ValueError): + hashlib.file_digest(None, "sha256") + + with self.assertRaises(ValueError): + with open(os_helper.TESTFN, "r") as f: + hashlib.file_digest(f, "sha256") + + with self.assertRaises(ValueError): + with open(os_helper.TESTFN, "wb") as f: + hashlib.file_digest(f, "sha256") + if __name__ == "__main__": unittest.main() From 508cf6b61dba29a4bc459784627baa3c75cf87a3 Mon Sep 17 00:00:00 2001 From: Andrey Maltsev Date: Sun, 9 Apr 2023 13:36:02 +0000 Subject: [PATCH 2/4] refactoring hashlib for tests --- Lib/hashlib.py | 316 +++++++++++++++++++++++++++++++++++++++ Lib/test/test_hashlib.py | 25 ++-- stdlib/src/blake2.rs | 20 +++ stdlib/src/hashlib.rs | 78 +++++----- stdlib/src/lib.rs | 16 +- stdlib/src/md5.rs | 15 ++ stdlib/src/sha1.rs | 15 ++ stdlib/src/sha256.rs | 20 +++ stdlib/src/sha3.rs | 46 ++++++ stdlib/src/sha512.rs | 20 +++ 10 files changed, 518 insertions(+), 53 deletions(-) create mode 100644 Lib/hashlib.py create mode 100644 stdlib/src/blake2.rs create mode 100644 stdlib/src/md5.rs create mode 100644 stdlib/src/sha1.rs create mode 100644 stdlib/src/sha256.rs create mode 100644 stdlib/src/sha3.rs create mode 100644 stdlib/src/sha512.rs diff --git a/Lib/hashlib.py b/Lib/hashlib.py new file mode 100644 index 0000000000..52a4f8a77b --- /dev/null +++ b/Lib/hashlib.py @@ -0,0 +1,316 @@ +#. Copyright (C) 2005-2010 Gregory P. Smith (greg@krypto.org) +# Licensed to PSF under a Contributor Agreement. +# + +__doc__ = """hashlib module - A common interface to many hash functions. + +new(name, data=b'', **kwargs) - returns a new hash object implementing the + given hash function; initializing the hash + using the given binary data. + +Named constructor functions are also available, these are faster +than using new(name): + +md5(), sha1(), sha224(), sha256(), sha384(), sha512(), blake2b(), blake2s(), +sha3_224, sha3_256, sha3_384, sha3_512, shake_128, and shake_256. + +More algorithms may be available on your platform but the above are guaranteed +to exist. See the algorithms_guaranteed and algorithms_available attributes +to find out what algorithm names can be passed to new(). + +NOTE: If you want the adler32 or crc32 hash functions they are available in +the zlib module. + +Choose your hash function wisely. Some have known collision weaknesses. +sha384 and sha512 will be slow on 32 bit platforms. + +Hash objects have these methods: + - update(data): Update the hash object with the bytes in data. Repeated calls + are equivalent to a single call with the concatenation of all + the arguments. + - digest(): Return the digest of the bytes passed to the update() method + so far as a bytes object. + - hexdigest(): Like digest() except the digest is returned as a string + of double length, containing only hexadecimal digits. + - copy(): Return a copy (clone) of the hash object. This can be used to + efficiently compute the digests of datas that share a common + initial substring. + +For example, to obtain the digest of the byte string 'Nobody inspects the +spammish repetition': + + >>> import hashlib + >>> m = hashlib.md5() + >>> m.update(b"Nobody inspects") + >>> m.update(b" the spammish repetition") + >>> m.digest() + b'\\xbbd\\x9c\\x83\\xdd\\x1e\\xa5\\xc9\\xd9\\xde\\xc9\\xa1\\x8d\\xf0\\xff\\xe9' + +More condensed: + + >>> hashlib.sha224(b"Nobody inspects the spammish repetition").hexdigest() + 'a4337bc45a8fc544c03f52dc550cd6e1e87021bc896588bd79e901e2' + +""" + +# This tuple and __get_builtin_constructor() must be modified if a new +# always available algorithm is added. +__always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', + 'blake2b', 'blake2s', + 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', + 'shake_128', 'shake_256') + + +algorithms_guaranteed = set(__always_supported) +algorithms_available = set(__always_supported) + +__all__ = __always_supported + ('new', 'algorithms_guaranteed', + 'algorithms_available', 'pbkdf2_hmac', 'file_digest') + + +__builtin_constructor_cache = {} + +# Prefer our blake2 implementation +# OpenSSL 1.1.0 comes with a limited implementation of blake2b/s. The OpenSSL +# implementations neither support keyed blake2 (blake2 MAC) nor advanced +# features like salt, personalization, or tree hashing. OpenSSL hash-only +# variants are available as 'blake2b512' and 'blake2s256', though. +__block_openssl_constructor = { + 'blake2b', 'blake2s', +} + +def __get_builtin_constructor(name): + cache = __builtin_constructor_cache + constructor = cache.get(name) + if constructor is not None: + return constructor + try: + if name in {'SHA1', 'sha1'}: + import _sha1 + cache['SHA1'] = cache['sha1'] = _sha1.sha1 + elif name in {'MD5', 'md5'}: + import _md5 + cache['MD5'] = cache['md5'] = _md5.md5 + elif name in {'SHA256', 'sha256', 'SHA224', 'sha224'}: + import _sha256 + cache['SHA224'] = cache['sha224'] = _sha256.sha224 + cache['SHA256'] = cache['sha256'] = _sha256.sha256 + elif name in {'SHA512', 'sha512', 'SHA384', 'sha384'}: + import _sha512 + cache['SHA384'] = cache['sha384'] = _sha512.sha384 + cache['SHA512'] = cache['sha512'] = _sha512.sha512 + elif name in {'blake2b', 'blake2s'}: + import _blake2 + cache['blake2b'] = _blake2.blake2b + cache['blake2s'] = _blake2.blake2s + elif name in {'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512'}: + import _sha3 + cache['sha3_224'] = _sha3.sha3_224 + cache['sha3_256'] = _sha3.sha3_256 + cache['sha3_384'] = _sha3.sha3_384 + cache['sha3_512'] = _sha3.sha3_512 + elif name in {'shake_128', 'shake_256'}: + import _sha3 + cache['shake_128'] = _sha3.shake_128 + cache['shake_256'] = _sha3.shake_256 + except ImportError: + pass # no extension module, this hash is unsupported.''' + + constructor = cache.get(name) + if constructor is not None: + return constructor + + raise ValueError('unsupported hash type ' + name) + + +def __get_openssl_constructor(name): + if name in __block_openssl_constructor: + # Prefer our builtin blake2 implementation. + return __get_builtin_constructor(name) + try: + # MD5, SHA1, and SHA2 are in all supported OpenSSL versions + # SHA3/shake are available in OpenSSL 1.1.1+ + f = getattr(_hashlib, 'openssl_' + name) + # Allow the C module to raise ValueError. The function will be + # defined but the hash not actually available. Don't fall back to + # builtin if the current security policy blocks a digest, bpo#40695. + f(usedforsecurity=False) + # Use the C function directly (very fast) + return f + except (AttributeError, ValueError): + return __get_builtin_constructor(name) + + +def __py_new(name, data=b'', **kwargs): + """new(name, data=b'', **kwargs) - Return a new hashing object using the + named algorithm; optionally initialized with data (which must be + a bytes-like object). + """ + return __get_builtin_constructor(name)(data, **kwargs) + + +def __hash_new(name, data=b'', **kwargs): + """new(name, data=b'') - Return a new hashing object using the named algorithm; + optionally initialized with data (which must be a bytes-like object). + """ + if name in __block_openssl_constructor: + # Prefer our builtin blake2 implementation. + return __get_builtin_constructor(name)(data, **kwargs) + try: + return _hashlib.new(name, data, **kwargs) + except ValueError: + # If the _hashlib module (OpenSSL) doesn't support the named + # hash, try using our builtin implementations. + # This allows for SHA224/256 and SHA384/512 support even though + # the OpenSSL library prior to 0.9.8 doesn't provide them. + return __get_builtin_constructor(name)(data) + + +try: + import _hashlib + new = __hash_new + __get_hash = __get_openssl_constructor + # TODO: RUSTPYTHON set in _hashlib instance PyFrozenSet algorithms_available + '''algorithms_available = algorithms_available.union( + _hashlib.openssl_md_meth_names)''' +except ImportError: + _hashlib = None + new = __py_new + __get_hash = __get_builtin_constructor + +try: + # OpenSSL's PKCS5_PBKDF2_HMAC requires OpenSSL 1.0+ with HMAC and SHA + from _hashlib import pbkdf2_hmac +except ImportError: + from warnings import warn as _warn + _trans_5C = bytes((x ^ 0x5C) for x in range(256)) + _trans_36 = bytes((x ^ 0x36) for x in range(256)) + + def pbkdf2_hmac(hash_name, password, salt, iterations, dklen=None): + """Password based key derivation function 2 (PKCS #5 v2.0) + + This Python implementations based on the hmac module about as fast + as OpenSSL's PKCS5_PBKDF2_HMAC for short passwords and much faster + for long passwords. + """ + _warn( + "Python implementation of pbkdf2_hmac() is deprecated.", + category=DeprecationWarning, + stacklevel=2 + ) + if not isinstance(hash_name, str): + raise TypeError(hash_name) + + if not isinstance(password, (bytes, bytearray)): + password = bytes(memoryview(password)) + if not isinstance(salt, (bytes, bytearray)): + salt = bytes(memoryview(salt)) + + # Fast inline HMAC implementation + inner = new(hash_name) + outer = new(hash_name) + blocksize = getattr(inner, 'block_size', 64) + if len(password) > blocksize: + password = new(hash_name, password).digest() + password = password + b'\x00' * (blocksize - len(password)) + inner.update(password.translate(_trans_36)) + outer.update(password.translate(_trans_5C)) + + def prf(msg, inner=inner, outer=outer): + # PBKDF2_HMAC uses the password as key. We can re-use the same + # digest objects and just update copies to skip initialization. + icpy = inner.copy() + ocpy = outer.copy() + icpy.update(msg) + ocpy.update(icpy.digest()) + return ocpy.digest() + + if iterations < 1: + raise ValueError(iterations) + if dklen is None: + dklen = outer.digest_size + if dklen < 1: + raise ValueError(dklen) + + dkey = b'' + loop = 1 + from_bytes = int.from_bytes + while len(dkey) < dklen: + prev = prf(salt + loop.to_bytes(4)) + # endianness doesn't matter here as long to / from use the same + rkey = from_bytes(prev) + for i in range(iterations - 1): + prev = prf(prev) + # rkey = rkey ^ prev + rkey ^= from_bytes(prev) + loop += 1 + dkey += rkey.to_bytes(inner.digest_size) + + return dkey[:dklen] + +try: + # OpenSSL's scrypt requires OpenSSL 1.1+ + from _hashlib import scrypt +except ImportError: + pass + + +def file_digest(fileobj, digest, /, *, _bufsize=2**18): + """Hash the contents of a file-like object. Returns a digest object. + + *fileobj* must be a file-like object opened for reading in binary mode. + It accepts file objects from open(), io.BytesIO(), and SocketIO objects. + The function may bypass Python's I/O and use the file descriptor *fileno* + directly. + + *digest* must either be a hash algorithm name as a *str*, a hash + constructor, or a callable that returns a hash object. + """ + # On Linux we could use AF_ALG sockets and sendfile() to archive zero-copy + # hashing with hardware acceleration. + if isinstance(digest, str): + digestobj = new(digest) + else: + digestobj = digest() + + if hasattr(fileobj, "getbuffer"): + # io.BytesIO object, use zero-copy buffer + digestobj.update(fileobj.getbuffer()) + return digestobj + + # Only binary files implement readinto(). + if not ( + hasattr(fileobj, "readinto") + and hasattr(fileobj, "readable") + and fileobj.readable() + ): + raise ValueError( + f"'{fileobj!r}' is not a file-like object in binary reading mode." + ) + + # binary file, socket.SocketIO object + # Note: socket I/O uses different syscalls than file I/O. + buf = bytearray(_bufsize) # Reusable buffer to reduce allocations. + view = memoryview(buf) + while True: + size = fileobj.readinto(buf) + if size == 0: + break # EOF + digestobj.update(view[:size]) + + return digestobj + + +for __func_name in __always_supported: + # try them all, some may not work due to the OpenSSL + # version not supporting that algorithm. + try: + globals()[__func_name] = __get_hash(__func_name) + except ValueError: + import logging + logging.exception('code for hash %s was not found.', __func_name) + + +# Cleanup locals() +del __always_supported, __func_name, __get_hash +del __py_new, __hash_new, __get_openssl_constructor diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index da748b2e78..b712a0e71d 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -215,15 +215,11 @@ def test_hash_array(self): else: c.hexdigest() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_algorithms_guaranteed(self): self.assertEqual(hashlib.algorithms_guaranteed, set(_algo for _algo in self.supported_hash_names if _algo.islower())) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_algorithms_available(self): self.assertTrue(set(hashlib.algorithms_guaranteed). issubset(hashlib.algorithms_available)) @@ -263,8 +259,6 @@ def test_unknown_hash(self): def test_new_upper_to_lower(self): self.assertEqual(hashlib.new("SHA256").name, "sha256") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_get_builtin_constructor(self): get_builtin_constructor = getattr(hashlib, '__get_builtin_constructor') @@ -406,6 +400,9 @@ def check_file_digest(self, name, data, hexdigest): for digest in digests: buf = io.BytesIO(data) buf.seek(0) + '''self.assertEqual( + dir(hashlib), None + )''' self.assertEqual( hashlib.file_digest(buf, digest).hexdigest(), hexdigest ) @@ -506,6 +503,8 @@ def test_extra_sha3(self): self.check_sha3('shake_128', 256, 1344, b'\x1f') self.check_sha3('shake_256', 512, 1088, b'\x1f') + # TODO: RUSTPYTHON + @unittest.expectedFailure @requires_blake2 def test_blocksize_name_blake2(self): self.check_blocksize_name('blake2b', 128, 64) @@ -749,6 +748,8 @@ def selftest_seq(length, seed): outer.update(keyed.digest()) return outer.hexdigest() + # TODO: RUSTPYTHON expect class, not function + @unittest.expectedFailure @requires_blake2 def test_blake2b(self): self.check_blake2(hashlib.blake2b, 16, 16, 64, 64, (1<<64)-1) @@ -770,6 +771,8 @@ def test_case_blake2b_1(self): "ba80a53f981c4d0d6a2797b69f12f6e94c212f14685ac4b74b12bb6fdbffa2d1"+ "7d87c5392aab792dc252d5de4533cc9518d38aa8dbf1925ab92386edd4009923") + # TODO: RUSTPYTHON expect class, not function + @unittest.expectedFailure @requires_blake2 def test_case_blake2b_all_parameters(self): # This checks that all the parameters work in general, and also that @@ -794,6 +797,8 @@ def test_blake2b_vectors(self): key = bytes.fromhex(key) self.check('blake2b', msg, md, key=key) + # TODO: RUSTPYTHON expect class, not function + @unittest.expectedFailure @requires_blake2 def test_blake2s(self): self.check_blake2(hashlib.blake2s, 8, 8, 32, 32, (1<<48)-1) @@ -813,6 +818,8 @@ def test_case_blake2s_1(self): self.check('blake2s', b"abc", "508c5e8c327c14e2e1a72ba34eeb452f37458b209ed63a294d999b4c86675982") + # TODO: RUSTPYTHON + @unittest.expectedFailure @requires_blake2 def test_case_blake2s_all_parameters(self): # This checks that all the parameters work in general, and also that @@ -1114,8 +1121,6 @@ def _test_pbkdf2_hmac(self, pbkdf2, supported): iterations=1, dklen=None) self.assertEqual(out, self.pbkdf2_results['sha1'][0][0]) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(builtin_hashlib is None, "test requires builtin_hashlib") def test_pbkdf2_hmac_py(self): with warnings_helper.check_warnings(): @@ -1168,14 +1173,10 @@ def test_scrypt(self): hashlib.scrypt(b'password', salt=b'salt', n=2, r=8, p=1, dklen=dklen) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_normalized_name(self): self.assertNotIn("blake2b512", hashlib.algorithms_available) self.assertNotIn("sha3-512", hashlib.algorithms_available) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_file_digest(self): data = b'a' * 65536 d1 = hashlib.sha256() diff --git a/stdlib/src/blake2.rs b/stdlib/src/blake2.rs new file mode 100644 index 0000000000..638069f670 --- /dev/null +++ b/stdlib/src/blake2.rs @@ -0,0 +1,20 @@ +// spell-checker:ignore usedforsecurity HASHXOF + +pub(crate) use _blake2::make_module; + +#[pymodule] +mod _blake2 { + use crate::hashlib::_hashlib::{BlakeHashArgs, HashWrapper, PyHasher}; + use crate::vm::{PyObjectRef, PyPayload, PyResult, VirtualMachine}; + use blake2::{Blake2b512, Blake2s256}; + + #[pyfunction(name = "blake2b")] + fn blake2b(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("blake2b", HashWrapper::new::(args.data)).into_pyobject(vm)) + } + + #[pyfunction(name = "blake2s")] + fn blake2s(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("blake2s", HashWrapper::new::(args.data)).into_pyobject(vm)) + } +} diff --git a/stdlib/src/hashlib.rs b/stdlib/src/hashlib.rs index 99c90480f4..3adcad6c1b 100644 --- a/stdlib/src/hashlib.rs +++ b/stdlib/src/hashlib.rs @@ -1,9 +1,9 @@ // spell-checker:ignore usedforsecurity HASHXOF -pub(crate) use hashlib::make_module; +pub(crate) use _hashlib::make_module; #[pymodule] -mod hashlib { +pub mod _hashlib { use crate::common::lock::PyRwLock; use crate::vm::{ builtins::{PyBytes, PyStrRef, PyTypeRef}, @@ -31,12 +31,11 @@ mod hashlib { usedforsecurity: bool, } - #[derive(FromArgs, Traverse)] + #[derive(FromArgs)] #[allow(unused)] - struct BlakeHashArgs { + pub struct BlakeHashArgs { #[pyarg(positional, optional)] - data: OptionalArg, - #[pytraverse(skip)] + pub data: OptionalArg, #[pyarg(named, default = "true")] usedforsecurity: bool, } @@ -50,12 +49,11 @@ mod hashlib { } } - #[derive(FromArgs, Traverse)] + #[derive(FromArgs)] #[allow(unused)] - struct HashArgs { + pub struct HashArgs { #[pyarg(any, optional)] - string: OptionalArg, - #[pytraverse(skip)] + pub string: OptionalArg, #[pyarg(named, default = "true")] usedforsecurity: bool, } @@ -84,11 +82,11 @@ mod hashlib { } #[pyattr] - #[pyclass(module = "hashlib", name = "HASH")] + #[pyclass(module = "_hashlib", name = "HASH")] #[derive(PyPayload)] - struct PyHasher { - name: String, - ctx: PyRwLock, + pub struct PyHasher { + pub name: String, + pub ctx: PyRwLock, } impl std::fmt::Debug for PyHasher { @@ -99,7 +97,7 @@ mod hashlib { #[pyclass] impl PyHasher { - fn new(name: &str, d: HashWrapper) -> Self { + pub fn new(name: &str, d: HashWrapper) -> Self { PyHasher { name: name.to_owned(), ctx: PyRwLock::new(d), @@ -108,7 +106,7 @@ mod hashlib { #[pyslot] fn slot_new(_cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error("cannot create 'hashlib.HASH' instances".into())) + Err(vm.new_type_error("cannot create '_hashlib.HASH' instances".into())) } #[pygetset] @@ -148,9 +146,9 @@ mod hashlib { } #[pyattr] - #[pyclass(module = "hashlib", name = "HASHXOF")] + #[pyclass(module = "_hashlib", name = "HASHXOF")] #[derive(PyPayload)] - struct PyHasherXof { + pub struct PyHasherXof { name: String, ctx: PyRwLock, } @@ -163,7 +161,7 @@ mod hashlib { #[pyclass] impl PyHasherXof { - fn new(name: &str, d: HashXofWrapper) -> Self { + pub fn new(name: &str, d: HashXofWrapper) -> Self { PyHasherXof { name: name.to_owned(), ctx: PyRwLock::new(d), @@ -172,7 +170,7 @@ mod hashlib { #[pyslot] fn slot_new(_cls: PyTypeRef, _args: FuncArgs, vm: &VirtualMachine) -> PyResult { - Err(vm.new_type_error("cannot create 'hashlib.HASHXOF' instances".into())) + Err(vm.new_type_error("cannot create '_hashlib.HASHXOF' instances".into())) } #[pygetset] @@ -232,90 +230,90 @@ mod hashlib { } } - #[pyfunction] + #[pyfunction(name = "openssl_md5")] fn md5(args: HashArgs) -> PyHasher { PyHasher::new("md5", HashWrapper::new::(args.string)) } - #[pyfunction] + #[pyfunction(name = "openssl_sha1")] fn sha1(args: HashArgs) -> PyHasher { PyHasher::new("sha1", HashWrapper::new::(args.string)) } - #[pyfunction] + #[pyfunction(name = "openssl_sha224")] fn sha224(args: HashArgs) -> PyHasher { PyHasher::new("sha224", HashWrapper::new::(args.string)) } - #[pyfunction] + #[pyfunction(name = "openssl_sha256")] fn sha256(args: HashArgs) -> PyHasher { PyHasher::new("sha256", HashWrapper::new::(args.string)) } - #[pyfunction] + #[pyfunction(name = "openssl_sha384")] fn sha384(args: HashArgs) -> PyHasher { PyHasher::new("sha384", HashWrapper::new::(args.string)) } - #[pyfunction] + #[pyfunction(name = "openssl_sha512")] fn sha512(args: HashArgs) -> PyHasher { PyHasher::new("sha512", HashWrapper::new::(args.string)) } - #[pyfunction] + #[pyfunction(name = "openssl_sha3_224")] fn sha3_224(args: HashArgs) -> PyHasher { PyHasher::new("sha3_224", HashWrapper::new::(args.string)) } - #[pyfunction] + #[pyfunction(name = "openssl_sha3_256")] fn sha3_256(args: HashArgs) -> PyHasher { PyHasher::new("sha3_256", HashWrapper::new::(args.string)) } - #[pyfunction] + #[pyfunction(name = "openssl_sha3_384")] fn sha3_384(args: HashArgs) -> PyHasher { PyHasher::new("sha3_384", HashWrapper::new::(args.string)) } - #[pyfunction] + #[pyfunction(name = "openssl_sha3_512")] fn sha3_512(args: HashArgs) -> PyHasher { PyHasher::new("sha3_512", HashWrapper::new::(args.string)) } - #[pyfunction] + #[pyfunction(name = "openssl_shake_128")] fn shake_128(args: HashArgs) -> PyHasherXof { PyHasherXof::new("shake_128", HashXofWrapper::new_shake_128(args.string)) } - #[pyfunction] + #[pyfunction(name = "openssl_shake_256")] fn shake_256(args: HashArgs) -> PyHasherXof { PyHasherXof::new("shake_256", HashXofWrapper::new_shake_256(args.string)) } - #[pyfunction] + #[pyfunction(name = "openssl_blake2b")] fn blake2b(args: BlakeHashArgs) -> PyHasher { PyHasher::new("blake2b", HashWrapper::new::(args.data)) } - #[pyfunction] + #[pyfunction(name = "openssl_blake2s")] fn blake2s(args: BlakeHashArgs) -> PyHasher { PyHasher::new("blake2s", HashWrapper::new::(args.data)) } - trait ThreadSafeDynDigest: DynClone + DynDigest + Sync + Send {} + pub trait ThreadSafeDynDigest: DynClone + DynDigest + Sync + Send {} impl ThreadSafeDynDigest for T where T: DynClone + DynDigest + Sync + Send {} clone_trait_object!(ThreadSafeDynDigest); /// Generic wrapper patching around the hashing libraries. #[derive(Clone)] - struct HashWrapper { + pub struct HashWrapper { block_size: usize, inner: Box, } impl HashWrapper { - fn new(data: OptionalArg) -> Self + pub fn new(data: OptionalArg) -> Self where D: ThreadSafeDynDigest + BlockSizeUser + Default + 'static, { @@ -348,13 +346,13 @@ mod hashlib { } #[derive(Clone)] - enum HashXofWrapper { + pub enum HashXofWrapper { Shake128(Shake128), Shake256(Shake256), } impl HashXofWrapper { - fn new_shake_128(data: OptionalArg) -> Self { + pub fn new_shake_128(data: OptionalArg) -> Self { let mut h = HashXofWrapper::Shake128(Shake128::default()); if let OptionalArg::Present(d) = data { d.with_ref(|bytes| h.update(bytes)); @@ -362,7 +360,7 @@ mod hashlib { h } - fn new_shake_256(data: OptionalArg) -> Self { + pub fn new_shake_256(data: OptionalArg) -> Self { let mut h = HashXofWrapper::Shake256(Shake256::default()); if let OptionalArg::Present(d) = data { d.with_ref(|bytes| h.update(bytes)); diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs index c78f744491..97aaa8b89b 100644 --- a/stdlib/src/lib.rs +++ b/stdlib/src/lib.rs @@ -13,7 +13,15 @@ mod contextvars; mod csv; mod dis; mod gc; + +mod blake2; mod hashlib; +mod md5; +mod sha1; +mod sha256; +mod sha3; +mod sha512; + mod json; #[cfg(not(any(target_os = "ios", target_os = "android", target_arch = "wasm32")))] mod locale; @@ -100,7 +108,13 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "_csv" => csv::make_module, "_dis" => dis::make_module, "gc" => gc::make_module, - "hashlib" => hashlib::make_module, + "_hashlib" => hashlib::make_module, + "_sha1" => sha1::make_module, + "_sha3" => sha3::make_module, + "_sha256" => sha256::make_module, + "_sha512" => sha512::make_module, + "_md5" => md5::make_module, + "_blake2" => blake2::make_module, "_json" => json::make_module, "math" => math::make_module, "pyexpat" => pyexpat::make_module, diff --git a/stdlib/src/md5.rs b/stdlib/src/md5.rs new file mode 100644 index 0000000000..387817ef0f --- /dev/null +++ b/stdlib/src/md5.rs @@ -0,0 +1,15 @@ +// spell-checker:ignore usedforsecurity HASHXOF + +pub(crate) use _md5::make_module; + +#[pymodule] +mod _md5 { + use crate::hashlib::_hashlib::{HashArgs, HashWrapper, PyHasher}; + use crate::vm::{PyObjectRef, PyPayload, PyResult, VirtualMachine}; + use md5::Md5; + + #[pyfunction(name = "md5")] + fn md5(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("md5", HashWrapper::new::(args.string)).into_pyobject(vm)) + } +} diff --git a/stdlib/src/sha1.rs b/stdlib/src/sha1.rs new file mode 100644 index 0000000000..cc50b76ed3 --- /dev/null +++ b/stdlib/src/sha1.rs @@ -0,0 +1,15 @@ +// spell-checker:ignore usedforsecurity HASHXOF + +pub(crate) use _sha1::make_module; + +#[pymodule] +mod _sha1 { + use crate::vm::{PyObjectRef, PyPayload, PyResult, VirtualMachine}; + use sha1::Sha1; + + use crate::hashlib::_hashlib::{HashArgs, HashWrapper, PyHasher}; + #[pyfunction(name = "sha1")] + fn sha1(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("sha1", HashWrapper::new::(args.string)).into_pyobject(vm)) + } +} diff --git a/stdlib/src/sha256.rs b/stdlib/src/sha256.rs new file mode 100644 index 0000000000..1695e32d76 --- /dev/null +++ b/stdlib/src/sha256.rs @@ -0,0 +1,20 @@ +// spell-checker:ignore usedforsecurity HASHXOF + +pub(crate) use _sha256::make_module; + +#[pymodule] +mod _sha256 { + use crate::hashlib::_hashlib::{HashArgs, HashWrapper, PyHasher}; + use crate::vm::{PyObjectRef, PyPayload, PyResult, VirtualMachine}; + use sha2::{Sha224, Sha256}; + + #[pyfunction(name = "sha224")] + fn sha224(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("sha224", HashWrapper::new::(args.string)).into_pyobject(vm)) + } + + #[pyfunction(name = "sha256")] + fn sha256(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("sha256", HashWrapper::new::(args.string)).into_pyobject(vm)) + } +} diff --git a/stdlib/src/sha3.rs b/stdlib/src/sha3.rs new file mode 100644 index 0000000000..76666a6cf2 --- /dev/null +++ b/stdlib/src/sha3.rs @@ -0,0 +1,46 @@ +// spell-checker:ignore usedforsecurity HASHXOF + +pub(crate) use _sha3::make_module; + +#[pymodule] +mod _sha3 { + use crate::hashlib::_hashlib::{HashArgs, HashWrapper, HashXofWrapper, PyHasher, PyHasherXof}; + use crate::vm::{PyObjectRef, PyPayload, PyResult, VirtualMachine}; + use sha3::{Sha3_224, Sha3_256, Sha3_384, Sha3_512}; + + #[pyfunction(name = "sha3_224")] + fn sha3_224(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("sha3_224", HashWrapper::new::(args.string)).into_pyobject(vm)) + } + + #[pyfunction(name = "sha3_256")] + fn sha3_256(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("sha3_256", HashWrapper::new::(args.string)).into_pyobject(vm)) + } + + #[pyfunction(name = "sha3_384")] + fn sha3_384(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("sha3_384", HashWrapper::new::(args.string)).into_pyobject(vm)) + } + + #[pyfunction(name = "sha3_512")] + fn sha3_512(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("sha3_512", HashWrapper::new::(args.string)).into_pyobject(vm)) + } + + #[pyfunction(name = "shake_128")] + fn shake_128(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok( + PyHasherXof::new("shake_128", HashXofWrapper::new_shake_128(args.string)) + .into_pyobject(vm), + ) + } + + #[pyfunction(name = "shake_256")] + fn shake_256(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok( + PyHasherXof::new("shake_256", HashXofWrapper::new_shake_256(args.string)) + .into_pyobject(vm), + ) + } +} diff --git a/stdlib/src/sha512.rs b/stdlib/src/sha512.rs new file mode 100644 index 0000000000..bb93fc9dba --- /dev/null +++ b/stdlib/src/sha512.rs @@ -0,0 +1,20 @@ +// spell-checker:ignore usedforsecurity HASHXOF + +pub(crate) use _sha512::make_module; + +#[pymodule] +mod _sha512 { + use crate::hashlib::_hashlib::{HashArgs, HashWrapper, PyHasher}; + use crate::vm::{PyObjectRef, PyPayload, PyResult, VirtualMachine}; + use sha2::{Sha384, Sha512}; + + #[pyfunction(name = "sha384")] + fn sha384(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("sha384", HashWrapper::new::(args.string)).into_pyobject(vm)) + } + + #[pyfunction(name = "sha512")] + fn sha512(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(PyHasher::new("sha512", HashWrapper::new::(args.string)).into_pyobject(vm)) + } +} From 514014f4fc3aefaa702da7164ba67bbde533ace8 Mon Sep 17 00:00:00 2001 From: Andrey Maltsev Date: Mon, 17 Apr 2023 10:07:07 +0000 Subject: [PATCH 3/4] add compare digest --- Cargo.lock | 10 ++++ Lib/test/test_enum.py | 4 +- Lib/test/test_hashlib.py | 12 +++-- Lib/test/test_hmac.py | 42 +++++++++++++++ stdlib/Cargo.toml | 1 + stdlib/src/blake2.rs | 17 +++--- stdlib/src/hashlib.rs | 108 +++++++++++++++++++++++++++------------ stdlib/src/lib.rs | 3 +- stdlib/src/md5.rs | 13 ++--- stdlib/src/sha1.rs | 13 ++--- stdlib/src/sha256.rs | 19 +++---- stdlib/src/sha3.rs | 54 +++++++++----------- 12 files changed, 186 insertions(+), 110 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 83cf55763c..23dcc4b2dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -950,6 +950,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "iana-time-zone" version = "0.1.53" @@ -2137,6 +2146,7 @@ dependencies = [ "foreign-types-shared", "gethostname", "hex", + "hmac", "itertools", "libc", "libsqlite3-sys", diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 053c7bff22..92cb26e458 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -2889,9 +2889,7 @@ class Color(StrMixin, AllMixin, IntFlag): self.assertEqual(Color.ALL.value, 7) self.assertEqual(str(Color.BLUE), 'blue') - # TODO: RUSTPYTHON - @unittest.expectedFailure - @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, inconsistent test result on Windows due to threading") + @unittest.skipIf(sys.platform == "win32" or sys.platform.startswith("linux"), "TODO: RUSTPYTHON, inconsistent test result on Windows due to threading") @threading_helper.reap_threads def test_unique_composite(self): # override __eq__ to be identity only diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index b712a0e71d..6f12aaa936 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -503,7 +503,7 @@ def test_extra_sha3(self): self.check_sha3('shake_128', 256, 1344, b'\x1f') self.check_sha3('shake_256', 512, 1088, b'\x1f') - # TODO: RUSTPYTHON + # TODO: RUSTPYTHON implement all blake2 params @unittest.expectedFailure @requires_blake2 def test_blocksize_name_blake2(self): @@ -748,7 +748,7 @@ def selftest_seq(length, seed): outer.update(keyed.digest()) return outer.hexdigest() - # TODO: RUSTPYTHON expect class, not function + # TODO: RUSTPYTHON add to constructor const value @unittest.expectedFailure @requires_blake2 def test_blake2b(self): @@ -771,7 +771,7 @@ def test_case_blake2b_1(self): "ba80a53f981c4d0d6a2797b69f12f6e94c212f14685ac4b74b12bb6fdbffa2d1"+ "7d87c5392aab792dc252d5de4533cc9518d38aa8dbf1925ab92386edd4009923") - # TODO: RUSTPYTHON expect class, not function + # TODO: RUSTPYTHON implement all blake2 fields @unittest.expectedFailure @requires_blake2 def test_case_blake2b_all_parameters(self): @@ -797,7 +797,7 @@ def test_blake2b_vectors(self): key = bytes.fromhex(key) self.check('blake2b', msg, md, key=key) - # TODO: RUSTPYTHON expect class, not function + # TODO: RUSTPYTHON add to constructor const value @unittest.expectedFailure @requires_blake2 def test_blake2s(self): @@ -818,7 +818,7 @@ def test_case_blake2s_1(self): self.check('blake2s', b"abc", "508c5e8c327c14e2e1a72ba34eeb452f37458b209ed63a294d999b4c86675982") - # TODO: RUSTPYTHON + # TODO: RUSTPYTHON implement all blake2 fields @unittest.expectedFailure @requires_blake2 def test_case_blake2s_all_parameters(self): @@ -1121,6 +1121,8 @@ def _test_pbkdf2_hmac(self, pbkdf2, supported): iterations=1, dklen=None) self.assertEqual(out, self.pbkdf2_results['sha1'][0][0]) + # TODO: RUSTPYTHON + @unittest.expectedFailure @unittest.skipIf(builtin_hashlib is None, "test requires builtin_hashlib") def test_pbkdf2_hmac_py(self): with warnings_helper.check_warnings(): diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 7cf99735ca..bc2e02528d 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -39,6 +39,8 @@ def wrapper(*args, **kwargs): class TestVectorsTestCase(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure def assert_hmac_internals( self, h, digest, hashname, digest_size, block_size ): @@ -48,6 +50,8 @@ def assert_hmac_internals( self.assertEqual(h.digest_size, digest_size) self.assertEqual(h.block_size, block_size) + # TODO: RUSTPYTHON + @unittest.expectedFailure def assert_hmac( self, key, data, digest, hashfunc, hashname, digest_size, block_size ): @@ -122,6 +126,8 @@ def assert_hmac( h, digest, hashname, digest_size, block_size ) + # TODO: RUSTPYTHON + @unittest.expectedFailure @hashlib_helper.requires_hashdigest('md5', openssl=True) def test_md5_vectors(self): # Test the HMAC module against test vectors from the RFC. @@ -164,6 +170,8 @@ def md5test(key, data, digest): b"and Larger Than One Block-Size Data"), "6f630fad67cda0ee1fb1f562db3aa53e") + # TODO: RUSTPYTHON + @unittest.expectedFailure @hashlib_helper.requires_hashdigest('sha1', openssl=True) def test_sha_vectors(self): def shatest(key, data, digest): @@ -323,18 +331,26 @@ def hmactest(key, data, hexdigests): '134676fb6de0446065c97440fa8c6a58', }) + # TODO: RUSTPYTHON + @unittest.expectedFailure @hashlib_helper.requires_hashdigest('sha224', openssl=True) def test_sha224_rfc4231(self): self._rfc4231_test_cases(hashlib.sha224, 'sha224', 28, 64) + # TODO: RUSTPYTHON + @unittest.expectedFailure @hashlib_helper.requires_hashdigest('sha256', openssl=True) def test_sha256_rfc4231(self): self._rfc4231_test_cases(hashlib.sha256, 'sha256', 32, 64) + # TODO: RUSTPYTHON + @unittest.expectedFailure @hashlib_helper.requires_hashdigest('sha384', openssl=True) def test_sha384_rfc4231(self): self._rfc4231_test_cases(hashlib.sha384, 'sha384', 48, 128) + # TODO: RUSTPYTHON + @unittest.expectedFailure @hashlib_helper.requires_hashdigest('sha512', openssl=True) def test_sha512_rfc4231(self): self._rfc4231_test_cases(hashlib.sha512, 'sha512', 64, 128) @@ -380,6 +396,8 @@ class ConstructorTestCase(unittest.TestCase): "6c845b47f52b3b47f6590c502db7825aad757bf4fadc8fa972f7cd2e76a5bdeb" ) + # TODO: RUSTPYTHON + @unittest.expectedFailure @hashlib_helper.requires_hashdigest('sha256') def test_normal(self): # Standard constructor call. @@ -402,6 +420,8 @@ def test_dot_new_with_str_key(self): with self.assertRaises(TypeError): h = hmac.new("key", digestmod='sha256') + # TODO: RUSTPYTHON + @unittest.expectedFailure @hashlib_helper.requires_hashdigest('sha256') def test_withtext(self): # Constructor call with text. @@ -411,6 +431,8 @@ def test_withtext(self): self.fail("Constructor call with text argument raised exception.") self.assertEqual(h.hexdigest(), self.expected) + # TODO: RUSTPYTHON + @unittest.expectedFailure @hashlib_helper.requires_hashdigest('sha256') def test_with_bytearray(self): try: @@ -420,6 +442,8 @@ def test_with_bytearray(self): self.fail("Constructor call with bytearray arguments raised exception.") self.assertEqual(h.hexdigest(), self.expected) + # TODO: RUSTPYTHON + @unittest.expectedFailure @hashlib_helper.requires_hashdigest('sha256') def test_with_memoryview_msg(self): try: @@ -428,6 +452,8 @@ def test_with_memoryview_msg(self): self.fail("Constructor call with memoryview msg raised exception.") self.assertEqual(h.hexdigest(), self.expected) + # TODO: RUSTPYTHON + @unittest.expectedFailure @hashlib_helper.requires_hashdigest('sha256') def test_withmodule(self): # Constructor call with text and digest module. @@ -436,6 +462,8 @@ def test_withmodule(self): except Exception: self.fail("Constructor call with hashlib.sha256 raised exception.") + # TODO: RUSTPYTHON + @unittest.expectedFailure @unittest.skipUnless(C_HMAC is not None, 'need _hashlib') def test_internal_types(self): # internal types like _hashlib.C_HMAC are not constructable @@ -443,6 +471,8 @@ def test_internal_types(self): with self.assertRaisesRegex(TypeError, "immutable type"): C_HMAC.value = None + # TODO: RUSTPYTHON + @unittest.expectedFailure @unittest.skipUnless(sha256_module is not None, 'need _sha256') def test_with_sha256_module(self): h = hmac.HMAC(b"key", b"hash this!", digestmod=sha256_module.sha256) @@ -455,6 +485,8 @@ def test_with_sha256_module(self): class SanityTestCase(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure @hashlib_helper.requires_hashdigest('sha256') def test_exercise_all_methods(self): # Exercising all methods once. @@ -496,6 +528,8 @@ def test_realcopy_old(self): "No real copy of the attribute 'outer'.") self.assertIs(h1._hmac, None) + # TODO: RUSTPYTHON + @unittest.expectedFailure @unittest.skipIf(_hashopenssl is None, "test requires _hashopenssl") @hashlib_helper.requires_hashdigest('sha256') def test_realcopy_hmac(self): @@ -504,6 +538,8 @@ def test_realcopy_hmac(self): h2 = h1.copy() self.assertTrue(id(h1._hmac) != id(h2._hmac)) + # TODO: RUSTPYTHON + @unittest.expectedFailure @hashlib_helper.requires_hashdigest('sha256') def test_equality(self): # Testing if the copy has the same digests. @@ -515,6 +551,8 @@ def test_equality(self): self.assertEqual(h1.hexdigest(), h2.hexdigest(), "Hexdigest of copy doesn't match original hexdigest.") + # TODO: RUSTPYTHON + @unittest.expectedFailure @hashlib_helper.requires_hashdigest('sha256') def test_equality_new(self): # Testing if the copy has the same digests with hmac.new(). @@ -532,6 +570,8 @@ def test_equality_new(self): class CompareDigestTestCase(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_hmac_compare_digest(self): self._test_compare_digest(hmac.compare_digest) if openssl_compare_digest is not None: @@ -542,6 +582,8 @@ def test_hmac_compare_digest(self): def test_operator_compare_digest(self): self._test_compare_digest(operator_compare_digest) + # TODO: RUSTPYTHON + @unittest.expectedFailure @unittest.skipIf(openssl_compare_digest is None, "test requires _hashlib") def test_openssl_compare_digest(self): self._test_compare_digest(openssl_compare_digest) diff --git a/stdlib/Cargo.toml b/stdlib/Cargo.toml index f53eb49211..edf288ab91 100644 --- a/stdlib/Cargo.toml +++ b/stdlib/Cargo.toml @@ -55,6 +55,7 @@ sha-1 = "0.10.0" sha2 = "0.10.2" sha3 = "0.10.1" blake2 = "0.10.4" +hmac = "0.12.1" ## unicode stuff unicode_names2 = { workspace = true } diff --git a/stdlib/src/blake2.rs b/stdlib/src/blake2.rs index 638069f670..9b7da3327c 100644 --- a/stdlib/src/blake2.rs +++ b/stdlib/src/blake2.rs @@ -4,17 +4,16 @@ pub(crate) use _blake2::make_module; #[pymodule] mod _blake2 { - use crate::hashlib::_hashlib::{BlakeHashArgs, HashWrapper, PyHasher}; - use crate::vm::{PyObjectRef, PyPayload, PyResult, VirtualMachine}; - use blake2::{Blake2b512, Blake2s256}; + use crate::hashlib::_hashlib::{local_blake2b, local_blake2s, BlakeHashArgs}; + use crate::vm::{PyPayload, PyResult, VirtualMachine}; - #[pyfunction(name = "blake2b")] - fn blake2b(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult { - Ok(PyHasher::new("blake2b", HashWrapper::new::(args.data)).into_pyobject(vm)) + #[pyfunction] + fn blake2b(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_blake2b(args).into_pyobject(vm)) } - #[pyfunction(name = "blake2s")] - fn blake2s(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult { - Ok(PyHasher::new("blake2s", HashWrapper::new::(args.data)).into_pyobject(vm)) + #[pyfunction] + fn blake2s(args: BlakeHashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_blake2s(args).into_pyobject(vm)) } } diff --git a/stdlib/src/hashlib.rs b/stdlib/src/hashlib.rs index 3adcad6c1b..6944c37f9d 100644 --- a/stdlib/src/hashlib.rs +++ b/stdlib/src/hashlib.rs @@ -7,7 +7,9 @@ pub mod _hashlib { use crate::common::lock::PyRwLock; use crate::vm::{ builtins::{PyBytes, PyStrRef, PyTypeRef}, - function::{ArgBytesLike, FuncArgs, OptionalArg}, + convert::ToPyObject, + function::{ArgBytesLike, ArgStrOrBytesLike, FuncArgs, OptionalArg}, + protocol::PyBuffer, PyObjectRef, PyPayload, PyResult, VirtualMachine, }; use blake2::{Blake2b512, Blake2s256}; @@ -19,14 +21,13 @@ pub mod _hashlib { use sha2::{Sha224, Sha256, Sha384, Sha512}; use sha3::{Sha3_224, Sha3_256, Sha3_384, Sha3_512, Shake128, Shake256}; - #[derive(FromArgs, Traverse)] + #[derive(FromArgs, Debug)] #[allow(unused)] struct NewHashArgs { #[pyarg(positional)] name: PyStrRef, #[pyarg(any, optional)] data: OptionalArg, - #[pytraverse(skip)] #[pyarg(named, default = "true")] usedforsecurity: bool, } @@ -49,7 +50,7 @@ pub mod _hashlib { } } - #[derive(FromArgs)] + #[derive(FromArgs, Debug)] #[allow(unused)] pub struct HashArgs { #[pyarg(any, optional)] @@ -97,7 +98,7 @@ pub mod _hashlib { #[pyclass] impl PyHasher { - pub fn new(name: &str, d: HashWrapper) -> Self { + fn new(name: &str, d: HashWrapper) -> Self { PyHasher { name: name.to_owned(), ctx: PyRwLock::new(d), @@ -161,7 +162,7 @@ pub mod _hashlib { #[pyclass] impl PyHasherXof { - pub fn new(name: &str, d: HashXofWrapper) -> Self { + fn new(name: &str, d: HashXofWrapper) -> Self { PyHasherXof { name: name.to_owned(), ctx: PyRwLock::new(d), @@ -212,94 +213,133 @@ pub mod _hashlib { #[pyfunction(name = "new")] fn hashlib_new(args: NewHashArgs, vm: &VirtualMachine) -> PyResult { match args.name.as_str().to_lowercase().as_str() { - "md5" => Ok(md5(args.into()).into_pyobject(vm)), - "sha1" => Ok(sha1(args.into()).into_pyobject(vm)), - "sha224" => Ok(sha224(args.into()).into_pyobject(vm)), - "sha256" => Ok(sha256(args.into()).into_pyobject(vm)), - "sha384" => Ok(sha384(args.into()).into_pyobject(vm)), - "sha512" => Ok(sha512(args.into()).into_pyobject(vm)), - "sha3_224" => Ok(sha3_224(args.into()).into_pyobject(vm)), - "sha3_256" => Ok(sha3_256(args.into()).into_pyobject(vm)), - "sha3_384" => Ok(sha3_384(args.into()).into_pyobject(vm)), - "sha3_512" => Ok(sha3_512(args.into()).into_pyobject(vm)), - "shake_128" => Ok(shake_128(args.into()).into_pyobject(vm)), - "shake_256" => Ok(shake_256(args.into()).into_pyobject(vm)), - "blake2b" => Ok(blake2b(args.into()).into_pyobject(vm)), - "blake2s" => Ok(blake2s(args.into()).into_pyobject(vm)), + "md5" => Ok(local_md5(args.into()).into_pyobject(vm)), + "sha1" => Ok(local_sha1(args.into()).into_pyobject(vm)), + "sha224" => Ok(local_sha224(args.into()).into_pyobject(vm)), + "sha256" => Ok(local_sha256(args.into()).into_pyobject(vm)), + "sha384" => Ok(local_sha384(args.into()).into_pyobject(vm)), + "sha512" => Ok(local_sha512(args.into()).into_pyobject(vm)), + "sha3_224" => Ok(local_sha3_224(args.into()).into_pyobject(vm)), + "sha3_256" => Ok(local_sha3_256(args.into()).into_pyobject(vm)), + "sha3_384" => Ok(local_sha3_384(args.into()).into_pyobject(vm)), + "sha3_512" => Ok(local_sha3_512(args.into()).into_pyobject(vm)), + "shake_128" => Ok(local_shake_128(args.into()).into_pyobject(vm)), + "shake_256" => Ok(local_shake_256(args.into()).into_pyobject(vm)), + "blake2b" => Ok(local_blake2b(args.into()).into_pyobject(vm)), + "blake2s" => Ok(local_blake2s(args.into()).into_pyobject(vm)), other => Err(vm.new_value_error(format!("Unknown hashing algorithm: {other}"))), } } #[pyfunction(name = "openssl_md5")] - fn md5(args: HashArgs) -> PyHasher { + pub fn local_md5(args: HashArgs) -> PyHasher { PyHasher::new("md5", HashWrapper::new::(args.string)) } #[pyfunction(name = "openssl_sha1")] - fn sha1(args: HashArgs) -> PyHasher { + pub fn local_sha1(args: HashArgs) -> PyHasher { PyHasher::new("sha1", HashWrapper::new::(args.string)) } #[pyfunction(name = "openssl_sha224")] - fn sha224(args: HashArgs) -> PyHasher { + pub fn local_sha224(args: HashArgs) -> PyHasher { PyHasher::new("sha224", HashWrapper::new::(args.string)) } #[pyfunction(name = "openssl_sha256")] - fn sha256(args: HashArgs) -> PyHasher { + pub fn local_sha256(args: HashArgs) -> PyHasher { PyHasher::new("sha256", HashWrapper::new::(args.string)) } #[pyfunction(name = "openssl_sha384")] - fn sha384(args: HashArgs) -> PyHasher { + pub fn local_sha384(args: HashArgs) -> PyHasher { PyHasher::new("sha384", HashWrapper::new::(args.string)) } #[pyfunction(name = "openssl_sha512")] - fn sha512(args: HashArgs) -> PyHasher { + pub fn local_sha512(args: HashArgs) -> PyHasher { PyHasher::new("sha512", HashWrapper::new::(args.string)) } #[pyfunction(name = "openssl_sha3_224")] - fn sha3_224(args: HashArgs) -> PyHasher { + pub fn local_sha3_224(args: HashArgs) -> PyHasher { PyHasher::new("sha3_224", HashWrapper::new::(args.string)) } #[pyfunction(name = "openssl_sha3_256")] - fn sha3_256(args: HashArgs) -> PyHasher { + pub fn local_sha3_256(args: HashArgs) -> PyHasher { PyHasher::new("sha3_256", HashWrapper::new::(args.string)) } #[pyfunction(name = "openssl_sha3_384")] - fn sha3_384(args: HashArgs) -> PyHasher { + pub fn local_sha3_384(args: HashArgs) -> PyHasher { PyHasher::new("sha3_384", HashWrapper::new::(args.string)) } #[pyfunction(name = "openssl_sha3_512")] - fn sha3_512(args: HashArgs) -> PyHasher { + pub fn local_sha3_512(args: HashArgs) -> PyHasher { PyHasher::new("sha3_512", HashWrapper::new::(args.string)) } #[pyfunction(name = "openssl_shake_128")] - fn shake_128(args: HashArgs) -> PyHasherXof { + pub fn local_shake_128(args: HashArgs) -> PyHasherXof { PyHasherXof::new("shake_128", HashXofWrapper::new_shake_128(args.string)) } #[pyfunction(name = "openssl_shake_256")] - fn shake_256(args: HashArgs) -> PyHasherXof { + pub fn local_shake_256(args: HashArgs) -> PyHasherXof { PyHasherXof::new("shake_256", HashXofWrapper::new_shake_256(args.string)) } #[pyfunction(name = "openssl_blake2b")] - fn blake2b(args: BlakeHashArgs) -> PyHasher { + pub fn local_blake2b(args: BlakeHashArgs) -> PyHasher { PyHasher::new("blake2b", HashWrapper::new::(args.data)) } #[pyfunction(name = "openssl_blake2s")] - fn blake2s(args: BlakeHashArgs) -> PyHasher { + pub fn local_blake2s(args: BlakeHashArgs) -> PyHasher { PyHasher::new("blake2s", HashWrapper::new::(args.data)) } + #[pyfunction] + fn compare_digest( + a: ArgStrOrBytesLike, + b: ArgStrOrBytesLike, + vm: &VirtualMachine, + ) -> PyResult { + fn is_str(arg: &ArgStrOrBytesLike) -> bool { + matches!(arg, ArgStrOrBytesLike::Str(_)) + } + + if is_str(&a) != is_str(&b) { + return Err(vm.new_type_error(format!( + "a bytes-like object is required, not '{}'", + b.as_object().class().name() + ))); + } + + let a_hash = a.borrow_bytes().to_vec(); + let b_hash = b.borrow_bytes().to_vec(); + + Ok((a_hash == b_hash).to_pyobject(vm)) + } + + #[derive(FromArgs, Debug)] + #[allow(unused)] + pub struct NewHMACHashArgs { + #[pyarg(positional)] + name: PyBuffer, + #[pyarg(any, optional)] + data: OptionalArg, + #[pyarg(named, default = "true")] + digestmod: bool, // TODO: RUSTPYTHON support functions & name functions + } + + #[pyfunction] + fn hmac_new(_args: NewHMACHashArgs, vm: &VirtualMachine) -> PyResult { + Err(vm.new_type_error("cannot create 'hmac' instances".into())) // TODO: RUSTPYTHON support hmac + } + pub trait ThreadSafeDynDigest: DynClone + DynDigest + Sync + Send {} impl ThreadSafeDynDigest for T where T: DynClone + DynDigest + Sync + Send {} diff --git a/stdlib/src/lib.rs b/stdlib/src/lib.rs index 97aaa8b89b..2a1a7d5ce0 100644 --- a/stdlib/src/lib.rs +++ b/stdlib/src/lib.rs @@ -20,7 +20,6 @@ mod md5; mod sha1; mod sha256; mod sha3; -mod sha512; mod json; #[cfg(not(any(target_os = "ios", target_os = "android", target_arch = "wasm32")))] @@ -112,7 +111,7 @@ pub fn get_module_inits() -> impl Iterator, StdlibInit "_sha1" => sha1::make_module, "_sha3" => sha3::make_module, "_sha256" => sha256::make_module, - "_sha512" => sha512::make_module, + // "_sha512" => sha512::make_module, // TODO: RUSPYTHON fix strange fail on vm: 'static type has not been initialized' "_md5" => md5::make_module, "_blake2" => blake2::make_module, "_json" => json::make_module, diff --git a/stdlib/src/md5.rs b/stdlib/src/md5.rs index 387817ef0f..833d217f5b 100644 --- a/stdlib/src/md5.rs +++ b/stdlib/src/md5.rs @@ -1,15 +1,12 @@ -// spell-checker:ignore usedforsecurity HASHXOF - pub(crate) use _md5::make_module; #[pymodule] mod _md5 { - use crate::hashlib::_hashlib::{HashArgs, HashWrapper, PyHasher}; - use crate::vm::{PyObjectRef, PyPayload, PyResult, VirtualMachine}; - use md5::Md5; + use crate::hashlib::_hashlib::{local_md5, HashArgs}; + use crate::vm::{PyPayload, PyResult, VirtualMachine}; - #[pyfunction(name = "md5")] - fn md5(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(PyHasher::new("md5", HashWrapper::new::(args.string)).into_pyobject(vm)) + #[pyfunction] + fn md5(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_md5(args).into_pyobject(vm)) } } diff --git a/stdlib/src/sha1.rs b/stdlib/src/sha1.rs index cc50b76ed3..3820e7d96a 100644 --- a/stdlib/src/sha1.rs +++ b/stdlib/src/sha1.rs @@ -1,15 +1,12 @@ -// spell-checker:ignore usedforsecurity HASHXOF - pub(crate) use _sha1::make_module; #[pymodule] mod _sha1 { - use crate::vm::{PyObjectRef, PyPayload, PyResult, VirtualMachine}; - use sha1::Sha1; + use crate::hashlib::_hashlib::{local_sha1, HashArgs}; + use crate::vm::{PyPayload, PyResult, VirtualMachine}; - use crate::hashlib::_hashlib::{HashArgs, HashWrapper, PyHasher}; - #[pyfunction(name = "sha1")] - fn sha1(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(PyHasher::new("sha1", HashWrapper::new::(args.string)).into_pyobject(vm)) + #[pyfunction] + fn sha1(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_sha1(args).into_pyobject(vm)) } } diff --git a/stdlib/src/sha256.rs b/stdlib/src/sha256.rs index 1695e32d76..bae22fa4cc 100644 --- a/stdlib/src/sha256.rs +++ b/stdlib/src/sha256.rs @@ -1,20 +1,17 @@ -// spell-checker:ignore usedforsecurity HASHXOF - pub(crate) use _sha256::make_module; #[pymodule] mod _sha256 { - use crate::hashlib::_hashlib::{HashArgs, HashWrapper, PyHasher}; - use crate::vm::{PyObjectRef, PyPayload, PyResult, VirtualMachine}; - use sha2::{Sha224, Sha256}; + use crate::hashlib::_hashlib::{local_sha224, local_sha256, HashArgs}; + use crate::vm::{PyPayload, PyResult, VirtualMachine}; - #[pyfunction(name = "sha224")] - fn sha224(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(PyHasher::new("sha224", HashWrapper::new::(args.string)).into_pyobject(vm)) + #[pyfunction] + fn sha224(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_sha224(args).into_pyobject(vm)) } - #[pyfunction(name = "sha256")] - fn sha256(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(PyHasher::new("sha256", HashWrapper::new::(args.string)).into_pyobject(vm)) + #[pyfunction] + fn sha256(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_sha256(args).into_pyobject(vm)) } } diff --git a/stdlib/src/sha3.rs b/stdlib/src/sha3.rs index 76666a6cf2..f0c1c5ef69 100644 --- a/stdlib/src/sha3.rs +++ b/stdlib/src/sha3.rs @@ -1,46 +1,40 @@ -// spell-checker:ignore usedforsecurity HASHXOF - pub(crate) use _sha3::make_module; #[pymodule] mod _sha3 { - use crate::hashlib::_hashlib::{HashArgs, HashWrapper, HashXofWrapper, PyHasher, PyHasherXof}; - use crate::vm::{PyObjectRef, PyPayload, PyResult, VirtualMachine}; - use sha3::{Sha3_224, Sha3_256, Sha3_384, Sha3_512}; - - #[pyfunction(name = "sha3_224")] - fn sha3_224(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(PyHasher::new("sha3_224", HashWrapper::new::(args.string)).into_pyobject(vm)) + use crate::hashlib::_hashlib::{ + local_sha3_224, local_sha3_256, local_sha3_384, local_sha3_512, local_shake_128, + local_shake_256, HashArgs, + }; + use crate::vm::{PyPayload, PyResult, VirtualMachine}; + + #[pyfunction] + fn sha3_224(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_sha3_224(args).into_pyobject(vm)) } - #[pyfunction(name = "sha3_256")] - fn sha3_256(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(PyHasher::new("sha3_256", HashWrapper::new::(args.string)).into_pyobject(vm)) + #[pyfunction] + fn sha3_256(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_sha3_256(args).into_pyobject(vm)) } - #[pyfunction(name = "sha3_384")] - fn sha3_384(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(PyHasher::new("sha3_384", HashWrapper::new::(args.string)).into_pyobject(vm)) + #[pyfunction] + fn sha3_384(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_sha3_384(args).into_pyobject(vm)) } - #[pyfunction(name = "sha3_512")] - fn sha3_512(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok(PyHasher::new("sha3_512", HashWrapper::new::(args.string)).into_pyobject(vm)) + #[pyfunction] + fn sha3_512(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_sha3_512(args).into_pyobject(vm)) } - #[pyfunction(name = "shake_128")] - fn shake_128(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok( - PyHasherXof::new("shake_128", HashXofWrapper::new_shake_128(args.string)) - .into_pyobject(vm), - ) + #[pyfunction] + fn shake_128(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_shake_128(args).into_pyobject(vm)) } - #[pyfunction(name = "shake_256")] - fn shake_256(args: HashArgs, vm: &VirtualMachine) -> PyResult { - Ok( - PyHasherXof::new("shake_256", HashXofWrapper::new_shake_256(args.string)) - .into_pyobject(vm), - ) + #[pyfunction] + fn shake_256(args: HashArgs, vm: &VirtualMachine) -> PyResult { + Ok(local_shake_256(args).into_pyobject(vm)) } } From 73e1eeb57e39006d3649f2e77ab66f6ddabb3ca8 Mon Sep 17 00:00:00 2001 From: Jeong YunWon Date: Sun, 23 Apr 2023 15:23:13 +0900 Subject: [PATCH 4/4] skip test_enum.test_unique_composite --- Lib/test/test_enum.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index 92cb26e458..1cccd27dee 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -2889,7 +2889,7 @@ class Color(StrMixin, AllMixin, IntFlag): self.assertEqual(Color.ALL.value, 7) self.assertEqual(str(Color.BLUE), 'blue') - @unittest.skipIf(sys.platform == "win32" or sys.platform.startswith("linux"), "TODO: RUSTPYTHON, inconsistent test result on Windows due to threading") + @unittest.skip("TODO: RUSTPYTHON, inconsistent test result due to threading") @threading_helper.reap_threads def test_unique_composite(self): # override __eq__ to be identity only