fixes #5611 -- use subtests for wycheproof tests for speed (#5616)

15 files changed, 65 insertions, 39 deletions

diff --git a/tests/conftest.py b/tests/conftest.py
index 77691c2d1..43debdd61 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -7,10 +7,7 @@ import pytest
 
 from cryptography.hazmat.backends.openssl import backend as openssl_backend
 
-from .utils import (
-    check_backend_support,
-    load_wycheproof_tests,
-)
+from .utils import check_backend_support
 
 
 def pytest_report_header(config):
@@ -26,17 +23,6 @@ def pytest_addoption(parser):
     parser.addoption("--wycheproof-root", default=None)
 
 
-def pytest_generate_tests(metafunc):
-    if "wycheproof" in metafunc.fixturenames:
-        wycheproof = metafunc.config.getoption("--wycheproof-root", skip=True)
-
-        testcases = []
-        marker = metafunc.definition.get_closest_marker("wycheproof_tests")
-        for path in marker.args:
-            testcases.extend(load_wycheproof_tests(wycheproof, path))
-        metafunc.parametrize("wycheproof", testcases)
-
-
 def pytest_runtest_setup(item):
     if openssl_backend._fips_enabled:
         for marker in item.iter_markers(name="skip_fips"):
diff --git a/tests/wycheproof/test_aes.py b/tests/wycheproof/test_aes.py
index 43e7164ce..c041b47e1 100644
--- a/tests/wycheproof/test_aes.py
+++ b/tests/wycheproof/test_aes.py
@@ -13,11 +13,12 @@ from cryptography.hazmat.primitives import padding
 from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
 from cryptography.hazmat.primitives.ciphers.aead import AESCCM, AESGCM
 
+from .utils import wycheproof_tests
 from ..hazmat.primitives.test_aead import _aead_supported
 
 
 @pytest.mark.requires_backend_interface(interface=CipherBackend)
-@pytest.mark.wycheproof_tests("aes_cbc_pkcs5_test.json")
+@wycheproof_tests("aes_cbc_pkcs5_test.json")
 def test_aes_cbc_pkcs5(backend, wycheproof):
     key = binascii.unhexlify(wycheproof.testcase["key"])
     iv = binascii.unhexlify(wycheproof.testcase["iv"])
@@ -45,7 +46,7 @@ def test_aes_cbc_pkcs5(backend, wycheproof):
 
 
 @pytest.mark.requires_backend_interface(interface=CipherBackend)
-@pytest.mark.wycheproof_tests("aes_gcm_test.json")
+@wycheproof_tests("aes_gcm_test.json")
 def test_aes_gcm(backend, wycheproof):
     key = binascii.unhexlify(wycheproof.testcase["key"])
     iv = binascii.unhexlify(wycheproof.testcase["iv"])
@@ -90,7 +91,7 @@ def test_aes_gcm(backend, wycheproof):
 
 
 @pytest.mark.requires_backend_interface(interface=CipherBackend)
-@pytest.mark.wycheproof_tests("aes_gcm_test.json")
+@wycheproof_tests("aes_gcm_test.json")
 def test_aes_gcm_aead_api(backend, wycheproof):
     key = binascii.unhexlify(wycheproof.testcase["key"])
     iv = binascii.unhexlify(wycheproof.testcase["iv"])
@@ -124,7 +125,7 @@ def test_aes_gcm_aead_api(backend, wycheproof):
     reason="Requires OpenSSL with AES-CCM support",
 )
 @pytest.mark.requires_backend_interface(interface=CipherBackend)
-@pytest.mark.wycheproof_tests("aes_ccm_test.json")
+@wycheproof_tests("aes_ccm_test.json")
 def test_aes_ccm_aead_api(backend, wycheproof):
     key = binascii.unhexlify(wycheproof.testcase["key"])
     iv = binascii.unhexlify(wycheproof.testcase["iv"])
diff --git a/tests/wycheproof/test_chacha20poly1305.py b/tests/wycheproof/test_chacha20poly1305.py
index 936fa8dc0..9c94c58f7 100644
--- a/tests/wycheproof/test_chacha20poly1305.py
+++ b/tests/wycheproof/test_chacha20poly1305.py
@@ -11,6 +11,7 @@ from cryptography.exceptions import InvalidTag
 from cryptography.hazmat.backends.interfaces import CipherBackend
 from cryptography.hazmat.primitives.ciphers.aead import ChaCha20Poly1305
 
+from .utils import wycheproof_tests
 from ..hazmat.primitives.test_aead import _aead_supported
 
 
@@ -19,8 +20,8 @@ from ..hazmat.primitives.test_aead import _aead_supported
     reason="Requires OpenSSL with ChaCha20Poly1305 support",
 )
 @pytest.mark.requires_backend_interface(interface=CipherBackend)
-@pytest.mark.wycheproof_tests("chacha20_poly1305_test.json")
-def test_chacha2poly1305(wycheproof):
+@wycheproof_tests("chacha20_poly1305_test.json")
+def test_chacha2poly1305(backend, wycheproof):
     key = binascii.unhexlify(wycheproof.testcase["key"])
     iv = binascii.unhexlify(wycheproof.testcase["iv"])
     aad = binascii.unhexlify(wycheproof.testcase["aad"])
diff --git a/tests/wycheproof/test_cmac.py b/tests/wycheproof/test_cmac.py
index 11a1d34b7..21ce9af88 100644
--- a/tests/wycheproof/test_cmac.py
+++ b/tests/wycheproof/test_cmac.py
@@ -12,9 +12,11 @@ from cryptography.hazmat.backends.interfaces import CMACBackend
 from cryptography.hazmat.primitives.ciphers.algorithms import AES
 from cryptography.hazmat.primitives.cmac import CMAC
 
+from .utils import wycheproof_tests
+
 
 @pytest.mark.requires_backend_interface(interface=CMACBackend)
-@pytest.mark.wycheproof_tests("aes_cmac_test.json")
+@wycheproof_tests("aes_cmac_test.json")
 def test_aes_cmac(backend, wycheproof):
     key = binascii.unhexlify(wycheproof.testcase["key"])
     msg = binascii.unhexlify(wycheproof.testcase["msg"])
diff --git a/tests/wycheproof/test_dsa.py b/tests/wycheproof/test_dsa.py
index 19d2ce486..1bcf31f02 100644
--- a/tests/wycheproof/test_dsa.py
+++ b/tests/wycheproof/test_dsa.py
@@ -11,6 +11,8 @@ from cryptography.exceptions import InvalidSignature
 from cryptography.hazmat.backends.interfaces import DSABackend
 from cryptography.hazmat.primitives import hashes, serialization
 
+from .utils import wycheproof_tests
+
 
 _DIGESTS = {
     "SHA-1": hashes.SHA1(),
@@ -20,7 +22,7 @@ _DIGESTS = {
 
 
 @pytest.mark.requires_backend_interface(interface=DSABackend)
-@pytest.mark.wycheproof_tests(
+@wycheproof_tests(
     "dsa_test.json",
     "dsa_2048_224_sha224_test.json",
     "dsa_2048_224_sha256_test.json",
diff --git a/tests/wycheproof/test_ecdh.py b/tests/wycheproof/test_ecdh.py
index a25337de1..a1a90c141 100644
--- a/tests/wycheproof/test_ecdh.py
+++ b/tests/wycheproof/test_ecdh.py
@@ -12,6 +12,7 @@ from cryptography.hazmat.backends.interfaces import EllipticCurveBackend
 from cryptography.hazmat.primitives import serialization
 from cryptography.hazmat.primitives.asymmetric import ec
 
+from .utils import wycheproof_tests
 from ..hazmat.primitives.test_ec import _skip_exchange_algorithm_unsupported
 
 
@@ -36,7 +37,7 @@ _CURVES = {
 
 
 @pytest.mark.requires_backend_interface(interface=EllipticCurveBackend)
-@pytest.mark.wycheproof_tests(
+@wycheproof_tests(
     "ecdh_test.json",
     "ecdh_brainpoolP224r1_test.json",
     "ecdh_brainpoolP256r1_test.json",
@@ -84,7 +85,7 @@ def test_ecdh(backend, wycheproof):
 
 
 @pytest.mark.requires_backend_interface(interface=EllipticCurveBackend)
-@pytest.mark.wycheproof_tests(
+@wycheproof_tests(
     "ecdh_secp224r1_ecpoint_test.json",
     "ecdh_secp256r1_ecpoint_test.json",
     "ecdh_secp384r1_ecpoint_test.json",
diff --git a/tests/wycheproof/test_ecdsa.py b/tests/wycheproof/test_ecdsa.py
index b1a98253d..e1bdbace6 100644
--- a/tests/wycheproof/test_ecdsa.py
+++ b/tests/wycheproof/test_ecdsa.py
@@ -12,6 +12,8 @@ from cryptography.hazmat.backends.interfaces import EllipticCurveBackend
 from cryptography.hazmat.primitives import hashes, serialization
 from cryptography.hazmat.primitives.asymmetric import ec
 
+from .utils import wycheproof_tests
+
 
 _DIGESTS = {
     "SHA-1": hashes.SHA1(),
@@ -27,7 +29,7 @@ _DIGESTS = {
 
 
 @pytest.mark.requires_backend_interface(interface=EllipticCurveBackend)
-@pytest.mark.wycheproof_tests(
+@wycheproof_tests(
     "ecdsa_test.json",
     "ecdsa_brainpoolP224r1_sha224_test.json",
     "ecdsa_brainpoolP256r1_sha256_test.json",
diff --git a/tests/wycheproof/test_eddsa.py b/tests/wycheproof/test_eddsa.py
index 13501a7ba..2de695f57 100644
--- a/tests/wycheproof/test_eddsa.py
+++ b/tests/wycheproof/test_eddsa.py
@@ -11,12 +11,14 @@ from cryptography.exceptions import InvalidSignature
 from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
 from cryptography.hazmat.primitives.asymmetric.ed448 import Ed448PublicKey
 
+from .utils import wycheproof_tests
+
 
 @pytest.mark.supported(
     only_if=lambda backend: backend.ed25519_supported(),
     skip_message="Requires OpenSSL with Ed25519 support",
 )
-@pytest.mark.wycheproof_tests("eddsa_test.json")
+@wycheproof_tests("eddsa_test.json")
 def test_ed25519_signature(backend, wycheproof):
     # We want to fail if/when wycheproof adds more edwards curve tests
     # so we can add them as well.
@@ -43,7 +45,7 @@ def test_ed25519_signature(backend, wycheproof):
     only_if=lambda backend: backend.ed448_supported(),
     skip_message="Requires OpenSSL with Ed448 support",
 )
-@pytest.mark.wycheproof_tests("ed448_test.json")
+@wycheproof_tests("ed448_test.json")
 def test_ed448_signature(backend, wycheproof):
     key = Ed448PublicKey.from_public_bytes(
         binascii.unhexlify(wycheproof.testgroup["key"]["pk"])
diff --git a/tests/wycheproof/test_hkdf.py b/tests/wycheproof/test_hkdf.py
index 0f553a919..4886be0fe 100644
--- a/tests/wycheproof/test_hkdf.py
+++ b/tests/wycheproof/test_hkdf.py
@@ -10,6 +10,8 @@ import pytest
 from cryptography.hazmat.primitives import hashes
 from cryptography.hazmat.primitives.kdf.hkdf import HKDF
 
+from .utils import wycheproof_tests
+
 
 _HASH_ALGORITHMS = {
     "HKDF-SHA-1": hashes.SHA1(),
@@ -19,7 +21,7 @@ _HASH_ALGORITHMS = {
 }
 
 
-@pytest.mark.wycheproof_tests(
+@wycheproof_tests(
     "hkdf_sha1_test.json",
     "hkdf_sha256_test.json",
     "hkdf_sha384_test.json",
diff --git a/tests/wycheproof/test_hmac.py b/tests/wycheproof/test_hmac.py
index 93433c1c1..bfc690795 100644
--- a/tests/wycheproof/test_hmac.py
+++ b/tests/wycheproof/test_hmac.py
@@ -10,6 +10,8 @@ import pytest
 from cryptography.exceptions import InvalidSignature
 from cryptography.hazmat.primitives import hashes, hmac
 
+from .utils import wycheproof_tests
+
 
 _HMAC_ALGORITHMS = {
     "HMACSHA1": hashes.SHA1(),
@@ -24,7 +26,7 @@ _HMAC_ALGORITHMS = {
 }
 
 
-@pytest.mark.wycheproof_tests(
+@wycheproof_tests(
     "hmac_sha1_test.json",
     "hmac_sha224_test.json",
     "hmac_sha256_test.json",
diff --git a/tests/wycheproof/test_keywrap.py b/tests/wycheproof/test_keywrap.py
index c04ac4907..6a8cfd08b 100644
--- a/tests/wycheproof/test_keywrap.py
+++ b/tests/wycheproof/test_keywrap.py
@@ -10,9 +10,11 @@ import pytest
 from cryptography.hazmat.backends.interfaces import CipherBackend
 from cryptography.hazmat.primitives import keywrap
 
+from .utils import wycheproof_tests
+
 
 @pytest.mark.requires_backend_interface(interface=CipherBackend)
-@pytest.mark.wycheproof_tests("kwp_test.json")
+@wycheproof_tests("kwp_test.json")
 def test_keywrap_with_padding(backend, wycheproof):
     wrapping_key = binascii.unhexlify(wycheproof.testcase["key"])
     key_to_wrap = binascii.unhexlify(wycheproof.testcase["msg"])
@@ -37,7 +39,7 @@ def test_keywrap_with_padding(backend, wycheproof):
 
 
 @pytest.mark.requires_backend_interface(interface=CipherBackend)
-@pytest.mark.wycheproof_tests("kw_test.json")
+@wycheproof_tests("kw_test.json")
 def test_keywrap(backend, wycheproof):
     wrapping_key = binascii.unhexlify(wycheproof.testcase["key"])
     key_to_wrap = binascii.unhexlify(wycheproof.testcase["msg"])
diff --git a/tests/wycheproof/test_rsa.py b/tests/wycheproof/test_rsa.py
index a3712cc75..5df466c50 100644
--- a/tests/wycheproof/test_rsa.py
+++ b/tests/wycheproof/test_rsa.py
@@ -12,6 +12,8 @@ from cryptography.hazmat.backends.interfaces import RSABackend
 from cryptography.hazmat.primitives import hashes, serialization
 from cryptography.hazmat.primitives.asymmetric import padding
 
+from .utils import wycheproof_tests
+
 
 _DIGESTS = {
     "SHA-1": hashes.SHA1(),
@@ -40,7 +42,7 @@ def should_verify(backend, wycheproof):
 
 
 @pytest.mark.requires_backend_interface(interface=RSABackend)
-@pytest.mark.wycheproof_tests(
+@wycheproof_tests(
     "rsa_signature_test.json",
     "rsa_signature_2048_sha224_test.json",
     "rsa_signature_2048_sha256_test.json",
@@ -91,7 +93,7 @@ def test_rsa_pkcs1v15_signature(backend, wycheproof):
             )
 
 
-@pytest.mark.wycheproof_tests("rsa_sig_gen_misc_test.json")
+@wycheproof_tests("rsa_sig_gen_misc_test.json")
 def test_rsa_pkcs1v15_signature_generation(backend, wycheproof):
     key = serialization.load_pem_private_key(
         wycheproof.testgroup["privateKeyPem"].encode(),
@@ -109,7 +111,7 @@ def test_rsa_pkcs1v15_signature_generation(backend, wycheproof):
 
 
 @pytest.mark.requires_backend_interface(interface=RSABackend)
-@pytest.mark.wycheproof_tests(
+@wycheproof_tests(
     "rsa_pss_2048_sha1_mgf1_20_test.json",
     "rsa_pss_2048_sha256_mgf1_0_test.json",
     "rsa_pss_2048_sha256_mgf1_32_test.json",
@@ -159,7 +161,7 @@ def test_rsa_pss_signature(backend, wycheproof):
 
 
 @pytest.mark.requires_backend_interface(interface=RSABackend)
-@pytest.mark.wycheproof_tests(
+@wycheproof_tests(
     "rsa_oaep_2048_sha1_mgf1sha1_test.json",
     "rsa_oaep_2048_sha224_mgf1sha1_test.json",
     "rsa_oaep_2048_sha224_mgf1sha224_test.json",
@@ -214,7 +216,7 @@ def test_rsa_oaep_encryption(backend, wycheproof):
             )
 
 
-@pytest.mark.wycheproof_tests(
+@wycheproof_tests(
     "rsa_pkcs1_2048_test.json",
     "rsa_pkcs1_3072_test.json",
     "rsa_pkcs1_4096_test.json",
diff --git a/tests/wycheproof/test_x25519.py b/tests/wycheproof/test_x25519.py
index 926a5e898..17aef36fe 100644
--- a/tests/wycheproof/test_x25519.py
+++ b/tests/wycheproof/test_x25519.py
@@ -12,12 +12,14 @@ from cryptography.hazmat.primitives.asymmetric.x25519 import (
     X25519PublicKey,
 )
 
+from .utils import wycheproof_tests
+
 
 @pytest.mark.supported(
     only_if=lambda backend: backend.x25519_supported(),
     skip_message="Requires OpenSSL with X25519 support",
 )
-@pytest.mark.wycheproof_tests("x25519_test.json")
+@wycheproof_tests("x25519_test.json")
 def test_x25519(backend, wycheproof):
     assert set(wycheproof.testgroup.items()) == {
         ("curve", "curve25519"),
diff --git a/tests/wycheproof/test_x448.py b/tests/wycheproof/test_x448.py
index 6c77457a7..8e7b32148 100644
--- a/tests/wycheproof/test_x448.py
+++ b/tests/wycheproof/test_x448.py
@@ -12,12 +12,14 @@ from cryptography.hazmat.primitives.asymmetric.x448 import (
     X448PublicKey,
 )
 
+from .utils import wycheproof_tests
+
 
 @pytest.mark.supported(
     only_if=lambda backend: backend.x448_supported(),
     skip_message="Requires OpenSSL with X448 support",
 )
-@pytest.mark.wycheproof_tests("x448_test.json")
+@wycheproof_tests("x448_test.json")
 def test_x448(backend, wycheproof):
     assert set(wycheproof.testgroup.items()) == {
         ("curve", "curve448"),
diff --git a/tests/wycheproof/utils.py b/tests/wycheproof/utils.py
new file mode 100644
index 000000000..eebbe7ce3
--- /dev/null
+++ b/tests/wycheproof/utils.py
@@ -0,0 +1,17 @@
+from ..utils import load_wycheproof_tests
+
+
+def wycheproof_tests(*paths):
+    def wrapper(func):
+        def run_wycheproof(backend, subtests, pytestconfig):
+            wycheproof_root = pytestconfig.getoption(
+                "--wycheproof-root", skip=True
+            )
+            for path in paths:
+                for test in load_wycheproof_tests(wycheproof_root, path):
+                    with subtests.test():
+                        func(backend, test)
+
+        return run_wycheproof
+
+    return wrapper