Merge pull request #1752 from Giskard-AI/feature/gsk-2581-modify-test-code-for-tests-in-an-existing-test-suite

GSK-2581 GSK-2587: Modify test code for tests in an existing test suite

Author: andreybavt

docs/reference/suite/index.rst

@@ -6,6 +6,10 @@ Test suite
    .. automethod:: __init__
    .. automethod:: run
    .. automethod:: add_test
+   .. automethod:: add_test
+   .. automethod:: remove_test
+   .. automethod:: upgrade_test
+   .. automethod:: update_test_params
    .. automethod:: upload
    .. automethod:: download
 

giskard/core/suite.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Optional, Tuple, Union
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 import inspect
 import logging
@@ -612,6 +612,35 @@ def add_test(
 
         return self
 
+    def upgrade_test(
+        self, test: GiskardTest, migrate_params_fn: Optional[Callable[[Dict[str, Any]], Dict[str, Any]]] = None
+    ) -> "Suite":
+        """Upgrade a test with a new version, the test being upgraded are matched using display_name tests property.
+
+        Parameters
+        ----------
+        test : GiskardTest
+            The newest version of a test to be upgraded
+        migrate_params_fn : Optional[Callable[[Dict[str, Any]], Dict[str, Any]]]
+            An optional callback used to migrate the old test params into the new params
+
+        Returns
+        -------
+        Suite
+            The current instance of the test suite to allow chained calls.
+
+        """
+
+        for test_to_upgrade in self.tests:
+            if test_to_upgrade.giskard_test.display_name != test.display_name:
+                continue
+
+            test_to_upgrade.giskard_test = test
+            if migrate_params_fn is not None:
+                test_to_upgrade.provided_inputs = migrate_params_fn(test_to_upgrade.provided_inputs.copy())
+
+        return self
+
     @singledispatchmethod
     def remove_test(self, arg):
         """Remove a test from the suite.

giskard/registry/giskard_test.py

@@ -112,6 +112,10 @@ def assert_(self):
 
             assert result.passed, message
 
+    @property
+    def display_name(self):
+        return self.meta.display_name
+
 
 Function = Callable[..., Result]
 

tests/test_programmatic.py

@@ -307,3 +307,81 @@ def test_with_custom_types():
 
     result = _test_with_custom_types(b=CustomClass(2), a=CustomClass(3)).execute()
     assert result
+
+
+def test_upgrade_test_with_migration():
+    @test(name="Named test")
+    def my_named_test(is_pass: bool):
+        return is_pass
+
+    suite = Suite().add_test(my_named_test(True), "True").add_test(my_named_test(False), "False")
+
+    assert len(suite.tests) == 2
+    assert suite.tests[0].test_id == "True"
+    assert suite.tests[0].giskard_test.meta.uuid == my_named_test.meta.uuid
+    assert suite.tests[0].provided_inputs == {"is_pass": True}
+    assert suite.tests[1].test_id == "False"
+    assert suite.tests[1].giskard_test.meta.uuid == my_named_test.meta.uuid
+    assert suite.tests[1].provided_inputs == {"is_pass": False}
+
+    @test(name="Named test")
+    def my_named_test_v2(passed: bool):
+        return passed
+
+    suite.upgrade_test(my_named_test_v2, lambda params: {"passed": params["is_pass"]})
+
+    assert len(suite.tests) == 2
+    assert suite.tests[0].test_id == "True"
+    assert suite.tests[0].giskard_test.meta.uuid == my_named_test_v2.meta.uuid
+    assert suite.tests[0].provided_inputs == {"passed": True}
+    assert suite.tests[1].test_id == "False"
+    assert suite.tests[1].giskard_test.meta.uuid == my_named_test_v2.meta.uuid
+    assert suite.tests[1].provided_inputs == {"passed": False}
+
+
+def test_upgrade_test_without():
+    @test(name="Named test")
+    def my_named_test(is_pass: bool):
+        return is_pass
+
+    suite = Suite().add_test(my_named_test(True), "True")
+
+    assert len(suite.tests) == 1
+    assert suite.tests[0].test_id == "True"
+    assert suite.tests[0].giskard_test.meta.uuid == my_named_test.meta.uuid
+    assert suite.tests[0].provided_inputs == {"is_pass": True}
+
+    @test(name="Named test")
+    def my_named_test_v2(passed: bool):
+        return passed
+
+    suite.upgrade_test(my_named_test_v2)
+
+    assert len(suite.tests) == 1
+    assert suite.tests[0].test_id == "True"
+    assert suite.tests[0].giskard_test.meta.uuid == my_named_test_v2.meta.uuid
+    assert suite.tests[0].provided_inputs == {"is_pass": True}
+
+
+def test_upgrade_test_not_matching():
+    @test(name="Named test")
+    def my_named_test(is_pass: bool):
+        return is_pass
+
+    suite = Suite().add_test(my_named_test(True), "True")
+
+    assert len(suite.tests) == 1
+    assert suite.tests[0].test_id == "True"
+    assert suite.tests[0].giskard_test.meta.uuid == my_named_test.meta.uuid
+    assert suite.tests[0].provided_inputs == {"is_pass": True}
+
+    @test(name="Named test V2")
+    def my_named_test_v2(passed: bool):
+        return passed
+
+    suite.upgrade_test(my_named_test_v2)
+
+    assert len(suite.tests) == 1
+    assert suite.tests[0].test_id == "True"
+    assert suite.tests[0].giskard_test.meta.uuid == my_named_test.meta.uuid
+    assert suite.tests[0].provided_inputs == {"is_pass": True}