From 4a85a4f55817904479881c3bf8a9c98a808912ae Mon Sep 17 00:00:00 2001 From: chakradhar886 Date: Fri, 15 May 2026 14:54:10 +0530 Subject: [PATCH 1/2] Mark capability-host tests as record_only test_capability_host_in_hub and test_capability_host_in_project use hardcoded resource group 'testrg' and workspace 'workspace' that only exist in the recorded cassette. Running live raises ResourceGroupNotFound. Mark them with @record_only() and document why. --- .../tests/latest/test_capability_host_scenarios.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/machinelearningservices/azext_mlv2/tests/latest/test_capability_host_scenarios.py b/src/machinelearningservices/azext_mlv2/tests/latest/test_capability_host_scenarios.py index 3301189e1de..5a478488d9f 100644 --- a/src/machinelearningservices/azext_mlv2/tests/latest/test_capability_host_scenarios.py +++ b/src/machinelearningservices/azext_mlv2/tests/latest/test_capability_host_scenarios.py @@ -8,11 +8,17 @@ # regenerated. # -------------------------------------------------------------------------- import yaml +from azure.cli.testsdk.scenario_tests.decorators import record_only + from azext_mlv2.tests.scenario_test_helper import MLBaseScenarioTest class CapabilityHostScenarioTest(MLBaseScenarioTest): + # Marked as record_only because the test uses hardcoded resource group + # 'testrg' and workspace 'workspace', which only exist in the recorded + # cassette. Running live raises ResourceGroupNotFound. + @record_only() def test_capability_host_in_hub(self) -> None: self.kwargs["cap_host_name"] = "test_host_hub" @@ -40,6 +46,10 @@ def test_capability_host_in_hub(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("cap_host_name", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg' and workspace 'workspace', which only exist in the recorded + # cassette. Running live raises ResourceGroupNotFound. + @record_only() def test_capability_host_in_project(self) -> None: self.kwargs["cap_host_name"] = "test_host_project" @@ -53,7 +63,7 @@ def test_capability_host_in_project(self) -> None: sleep(120) # This sleep is only required for fresh recording of cassette capability_host_obj_show = self.cmd( - "az ml capability-host show -g tsetrg -w workspacename -n {cap_host_name}" + "az ml capability-host show -g testrg -w workspace -n {cap_host_name}" ) capability_host_obj_show = yaml.safe_load(capability_host_obj_show.output) From a0b21462412d2e5932411c1288ae491e393939ce Mon Sep 17 00:00:00 2001 From: chakradhar886 Date: Fri, 15 May 2026 16:31:10 +0530 Subject: [PATCH 2/2] Mark mlv2 tests with hardcoded resources as record_only Many tests in machinelearningservices/azext_mlv2/tests/latest reference hardcoded resource groups (testrg, test-rg-000006, test-cli-rg), workspaces (testworkspace), or registries (test-cli-reg, testFeed, bani-reg-wentral) that only exist in their recorded cassettes. Running them in --live mode raises ResourceGroupNotFound, ResourceNotFound, or AuthorizationFailed. Add @record_only() to those tests (and import the decorator) so they are skipped in live mode and continue to pass via cassette playback. Also document the rationale in a comment above each decorator. --- .../latest/test_batch_endpoint_scenarios.py | 99 ++++++++++- .../tests/latest/test_component_scenarios.py | 65 ++++++++ .../tests/latest/test_compute_scenarios.py | 33 ++++ .../tests/latest/test_datastore_scenarios.py | 33 ++++ .../test_deployment_template_scenarios.py | 51 +++++- .../latest/test_environment_scenarios.py | 49 ++++++ .../latest/test_feature_set_scenarios.py | 7 +- .../test_feature_store_entity_scenarios.py | 5 + .../latest/test_feature_store_scenarios.py | 9 + .../tests/latest/test_job_scenarios.py | 157 ++++++++++++++++++ .../test_k8s_online_endpoint_scenarios.py | 21 +++ .../tests/latest/test_model_scenarios.py | 69 ++++++++ .../latest/test_online_endpoint_scenarios.py | 149 +++++++++++++++++ .../tests/latest/test_registry_scenarios.py | 17 ++ .../tests/latest/test_workspace_scenarios.py | 13 ++ 15 files changed, 774 insertions(+), 3 deletions(-) diff --git a/src/machinelearningservices/azext_mlv2/tests/latest/test_batch_endpoint_scenarios.py b/src/machinelearningservices/azext_mlv2/tests/latest/test_batch_endpoint_scenarios.py index c5155df7774..e926774a19a 100644 --- a/src/machinelearningservices/azext_mlv2/tests/latest/test_batch_endpoint_scenarios.py +++ b/src/machinelearningservices/azext_mlv2/tests/latest/test_batch_endpoint_scenarios.py @@ -1,4 +1,4 @@ -# -------------------------------------------------------------------------- +# -------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for # license information. @@ -14,9 +14,14 @@ from knack.util import CLIError from ..util import private_flag +from azure.cli.testsdk.scenario_tests.decorators import record_only class BatchEndpointScenarioTest(MLBaseScenarioTest): + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_batch_endpoint(self) -> None: endpoint_name_suffix = "-1" # Updating dictionary with the job name @@ -57,6 +62,10 @@ def test_batch_endpoint(self) -> None: self.kwargs.pop("batch_endpoint_name_1", None) # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_batch_deployment(self) -> None: endpoint_name_suffix = "-2" @@ -115,6 +124,10 @@ def test_batch_deployment(self) -> None: self.kwargs.pop("batch_deployment_name_2", None) # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_batch_deployment_endpoint_name_casing(self) -> None: endpoint_name_suffix = "-4" @@ -173,6 +186,10 @@ def test_batch_deployment_endpoint_name_casing(self) -> None: self.kwargs.pop("batch_endpoint_name_3", None) self.kwargs.pop("batch_deployment_name_3", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Can't re-record due to invalid EnvironmentId (2376035)") def test_batch_deployment_output_file(self) -> None: endpoint_name_suffix = "-41" @@ -209,6 +226,10 @@ def test_batch_deployment_output_file(self) -> None: self.kwargs.pop("batch_endpoint_name_4", None) self.kwargs.pop("batch_deployment_name_4", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Can't re-record due to missing image (2376067)") def test_batch_endpoint_invoke(self) -> None: # az ml batch-endpoint update --name test-batch-ept-122342-11 --set defaults.deployment_name=test-batch-dep-000017 @@ -267,6 +288,10 @@ def test_batch_endpoint_invoke(self) -> None: self.kwargs.pop("batch_endpoint_name_5", None) self.kwargs.pop("batch_deployment_name_5", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Can't re-record due to invalid EnvironmentId (2376035)") def test_batch_endpoint_update(self) -> None: endpoint_name_suffix = "-62" @@ -313,6 +338,10 @@ def test_batch_endpoint_update(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("batch_endpoint_name_6", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Can't re-record due to missing image (2376067)") def test_batch_deployment_update_with_file_and_set(self) -> None: """ @@ -345,6 +374,10 @@ def test_batch_deployment_update_with_file_and_set(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("batch_endpoint_name_7", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_batch_endpoint_invoke_invalid_deployment(self) -> None: endpoint_name_suffix = "-8" # Updating dictionary with the job name @@ -366,12 +399,20 @@ def test_batch_endpoint_invoke_invalid_deployment(self) -> None: self.kwargs.pop("batch_endpoint_name_8", None) self.kwargs.pop("batch_deployment_name_8", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_batch_deployment_wrong_endpoint(self) -> None: with pytest.raises(Exception): self.cmd( "az ml batch-deployment create -f ./src/machinelearningservices/azext_mlv2/tests/test_configs/deployments/batch/batch_deployment_1.yaml -n {batchDeploymentName} -e {environmentName} -g testrg -w testworkspace" ) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_batch_endpoint_list(self) -> None: batch_endpoint_list = self.cmd("az ml batch-endpoint list -g testrg -w testworkspace") batch_endpoint_list = yaml.safe_load(batch_endpoint_list.output) @@ -390,6 +431,10 @@ def test_batch_endpoint_list(self) -> None: assert "properties" not in endpoint assert "tags" not in endpoint + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_batch_deployment_list(self) -> None: batch_deployment_list = self.cmd("az ml batch-deployment list -e batch-endpoint-test -g testrg -w testworkspace") batch_deployment_list = yaml.safe_load(batch_deployment_list.output) @@ -417,6 +462,10 @@ def test_batch_deployment_list(self) -> None: assert "resources" not in deployment assert "retry_settings" not in deployment + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_batch_deployment_list_jobs(self) -> None: batch_deployment_list_jobs = self.cmd("az ml batch-deployment list-jobs -g testrg -w testworkspace -e bla1 --name blue") batch_deployment_list_jobs = yaml.safe_load(batch_deployment_list_jobs.output) @@ -431,6 +480,10 @@ def test_batch_deployment_list_jobs(self) -> None: assert "outputFileName" not in deployment assert "tags" not in deployment + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_batch_endpoint_update_with_set_and_file(self) -> None: endpoint_name_suffix = "-9" # Updating dictionary with the job name @@ -458,6 +511,10 @@ def test_batch_endpoint_update_with_set_and_file(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("batch_endpoint_name_9", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_batch_endpoint_update_non_existent_endpoint(self) -> None: with pytest.raises( Exception, match=r"Message: The Resource.*test-batch.* under resource group .* was not found." @@ -465,6 +522,10 @@ def test_batch_endpoint_update_non_existent_endpoint(self) -> None: batch_endpoint_update = self.cmd("az ml batch-endpoint update -n test-batch --set tags.abc=456 tags.t1=v2 -g testrg -w testworkspace") assert "Code: ResourceNotFound" in str(exp.value) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Can't re-record due to invalid EnvironmentId (2376035)") def test_batch_endpoint_invoke_with_uri_file(self) -> None: endpoint_name_suffix = "-10" @@ -501,6 +562,10 @@ def test_batch_endpoint_invoke_with_uri_file(self) -> None: self.kwargs.pop("batch_endpoint_name_10", None) self.kwargs.pop("batch_deployment_name_10", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Can't re-record due to invalid EnvironmentId (2376035)") def test_batch_endpoint_invoke_with_uri_folder(self) -> None: endpoint_name_suffix = "-11" @@ -536,6 +601,10 @@ def test_batch_endpoint_invoke_with_uri_folder(self) -> None: self.kwargs.pop("batch_endpoint_name_11", None) self.kwargs.pop("batch_deployment_name_11", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Can't re-record due to invalid EnvironmentId (2376035)") def test_batch_endpoint_invoke_with_invalid_input_type(self) -> None: endpoint_name_suffix = "-12" @@ -565,6 +634,10 @@ def test_batch_endpoint_invoke_with_invalid_input_type(self) -> None: self.kwargs.pop("batch_endpoint_name_12", None) self.kwargs.pop("batch_deployment_name_12", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Can't re-record due to missing image (2376067)") def test_batch_endpoint_invoke_with_datastore_paths(self) -> None: endpoint_name_suffix = "-13" @@ -614,6 +687,10 @@ def test_batch_endpoint_invoke_with_datastore_paths(self) -> None: self.kwargs.pop("batch_endpoint_name_13", None) self.kwargs.pop("batch_deployment_name_13", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Could not push test yamls into repo. TOD: task 2007149 will either re-enable or remove") def test_batch_deployment_ast_scoring_script_validation(self) -> None: endpoint_name_suffix = "-14" @@ -656,6 +733,10 @@ def test_batch_deployment_ast_scoring_script_validation(self) -> None: self.kwargs.pop("batch_deployment_name_14", None) self.kwargs.pop("batch_endpoint_name_14", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Can't re-record due to invalid EnvironmentId (2376035)") def test_batch_endpoint_with_invalid_datastore_path(self) -> None: endpoint_name_suffix = "-15" @@ -687,6 +768,10 @@ def test_batch_endpoint_with_invalid_datastore_path(self) -> None: self.kwargs.pop("batch_endpoint_name_15", None) self.kwargs.pop("batch_deployment_name_15", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Could not re-record test despite test running succesfully") def test_batch_deployment_component_create_show(self) -> None: endpoint_name_suffix = "-16" @@ -725,6 +810,10 @@ def test_batch_deployment_component_create_show(self) -> None: self.kwargs.pop("batch_endpoint_name_16", None) self.kwargs.pop("batch_deployment_name_16", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Could not push test yamls into repo. TOD: task 2007149 will either re-enable or remove") def test_batch_deployment_component_invoke(self) -> None: endpoint_name_suffix = "-17" @@ -779,6 +868,10 @@ def test_batch_deployment_component_invoke(self) -> None: self.kwargs.pop("batch_endpoint_name_17", None) self.kwargs.pop("batch_deployment_name_17", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_batch_component_deployment_local_component_create(self) -> None: with private_flag(): endpoint_name_suffix = "-18" @@ -822,6 +915,10 @@ def test_batch_component_deployment_local_component_create(self) -> None: self.kwargs.pop("batch_endpoint_name_18", None) self.kwargs.pop("batch_deployment_name_18", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Could not push test yamls into repo. TOD: task 2007149 will either re-enable or remove") def test_batch_deployment_component_invoke_input_types(self) -> None: endpoint_name_suffix = "-19" diff --git a/src/machinelearningservices/azext_mlv2/tests/latest/test_component_scenarios.py b/src/machinelearningservices/azext_mlv2/tests/latest/test_component_scenarios.py index 076031c4c9a..b39e824cddb 100644 --- a/src/machinelearningservices/azext_mlv2/tests/latest/test_component_scenarios.py +++ b/src/machinelearningservices/azext_mlv2/tests/latest/test_component_scenarios.py @@ -24,6 +24,7 @@ from azure.cli.testsdk.scenario_tests.decorators import AllowLargeResponse from ..util import private_flag +from azure.cli.testsdk.scenario_tests.decorators import record_only @pytest.fixture() @@ -68,6 +69,10 @@ def assert_component_create_get_list(self, path, version, _type): assert len(yaml.safe_load(components.output)) != 0 return component_obj + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @AllowLargeResponse() def test_component(self): component_obj = self.assert_component_create_get_list( @@ -105,6 +110,10 @@ def test_component(self): component_restore_obj = self.cmd("az ml component restore -n {componentName} -g testrg -w testworkspace") assert component_restore_obj.output == "" + # Marked as record_only because the test uses hardcoded resource group + # 'testrg' (via assert_component_create_get_list helper), which only + # exists in the recorded cassette. Running live raises AuthorizationFailed. + @record_only() @AllowLargeResponse() def test_automl_component(self): self.assert_component_create_get_list( @@ -113,32 +122,56 @@ def test_automl_component(self): _type="automl", ) + # Marked as record_only because the test uses hardcoded resources + # (resource group, registry, etc.) that only exist in the recorded + # cassette. Running live raises ResourceNotFound or similar errors. + @record_only() def test_component_show_registry(self) -> None: env_obj = self.cmd("az ml component show -n 'HelloWorldRegistry' -v 0.0.1 --registry-name testFeed") env_obj = yaml.safe_load(env_obj.output) assert env_obj["id"] == "azureml://registries/testFeed/components/HelloWorldRegistry/versions/0.0.1" assert env_obj["name"] == "HelloWorldRegistry" + # Marked as record_only because the test uses hardcoded resources + # (resource group, registry, etc.) that only exist in the recorded + # cassette. Running live raises ResourceNotFound or similar errors. + @record_only() def test_component_show_without_version_label(self) -> None: with pytest.raises(ClientRequestError) as error_info: env_obj = self.cmd("az ml component show -n 'HelloWorldRegistry' --registry-name testFeed") assert "Must provide either version or label." in str(error_info) + # Marked as record_only because the test uses hardcoded resources + # (resource group, registry, etc.) that only exist in the recorded + # cassette. Running live raises ResourceNotFound or similar errors. + @record_only() def test_component_list_registry(self) -> None: env_obj = self.cmd("az ml component list -n 'HelloWorldRegistry' --registry-name testFeed") env_obj = yaml.safe_load(env_obj.output) assert len(env_obj) == 1 + # Marked as record_only because the test uses hardcoded resources + # (resource group, registry, etc.) that only exist in the recorded + # cassette. Running live raises ResourceNotFound or similar errors. + @record_only() def test_component_container_list_registry(self) -> None: env_obj = self.cmd("az ml component list --registry-name testFeed") env_obj = yaml.safe_load(env_obj.output) assert len(env_obj) > 1 + # Marked as record_only because the test uses hardcoded resources + # (resource group, registry, etc.) that only exist in the recorded + # cassette. Running live raises ResourceNotFound or similar errors. + @record_only() def test_component_archive_registry(self) -> None: env_obj = self.cmd("az ml component archive -n batchscore -v 1.0.12 --registry-name bani-reg-wentral") env_obj = yaml.safe_load(env_obj.output) assert env_obj is None + # Marked as record_only because the test uses hardcoded resources + # (resource group, registry, etc.) that only exist in the recorded + # cassette. Running live raises ResourceNotFound or similar errors. + @record_only() def test_component_restore_registry(self) -> None: env_obj = self.cmd("az ml component restore -n batchscore -v 1.0.12 --registry-name bani-reg-wentral") env_obj = yaml.safe_load(env_obj.output) @@ -151,6 +184,10 @@ def get_result_obj(self, cmd_str: str) -> Dict[str, Any]: else: return {} + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_component_validate_on_load(self) -> None: for target_file, target_field, target_message in [ @@ -170,6 +207,10 @@ def test_component_validate_on_load(self) -> None: self.cmd("az ml component validate -g testrg -w testworkspace --file {}".format(target_file)) assert target_message in str(err.value) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_component_validate(self) -> None: for target_file, target_field, target_message in [ @@ -208,6 +249,10 @@ def test_component_validate(self) -> None: target_field_found = True assert target_field_found, "target field {} not found in\n{}".format(target_field, json.dumps(messages)) + # Marked as record_only because the test uses hardcoded resources + # (resource group, registry, etc.) that only exist in the recorded + # cassette. Running live raises ResourceNotFound or similar errors. + @record_only() def test_component_with_short_resource_id(self) -> None: with private_flag(): with pytest.raises((ValidationException, CLIError)) as exc_info: @@ -220,6 +265,10 @@ def test_component_with_short_resource_id(self) -> None: in str(exception_raised) ) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_component_git_path(self) -> None: with private_flag(): component_obj = self.cmd( @@ -234,6 +283,10 @@ def test_component_git_path(self) -> None: assert "creation_context" in component_obj assert "code" in component_obj + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_component_create_skip_validation(self) -> None: create_component_command = "az ml component create -f ./src/machinelearningservices/azext_mlv2/tests/test_configs/components/helloworld_component.yml -g testrg -w testworkspace" @@ -245,6 +298,10 @@ def test_component_create_skip_validation(self) -> None: self.cmd(create_component_command) mock_validate.assert_called_once() + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_enable_internal_component(self): from azure.ai.ml.constants._common import AZUREML_INTERNAL_COMPONENTS_ENV_VAR @@ -270,6 +327,10 @@ def test_enable_internal_component(self): validation_result = json.loads(execution_result.output) assert validation_result["result"] == "Succeeded" + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="This test is not working yet") def test_show_internal_component(self): from azure.ai.ml.constants import AZUREML_INTERNAL_COMPONENTS_ENV_VAR @@ -299,6 +360,10 @@ def test_show_internal_component(self): ) ) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg' (via assert_component_create_get_list helper), which only + # exists in the recorded cassette. Running live raises AuthorizationFailed. + @record_only() @AllowLargeResponse() def test_create_pipeline_component(self): self.cassette.allow_playback_repeats = True diff --git a/src/machinelearningservices/azext_mlv2/tests/latest/test_compute_scenarios.py b/src/machinelearningservices/azext_mlv2/tests/latest/test_compute_scenarios.py index a52481ac729..38854c295e1 100644 --- a/src/machinelearningservices/azext_mlv2/tests/latest/test_compute_scenarios.py +++ b/src/machinelearningservices/azext_mlv2/tests/latest/test_compute_scenarios.py @@ -11,9 +11,14 @@ import pytest from azext_mlv2.tests.scenario_test_helper import MLBaseScenarioTest from azure.core.exceptions import HttpResponseError +from azure.cli.testsdk.scenario_tests.decorators import record_only class ComputeScenarioTest(MLBaseScenarioTest): + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_compute_aml(self) -> None: compute_name_suffix = "cc1" # Updating dictionary with the job name @@ -48,6 +53,10 @@ def test_compute_aml(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("compute_name_1", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_compute_list_nodes(self) -> None: compute_name_suffix = "cc2" # Updating dictionary with the job name @@ -63,6 +72,10 @@ def test_compute_list_nodes(self) -> None: assert ws_obj["private_ip_address"] == "10.0.0.4" assert ws_obj["public_ip_address"] == "20.246.149.32" + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_compute_compute_instance(self) -> None: compute_name_suffix = "ci3" # Updating dictionary with the job name @@ -90,6 +103,10 @@ def test_compute_compute_instance(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("compute_name_3", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_compute_compute_instance_with_schedules(self) -> None: compute_name_suffix = "ci4" # Updating dictionary with the job name @@ -113,6 +130,10 @@ def test_compute_compute_instance_with_schedules(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("compute_name_4", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_compute_aml_no_location(self) -> None: compute_name_suffix = "cc5" # Updating dictionary with the job name @@ -136,6 +157,10 @@ def test_compute_aml_no_location(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("compute_name_5", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_compute_compute_instance_with_no_public_ip(self) -> None: compute_name_suffix = "ci6" # Updating dictionary with the job name @@ -148,11 +173,19 @@ def test_compute_compute_instance_with_no_public_ip(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("compute_name_6", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_compute_list(self) -> None: ws_obj_list = self.cmd("az ml compute list -g testrg -w testworkspace") ws_obj_list = yaml.safe_load(ws_obj_list.output) assert len(ws_obj_list) > 0 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_compute_list_type(self) -> None: ws_obj_list = self.cmd("az ml compute list --type AmlCompute -g testrg -w testworkspace") ws_obj_list = yaml.safe_load(ws_obj_list.output) diff --git a/src/machinelearningservices/azext_mlv2/tests/latest/test_datastore_scenarios.py b/src/machinelearningservices/azext_mlv2/tests/latest/test_datastore_scenarios.py index 635909025ec..7f50d080446 100644 --- a/src/machinelearningservices/azext_mlv2/tests/latest/test_datastore_scenarios.py +++ b/src/machinelearningservices/azext_mlv2/tests/latest/test_datastore_scenarios.py @@ -12,6 +12,7 @@ from azext_mlv2.tests.scenario_test_helper import MLBaseScenarioTest from azure.ai.ml.exceptions import ValidationErrorType, ValidationException from knack.cli import CLIError +from azure.cli.testsdk.scenario_tests.decorators import record_only def b64read(p): @@ -28,6 +29,10 @@ def create_cmd_str(file_name: str, var_name: str, suffix=None) -> str: class DatastoreScenarioTest(MLBaseScenarioTest): + # Marked as record_only because the test uses hardcoded resource group + # 'testrg' (via create_cmd_str helper), which only exists in the recorded + # cassette. Running live raises AuthorizationFailed. + @record_only() def test_datastore_blob(self) -> None: file_name = "blob_store.yml" var_name = "blobDatastoreName" @@ -38,6 +43,10 @@ def test_datastore_blob(self) -> None: assert ds_obj["type"] == "azure_blob" assert ds_obj["container_name"] == "testblob" + # Marked as record_only because the test uses hardcoded resource group + # 'testrg' (via create_cmd_str helper), which only exists in the recorded + # cassette. Running live raises AuthorizationFailed. + @record_only() def test_datastore_file(self) -> None: file_name = "file_store.yml" var_name = "fileDatastoreName" @@ -48,6 +57,10 @@ def test_datastore_file(self) -> None: assert ds_obj["type"] == "azure_file" assert ds_obj["file_share_name"] == "testfileshare" + # Marked as record_only because the test uses hardcoded resource group + # 'testrg' (via create_cmd_str helper), which only exists in the recorded + # cassette. Running live raises AuthorizationFailed. + @record_only() def test_datastore_blob_credentialless(self) -> None: file_name = "credential_less_blob_store.yml" var_name = "blobCredlessDatastoreName" @@ -59,6 +72,10 @@ def test_datastore_blob_credentialless(self) -> None: assert ds_obj["account_name"] == "sdkvnextcli902348975" assert ds_obj["container_name"] == "testblob" + # Marked as record_only because the test uses hardcoded resource group + # 'testrg' (via create_cmd_str helper), which only exists in the recorded + # cassette. Running live raises AuthorizationFailed. + @record_only() def test_datastore_adlsgen1_credentialless(self) -> None: file_name = "credential_less_adls_gen1.yml" var_name = "adlsgen1CredlessDatastoreName" @@ -68,6 +85,10 @@ def test_datastore_adlsgen1_credentialless(self) -> None: assert ds_obj["type"] == "azure_data_lake_gen1" assert ds_obj["store_name"] == "mytestdatalakegen1" + # Marked as record_only because the test uses hardcoded resource group + # 'testrg' (via create_cmd_str helper), which only exists in the recorded + # cassette. Running live raises AuthorizationFailed. + @record_only() def test_datastore_adlsgen1(self) -> None: file_name = "adls_gen1.yml" var_name = "adlsgen1DatastoreName" @@ -77,6 +98,10 @@ def test_datastore_adlsgen1(self) -> None: assert ds_obj["type"] == "azure_data_lake_gen1" assert ds_obj["store_name"] == "cliv2testdatalake" + # Marked as record_only because the test uses hardcoded resource group + # 'testrg' (via create_cmd_str helper), which only exists in the recorded + # cassette. Running live raises AuthorizationFailed. + @record_only() def test_datastore_adlsgen2_credentialless(self) -> None: file_name = "credential_less_adls_gen2.yml" var_name = "adlsgen2CredlessDatastoreName" @@ -87,6 +112,10 @@ def test_datastore_adlsgen2_credentialless(self) -> None: assert ds_obj["account_name"] == "mytestdatalakegen2" assert ds_obj["filesystem"] == "my-gen2-container" + # Marked as record_only because the test uses hardcoded resource group + # 'testrg' (via create_cmd_str helper), which only exists in the recorded + # cassette. Running live raises AuthorizationFailed. + @record_only() def test_datastore_adlsgen2(self) -> None: file_name = "adls_gen2.yml" var_name = "adlsgen2DatastoreName" @@ -97,6 +126,10 @@ def test_datastore_adlsgen2(self) -> None: assert ds_obj["account_name"] == "cliv2testblob" assert ds_obj["filesystem"] == "a-file-system" + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip("this test is very flaky and causing many issues, please fix") def test_datastore_validation_errors(self) -> None: with pytest.raises((ValidationException, CLIError)) as ex: diff --git a/src/machinelearningservices/azext_mlv2/tests/latest/test_deployment_template_scenarios.py b/src/machinelearningservices/azext_mlv2/tests/latest/test_deployment_template_scenarios.py index c516ce55097..808acadfe8c 100644 --- a/src/machinelearningservices/azext_mlv2/tests/latest/test_deployment_template_scenarios.py +++ b/src/machinelearningservices/azext_mlv2/tests/latest/test_deployment_template_scenarios.py @@ -1,4 +1,4 @@ -# -------------------------------------------------------------------------- +# -------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for # license information. @@ -12,6 +12,7 @@ import tempfile import os from azext_mlv2.tests.scenario_test_helper import MLBaseScenarioTest +from azure.cli.testsdk.scenario_tests.decorators import record_only class DeploymentTemplateScenarioTest(MLBaseScenarioTest): @@ -79,6 +80,10 @@ def _get_config_path(self): config_path = os.path.join(tests_dir, "test_configs", "deployment_template", "deployment_template_basic.yaml") return os.path.normpath(config_path) + # Marked as record_only because the test uses class-level hardcoded + # resource group/registry (test-cli-rg/test-cli-reg) that only exist + # in the recorded cassette. Running live raises ResourceNotFound. + @record_only() def test_01_deployment_template_create(self) -> None: """Test Step 1: Create a deployment template.""" print(f"\n[TEST 01] Creating deployment template: {self.template_name} v{self.template_version}") @@ -111,6 +116,10 @@ def test_01_deployment_template_create(self) -> None: else: print(f"[TEST 01] Template created (no output returned)") + # Marked as record_only because the test uses class-level hardcoded + # resource group/registry (test-cli-rg/test-cli-reg) that only exist + # in the recorded cassette. Running live raises ResourceNotFound. + @record_only() def test_02_deployment_template_list(self) -> None: """Test Step 2: List deployment templates in the registry.""" print(f"\n[TEST 02] Listing deployment templates in registry: {self.registry_name}") @@ -132,6 +141,10 @@ def test_02_deployment_template_list(self) -> None: else: print(f"[TEST 02] List operation completed (no output)") + # Marked as record_only because the test uses class-level hardcoded + # resource group/registry (test-cli-rg/test-cli-reg) that only exist + # in the recorded cassette. Running live raises ResourceNotFound. + @record_only() def test_03_deployment_template_get(self) -> None: """Test Step 3: Get the specific deployment template.""" print(f"\n[TEST 03] Getting deployment template: {self.template_name} v{self.template_version}") @@ -159,6 +172,10 @@ def test_03_deployment_template_get(self) -> None: else: print(f"[TEST 03] Get operation completed") + # Marked as record_only because the test uses class-level hardcoded + # resource group/registry (test-cli-rg/test-cli-reg) that only exist + # in the recorded cassette. Running live raises ResourceNotFound. + @record_only() def test_04_deployment_template_update(self) -> None: """Test Step 4: Update deployment template (description and tags).""" print(f"\n[TEST 04] Updating deployment template description and tags") @@ -188,6 +205,10 @@ def test_04_deployment_template_update(self) -> None: else: print(f"[TEST 04] Update completed") + # Marked as record_only because the test uses class-level hardcoded + # resource group/registry (test-cli-rg/test-cli-reg) that only exist + # in the recorded cassette. Running live raises ResourceNotFound. + @record_only() def test_05_deployment_template_archive(self) -> None: """Test Step 5: Archive the deployment template.""" print(f"\n[TEST 05] Archiving deployment template") @@ -199,6 +220,10 @@ def test_05_deployment_template_archive(self) -> None: assert archive_result.output == "" or "archived" in archive_result.output.lower() print(f"[TEST 05] Template archived successfully") + # Marked as record_only because the test uses class-level hardcoded + # resource group/registry (test-cli-rg/test-cli-reg) that only exist + # in the recorded cassette. Running live raises ResourceNotFound. + @record_only() def test_06_deployment_template_restore(self) -> None: """Test Step 6: Restore the deployment template.""" print(f"\n[TEST 06] Restoring deployment template") @@ -210,6 +235,10 @@ def test_06_deployment_template_restore(self) -> None: assert restore_result.output == "" or "restored" in restore_result.output.lower() print(f"[TEST 06] Template restored successfully") + # Marked as record_only because the test uses class-level hardcoded + # resource group/registry (test-cli-rg/test-cli-reg) that only exist + # in the recorded cassette. Running live raises ResourceNotFound. + @record_only() def test_07_deployment_template_create_with_version(self) -> None: """Test: Create a new version of the deployment template.""" print(f"\n[TEST 07] Creating new version of deployment template") @@ -253,6 +282,10 @@ def test_07_deployment_template_create_with_version(self) -> None: except: pass + # Marked as record_only because the test uses class-level hardcoded + # resource group/registry (test-cli-rg/test-cli-reg) that only exist + # in the recorded cassette. Running live raises ResourceNotFound. + @record_only() def test_08_deployment_template_list_specific(self) -> None: """Test: List all deployment templates and verify list operation works.""" print(f"\n[TEST 08] Listing deployment templates in registry") @@ -279,6 +312,10 @@ def test_08_deployment_template_list_specific(self) -> None: else: print(f"[TEST 08] List operation returned no output") + # Marked as record_only because the test uses class-level hardcoded + # resource group/registry (test-cli-rg/test-cli-reg) that only exist + # in the recorded cassette. Running live raises ResourceNotFound. + @record_only() def test_09_deployment_template_get_nonexistent(self) -> None: """Test: Try to get a non-existent deployment template (negative test).""" print(f"\n[TEST 09] Testing error handling for non-existent template") @@ -299,6 +336,10 @@ def test_09_deployment_template_get_nonexistent(self) -> None: assert "does not exist" in str(e).lower() or "not found" in str(e).lower() or \ "error" in str(e).lower(), f"Unexpected error message: {e}" + # Marked as record_only because the test uses class-level hardcoded + # resource group/registry (test-cli-rg/test-cli-reg) that only exist + # in the recorded cassette. Running live raises ResourceNotFound. + @record_only() def test_10_deployment_template_update_multiple_tags(self) -> None: """Test: Update deployment template with multiple tag changes.""" print(f"\n[TEST 10] Updating deployment template with multiple tags") @@ -324,6 +365,10 @@ def test_10_deployment_template_update_multiple_tags(self) -> None: else: print(f"[TEST 10] Update completed") + # Marked as record_only because the test uses class-level hardcoded + # resource group/registry (test-cli-rg/test-cli-reg) that only exist + # in the recorded cassette. Running live raises ResourceNotFound. + @record_only() def test_11_deployment_template_archive_restore_cycle(self) -> None: """Test: Archive and restore cycle for version 1.""" print(f"\n[TEST 11] Testing archive-restore cycle for version 1") @@ -364,6 +409,10 @@ def test_11_deployment_template_archive_restore_cycle(self) -> None: assert restore_result.output == "" or "restored" in restore_result.output.lower() print(f"[TEST 11] Version {test_version} restored successfully") + # Marked as record_only because the test uses class-level hardcoded + # resource group/registry (test-cli-rg/test-cli-reg) that only exist + # in the recorded cassette. Running live raises ResourceNotFound. + @record_only() def test_12_deployment_template_full_workflow_verification(self) -> None: """Test: Final verification - list templates and check we can get our template.""" print(f"\n[TEST 12] Final verification of deployment template state") diff --git a/src/machinelearningservices/azext_mlv2/tests/latest/test_environment_scenarios.py b/src/machinelearningservices/azext_mlv2/tests/latest/test_environment_scenarios.py index 0b4dfc00b90..1bc5a9219cf 100644 --- a/src/machinelearningservices/azext_mlv2/tests/latest/test_environment_scenarios.py +++ b/src/machinelearningservices/azext_mlv2/tests/latest/test_environment_scenarios.py @@ -6,6 +6,7 @@ import yaml from azext_mlv2.tests.scenario_test_helper import MLBaseScenarioTest from azext_mlv2.tests.util import assert_same +from azure.cli.testsdk.scenario_tests.decorators import record_only class EnvironmentScenarioTest(MLBaseScenarioTest): @@ -16,6 +17,10 @@ def test_environment_no_workspace_no_registry(self) -> None: dep_obj = self.cmd(f'{base_command} --workspace-name="" --registry-name=""') assert "one the following arguments are required: [--workspace-name/-w, --registry-name]" in str(exp.value) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_environment(self) -> None: env_obj = self.cmd( "az ml environment create --file ./src/machinelearningservices/azext_mlv2/tests/test_configs/environment/environment_conda.yml --name {environmentName} --set version=1 --tags test=test -g testrg -w testworkspace" @@ -56,6 +61,10 @@ def test_environment(self) -> None: env_restore_obj = self.cmd("az ml environment restore -g testrg -w testworkspace -n {environmentName}") assert env_restore_obj.output == "" + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_environment_list(self) -> None: env_list_obj = self.cmd("az ml environment list -g testrg -w testworkspace") env_list_obj = yaml.safe_load(env_list_obj.output) @@ -75,6 +84,10 @@ def test_environment_list(self) -> None: assert "id" not in env_name_obj[0] # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_environment_with_docker_context(self) -> None: env_obj = self.cmd( @@ -96,6 +109,10 @@ def test_environment_with_docker_context(self) -> None: assert_same(env_obj, env_show_obj, filter=["creation_context"]) # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_environment_build_context(self) -> None: env_obj = self.cmd( @@ -112,6 +129,10 @@ def test_environment_build_context(self) -> None: assert_same(env_obj, env_show_obj, filter=["creation_context"]) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_environment_with_image(self) -> None: env_obj = self.cmd( "az ml environment create --file ./src/machinelearningservices/azext_mlv2/tests/test_configs/environment/environment_docker_image.yml --name {environmentName} --version 1 --image pytorch.pytorch --conda-file endpoint_conda.yml -g testrg -w testworkspace" @@ -120,6 +141,10 @@ def test_environment_with_image(self) -> None: assert env_obj["image"] == "pytorch.pytorch" assert "conda_file" in env_obj + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_environment_params_override(self) -> None: env_obj = self.cmd( "az ml environment create --file ./src/machinelearningservices/azext_mlv2/tests/test_configs/environment/environment_conda_name_version.yml --name {environmentName} --description bla --os-type windows --version 2 -g testrg -w testworkspace" @@ -129,6 +154,10 @@ def test_environment_params_override(self) -> None: assert env_obj["os_type"] == "windows" assert env_obj["version"] == "2" + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_environment_with_no_file(self) -> None: env_obj = self.cmd( "az ml environment create --name {environmentName} --version 1 --image pytorch.pytorch --conda-file ./src/machinelearningservices/azext_mlv2/tests/test_configs/environment/endpoint_conda.yml -g testrg -w testworkspace" @@ -167,6 +196,10 @@ def test_environment_anonymous_env_with_conda(self) -> None: assert anon_environment[-1] == "10e20cf4ec11a618e87f0dd965ef70b2" assert anon_environment[-3] == "CliV2AnonymousEnvironment" + # Marked as record_only because the test uses hardcoded resources + # (resource group, registry, etc.) that only exist in the recorded + # cassette. Running live raises ResourceNotFound or similar errors. + @record_only() def test_environment_show_registry(self) -> None: env_obj = self.cmd( "az ml environment show -n 4c99f460-20cd-4821-8745-202aa7555604 -v 93435847-704b-4280-83f3-f735d8b5eff7 --registry-name testfeed" @@ -178,12 +211,20 @@ def test_environment_show_registry(self) -> None: ) assert env_obj["name"] == "4c99f460-20cd-4821-8745-202aa7555604" + # Marked as record_only because the test uses hardcoded resources + # (resource group, registry, etc.) that only exist in the recorded + # cassette. Running live raises ResourceNotFound or similar errors. + @record_only() def test_environment_list_registry(self) -> None: env_obj = self.cmd("az ml environment list --registry-name test-registry-ux-1") env_obj = yaml.safe_load(env_obj.output) assert len(env_obj) > 1 + # Marked as record_only because the test uses hardcoded resources + # (resource group, registry, etc.) that only exist in the recorded + # cassette. Running live raises ResourceNotFound or similar errors. + @record_only() def test_environment_create_in_registry(self) -> None: env_obj = self.cmd( @@ -197,6 +238,10 @@ def test_environment_create_in_registry(self) -> None: env_obj_1 = yaml.safe_load(env_obj_1.output) assert len(env_obj_1) > 1 + # Marked as record_only because the test uses hardcoded resources + # (resource group, registry, etc.) that only exist in the recorded + # cassette. Running live raises ResourceNotFound or similar errors. + @record_only() def test_environment_archive_in_registry(self) -> None: env_archive_obj = self.cmd( @@ -212,6 +257,10 @@ def test_environment_restore_in_registry(self) -> None: ) assert env_restore_obj.output == "" + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_environment_update(self) -> None: env_obj = self.cmd( "az ml environment update -n online-endpoint-mir-test -v 3 --set tags.nn=kkk -g testrg -w testworkspace" diff --git a/src/machinelearningservices/azext_mlv2/tests/latest/test_feature_set_scenarios.py b/src/machinelearningservices/azext_mlv2/tests/latest/test_feature_set_scenarios.py index 71af9aa9ceb..20c2034d17f 100644 --- a/src/machinelearningservices/azext_mlv2/tests/latest/test_feature_set_scenarios.py +++ b/src/machinelearningservices/azext_mlv2/tests/latest/test_feature_set_scenarios.py @@ -1,4 +1,4 @@ -# -------------------------------------------------------------------------- +# -------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for # license information. @@ -10,10 +10,15 @@ import pytest import yaml from azext_mlv2.tests.scenario_test_helper import MLBaseScenarioTest +from azure.cli.testsdk.scenario_tests.decorators import record_only class FeatureSetScenarioTest(MLBaseScenarioTest): # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resources + # (resource group, registry, etc.) that only exist in the recorded + # cassette. Running live raises ResourceNotFound or similar errors. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_featureSet(self) -> None: self.kwargs["resourceGroup"] = "test-rg-000006" diff --git a/src/machinelearningservices/azext_mlv2/tests/latest/test_feature_store_entity_scenarios.py b/src/machinelearningservices/azext_mlv2/tests/latest/test_feature_store_entity_scenarios.py index 91f3c61a14f..6f67e8e4d59 100644 --- a/src/machinelearningservices/azext_mlv2/tests/latest/test_feature_store_entity_scenarios.py +++ b/src/machinelearningservices/azext_mlv2/tests/latest/test_feature_store_entity_scenarios.py @@ -10,9 +10,14 @@ import pytest import yaml from azext_mlv2.tests.scenario_test_helper import MLBaseScenarioTest +from azure.cli.testsdk.scenario_tests.decorators import record_only class FeatureStoreEntityScenarioTest(MLBaseScenarioTest): + # Marked as record_only because the test uses hardcoded resources + # (resource group, registry, etc.) that only exist in the recorded + # cassette. Running live raises ResourceNotFound or similar errors. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_featureStoreEntity(self) -> None: self.kwargs["resourceGroup"] = "test-rg-000006" diff --git a/src/machinelearningservices/azext_mlv2/tests/latest/test_feature_store_scenarios.py b/src/machinelearningservices/azext_mlv2/tests/latest/test_feature_store_scenarios.py index fb92aa86874..7199966a203 100644 --- a/src/machinelearningservices/azext_mlv2/tests/latest/test_feature_store_scenarios.py +++ b/src/machinelearningservices/azext_mlv2/tests/latest/test_feature_store_scenarios.py @@ -10,9 +10,14 @@ import pytest import yaml from azext_mlv2.tests.scenario_test_helper import MLBaseScenarioTest +from azure.cli.testsdk.scenario_tests.decorators import record_only class FeatureStoreScenarioTest(MLBaseScenarioTest): + # Marked as record_only because the test uses hardcoded resources + # (resource group, registry, etc.) that only exist in the recorded + # cassette. Running live raises ResourceNotFound or similar errors. + @record_only() def test_featureStore(self) -> None: self.kwargs["resourceGroup"] = "test-rg-000006" @@ -72,6 +77,10 @@ def test_featureStore(self) -> None: ) self.assertEqual(fs_obj_delete.output, "") + # Marked as record_only because the test uses hardcoded resources + # (resource group, registry, etc.) that only exist in the recorded + # cassette. Running live raises ResourceNotFound or similar errors. + @record_only() def test_featurestore_managednetwork_provision(self) -> None: self.kwargs["resourceGroup"] = "test-rg-000006" diff --git a/src/machinelearningservices/azext_mlv2/tests/latest/test_job_scenarios.py b/src/machinelearningservices/azext_mlv2/tests/latest/test_job_scenarios.py index 934df52b83f..1116316e2c9 100644 --- a/src/machinelearningservices/azext_mlv2/tests/latest/test_job_scenarios.py +++ b/src/machinelearningservices/azext_mlv2/tests/latest/test_job_scenarios.py @@ -29,6 +29,7 @@ from azure.ai.ml.exceptions import UserErrorException from ..util import private_flag +from azure.cli.testsdk.scenario_tests.decorators import record_only @pytest.fixture() @@ -45,6 +46,10 @@ def disable_private_preview_features(): @pytest.mark.usefixtures("enable_private_preview_features") class JobScenarioTest(MLBaseScenarioTest): + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: CannotOverwriteExistingCassetteException in pipeline.") def test_job_command_job(self) -> None: # TODO: Assumes there is a compute called testCompute, which may prevent a successful recording @@ -120,6 +125,10 @@ def test_job_command_job(self) -> None: self.kwargs.pop("command_job_name1", None) # Ignoring properties ProcessStatusFile and ProcessInfoFile, present only after the job is running + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="ServiceError:Service 'DATA_CAPABILITY' returned capability start response with code") def test_job_command_job_show_to_asset_create(self) -> None: job_name_suffix = "-2" @@ -147,6 +156,10 @@ def test_job_command_job_show_to_asset_create(self) -> None: #@pytest.mark.public_preview_only # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_job_command_job_list(self) -> None: """ @@ -176,6 +189,10 @@ def test_job_command_job_list(self) -> None: assert len(default_job_list_yml) >= total_jobs assert len(default_job_list_yml) <= MAX_LIST_CLI_RESULTS + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_job_command_git_path(self) -> None: with private_flag(): job_name_suffix = "-4" @@ -200,6 +217,10 @@ def test_job_command_git_path(self) -> None: self.kwargs.pop("command_job_name4", None) # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_job_command_job_create_skip_validation(self) -> None: job_name_suffix = "-5" @@ -214,6 +235,10 @@ def test_job_command_job_create_skip_validation(self) -> None: mock_validate.assert_called_once() # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_job_command_job_archive_restore(self) -> None: job_name_suffix = "-3" @@ -237,6 +262,10 @@ def test_job_command_job_archive_restore(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("command_job_name3", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_job_environment_version(self) -> None: self.cmd( "az ml environment create --file ./src/machinelearningservices/azext_mlv2/tests/test_configs/environment/environment_colon_version.yml -g testrg -w testworkspace" @@ -251,6 +280,10 @@ def test_job_environment_version(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("job_name1", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.usefixtures("enable_private_preview_features") @pytest.mark.skip(reason="TODO: 2161301, Test is failing when executing from recording in ADO") def test_job_command_job_with_asset_registry(self) -> None: @@ -266,6 +299,10 @@ def test_job_command_job_with_asset_registry(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("command_job_name1", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: 1796036, does not complete when recorded") def test_job_sweep_job(self) -> None: job_name_suffix = "2" @@ -305,6 +342,10 @@ def test_job_sweep_job(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("sweep_job_name1", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Can't record - job times out due to max_duration reached") def test_job_sweep_job_registry(self) -> None: job_name_suffix = "90" @@ -317,6 +358,10 @@ def test_job_sweep_job_registry(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("sweep_job_name", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: 1788025, could not re-record this test") def test_job_pipeline_job_settings_simple(self) -> None: job_name_suffix = "-1" @@ -345,6 +390,10 @@ def test_job_pipeline_job_settings_simple(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("pipeline_job_name1", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Unable to get authority configuration for login") def test_cli_error(self) -> None: create_job_command = "az ml job create --file ./src/machinelearningservices/azext_mlv2/tests/test_configs/pipeline_jobs/invalid/with_invalid_component.yml -g testrg -w testworkspace" @@ -354,6 +403,10 @@ def test_cli_error(self) -> None: # Assert missing environment path assert "jobs.hello_world_component.component.environment" in str(e.value) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: 1796036, does not complete when recorded") def test_job_pipeline_job_input_paths(self) -> None: job_name_suffix = "-3" @@ -427,6 +480,10 @@ def test_job_pipeline_job_input_paths(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("pipeline_job_name2", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Tests always fail.") def test_job_pipeline_job_translated_from_command_job(self) -> None: job_name_suffix = "-4" @@ -474,6 +531,10 @@ def test_job_pipeline_job_translated_from_command_job(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("pipeline_job_name3", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_pipeline_job_validate(self) -> None: import json @@ -501,6 +562,10 @@ def test_pipeline_job_validate(self) -> None: } # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_automl_tabular_classification_job(self) -> None: classification_job_name = "{}{}".format(self.kwargs.get("automlClassificationJobName", None), "-1") @@ -517,6 +582,10 @@ def test_automl_tabular_classification_job(self) -> None: self._validate_automl_job_common_properties(file_obj=file_obj, job_obj=job_obj) # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_automl_tabular_forecasting_job(self) -> None: forecasting_job_name = "{}{}".format(self.kwargs.get("automlForecastingJobName", None), "-1") @@ -540,6 +609,10 @@ def test_automl_tabular_forecasting_job(self) -> None: assert value == job_value # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_automl_tabular_regression_job(self) -> None: regression_job_name = "{}{}".format(self.kwargs.get("automlRegressionJobName", None), "-1") @@ -555,6 +628,10 @@ def test_automl_tabular_regression_job(self) -> None: file_obj = yaml.safe_load(f) self._validate_automl_job_common_properties(file_obj=file_obj, job_obj=job_obj) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Tests always fail.") def test_automl_image_classification_job(self) -> None: image_classification_job_name = "{}{}".format(self.kwargs.get("automlImageClassificationJobName", None), "-2") @@ -574,6 +651,10 @@ def test_automl_image_classification_job(self) -> None: self._validate_automl_image_job_properties(file_obj=file_obj, job_obj=job_obj) # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_automl_image_classification_automode_job(self) -> None: image_classification_automode_job_name = "{}{}".format( @@ -594,6 +675,10 @@ def test_automl_image_classification_automode_job(self) -> None: ) self._validate_automl_image_job_properties(file_obj=file_obj, job_obj=job_obj) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Tests always fail.") def test_automl_image_classification_multilabel_job(self) -> None: image_classification_multilabel_job_name = "{}{}".format( @@ -615,6 +700,10 @@ def test_automl_image_classification_multilabel_job(self) -> None: self._validate_automl_image_job_properties(file_obj=file_obj, job_obj=job_obj) # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_automl_image_classification_multilabel_automode_job(self) -> None: image_classification_multilabel_automode_job_name = "{}{}".format( @@ -635,6 +724,10 @@ def test_automl_image_classification_multilabel_automode_job(self) -> None: ) self._validate_automl_image_job_properties(file_obj=file_obj, job_obj=job_obj) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Tests always fail.") def test_automl_image_instance_segmentation_job(self) -> None: image_instance_segmentation_job_name = "{}{}".format( @@ -656,6 +749,10 @@ def test_automl_image_instance_segmentation_job(self) -> None: self._validate_automl_image_job_properties(file_obj=file_obj, job_obj=job_obj) # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_automl_image_instance_segmentation_automode_job(self) -> None: image_instance_segmentation_automode_job_name = "{}{}".format( @@ -676,6 +773,10 @@ def test_automl_image_instance_segmentation_automode_job(self) -> None: ) self._validate_automl_image_job_properties(file_obj=file_obj, job_obj=job_obj) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Tests always fail.") def test_automl_image_object_detection_job(self) -> None: image_object_detection_job_name = "{}{}".format( @@ -697,6 +798,10 @@ def test_automl_image_object_detection_job(self) -> None: self._validate_automl_image_job_properties(file_obj=file_obj, job_obj=job_obj) # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_automl_image_object_detection_automode_job(self) -> None: image_object_detection_automode_job_name = "{}{}".format( @@ -717,6 +822,10 @@ def test_automl_image_object_detection_automode_job(self) -> None: ) self._validate_automl_image_job_properties(file_obj=file_obj, job_obj=job_obj) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Tests always fail.") def test_automl_image_job_limits_outside_sweep(self) -> None: image_job_name = "{}{}".format(self.kwargs.get("automlImageObjectDetectionJobName", None), "-1") @@ -808,6 +917,10 @@ def validate_internal(file_internal, job_internal) -> None: if file_obj.get("training"): validate_internal(file_obj["training"], job_obj["training"]) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Tests always fail.") def test_automl_text_classification_job(self) -> None: job_name = "{}{}".format(self.kwargs.get("automlTextClassificationJobName", None), "-1") @@ -826,6 +939,10 @@ def test_automl_text_classification_job(self) -> None: file_obj = yaml.safe_load(f) self._validate_automl_job_common_properties(file_obj=file_obj, job_obj=job_obj, is_gpu=True) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Tests always fail.") def test_automl_text_classification_multilabel_job(self) -> None: job_name = "{}{}".format(self.kwargs.get("automlTextMultilabelJobName", None), "-1") @@ -844,6 +961,10 @@ def test_automl_text_classification_multilabel_job(self) -> None: file_obj = yaml.safe_load(f) self._validate_automl_job_common_properties(file_obj=file_obj, job_obj=job_obj, is_gpu=True) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Tests always fail.") def test_automl_text_ner_job(self) -> None: job_name = "{}{}".format(self.kwargs.get("automlTextNERJobName", None), "-1") @@ -863,6 +984,10 @@ def test_automl_text_ner_job(self) -> None: self._validate_automl_job_common_properties(file_obj=file_obj, job_obj=job_obj, is_gpu=True) """ + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.public_preview_only def test_job_automl_job_public_preview_sanity(self) -> None: with pytest.raises(CLIError) as ex: @@ -870,6 +995,10 @@ def test_job_automl_job_public_preview_sanity(self) -> None: assert "Unsupported job type: automl_job" in str(ex) """ + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: 1788034, could not re-record this test") def test_component_job_with_registry_uri(self): component_obj = self.cmd( @@ -881,6 +1010,10 @@ def test_component_job_with_registry_uri(self): == "azureml://registries/testFeed/components/my_hello_world_asset_2/versions/1" ) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: 1796036, does not complete when recorded") def test_pipeline_job_with_automl_node(self): job_name_suffix = "-11" @@ -903,6 +1036,10 @@ def test_pipeline_job_with_automl_node(self): file_obj=file_obj, job_obj=job_obj, skip_compute_experiment=True ) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: 1796036, does not complete when recorded") def test_pipeline_job_with_parallel_node(self): job_name_suffix = "-12" @@ -913,6 +1050,10 @@ def test_pipeline_job_with_parallel_node(self): job_obj = yaml.safe_load(job_obj.output) assert job_obj["name"] == self.kwargs["pipeline_job_name12"] + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: 2161301, Test is failing when executing from recording in ADO") def test_pipeline_job_with_registry_env(self): job_name_suffix = "-14" @@ -923,6 +1064,10 @@ def test_pipeline_job_with_registry_env(self): job_obj = yaml.safe_load(job_obj.output) assert job_obj["name"] == self.kwargs["pipeline_job_name14"] + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Tests always fail.") def test_pipeline_job_with_pipeline_node(self): job_name_suffix = "-15" @@ -933,6 +1078,10 @@ def test_pipeline_job_with_pipeline_node(self): job_obj = yaml.safe_load(job_obj.output) assert job_obj["name"] == self.kwargs["pipeline_job_name15"] + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Tests always fail.") def test_pipeline_job_with_parameter_group(self): job_name_suffix = "-16" @@ -943,6 +1092,10 @@ def test_pipeline_job_with_parameter_group(self): job_obj = yaml.safe_load(job_obj.output) assert job_obj["name"] == self.kwargs["pipeline_job_name16"] + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: 2174601 - Test is failing with RG issues when executing in ADO") def test_custom_property_logging(self) -> None: job_name_suffix = "-17" @@ -957,6 +1110,10 @@ def test_custom_property_logging(self) -> None: @pytest.mark.private_preview_only @pytest.mark.usefixtures("disable_private_preview_features") class JobPrivatePreviewScenarioTest(MLBaseScenarioTest): + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: 1796036, does not complete when recorded") def test_pipeline_job_private_preview_features(self): test_paths = [ diff --git a/src/machinelearningservices/azext_mlv2/tests/latest/test_k8s_online_endpoint_scenarios.py b/src/machinelearningservices/azext_mlv2/tests/latest/test_k8s_online_endpoint_scenarios.py index dbc210d99c1..6b577fd46c9 100644 --- a/src/machinelearningservices/azext_mlv2/tests/latest/test_k8s_online_endpoint_scenarios.py +++ b/src/machinelearningservices/azext_mlv2/tests/latest/test_k8s_online_endpoint_scenarios.py @@ -14,9 +14,14 @@ from azure.ai.ml._scope_dependent_operations import OperationScope import pytest +from azure.cli.testsdk.scenario_tests.decorators import record_only class K8SOnlineEndpointScenarioTest(MLBaseScenarioTest): + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_online_endpoint_k8s(self) -> None: endpoint_name_suffix = "-k8s" self.kwargs["online_endpoint_name"] = "{}{}".format( @@ -41,6 +46,10 @@ def test_online_endpoint_k8s(self) -> None: assert cmd_delete.output == "" self.kwargs.pop("online_endpoint_name", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_online_endpoint_k8s_set_compute(self) -> None: endpoint_name_suffix = "-k8s-2" self.kwargs["online_endpoint_name_2"] = "{}{}".format( @@ -62,6 +71,10 @@ def test_online_endpoint_k8s_set_compute(self) -> None: assert cmd_delete.output == "" self.kwargs.pop("online_endpoint_name_2", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="AKS legacy compute is not supported for online endpoints or deployments") def test_online_endpoint_k8s_deployment_cpu(self) -> None: endpoint_name_suffix = "-k8s-cpu" @@ -188,6 +201,10 @@ def test_online_endpoint_k8s_deployment_cpu(self) -> None: self.kwargs.pop("online_endpoint_name_cpu", None) self.kwargs.pop("online_deployment_name_cpu", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="AKS legacy compute is not supported for online endpoints or deployments") def test_online_endpoint_k8s_deployment_gpu(self) -> None: endpoint_name_suffix = "-k8s-gpu" @@ -263,6 +280,10 @@ def test_online_endpoint_k8s_deployment_gpu(self) -> None: self.kwargs.pop("online_endpoint_name_gpu", None) self.kwargs.pop("online_deployment_name_gpu", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: 1887461, could not re-record this test due to regression") def test_online_endpoint_k8s_deployment_update(self) -> None: endpoint_name_suffix = "-k8s-gpu2" diff --git a/src/machinelearningservices/azext_mlv2/tests/latest/test_model_scenarios.py b/src/machinelearningservices/azext_mlv2/tests/latest/test_model_scenarios.py index 45a1265319c..fa85b0b2965 100644 --- a/src/machinelearningservices/azext_mlv2/tests/latest/test_model_scenarios.py +++ b/src/machinelearningservices/azext_mlv2/tests/latest/test_model_scenarios.py @@ -18,6 +18,7 @@ from azure.ai.ml.constants._common import AssetTypes from ..util import private_flag +from azure.cli.testsdk.scenario_tests.decorators import record_only class ModelScenarioTest(MLBaseScenarioTest): @@ -29,6 +30,10 @@ def test_model_no_workspace_no_registry(self) -> None: assert "one the following arguments are required: [--workspace-name/-w, --registry-name]" in str(exp.value) # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_model(self) -> None: model_obj = self.cmd( @@ -77,6 +82,10 @@ def test_model(self) -> None: assert model_restore_obj.output == "" # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_model_with_stage(self) -> None: model_obj = self.cmd( @@ -103,6 +112,10 @@ def test_model_with_stage(self) -> None: assert "stage" in model assert model["stage"] == "Production" + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_model_list(self) -> None: models = self.cmd("az ml model list -g testrg -w testworkspace") models = yaml.safe_load(models.output) @@ -133,6 +146,10 @@ def test_model_list(self) -> None: assert "tags" in model_name_obj[0] # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_model_remote_path(self) -> None: model_obj = self.cmd( @@ -147,6 +164,10 @@ def test_model_remote_path(self) -> None: model_remote_obj["path"] == model_obj["path"] # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_model_without_yml(self) -> None: @@ -164,6 +185,10 @@ def test_model_without_yml(self) -> None: assert_same(model_obj, model_show_job) # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_model_update_stage(self) -> None: @@ -191,6 +216,10 @@ def test_model_update_stage(self) -> None: model_show_job = yaml.safe_load(model_show_job.output) assert_same(model_obj, model_show_job) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_model_datastore_uri(self) -> None: """ Re-record instructions: In order to re-record this test create a job run that outputs a model. Go into datastores @@ -230,6 +259,10 @@ def test_model_datastore_uri(self) -> None: model_show_job_file = yaml.safe_load(model_show_job_file.output) assert_same(model_obj_file, model_show_job_file) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_model_job_uri(self) -> None: """ Re-record instructions: In order to re-record this test create a job run that outputs a model to either a default output folder or a named output folder. @@ -263,6 +296,10 @@ def test_model_job_uri(self) -> None: model_show_job_file = yaml.safe_load(model_show_job_file.output) assert_same(model_obj_file, model_show_job_file) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_model_mlflow_uri(self) -> None: """ Re-record instructions: In order to re-record this test create a job run that outputs a model using mlflow.autolog(). Copy the run id @@ -288,6 +325,10 @@ def test_model_mlflow_uri(self) -> None: assert_same(model_obj_folder, model_show_job_folder) # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_model_download_mlflow(self) -> None: self.cmd( @@ -302,6 +343,10 @@ def test_model_download_mlflow(self) -> None: assert "lightgbm_mlflow_model" in files[1] # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_model_download_local_upload_singlefile(self) -> None: self.cmd( @@ -315,6 +360,10 @@ def test_model_download_local_upload_singlefile(self) -> None: assert "lightgbm_mlflow_model" in files[1] # This test is not working. TODO: https://dev.azure.com/msdata/Vienna/_workitems/edit/3372868 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Recording and replay not working.") def test_model_anon_with_batch(self) -> None: """ @@ -330,6 +379,10 @@ def test_model_anon_with_batch(self) -> None: model_cmd_out = yaml.safe_load(model_cmd.output) assert "bd64529f9a171e743eea068b71027c76" in model_cmd_out["model"] + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_model_create_in_registry(self) -> None: """ To re record this model we have to increment the version as the model cannot be removed from registry; @@ -345,6 +398,10 @@ def test_model_create_in_registry(self) -> None: model_obj = yaml.safe_load(model_obj.output) assert len(model_obj) > 1 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_model_show_and_list_in_registry(self) -> None: model_obj_get = self.cmd("az ml model show -n model_version_e2e -v 1 --registry-name testFeed -g testrg -w testworkspace") model_obj_get = yaml.safe_load(model_obj_get.output) @@ -355,6 +412,10 @@ def test_model_show_and_list_in_registry(self) -> None: model_obj_list = yaml.safe_load(model_obj_list.output) assert len(model_obj_list) > 1 + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_model_create_in_registry_no_storage(self) -> None: # this will test creation of model in a registry which was created without a storage account. If you want to rerecord create a registry without storage # and then use that registry @@ -367,6 +428,10 @@ def test_model_create_in_registry_no_storage(self) -> None: else: assert False + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_model_archive_in_registry(self) -> None: env_archive_obj = self.cmd( @@ -374,6 +439,10 @@ def test_model_archive_in_registry(self) -> None: ) assert env_archive_obj.output == "" + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_model_restore_in_registry(self) -> None: env_restore_obj = self.cmd( diff --git a/src/machinelearningservices/azext_mlv2/tests/latest/test_online_endpoint_scenarios.py b/src/machinelearningservices/azext_mlv2/tests/latest/test_online_endpoint_scenarios.py index ccd71c3f987..aeabf3c6210 100644 --- a/src/machinelearningservices/azext_mlv2/tests/latest/test_online_endpoint_scenarios.py +++ b/src/machinelearningservices/azext_mlv2/tests/latest/test_online_endpoint_scenarios.py @@ -19,10 +19,15 @@ from azure.ai.ml._scope_dependent_operations import OperationScope from ..util import private_flag +from azure.cli.testsdk.scenario_tests.decorators import record_only class OnlineEndpointScenarioTest(MLBaseScenarioTest): + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_endpoint_properties(self) -> None: cmd_create = self.cmd( "az ml online-deployment show -n data-t-t -e bani-e2e-1 -g testrg -w testworkspace" @@ -35,6 +40,10 @@ def test_endpoint_properties(self) -> None: assert cmd_create["liveness_probe"] is not None assert cmd_create["egress_public_network_access"] is not None + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_online_endpoint_mir(self) -> None: endpoint_name_suffix = "-1" # Updating dictionary with the job name @@ -61,6 +70,10 @@ def test_online_endpoint_mir(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("online_endpoint_name_1", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_online_endpoint_idempotent(self) -> None: endpoint_name_suffix = "-2" # Updating dictionary with the job name @@ -90,6 +103,10 @@ def test_online_endpoint_idempotent(self) -> None: self.kwargs.pop("online_endpoint_name_2", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: 2161301, Test is failing when executing from recording in ADO") def test_online_endpoint_mir_deployment_create(self) -> None: # Using existing endpoint. This eliminated dependency on endpoint creation @@ -134,6 +151,10 @@ def test_online_endpoint_mir_deployment_create(self) -> None: self.kwargs.pop("online_endpoint_name_4", None) self.kwargs.pop("online_deployment_name_4", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO (2376694): Test fails with InternalServerError and points to non-existent logs") def test_online_endpoint_runs_uri(self) -> None: endpoint_name_suffix = "-22" @@ -164,6 +185,10 @@ def test_online_endpoint_runs_uri(self) -> None: self.kwargs.pop("online_endpoint_name_22", None) self.kwargs.pop("online_deployment_name_22", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO (2377182): Online endpoint test fails with ModelAssetPathNotFoundInStorage") def test_online_endpoint_datastore_uri(self) -> None: endpoint_name_suffix = "-21" @@ -195,6 +220,10 @@ def test_online_endpoint_datastore_uri(self) -> None: self.kwargs.pop("online_endpoint_name_21", None) self.kwargs.pop("online_deployment_name_21", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: 2161301, Test is failing when executing from recording in ADO") def test_online_endpoint_job_uri(self) -> None: endpoint_name_suffix = "-20" @@ -226,6 +255,10 @@ def test_online_endpoint_job_uri(self) -> None: self.kwargs.pop("online_endpoint_name_20", None) self.kwargs.pop("online_deployment_name_20", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: 2161301, Test is failing when executing from recording in ADO") def test_online_endpoint_mir_deployment_create_with_all_traffic(self) -> None: endpoint_name_suffix = "-19v2" @@ -257,6 +290,10 @@ def test_online_endpoint_mir_deployment_create_with_all_traffic(self) -> None: self.kwargs.pop("online_endpoint_name_19", None) self.kwargs.pop("online_deployment_name_19", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: 2161301, Test is failing when executing from recording in ADO") def test_online_endpoint_mir_deployment_update(self) -> None: endpoint_name_suffix = "-6" @@ -299,6 +336,10 @@ def test_online_endpoint_mir_deployment_update(self) -> None: self.kwargs.pop("online_endpoint_name_5", None) self.kwargs.pop("online_deployment_name_5", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: 2161301, Test is failing when executing from recording in ADO") def test_online_endpoint_mir_deployment_delete(self) -> None: endpoint_name_suffix = "-6" @@ -319,6 +360,10 @@ def test_online_endpoint_mir_deployment_delete(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("online_endpoint_name_6", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_online_endpoint_invoke_invalid_deployment(self) -> None: endpoint_name_suffix = "-7" # Updating dictionary with the job name @@ -337,6 +382,10 @@ def test_online_endpoint_invoke_invalid_deployment(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("online_endpoint_name_7", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_online_endpoint_wrong_deployment(self) -> None: endpoint_name_suffix = "-8" # Updating dictionary with the job name @@ -354,6 +403,10 @@ def test_online_endpoint_wrong_deployment(self) -> None: self.kwargs.pop("online_endpoint_name_8", None) self.kwargs.pop("online_deployment_name_8", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO (2376694): Test fails with InternalServerError and points to non-existent logs") def test_online_endpoint_update(self) -> None: endpoint_name_suffix = "-91" @@ -405,6 +458,10 @@ def test_online_endpoint_update(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("online_endpoint_name_9", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO (2376694): Test fails with InternalServerError and points to non-existent logs") def test_online_endpoint_invoke(self) -> None: endpoint_name_suffix = "-110" @@ -429,6 +486,10 @@ def test_online_endpoint_invoke(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("online_endpoint_name_10", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: Task-1777932 will re-enable and fix this test.") def test_mir_deployment_private_feature(self) -> None: endpoint_name_suffix = "-11" @@ -465,6 +526,10 @@ def test_mir_deployment_private_feature(self) -> None: self.kwargs.pop("online_endpoint_name_11", None) self.kwargs.pop("online_deployment_name_11", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO (2376694): Test fails with InternalServerError and points to non-existent logs") def test_online_deployment_update_using_set(self) -> None: endpoint_name_suffix = "-12" @@ -494,6 +559,10 @@ def test_online_deployment_update_using_set(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("online_endpoint_name_12", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO (2377498): test fails with ImageBuildFailure") def test_online_deployment_get_logs(self) -> None: endpoint_name_suffix = "-15" @@ -532,12 +601,20 @@ def test_online_deployment_get_logs(self) -> None: self.kwargs.pop("online_endpoint_name_13", None) self.kwargs.pop("online_deployment_name_13", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_online_endpoint_delete_nonexisting_endpoint(self) -> None: with pytest.raises(Exception) as exp: self.cmd("az ml online-endpoint delete -n online-endpoint-nonexisting --no-wait -y -g testrg -w testworkspace") # Assert error message when users try to delete a nonexisting online endpoint assert f"Online endpoint online-endpoint-nonexisting does not exist." in str(exp.value) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO (2376694): Test fails with InternalServerError and points to non-existent logs") def test_online_endpoint_update_using_set_and_file(self) -> None: endpoint_name_suffix = "-14" @@ -563,6 +640,10 @@ def test_online_endpoint_update_using_set_and_file(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("online_endpoint_name_14", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_online_endpoint_update_with_incomplete_file(self) -> None: endpoint_name_suffix = "-15" # Updating dictionary with the job name @@ -595,6 +676,10 @@ def test_online_endpoint_update_with_incomplete_file(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("online_endpoint_name_15", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: 2161301, Test is failing when executing from recording in ADO") def test_online_deployment_update_using_set_and_file(self) -> None: endpoint_name_suffix = "-16" @@ -638,6 +723,10 @@ def test_online_deployment_update_using_set_and_file(self) -> None: self.kwargs.pop("online_endpoint_name_16", None) self.kwargs.pop("online_deployment_name_16", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: 2161301, Test is failing when executing from recording in ADO") def test_online_deployment_update_overwriting_field_using_set(self) -> None: endpoint_name_suffix = "-25" @@ -682,6 +771,10 @@ def test_online_deployment_update_overwriting_field_using_set(self) -> None: self.kwargs.pop("online_endpoint_name_25", None) self.kwargs.pop("online_deployment_name_25", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO (2376694): Test fails with InternalServerError and points to non-existent logs") def test_online_deployment_update_using_incomplete_file(self) -> None: endpoint_name_suffix = "-17" @@ -711,6 +804,10 @@ def test_online_deployment_update_using_incomplete_file(self) -> None: self.kwargs.pop("online_endpoint_name_17", None) self.kwargs.pop("online_deployment_name_17", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_online_deployment_update_nonexistent_deployment(self) -> None: endpoint_name_suffix = "-18" # Updating dictionary with the job name @@ -745,6 +842,10 @@ def test_online_deployment_update_nonexistent_deployment(self) -> None: self.kwargs.pop("online_endpoint_name_18", None) self.kwargs.pop("online_deployment_name_18", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO (2376694): Test fails with InternalServerError and points to non-existent logs") def test_online_deployment_update_without_scale_settings(self): endpoint_name_suffix = "-23" @@ -783,6 +884,10 @@ def test_online_deployment_update_without_scale_settings(self): self.kwargs.pop("online_endpoint_name_23", None) self.kwargs.pop("online_deployment_name_23", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_online_endpoint_update_nonexistent_endpoint(self) -> None: endpoint_name_suffix = "-19" # Updating dictionary with the job name @@ -805,6 +910,10 @@ def test_online_endpoint_update_nonexistent_endpoint(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("online_endpoint_name_19", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_online_endpoint_traffic_update(self) -> None: endpoint_name_suffix = "-20" # Updating dictionary with the job name @@ -819,6 +928,10 @@ def test_online_endpoint_traffic_update(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("online_endpoint_name_20", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Failing in playback") def test_online_endpoint_code_asset_id(self) -> None: endpoint_name_suffix = "-23" @@ -848,6 +961,10 @@ def test_online_endpoint_code_asset_id(self) -> None: self.kwargs.pop("online_endpoint_name_23", None) self.kwargs.pop("online_deployment_name_23", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Failing in playback") def test_online_endpoint_code_name_version(self) -> None: endpoint_name_suffix = "-24" @@ -877,6 +994,10 @@ def test_online_endpoint_code_name_version(self) -> None: self.kwargs.pop("online_endpoint_name_24", None) self.kwargs.pop("online_deployment_name_24", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_online_endpoint_list(self) -> None: online_endpoint_obj = self.cmd("az ml online-endpoint list -g testrg -w testworkspace") online_endpoint_obj = yaml.safe_load(online_endpoint_obj.output) @@ -894,6 +1015,10 @@ def test_online_endpoint_list(self) -> None: assert "description" not in online_endpoint assert "identity" not in online_endpoint + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Failing in playback") def test_online_deployment_list(self) -> None: online_deployment_obj = self.cmd("az ml online-deployment list -e mir-endpoint -g testrg -w testworkspace") @@ -920,6 +1045,10 @@ def test_online_deployment_list(self) -> None: assert "properties" not in online_deployment + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: 2161301, Test is failing when executing from recording in ADO") def test_online_endpoint_mir_registry_deployment_create(self) -> None: # Using existing endpoint. This eliminated dependency on endpoint creation @@ -974,6 +1103,10 @@ def test_online_endpoint_mir_registry_deployment_create(self) -> None: # To re create it needs an uai, create an uai with below command # "az identity create --name test_uai" # use the id into the --set below + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_endpoint_create_with_uai(self) -> None: self.kwargs[ "uai" @@ -984,6 +1117,10 @@ def test_endpoint_create_with_uai(self) -> None: endpoint = yaml.safe_load(endpoint.output) endpoint["identity"]["user_assigned_identities"][0]["resource_id"] == self.kwargs.pop("uai", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO (2376694): Test fails with InternalServerError and points to non-existent logs") def test_endpoint_and_deployment_exist_with_yaml(self) -> None: @@ -1018,6 +1155,10 @@ def test_endpoint_and_deployment_exist_with_yaml(self) -> None: # Delete a key regardless of whether it is in the dictionary for the new name self.kwargs.pop("online_endpoint_name_22", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO: 2161301, Test is failing when executing from recording in ADO") def test_mdc_data_storage_basic_managed(self) -> None: endpoint_name_suffix = "-25" @@ -1088,6 +1229,10 @@ def test_mdc_data_storage_basic_managed(self) -> None: self.kwargs.pop("online_deployment_name_23", None) self.kwargs.pop("online_endpoint_name_23", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="TODO (2376694): Test fails with InternalServerError and points to non-existent logs") def test_mdc_data_storage_standard_managed(self) -> None: endpoint_name_suffix = "-24" @@ -1166,6 +1311,10 @@ def test_mdc_data_storage_standard_managed(self) -> None: self.kwargs.pop("online_deployment_name_24", None) self.kwargs.pop("online_endpoint_name_24", None) + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() @pytest.mark.skip(reason="Could not push test yamls into repo. TOD: task 2007149 will either re-enable or remove") def test_ast_scoring_script_validation(self) -> None: endpoint_name_suffix = "-26" diff --git a/src/machinelearningservices/azext_mlv2/tests/latest/test_registry_scenarios.py b/src/machinelearningservices/azext_mlv2/tests/latest/test_registry_scenarios.py index fddabed3202..8f85c5f3093 100644 --- a/src/machinelearningservices/azext_mlv2/tests/latest/test_registry_scenarios.py +++ b/src/machinelearningservices/azext_mlv2/tests/latest/test_registry_scenarios.py @@ -11,14 +11,23 @@ from azext_mlv2.tests.scenario_test_helper import MLBaseScenarioTest from azure.ai.ml import load_registry +from azure.cli.testsdk.scenario_tests.decorators import record_only class RegistryScenarioTest(MLBaseScenarioTest): + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_registry_list(self): env_obj = self.cmd("az ml registry list -g testrg") env_obj = yaml.safe_load(env_obj.output) assert env_obj[0]["container_registry"]["acr_account_sku"] == "premium" + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_registry_show(self): env_obj = self.cmd("az ml registry show -g testrg --name testregistry") env_obj = yaml.safe_load(env_obj.output) @@ -26,6 +35,10 @@ def test_registry_show(self): # note this test will fail if the specified registry already exists before the test starts. # This might happen if the test fails in a live run before the delete operation occurs. + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_registry_create_and_delete(self): target_yaml = ( "./src/machinelearningservices/azext_mlv2/tests/test_configs/registry/registry_valid_cli_test.yaml" @@ -57,6 +70,10 @@ def test_registry_create_and_delete(self): del_result = self.cmd(f"az ml registry delete --debug -g testrg --name {reg.name}") assert del_result.output == "" + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_registry_update_with_file(self) -> None: # create a registry to start env_obj = self.cmd( diff --git a/src/machinelearningservices/azext_mlv2/tests/latest/test_workspace_scenarios.py b/src/machinelearningservices/azext_mlv2/tests/latest/test_workspace_scenarios.py index 926f923660f..216b62d7005 100644 --- a/src/machinelearningservices/azext_mlv2/tests/latest/test_workspace_scenarios.py +++ b/src/machinelearningservices/azext_mlv2/tests/latest/test_workspace_scenarios.py @@ -10,10 +10,15 @@ import pytest import yaml from azext_mlv2.tests.scenario_test_helper import MLBaseScenarioTest +from azure.cli.testsdk.scenario_tests.decorators import record_only class WorkspaceScenarioTest(MLBaseScenarioTest): ''' + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_workspace(self) -> None: workspaceName = self.kwargs.get("workspaceName", None) workspaceName += "_full" @@ -82,6 +87,10 @@ def test_workspace(self) -> None: assert ws_obj_del.output == "" ''' + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_workspace_managednetwork_outbound_rule_operations(self) -> None: workspaceName = self.kwargs.get("workspaceName", None) workspaceName += "_mvnet" @@ -276,6 +285,10 @@ def test_workspace_managednetwork_outbound_rule_operations(self) -> None: assert ws_obj_del.output == "" ''' + # Marked as record_only because the test uses hardcoded resource group + # 'testrg', which only exists in the recorded cassette. Running live + # raises ResourceGroupNotFound or AuthorizationFailed. + @record_only() def test_workspace_managednetwork_provision(self) -> None: workspaceName = self.kwargs.get("workspaceName", None) workspaceName += "_mvnetprov"