{"meta":{"start_date":"2026-04-02","end_date":"2026-04-02","start_at":"2026-04-02T00:00:00Z","end_at":"2026-04-03T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T19:06:09Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":13,"failed_runs":11,"raw_occurrences":87,"matched_occurrences":87,"runs_affected":11},"rows":[{"environment":"prod","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":22,"runs_affected":4,"failed_runs":4,"run_impact_percent":30.76923076923077,"anchor_occurrences":22,"after_last_push_count":23,"trend_counts":[1,0,0,0,0,0,22],"trend_range":"2026-03-27..2026-04-02 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-02","2026-03-23"],"prior_runs_affected":7,"prior_last_seen_at":"2026-03-27T04:16:09Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":3}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012c8540\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-t869kp\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-qnsckw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00074f9a0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0012c8540\u003e: \n failed waiting for hcpCluster=\"admin-cred-lifecycle-t869kp\" in resourcegroup=\"admin-credential-lifecycle-test-qnsckw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"admin-cred-lifecycle-t869kp\\\" in resourcegroup=\\\"admin-credential-lifecycle-test-qnsckw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc00074f9a0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-t869kp in resource group admin-credential-lifecycle-test-qnsckw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000dd4e70\u003e: \n failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-2xbb2m\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-2xbb2m, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-2xbb2m\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-2xbb2m, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010ef9b0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-2xbb2m\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:94]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000dd4e70\u003e: \n failed waiting for hcpCluster=\"patch-name-cluster\" in resourcegroup=\"patch-name-2xbb2m\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-2xbb2m, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"patch-name-cluster\\\" in resourcegroup=\\\"patch-name-2xbb2m\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-2xbb2m, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc0010ef9b0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster patch-name-cluster in resource group patch-name-2xbb2m\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a1cc60\u003e: \n failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-nm2pxw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-nm2pxw, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-nm2pxw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-nm2pxw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a7e4e0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-nm2pxw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc000a1cc60\u003e: \n failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-nm2pxw\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-nm2pxw, error: context deadline exceeded\n {\n msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-nm2pxw\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-nm2pxw, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000a7e4e0\u003e{\n s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-nm2pxw\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"0a5a5c3cd727714b958f5d99c98b66e348e3db4111454dec8b1404698f2e0794","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"602da1c819b7275183aaa1febbf8be89b01df162a3578abadddbbccd46f2c239","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"6621bf1de0d69411a9065b77e5668e42eef503bb9ffcc29837584f3701bf7719","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"77bc18f4976eeb2ac0f4331febecd092cfe9f711cc268f53af542f3b3632ccb2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"a54f1c852f9da1d8f55d2f99b680bffc752682b546f9f17e7fc3eb3655aef597","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"fb486152deca0eba09a175d7862f2d749ccb1ae46463ef20cabad4e7df2b2aaa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"61a0a590b848614b037b56bafbe413c7ebc46d25f3f5fd367783a83492aabae1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"70c3aed9dcf9d4690d316f75b5f6293fb9b255dea6bc67906dc9905abcacb821","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"8bb7de9b10491a5ec457f50056f87d1f63a31c9cc3e80935e93e3ba4023e2164","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"95d51df2e9bb8e2ab7173a4b7e98f477d5bdd1867f55a5914ac8a5bcf2017b4e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"a2c853e8faaf6b186309d06a6136d2f6b89badfa8f6172ae61fe9adc0c1458c0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"c29effccab9c07c6d8bad7280555e5a4daafa065e1f598a5672df600d11435fb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"f2e9ab95d59de09cd7c68b9d45603f48dcc7ae70fd06bf026e6a73c00e33f05c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"f41b3019192128e043d558d99fb921bcaaf70089e62332adee984faa6ebca7e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","signature_id":"3994952cbbf87a51591ede9e6c056fa912c759e66404e2786708b117d1988eb5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","signature_id":"44c962b6b7ee4eda614e728b25fe72f7f819bd96a0eafa86267710f755d3954d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","signature_id":"54c9710ba8898fedef07e0e373e1e9bec177d9d7a3a262f4872e5a0e23e67e76","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","signature_id":"a5acc838efa40f2e3015b2f5656215fc959732bf821c4f98925ba8e8460b95e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","signature_id":"a693b54b61382ae49d313d16b61863448c5c697a2d7e8a751f27a72c483613c4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039782869789839360","occurred_at":"2026-04-02T19:11:48Z","signature_id":"17372c8fccbca8cace580c5893ad858a6d2524fe5aee1dcf6598b8835c3a2b20","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039782869789839360","occurred_at":"2026-04-02T19:11:48Z","signature_id":"3dbc76b005b4e15f86bd5dc3fdc4f60aceb0cbe928754717e37b1fa61707d06e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039782869789839360","occurred_at":"2026-04-02T19:11:48Z","signature_id":"96ca622f1e934bff69e33dc1c686d3fd304f04accb666698f76f6d12378d24f4","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":18,"runs_affected":9,"failed_runs":9,"run_impact_percent":69.23076923076923,"anchor_occurrences":18,"after_last_push_count":24,"trend_counts":[0,0,0,1,0,0,18],"trend_range":"2026-03-27..2026-04-02 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-02","2026-03-09","2026-03-16"],"prior_runs_affected":22,"prior_last_seen_at":"2026-03-19T02:48:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:51]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:51]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039774495035625472","occurred_at":"2026-04-02T18:38:31Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039774495035625472","occurred_at":"2026-04-02T18:38:31Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039766803189600256","occurred_at":"2026-04-02T18:07:57Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039766803189600256","occurred_at":"2026-04-02T18:07:57Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039741904471461888","occurred_at":"2026-04-02T16:29:01Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039741904471461888","occurred_at":"2026-04-02T16:29:01Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039734796569022464","occurred_at":"2026-04-02T16:00:42Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039734796569022464","occurred_at":"2026-04-02T16:00:42Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039733603084013568","occurred_at":"2026-04-02T15:56:01Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039733603084013568","occurred_at":"2026-04-02T15:56:01Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039651240933068800","occurred_at":"2026-04-02T10:28:45Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039651240933068800","occurred_at":"2026-04-02T10:28:45Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039651240933068800","occurred_at":"2026-04-02T10:28:45Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039651240933068800","occurred_at":"2026-04-02T10:28:45Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":18,"runs_affected":5,"failed_runs":5,"run_impact_percent":38.46153846153846,"anchor_occurrences":18,"after_last_push_count":23,"trend_counts":[0,0,0,0,0,0,18],"trend_range":"2026-03-27..2026-04-02 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-02","2026-03-16","2026-03-23"],"prior_runs_affected":16,"prior_last_seen_at":"2026-03-26T14:57:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1}],"full_error_samples":["Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775174130 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to update node pool labels and taints [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:39\n STEP: creating a resource group @ 04/02/26 23:55:30.12\n \"ts\"=\"2026-04-02 23:55:30.120343\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-labels-taints-vvn7hc\"\n STEP: creating cluster parameters @ 04/02/26 23:55:34.115\n STEP: creating customer resources @ 04/02/26 23:55:34.116\n STEP: creating the HCP cluster @ 04/02/26 23:56:27.658\n \"ts\"=\"2026-04-02 23:56:27.658606\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"cluster-np-labels-taints\" \"resourceGroup\"=\"rg-np-labels-taints-vvn7hc\"\n STEP: getting credentials @ 04/03/26 00:05:46.538\n STEP: creating the node pool with initial labels and taints @ 04/03/26 00:07:53.808\n STEP: waiting for nodes to be ready @ 04/03/26 00:16:31.838\n STEP: verifying initial labels are present on nodes @ 04/03/26 00:16:31.986\n STEP: verifying initial taints are present on nodes @ 04/03/26 00:16:32.013\n STEP: updating nodepool with new taints and scaling up @ 04/03/26 00:16:32.013\n STEP: verifying nodes are scaled to the expected count @ 04/03/26 00:29:03.022\n STEP: verifying new taint is present on newly created node @ 04/03/26 00:29:03.167\n STEP: updating nodepool with a new label and scaling up @ 04/03/26 00:29:03.194\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775174130 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to update node pool labels and taints [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:39\n STEP: creating a resource group @ 04/02/26 23:55:30.12\n \"ts\"=\"2026-04-02 23:55:30.120343\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-labels-taints-vvn7hc\"\n STEP: creating cluster parameters @ 04/02/26 23:55:34.115\n STEP: creating customer resources @ 04/02/26 23:55:34.116\n STEP: creating the HCP cluster @ 04/02/26 23:56:27.658\n \"ts\"=\"2026-04-02 23:56:27.658606\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"cluster-np-labels-taints\" \"resourceGroup\"=\"rg-np-labels-taints-vvn7hc\"\n STEP: getting credentials @ 04/03/26 00:05:46.538\n STEP: creating the node pool with initial labels and taints @ 04/03/26 00:07:53.808\n STEP: waiting for nodes to be ready @ 04/03/26 00:16:31.838\n STEP: verifying initial labels are present on nodes @ 04/03/26 00:16:31.986\n STEP: verifying initial taints are present on nodes @ 04/03/26 00:16:32.013\n STEP: updating nodepool with new taints and scaling up @ 04/03/26 00:16:32.013\n STEP: verifying nodes are scaled to the expected count @ 04/03/26 00:29:03.022\n STEP: verifying new taint is present on newly created node @ 04/03/26 00:29:03.167\n STEP: updating nodepool with a new label and scaling up @ 04/03/26 00:29:03.194\nCommand Error: signal: killed\nDeserializaion Error: no output from command","Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775173098 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n STEP: creating a resource group @ 04/03/26 00:04:23.094\n \"ts\"=\"2026-04-03 00:04:23.094924\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n STEP: creating cluster parameters @ 04/03/26 00:04:26.964\n STEP: creating customer resources @ 04/03/26 00:04:26.964\n STEP: creating the HCP cluster @ 04/03/26 00:05:20.217\n \"ts\"=\"2026-04-03 00:05:20.217661\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-autoscale-cluster\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n STEP: creating nodepool parameters with autoscaling enabled @ 04/03/26 00:18:41.499\n STEP: creating the autoscaling nodepool @ 04/03/26 00:18:41.499\n STEP: verifying the cluster has default autoscaling parameters @ 04/03/26 00:31:21.659\n \"ts\"=\"2026-04-03 00:31:21.802840\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-03 00:31:21.802917\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-autoscale-cluster-pdldv9\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n \"ts\"=\"2026-04-03 00:31:21.916762\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-autoscale-cluster-pdldv9\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-03 00:31:22.918087\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n \"ts\"=\"2026-04-03 00:31:22.918195\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n \"ts\"=\"2026-04-03 00:31:22.918256\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-03 00:31:22.919731\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-132\"\n \"ts\"=\"2026-04-03 00:31:22.919749\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n \"ts\"=\"2026-04-03 00:31:23.016046\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\" \"deployment\"=\"pooledNonMsiScopedAssignments-ssrwqrnhyyabs\"\n \"ts\"=\"2026-04-03 00:31:23.121326\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-132\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-03 00:31:23.128889\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\" \"deployment\"=\"customer-infra-np-autoscale-cluster-pdldv9\"\n \"ts\"=\"2026-04-03 00:31:23.288779\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-132\" \"deployment\"=\"msi-deployment-132\"\n \"ts\"=\"2026-04-03 00:31:23.467072\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-03 00:31:23.467203\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-03 00:31:23.467319\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n \"ts\"=\"2026-04-03 00:50:52.078081\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n \"ts\"=\"2026-04-03 00:50:52.078111\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775173098 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled [PreLaunchSetup:None, Importance:Medium, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:36\n STEP: creating a resource group @ 04/03/26 00:04:23.094\n \"ts\"=\"2026-04-03 00:04:23.094924\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n STEP: creating cluster parameters @ 04/03/26 00:04:26.964\n STEP: creating customer resources @ 04/03/26 00:04:26.964\n STEP: creating the HCP cluster @ 04/03/26 00:05:20.217\n \"ts\"=\"2026-04-03 00:05:20.217661\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-autoscale-cluster\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n STEP: creating nodepool parameters with autoscaling enabled @ 04/03/26 00:18:41.499\n STEP: creating the autoscaling nodepool @ 04/03/26 00:18:41.499\n STEP: verifying the cluster has default autoscaling parameters @ 04/03/26 00:31:21.659\n \"ts\"=\"2026-04-03 00:31:21.802840\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n \"ts\"=\"2026-04-03 00:31:21.802917\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-autoscale-cluster-pdldv9\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n \"ts\"=\"2026-04-03 00:31:21.916762\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-autoscale-cluster-pdldv9\" \"resourceGroup\"=\"\"\n \"ts\"=\"2026-04-03 00:31:22.918087\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n \"ts\"=\"2026-04-03 00:31:22.918195\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-3ace844abea5a4d3a28ffc64801a1d716d83f775e2e8d3dceadd0cb72b88474b.yaml.gz\"\n \"ts\"=\"2026-04-03 00:31:22.918256\" \"level\"=0 \"msg\"=\"collecting debug info\"\n \"ts\"=\"2026-04-03 00:31:22.919731\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-132\"\n \"ts\"=\"2026-04-03 00:31:22.919749\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n \"ts\"=\"2026-04-03 00:31:23.016046\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\" \"deployment\"=\"pooledNonMsiScopedAssignments-ssrwqrnhyyabs\"\n \"ts\"=\"2026-04-03 00:31:23.121326\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-132\" \"deployment\"=\"pooledMsiScopedAssignments\"\n \"ts\"=\"2026-04-03 00:31:23.128889\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\" \"deployment\"=\"customer-infra-np-autoscale-cluster-pdldv9\"\n \"ts\"=\"2026-04-03 00:31:23.288779\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-132\" \"deployment\"=\"msi-deployment-132\"\n \"ts\"=\"2026-04-03 00:31:23.467072\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n \"ts\"=\"2026-04-03 00:31:23.467203\" \"level\"=0 \"msg\"=\"deleting created resources\"\n \"ts\"=\"2026-04-03 00:31:23.467319\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n \"ts\"=\"2026-04-03 00:50:52.078081\" \"level\"=0 \"msg\"=\"no left behind managed resource groups found\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\n \"ts\"=\"2026-04-03 00:50:52.078111\" \"level\"=0 \"msg\"=\"deleting resource group\" \"resourceGroup\"=\"np-autoscaling-d2zlbp\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","Running Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775174183 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n STEP: creating resource group @ 04/03/26 00:04:23.463\n \"ts\"=\"2026-04-03 00:04:23.463319\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-7s5scz-b2nc6w\"\n STEP: creating cluster parameters at control plane version @ 04/03/26 00:04:27.591\n STEP: creating customer resources @ 04/03/26 00:04:27.714\n STEP: creating the HCP cluster with version 4.21.7 @ 04/03/26 00:05:20.496\n \"ts\"=\"2026-04-03 00:05:20.496874\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-7s5scz\" \"resourceGroup\"=\"rg-np-version-upgrade-7s5scz-b2nc6w\"\n STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/03/26 00:18:42.133\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/03/26 00:30:11.111\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/03/26 00:33:47.481\n STEP: capturing node release images before upgrade @ 04/03/26 00:33:47.596\n STEP: triggering nodepool upgrade to version 4.21.7 @ 04/03/26 00:33:47.644\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/03/26 00:34:39.143\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite: - /opt/app-root/src/github.com/Azure/ARO-HCP\n ============================================================\n Random Seed: 1775174183 - will randomize all specs\n\n Will run 1 of 1 specs\n ------------------------------\n Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n STEP: creating resource group @ 04/03/26 00:04:23.463\n \"ts\"=\"2026-04-03 00:04:23.463319\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-7s5scz-b2nc6w\"\n STEP: creating cluster parameters at control plane version @ 04/03/26 00:04:27.591\n STEP: creating customer resources @ 04/03/26 00:04:27.714\n STEP: creating the HCP cluster with version 4.21.7 @ 04/03/26 00:05:20.496\n \"ts\"=\"2026-04-03 00:05:20.496874\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-7s5scz\" \"resourceGroup\"=\"rg-np-version-upgrade-7s5scz-b2nc6w\"\n STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/03/26 00:18:42.133\n STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/03/26 00:30:11.111\n STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/03/26 00:33:47.481\n STEP: capturing node release images before upgrade @ 04/03/26 00:33:47.596\n STEP: triggering nodepool upgrade to version 4.21.7 @ 04/03/26 00:33:47.644\n STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/03/26 00:34:39.143\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"2173f70b91c6bd7f5e4e4471684b57b09412d9d569b1fbe8d10eef8d55358155","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"3bdd4518e35aa05d1f68a1d08e5f52cff965914b56611388669abd719d32cb58","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"ff566a0f193f48a9ea966eddaa287843632a1772b7c0ae15a93934857c859a81","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"1ff548cb71a313175df858c7795325be9045905a3111e7e9176defc19836ede5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039782869789839360","occurred_at":"2026-04-02T19:11:48Z","signature_id":"69b97748a98721d03376b047e26a8ab738922dee3e3915026ee9110e4927c1ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039782869789839360","occurred_at":"2026-04-02T19:11:48Z","signature_id":"a24f878cc7a5f55b9e221a3806e22a12eb532f19da6bbd44a0b8d98d0ae9e30b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039766803189600256","occurred_at":"2026-04-02T18:07:57Z","signature_id":"0d6d738f471638f58110a043a102f05c2da76aaba189c4a5e7aa46163da27f0d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"10a831987a4be893a9ce87869d689cc985a10c8e1bb02e94c90780c844b62b5d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"343b69b9abdb0dbbbe18730a094b99c94da5fe39b0bfe0bc8d50516f80da0fc6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"544adc8e9ab97df3076f3902b260c61ba87e23a109d40f5068d246ae8edf5702","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"5f61202956ac87e3d93f10c8a415a8d113dfe2e5738870debb3c16c7d7ea925d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"879079e6ffdb78b0cfdd72cea6ac98ec70779604b0c6ae9831a74dc3d4e76e14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"b4fe70556501746681ecb3c63407563373fd3957d8ba80d71df1e8acfa098cf1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"c4518cf266a9ae8b9147a4bc9108884198c498a4e0ee55b158614588b552026d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"c916d0a06bab1834b33b0882eec112cc16fb97ff309aae36491c5cdfa36ac14d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"d284e527b2816b754418e6bc1461dca78d18ee8332b2f10fc25ac8eb7a7b7695","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"f57ec76b632f5772ac7a0f8849727df031d5ca56f807171233b950f03ff6df85","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"f71f8c82e2c81fb9ce9bd2b9c751d34e51f6f04d77b8d2ae0b9f4bedf99710eb","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"120e48adced5265e3c99ccc28967cb2a7471c544770275711c2aee3f2d3418be","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zhd77m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":12,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":12,"after_last_push_count":12,"trend_counts":[0,0,0,0,0,0,12],"trend_range":"2026-03-27..2026-04-02 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-02","2026-03-16","2026-03-23"],"prior_runs_affected":18,"prior_last_seen_at":"2026-03-28T02:00:14Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:136]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009c8c90\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zhd77m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-zhd77m\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000d3ab60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m\",\n },\n \u003c*fmt.wrapErrors | 0xc0009c8c60\u003e{\n msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-zhd77m\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000d3ab60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:136]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0009c8c90\u003e: \n failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-zhd77m\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster idms-e2e-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-zhd77m\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000d3ab60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m\",\n },\n \u003c*fmt.wrapErrors | 0xc0009c8c60\u003e{\n msg: \"failed waiting for cluster=\\\"idms-e2e-hcp-cluster\\\" in resourcegroup=\\\"idms-zhd77m\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000d3ab60\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223FromParam for cluster idms-e2e-hcp-cluster in resource group idms-zhd77m\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015f8f90\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-j9ccbj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj, error: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-j9ccbj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001634e80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj\",\n },\n \u003c*fmt.wrapErrors | 0xc0015f8f60\u003e{\n msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-j9ccbj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001634e80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0015f8f90\u003e: \n failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj, error: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-j9ccbj\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj, error: context deadline exceeded\n {\n msg: \"failed to create HCP cluster disabled-image-registry-hcp-cluster, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj, error: failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-j9ccbj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001634e80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj\",\n },\n \u003c*fmt.wrapErrors | 0xc0015f8f60\u003e{\n msg: \"failed waiting for cluster=\\\"disabled-image-registry-hcp-cluster\\\" in resourcegroup=\\\"disabled-image-registry-j9ccbj\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc001634e80\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPClusterFromParam for cluster disabled-image-registry-hcp-cluster in resource group disabled-image-registry-j9ccbj\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n },\n ],\n }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:135]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010d6660\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-mxj67k\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-mxj67k, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-mxj67k\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-mxj67k, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fcc6d0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-mxj67k\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:135]: Unexpected error:\n \u003c*fmt.wrapErrors | 0xc0010d6660\u003e: \n failed waiting for cluster=\"private-kv-cluster\" in resourcegroup=\"private-keyvault-mxj67k\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-mxj67k, error: context deadline exceeded\n {\n msg: \"failed waiting for cluster=\\\"private-kv-cluster\\\" in resourcegroup=\\\"private-keyvault-mxj67k\\\" to finish creating, caused by: timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-mxj67k, error: context deadline exceeded\",\n errs: [\n \u003c*errors.errorString | 0xc000fcc6d0\u003e{\n s: \"timeout '45.000000' minutes exceeded during CreateHCPCluster20251223AndWait for cluster private-kv-cluster in resource group private-keyvault-mxj67k\",\n },\n \u003ccontext.deadlineExceededError\u003e{},\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"0712c50b4b58d80c7fa67f9c8b147184116e65cd92f9e399e67019ad4489ec1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"0c557fcf018f9a61194a38b090f24f00dd3e276e1f2d7cfd13ebc026187bd706","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"28ca8e03fd5320e3f8efb638a095e3b0dd77134ad11a7b771536a46c0242eb16","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"5b24d397eafd651742199dea48c22fa110f454db09b76c3e8ee7360cd5472994","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"6d289eb97cdf8c771abc718ea433d15081de7b5463711d7aa5396be1ace3c0a4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"88e82f944c139e6cc19b557b3a082418af51be333b6988578d68f6c17833d85a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"8c013b50bbb0e285088843f7ca56a5d04d08e5252671dcc9fa791d1c881f1bb6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"a7cfda201a2b66ae296ad118419320731cc0627a8536ef129427b5ac3d420469","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"bc84585441cba890625bb3af9f5f77ae221485dd55718ed23036a13e579acb7c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"c6bc5c31aec45d33666384710b83a3f2ff5b4f5da157f7118eaf6285453023bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"d77a128e9ccb342aa703c1ffc152a70ad5d80aaab8aaba6ca7703762895ebdcf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"e63c24cd1d8891d1ce96daa3f00c9145ffedccd612aa864496c0aca698a66afe","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":7,"runs_affected":3,"failed_runs":3,"run_impact_percent":23.076923076923077,"anchor_occurrences":7,"after_last_push_count":7,"trend_counts":[0,0,0,0,0,0,7],"trend_range":"2026-03-27..2026-04-02 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:118]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000696dc0\u003e: \n failed creating external auth \"ea-list\" in resourcegroup=\"ea-list-zdqkwr\" for cluster=\"ea-listmrjvrf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-zdqkwr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listmrjvrf/externalAuths/ea-list\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed creating external auth \\\"ea-list\\\" in resourcegroup=\\\"ea-list-zdqkwr\\\" for cluster=\\\"ea-listmrjvrf\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-zdqkwr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listmrjvrf/externalAuths/ea-list\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0015f1560\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Request-Id\": [\n \"166169b8-0520-43f1-8408-4075317b3e22\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T234332Z:cbd8afe4-1516-446c-bfca-e27700abfeb7\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Envoy-Upstream-Service-Time\": [\"238\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/0aa7dca7-ac55-417f-a9ed-45134e5db709\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Msedge-Ref\": [\n \"Ref A: A7DD4C4BA3564926A620C3D8ADA15FE4 Ref B: CHI261140511054 Ref C: 2026-04-02T23:43:32Z\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"cbd8afe4-1516-446c-bfca-e27700abfeb7\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Content-Length\": [\"107\"],\n \"Content-Type\": [\"application/json\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 23:43:32 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000696d00\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_list_and_verify.go:118]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000696dc0\u003e: \n failed creating external auth \"ea-list\" in resourcegroup=\"ea-list-zdqkwr\" for cluster=\"ea-listmrjvrf\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-zdqkwr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listmrjvrf/externalAuths/ea-list\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed creating external auth \\\"ea-list\\\" in resourcegroup=\\\"ea-list-zdqkwr\\\" for cluster=\\\"ea-listmrjvrf\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ea-list-zdqkwr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-listmrjvrf/externalAuths/ea-list\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0015f1560\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Request-Id\": [\n \"166169b8-0520-43f1-8408-4075317b3e22\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T234332Z:cbd8afe4-1516-446c-bfca-e27700abfeb7\",\n ],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"X-Envoy-Upstream-Service-Time\": [\"238\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/0aa7dca7-ac55-417f-a9ed-45134e5db709\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Msedge-Ref\": [\n \"Ref A: A7DD4C4BA3564926A620C3D8ADA15FE4 Ref B: CHI261140511054 Ref C: 2026-04-02T23:43:32Z\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Ms-Correlation-Request-Id\": [\n \"cbd8afe4-1516-446c-bfca-e27700abfeb7\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Content-Length\": [\"107\"],\n \"Content-Type\": [\"application/json\"],\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 23:43:32 GMT\",\n ],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000696d00\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a21680\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-7z8hbg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-7z8hbg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000845d40\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"154\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 23:47:08 GMT\",\n ],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"a6270c0d-5904-4199-be64-b4731e31f158\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/25b0928e-3d5f-45fb-ab35-db007159f40d\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"83a37837-14b6-447f-9ee2-d29ececcd65c\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T234708Z:83a37837-14b6-447f-9ee2-d29ececcd65c\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 18AE25CEB3C847B2B0880A4661A960C2 Ref B: CHI261140511054 Ref C: 2026-04-02T23:47:08Z\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"Content-Length\": [\"107\"],\n \"Content-Type\": [\"application/json\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a215c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"ma...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000a21680\u003e: \n failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-7z8hbg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster/requestAdminCredential\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-7z8hbg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000845d40\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Envoy-Upstream-Service-Time\": [\"154\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 23:47:08 GMT\",\n ],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"a6270c0d-5904-4199-be64-b4731e31f158\",\n ],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/25b0928e-3d5f-45fb-ab35-db007159f40d\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"83a37837-14b6-447f-9ee2-d29ececcd65c\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T234708Z:83a37837-14b6-447f-9ee2-d29ececcd65c\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Expires\": [\"-1\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 18AE25CEB3C847B2B0880A4661A960C2 Ref B: CHI261140511054 Ref C: 2026-04-02T23:47:08Z\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"Content-Length\": [\"107\"],\n \"Content-Type\": [\"application/json\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc000a215c0\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"POST\",\n URL: {\n Scheme: \"https\",\n Opaque: \"\",\n User: nil,\n Host: \"ma...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:172]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00138d620\u003e: \n failed creating external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-9tvzvg\" for cluster=\"ea-cluster\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-9tvzvg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed creating external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-9tvzvg\\\" for cluster=\\\"ea-cluster\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-9tvzvg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000985ad0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"39fbee8f-3cf5-48e0-b960-f67dbe52bffb\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 61F6A3FD78CC4CD8BFF56C31AA11F0E1 Ref B: CH1AA2020603031 Ref C: 2026-04-02T23:58:09Z\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 23:58:09 GMT\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"260\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/90122db5-9edb-46e1-a78d-3e8a94c781aa\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"ad53fc85-9a2b-46de-b180-070eb50b5a68\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T235809Z:ad53fc85-9a2b-46de-b180-070eb50b5a68\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00138d560\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:172]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00138d620\u003e: \n failed creating external auth \"external-auth\" in resourcegroup=\"external-auth-cluster-9tvzvg\" for cluster=\"ea-cluster\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-9tvzvg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\n --------------------------------------------------------------------------------\n RESPONSE 500: 500 Internal Server Error\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Internal server error.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed creating external auth \\\"external-auth\\\" in resourcegroup=\\\"external-auth-cluster-9tvzvg\\\" for cluster=\\\"ea-cluster\\\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-9tvzvg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/externalAuths/external-auth\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Internal server error.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc000985ad0\u003e{\n ErrorCode: \"InternalServerError\",\n StatusCode: 500,\n RawResponse: {\n Status: \"500 Internal Server Error\",\n StatusCode: 500,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"Expires\": [\"-1\"],\n \"X-Ms-Error-Code\": [\n \"InternalServerError\",\n ],\n \"X-Ms-Request-Id\": [\n \"39fbee8f-3cf5-48e0-b960-f67dbe52bffb\",\n ],\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n \"X-Msedge-Ref\": [\n \"Ref A: 61F6A3FD78CC4CD8BFF56C31AA11F0E1 Ref B: CH1AA2020603031 Ref C: 2026-04-02T23:58:09Z\",\n ],\n \"Date\": [\n \"Thu, 02 Apr 2026 23:58:09 GMT\",\n ],\n \"X-Envoy-Upstream-Service-Time\": [\"260\"],\n \"X-Ms-Failure-Cause\": [\"service\"],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"107\"],\n \"X-Ms-Operation-Identifier\": [\n \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/canadacentral/90122db5-9edb-46e1-a78d-3e8a94c781aa\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"ad53fc85-9a2b-46de-b180-070eb50b5a68\",\n ],\n \"X-Ms-Routing-Request-Id\": [\n \"CANADACENTRAL:20260402T235809Z:ad53fc85-9a2b-46de-b180-070eb50b5a68\",\n ],\n \"X-Cache\": [\"CONFIG_NOCACHE\"],\n \"Cache-Control\": [\"no-cache\"],\n \"Content-Type\": [\"application/json\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n \"X-Content-Type-Options\": [\"nosniff\"],\n },\n Body: \u003c*exported.nopClosingBytesReader | 0xc00138d560\u003e{\n s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n i: 0,\n },\n ContentLength: 107,\n TransferEncoding: nil,\n Close: false,\n Uncompressed: false,\n Trailer: nil,\n Request: {\n Method: \"PUT\",\n URL: {\n Scheme: \"https\",\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"0d24c8a507e608049ef5aef85ec5ea1a9f166d23aa217ff6cdbbc3789165c604","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"a0048cee302965dfeb513ef4fabde7392fc0ee3edd122e97e6c485f5ee38f30f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"bb803c8293f9cd63e51d5b6e61986b87a54a6d5bac398af30f1214398c7c2f83","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"49af8d5d6289c5f9653a0cd5b17556a7b822922c916541f0d601e3913728bc43","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","signature_id":"4c7844e62748bc91078d395584d560c1e7d7170e1afc3d5c2015d760a9626096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","signature_id":"d478f3c070bdbbdf6cce6725f9b35024fea57ed2435ef8346a8feef92e7f5381","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039801784163110912","occurred_at":"2026-04-02T20:26:55Z","signature_id":"f1e0d24bd3fa73c5580d0a8e6922db4fef593d3a6d46793c243e5e71c5f53b67","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"c36ae336b29ede965426348568977f4287cdb5ff3cdeab9c4ba7530a7df03c1a","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationFailed; detail message The client '\u003cuuid\u003e' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":3,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-03-27..2026-04-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010502c0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-8xlw7k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\",\n \"name\": \"3484e971-8275-4ae3-a76f-65d592316cf3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T01:29:14.718319727Z\",\n \"endTime\": \"2026-04-02T01:30:18.264779861Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-8xlw7k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"name\\\": \\\"3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:29:14.718319727Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:30:18.264779861Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010502a0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-8xlw7k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"name\\\": \\\"3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:29:14.718319727Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:30:18.264779861Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b5319333...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:166]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010502c0\u003e: \n failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\"node-limit-cluster\" in resourcegroup=\"node-limit-test-8xlw7k\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\",\n \"name\": \"3484e971-8275-4ae3-a76f-65d592316cf3\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T01:29:14.718319727Z\",\n \"endTime\": \"2026-04-02T01:30:18.264779861Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster node-limit-cluster: failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-8xlw7k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"name\\\": \\\"3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:29:14.718319727Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:30:18.264779861Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010502a0\u003e{\n msg: \"failed waiting for cluster=\\\"node-limit-cluster\\\" in resourcegroup=\\\"node-limit-test-8xlw7k\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"name\\\": \\\"3484e971-8275-4ae3-a76f-65d592316cf3\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:29:14.718319727Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:30:18.264779861Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-8xlw7k/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b5319333...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00078d420\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-p4zvhp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\",\n \"name\": \"f32ace59-a2fb-42f6-9421-b40f5132fd29\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T00:27:03.073725945Z\",\n \"endTime\": \"2026-04-02T00:27:41.510479828Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-p4zvhp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"name\\\": \\\"f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:27:03.073725945Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:27:41.510479828Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00078d3c0\u003e{\n msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-p4zvhp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"name\\\": \\\"f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:27:03.073725945Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:27:41.510479828Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not ha...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:80]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00078d420\u003e: \n failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-p4zvhp\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\",\n \"name\": \"f32ace59-a2fb-42f6-9421-b40f5132fd29\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T00:27:03.073725945Z\",\n \"endTime\": \"2026-04-02T00:27:41.510479828Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster hcp-cluster-np-128: failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-p4zvhp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"name\\\": \\\"f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:27:03.073725945Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:27:41.510479828Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00078d3c0\u003e{\n msg: \"failed waiting for cluster=\\\"hcp-cluster-np-128\\\" in resourcegroup=\\\"clusternp128-p4zvhp\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"name\\\": \\\"f32ace59-a2fb-42f6-9421-b40f5132fd29\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:27:03.073725945Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:27:41.510479828Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-p4zvhp/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not ha...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000894f20\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-8sxmq6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\",\n \"name\": \"a300f2ba-df79-4404-a4a1-aae22a4d2587\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T00:22:54.935266426Z\",\n \"endTime\": \"2026-04-02T00:23:25.288802016Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-8sxmq6\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"name\\\": \\\"a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:22:54.935266426Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:23:25.288802016Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000894f00\u003e{\n msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-8sxmq6\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"name\\\": \\\"a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:22:54.935266426Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:23:25.288802016Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authori...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:160]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000894f20\u003e: \n failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\"patch-tags-cluster\" in resourcegroup=\"patch-tags-8sxmq6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\",\n \"name\": \"a300f2ba-df79-4404-a4a1-aae22a4d2587\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T00:22:54.935266426Z\",\n \"endTime\": \"2026-04-02T00:23:25.288802016Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster patch-tags-cluster: failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-8sxmq6\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"name\\\": \\\"a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:22:54.935266426Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:23:25.288802016Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000894f00\u003e{\n msg: \"failed waiting for cluster=\\\"patch-tags-cluster\\\" in resourcegroup=\\\"patch-tags-8sxmq6\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"name\\\": \\\"a300f2ba-df79-4404-a4a1-aae22a4d2587\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T00:22:54.935266426Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T00:23:25.288802016Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-8sxmq6/providers/Microsoft.Network/networkSecurityGroups/customer-nsg\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '05d38701-f93a-4033-9e9e-2d387e154e87' with object id '3bb4466b-2c3a-411a-bdf6-ab9062e5a2e8' does not have authori...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"760e7b83d354480bf49188911ed5c051f4b34e00e2943fc1a4213892da050339","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"8eb97e22943903cb8d934c6e5984a6ccdef7fd32962faac882fba587161198b5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"bcb22337065c03471ff7a0d8c4db52818ec22b275d83d1ee92f21bc5c25bc706","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout","failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":15.384615384615385,"anchor_occurrences":2,"after_last_push_count":4,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-03-27..2026-04-02 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-23"],"prior_runs_affected":4,"prior_last_seen_at":"2026-03-26T14:57:05Z","contributing_tests":[{"failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":2}],"full_error_samples":["{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-03T00:52:05Z\"}","{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-02T02:21:49Z\"}"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039837667201388544","occurred_at":"2026-04-02T22:49:32Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f857de2c5e56e72217fe7d44719f4c942b4d08fb11481e22c3bb2c99ba18642b","failure_pattern":"Cluster provisioning failed","search_query":"Cluster provisioning failed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,1,0,0,1],"trend_range":"2026-03-27..2026-04-02 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-02"],"prior_runs_affected":3,"prior_last_seen_at":"2026-03-04T11:18:47Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:153]: Cluster provisioning failed\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:153]: Cluster provisioning failed"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"14c80964b9874c9081110be783c8e717ab9db60e5a66ec45327b015be13e84aa","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"d78c0bca89c95459375355a5923414dd1603fcc7a1039f0264667a1b20c77dc3","failure_pattern":"ERROR CODE: DeploymentNotFound","search_query":"ERROR CODE: DeploymentNotFound","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-27..2026-04-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:109]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010281e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-subnet-reuse-fnhchx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: DeploymentNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentNotFound\",\n \"message\": \"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-subnet-reuse-fnhchx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010281a0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-cluster-subnet-reuse-fnhchx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001028180\u003e{\n msg: \"failed waiting for deployment \\\"mi-cluster-subnet-reuse-fnhchx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0006421b0\u003e{\n ErrorCode: \"DeploymentNotFound\",\n StatusCode: 404,\n RawResponse: {\n Status: \"404 Not Found\",\n StatusCode: 404,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3748\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 18:13:11 GMT\",\n ],\n \"X-Ms-Failure-Cause\": [\"gateway\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"248\"],\n \"X-Ms-Request-Id\": [\n \"4af1ec23-5e77-436c-a930-c88c039bbaf2\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"4af1ec23-5e77-436c-a930-c88c039bbaf2\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"115\"],\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nsg_subnet_reuse.go:109]: Unexpected error:\n \u003c*fmt.wrapError | 0xc0010281e0\u003e: \n failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-cluster-subnet-reuse-fnhchx\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\n --------------------------------------------------------------------------------\n RESPONSE 404: 404 Not Found\n ERROR CODE: DeploymentNotFound\n --------------------------------------------------------------------------------\n {\n \"error\": {\n \"code\": \"DeploymentNotFound\",\n \"message\": \"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create managed identities: failed to create managed identities: failed waiting for deployment \\\"mi-cluster-subnet-reuse-fnhchx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc0010281a0\u003e{\n msg: \"failed to create managed identities: failed waiting for deployment \\\"mi-cluster-subnet-reuse-fnhchx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc001028180\u003e{\n msg: \"failed waiting for deployment \\\"mi-cluster-subnet-reuse-fnhchx\\\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-cluster-subnet-reuse-fnhchx/operationStatuses/08584264532945813091\\n--------------------------------------------------------------------------------\\nRESPONSE 404: 404 Not Found\\nERROR CODE: DeploymentNotFound\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"DeploymentNotFound\\\",\\n \\\"message\\\": \\\"Deployment 'mi-cluster-subnet-reuse-fnhchx' could not be found.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*exported.ResponseError | 0xc0006421b0\u003e{\n ErrorCode: \"DeploymentNotFound\",\n StatusCode: 404,\n RawResponse: {\n Status: \"404 Not Found\",\n StatusCode: 404,\n Proto: \"HTTP/2.0\",\n ProtoMajor: 2,\n ProtoMinor: 0,\n Header: {\n \"X-Ms-Ratelimit-Remaining-Subscription-Global-Reads\": [\"3748\"],\n \"Date\": [\n \"Thu, 02 Apr 2026 18:13:11 GMT\",\n ],\n \"X-Ms-Failure-Cause\": [\"gateway\"],\n \"X-Ms-Ratelimit-Remaining-Subscription-Reads\": [\"248\"],\n \"X-Ms-Request-Id\": [\n \"4af1ec23-5e77-436c-a930-c88c039bbaf2\",\n ],\n \"X-Ms-Correlation-Request-Id\": [\n \"4af1ec23-5e77-436c-a930-c88c039bbaf2\",\n ],\n \"Strict-Transport-Security\": [\n \"max-age=31536000; includeSubDomains\",\n ],\n \"Cache-Control\": [\"no-cache\"],\n \"Pragma\": [\"no-cache\"],\n \"Content-Length\": [\"115\"],\n ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039766803189600256","occurred_at":"2026-04-02T18:07:57Z","signature_id":"e39559634ab284bff6218016b0a95f285d7a8af2e09376008215d96e16037f4e","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"3faf68220dd7536a7ab4746fe0c4241f0ba52074a7b568e4e5cd42f42f541cba","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationFailed; detail message The client '\u003cuuid\u003e' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-27..2026-04-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000bea800\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-z8jp8j: failed waiting for cluster=\"np-version-upgrade-cluster-z8jp8j\" in resourcegroup=\"rg-np-version-upgrade-z8jp8j-brqz27\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\",\n \"name\": \"3ad28075-f0f4-45dd-b143-f32346eb45d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T01:33:37.647438148Z\",\n \"endTime\": \"2026-04-02T01:34:08.890577638Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg-np-upgrade-z8jp8j' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-version-upgrade-cluster-z8jp8j: failed waiting for cluster=\\\"np-version-upgrade-cluster-z8jp8j\\\" in resourcegroup=\\\"rg-np-version-upgrade-z8jp8j-brqz27\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"name\\\": \\\"3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:33:37.647438148Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:34:08.890577638Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-np-upgrade-z8jp8j' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000bea7c0\u003e{\n msg: \"failed waiting for cluster=\\\"np-version-upgrade-cluster-z8jp8j\\\" in resourcegroup=\\\"rg-np-version-upgrade-z8jp8j-brqz27\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"name\\\": \\\"3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:33:37.647438148Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:34:08.890577638Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-np-upgrade-z8jp8j' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:132]: Unexpected error:\n \u003c*fmt.wrapError | 0xc000bea800\u003e: \n failed to create HCP cluster np-version-upgrade-cluster-z8jp8j: failed waiting for cluster=\"np-version-upgrade-cluster-z8jp8j\" in resourcegroup=\"rg-np-version-upgrade-z8jp8j-brqz27\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\",\n \"name\": \"3ad28075-f0f4-45dd-b143-f32346eb45d0\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T01:33:37.647438148Z\",\n \"endTime\": \"2026-04-02T01:34:08.890577638Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"failed to get network security group 'customer-nsg-np-upgrade-z8jp8j' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster np-version-upgrade-cluster-z8jp8j: failed waiting for cluster=\\\"np-version-upgrade-cluster-z8jp8j\\\" in resourcegroup=\\\"rg-np-version-upgrade-z8jp8j-brqz27\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"name\\\": \\\"3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:33:37.647438148Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:34:08.890577638Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-np-upgrade-z8jp8j' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client 'dc7ea24e-36a3-498b-8f49-89d5b180a0d0' with object id 'cc5bfbf9-edb5-415d-8cef-d5b531933373' does not have authorization to perform action 'Microsoft.Network/networkSecurityGroups/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc000bea7c0\u003e{\n msg: \"failed waiting for cluster=\\\"np-version-upgrade-cluster-z8jp8j\\\" in resourcegroup=\\\"rg-np-version-upgrade-z8jp8j-brqz27\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"name\\\": \\\"3ad28075-f0f4-45dd-b143-f32346eb45d0\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:33:37.647438148Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:34:08.890577638Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"failed to get network security group 'customer-nsg-np-upgrade-z8jp8j' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-z8jp8j-brqz27/providers/Microsoft.Network/networkSecurityGroups/customer-nsg-np-upgrade-z8jp8j\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"20cc6f67b762823d979736e097ec3b236ea3bc2eaa4a0f6f645004250bc4c285","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"2b16feefd927dd52de60b3d699308db2fee6e43cd349d94e0653b4b9fce6f419","failure_pattern":"ERROR CODE: InternalServerError; detail code AuthorizationFailed; detail message The client '\u003cuuid\u003e' with object id '\u003cuuid\u003e' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/\u003cresource\u003e' over scope '/subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn' or the scope is invalid. If access was recently granted, please refresh your credentials.; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-27..2026-04-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Control plane automated z-stream upgrade with candidate channel should perform an automated control plane z-stream upgrade for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00131a180\u003e: \n failed to create HCP cluster cluster-zstream-4-21-xvw7fn: failed waiting for cluster=\"cluster-zstream-4-21-xvw7fn\" in resourcegroup=\"rg-zstream-upgrade-4-21-qf98rq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\",\n \"name\": \"66f6aa6d-0e58-4cf7-a817-132dc9b1db23\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T01:34:57.397134314Z\",\n \"endTime\": \"2026-04-02T01:36:07.715905327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to get virtual network 'customer-vnet-zstream-xvw7fn' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client '0c358081-82b1-4f65-97bc-be8647a5b112' with object id 'b3c21201-bf1b-4718-b697-526dbc98e7e9' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-xvw7fn: failed waiting for cluster=\\\"cluster-zstream-4-21-xvw7fn\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-qf98rq\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"name\\\": \\\"66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:34:57.397134314Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:36:07.715905327Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-xvw7fn' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '0c358081-82b1-4f65-97bc-be8647a5b112' with object id 'b3c21201-bf1b-4718-b697-526dbc98e7e9' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00131a160\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-xvw7fn\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-qf98rq\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"name\\\": \\\"66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:34:57.397134314Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:36:07.715905327Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-xvw7fn' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:137]: Unexpected error:\n \u003c*fmt.wrapError | 0xc00131a180\u003e: \n failed to create HCP cluster cluster-zstream-4-21-xvw7fn: failed waiting for cluster=\"cluster-zstream-4-21-xvw7fn\" in resourcegroup=\"rg-zstream-upgrade-4-21-qf98rq\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\n --------------------------------------------------------------------------------\n RESPONSE 200: 200 OK\n ERROR CODE: InternalServerError\n --------------------------------------------------------------------------------\n {\n \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\",\n \"name\": \"66f6aa6d-0e58-4cf7-a817-132dc9b1db23\",\n \"status\": \"Failed\",\n \"startTime\": \"2026-04-02T01:34:57.397134314Z\",\n \"endTime\": \"2026-04-02T01:36:07.715905327Z\",\n \"error\": {\n \"code\": \"InternalServerError\",\n \"message\": \"Failed to get virtual network 'customer-vnet-zstream-xvw7fn' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn\\n--------------------------------------------------------------------------------\\nRESPONSE 403: 403 Forbidden\\nERROR CODE: AuthorizationFailed\\n--------------------------------------------------------------------------------\\n{\\n \\\"error\\\": {\\n \\\"code\\\": \\\"AuthorizationFailed\\\",\\n \\\"message\\\": \\\"The client '0c358081-82b1-4f65-97bc-be8647a5b112' with object id 'b3c21201-bf1b-4718-b697-526dbc98e7e9' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\"\n }\n }\n --------------------------------------------------------------------------------\n \n {\n msg: \"failed to create HCP cluster cluster-zstream-4-21-xvw7fn: failed waiting for cluster=\\\"cluster-zstream-4-21-xvw7fn\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-qf98rq\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"name\\\": \\\"66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:34:57.397134314Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:36:07.715905327Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-xvw7fn' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\\\\\\": \\\\\\\"The client '0c358081-82b1-4f65-97bc-be8647a5b112' with object id 'b3c21201-bf1b-4718-b697-526dbc98e7e9' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/read' over scope '/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn' or the scope is invalid. If access was recently granted, please refresh your credentials.\\\\\\\"\\\\n }\\\\n}\\\\n--------------------------------------------------------------------------------\\\\n\\\"\\n }\\n}\\n--------------------------------------------------------------------------------\\n\",\n err: \u003c*fmt.wrapError | 0xc00131a160\u003e{\n msg: \"failed waiting for cluster=\\\"cluster-zstream-4-21-xvw7fn\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-21-qf98rq\\\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n \\\"id\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/eastus2euap/hcpOperationStatuses/66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"name\\\": \\\"66f6aa6d-0e58-4cf7-a817-132dc9b1db23\\\",\\n \\\"status\\\": \\\"Failed\\\",\\n \\\"startTime\\\": \\\"2026-04-02T01:34:57.397134314Z\\\",\\n \\\"endTime\\\": \\\"2026-04-02T01:36:07.715905327Z\\\",\\n \\\"error\\\": {\\n \\\"code\\\": \\\"InternalServerError\\\",\\n \\\"message\\\": \\\"Failed to get virtual network 'customer-vnet-zstream-xvw7fn' details: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-21-qf98rq/providers/Microsoft.Network/virtualNetworks/customer-vnet-zstream-xvw7fn\\\\n--------------------------------------------------------------------------------\\\\nRESPONSE 403: 403 Forbidden\\\\nERROR CODE: AuthorizationFailed\\\\n--------------------------------------------------------------------------------\\\\n{\\\\n \\\\\\\"error\\\\\\\": {\\\\n \\\\\\\"code\\\\\\\": \\\\\\\"AuthorizationFailed\\\\\\\",\\\\n \\\\\\\"message\\...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039497738340536320","occurred_at":"2026-04-02T00:18:47Z","signature_id":"f5b50909a454a74f214ea8348c5bbbdd39c94bd8e1438791df15454e3358b666","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"802998fa7e45e8509bd2cc7f1eb9749f5e3559041c2c929960e3ec4481056396","failure_pattern":"found 1 managed resource groups left behind HCP clusters in cluster-listing-g7w797","search_query":"occurred","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at the resource group level","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":7.6923076923076925,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-03-27..2026-04-02 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to list HCP clusters without node pools at the resource group level","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246]: Unexpected error:\n \u003c*errors.joinError | 0xc001010078\u003e: \n found 1 managed resource groups left behind HCP clusters in cluster-listing-g7w797\n {\n errs: [\n \u003c*errors.errorString | 0xc00185b800\u003e{\n s: \"found 1 managed resource groups left behind HCP clusters in cluster-listing-g7w797\",\n },\n ],\n }\noccurred\nfail [github.com/Azure/ARO-HCP/test/util/framework/per_test_framework.go:246]: Unexpected error:\n \u003c*errors.joinError | 0xc001010078\u003e: \n found 1 managed resource groups left behind HCP clusters in cluster-listing-g7w797\n {\n errs: [\n \u003c*errors.errorString | 0xc00185b800\u003e{\n s: \"found 1 managed resource groups left behind HCP clusters in cluster-listing-g7w797\",\n },\n ],\n }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039815819470311424","occurred_at":"2026-04-02T21:22:43Z","signature_id":"bfda7f6f00a6578d4c5d8c5848019b1684cf7315d6a0363d79eb51f8fca100db","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}