{"meta":{"start_date":"2026-04-03","end_date":"2026-04-03","start_at":"2026-04-03T00:00:00Z","end_at":"2026-04-04T00:00:00Z","timezone":"UTC","generated_at":"2026-08-26T18:10:03Z","environments":["prod"]},"environments":[{"environment":"prod","summary":{"total_runs":5,"failed_runs":5,"raw_occurrences":41,"matched_occurrences":41,"runs_affected":5},"rows":[{"environment":"prod","failure_pattern_id":"4b353f527c4ee510a2731f38db17c1491687bbf0c051882c44d898e766fdc677","failure_pattern":"timeout during GetAdminRESTConfigForHCPCluster while waiting for hcpcluster creds","search_query":"failed waiting for hcpCluster=\"admin-cred-lifecycle-5wntpm\" in resourcegroup=\"admin-credential-lifecycle-test-5h6742\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster admin-cred-lifecycle-5wntpm in resource group admin-credential-lifecycle-test-5h6742, error: context deadline exceeded","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":18,"runs_affected":4,"failed_runs":4,"run_impact_percent":80,"anchor_occurrences":18,"after_last_push_count":41,"trend_counts":[0,0,0,0,0,22,18],"trend_range":"2026-03-28..2026-04-03 UTC","prior_windows_present":2,"prior_window_starts":["2026-03-02","2026-03-23"],"prior_runs_affected":11,"prior_last_seen_at":"2026-03-27T04:16:09Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP stable channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.19","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.19","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:126]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000eb6a20\u003e: \n    failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-l56s4b\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-l56s4b, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-l56s4b\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-l56s4b, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000905dd0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-l56s4b\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cilium.go:126]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc000eb6a20\u003e: \n    failed waiting for hcpCluster=\"cilium-cluster\" in resourcegroup=\"e2e-cilium-l56s4b\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-l56s4b, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cilium-cluster\\\" in resourcegroup=\\\"e2e-cilium-l56s4b\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-l56s4b, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc000905dd0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cilium-cluster in resource group e2e-cilium-l56s4b\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011b0810\u003e: \n    failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-2v4phv\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-2v4phv, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-2v4phv\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-2v4phv, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0009a5550\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-2v4phv\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:157]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0011b0810\u003e: \n    failed waiting for hcpCluster=\"cluster-ver-4-19\" in resourcegroup=\"rg-cluster-back-version-2v4phv\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-2v4phv, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-ver-4-19\\\" in resourcegroup=\\\"rg-cluster-back-version-2v4phv\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-2v4phv, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0009a5550\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-ver-4-19 in resource group rg-cluster-back-version-2v4phv\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013f0840\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-19-f4hz5d\" in resourcegroup=\"rg-zstream-upgrade-4-19-mf7hcn\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-f4hz5d in resource group rg-zstream-upgrade-4-19-mf7hcn, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-f4hz5d\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-mf7hcn\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-f4hz5d in resource group rg-zstream-upgrade-4-19-mf7hcn, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0015a34a0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-f4hz5d in resource group rg-zstream-upgrade-4-19-mf7hcn\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:151]: Unexpected error:\n    \u003c*fmt.wrapErrors | 0xc0013f0840\u003e: \n    failed waiting for hcpCluster=\"cluster-zstream-4-19-f4hz5d\" in resourcegroup=\"rg-zstream-upgrade-4-19-mf7hcn\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-f4hz5d in resource group rg-zstream-upgrade-4-19-mf7hcn, error: context deadline exceeded\n    {\n        msg: \"failed waiting for hcpCluster=\\\"cluster-zstream-4-19-f4hz5d\\\" in resourcegroup=\\\"rg-zstream-upgrade-4-19-mf7hcn\\\" to finish getting creds, caused by: timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-f4hz5d in resource group rg-zstream-upgrade-4-19-mf7hcn, error: context deadline exceeded\",\n        errs: [\n            \u003c*errors.errorString | 0xc0015a34a0\u003e{\n                s: \"timeout '10.000000' minutes exceeded during GetAdminRESTConfigForHCPCluster for cluster cluster-zstream-4-19-f4hz5d in resource group rg-zstream-upgrade-4-19-mf7hcn\",\n            },\n            \u003ccontext.deadlineExceededError\u003e{},\n        ],\n    }\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"978b5249588852e82b6e0319ffa1c4ff491d36b8733adfe165bd31b6a37ca120","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"cddfa084aa297044efcb077e7e99863ef8707f89d1167c5e6999483547025bd0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"f54f60621797a03b537810c47c073ed6e6a3b5a9e129109970782f85585da577","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040090253817024512","occurred_at":"2026-04-03T15:33:14Z","signature_id":"a101d653698556fe26c5748c10a638d54b4b0d1786393818b489710e7457d0e2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","signature_id":"13c51be1381e0dcdda4cd451026a1c0c513d2b1c12356428de97fbb810adac22","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","signature_id":"2d171a77f2efb9edea4c7031a4c5d4e19f877ce189da05ccb5c22454d8e0dc36","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","signature_id":"6aed24b9937e32b551ffc5f89291baa02e68d2ab1f1955dc02cb3cedc3b73787","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","signature_id":"9068b60dd134a34b45e2c4ca4bac89861a7aff0af4a58c254886eadfddc1f236","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","signature_id":"9623212ee8ee787df04708ce9e32276e48e27d5b6b6f225261f64c8fa2982fe4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","signature_id":"a30104af4d916dd7cbfec3de76f31032165c563f553ec6306160c9eed3e1a2b9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"1798b4cb1a37816e867fca622bd7878502beca3293e01e7628bd7340f11a75c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"2db8f4dcd8799aa52814f81458b5d3d6bfe958d2aabd8c10d467e2405ffd0cbf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"501ec29b0bb62d873bf180fc4250d995a59d71226a623c67a7cb58747dd86494","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"90c9cdb50ab4b9b54916bc8225453c71c231afe899393e6702c0830fcf5fda85","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"a8d8ea4155e25db792b9e78559057064deaac8c45a6fd201e17b15a080b1a1dc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"c514ce210dc57018167504b68244b55a840f03fd60d71cec6b16d5a3957651d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"cc9896340c43f9cc28fd732671e71c508eb3570fc2132dd3a1db1ed99255adad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"e1fcf0aa66a65d43263fd7eb810b789a04625b5e903e70a78d19b18184a6fe25","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"dd1f947eca3fe3915ce4373a1ab3e42da9e7a01ae8596ca121c054e99a3879b6","failure_pattern":"Interrupted by User","search_query":"Interrupted by User","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":8,"runs_affected":5,"failed_runs":5,"run_impact_percent":100,"anchor_occurrences":8,"after_last_push_count":32,"trend_counts":[0,0,1,0,0,18,8],"trend_range":"2026-03-28..2026-04-03 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-02","2026-03-09","2026-03-16"],"prior_runs_affected":27,"prior_last_seen_at":"2026-03-19T02:48:42Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [:0]: Interrupted by User\nfail [:0]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:51]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:51]: Interrupted by User","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:42]: Interrupted by User\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:42]: Interrupted by User"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040090253817024512","occurred_at":"2026-04-03T15:33:14Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","signature_id":"e6da33d04a54f33cfcf6737adce240a35f387bc578cc7378a48365638a71d12d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039885780922404864","occurred_at":"2026-04-03T02:00:44Z","signature_id":"c1bcce226b75a44e306b342dfa0dc30702c45c04c3d3a51ffc00fd88decaba59","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5ad74b449077ea2e582fe85a0913093cd38cf8d7ca684b6d3c8071de661009df","failure_pattern":"ERROR CODE: InternalServerError; context ; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":7,"runs_affected":3,"failed_runs":3,"run_impact_percent":60,"anchor_occurrences":7,"after_last_push_count":14,"trend_counts":[0,0,0,0,0,7,7],"trend_range":"2026-03-28..2026-04-03 UTC","prior_windows_present":0,"prior_runs_affected":6,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000988ce0\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-bx2wzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-cq5b22/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-bx2wzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-cq5b22/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc000def680\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: DAC0D39B3291499EB510E196A8464291 Ref B: CHI261140504052 Ref C: 2026-04-03T19:47:34Z\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"bbeb88a5-59d3-41b8-9d97-5ea6acb0a3e2\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"497\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Date\": [\n                        \"Fri, 03 Apr 2026 19:47:34 GMT\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Content-Length\": [\"107\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/switzerlandnorth/704ea369-be04-459b-a6e0-8571e30af23b\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"5d4a3adb-fb62-43e1-82b6-3016b7ae4e34\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"SWITZERLANDNORTH:20260403T194735Z:5d4a3adb-fb62-43e1-82b6-3016b7ae4e34\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"Expires\": [\"-1\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc000988c20\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_credential_lifecycle.go:174]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000988ce0\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-bx2wzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-cq5b22/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/admin-credential-lifecycle-test-bx2wzm/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/admin-cred-lifecycle-cq5b22/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc000def680\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: DAC0D39B3291499EB510E196A8464291 Ref B: CHI261140504052 Ref C: 2026-04-03T19:47:34Z\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"bbeb88a5-59d3-41b8-9d97-5ea6acb0a3e2\",\n                    ],\n                    \"X-Envoy-Upstream-Service-Time\": [\"497\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"Date\": [\n                        \"Fri, 03 Apr 2026 19:47:34 GMT\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Content-Length\": [\"107\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/switzerlandnorth/704ea369-be04-459b-a6e0-8571e30af23b\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"5d4a3adb-fb62-43e1-82b6-3016b7ae4e34\",\n                    ],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"SWITZERLANDNORTH:20260403T194735Z:5d4a3adb-fb62-43e1-82b6-3016b7ae4e34\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"Expires\": [\"-1\"],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc000988c20\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0004e0160\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-w8b9cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-w8b9cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc0015f0180\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"ea03dd36-95b8-47c7-90e1-a8dc79a919d8\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"09d15fc6-6c8e-4784-845a-952eadfe9277\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"SWITZERLANDNORTH:20260403T193832Z:ea03dd36-95b8-47c7-90e1-a8dc79a919d8\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: EFBD4EB481774F60AB6EA1EDCD5ECFAF Ref B: CHI261140508025 Ref C: 2026-04-03T19:38:32Z\",\n                    ],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"80\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/switzerlandnorth/6e183501-6da1-4585-8100-e8772f21fd65\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"Date\": [\n                        \"Fri, 03 Apr 2026 19:38:32 GMT\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0004e0000\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                     ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:103]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc0004e0160\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-w8b9cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/external-auth-cluster-w8b9cs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ea-cluster/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc0015f0180\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Pragma\": [\"no-cache\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"ea03dd36-95b8-47c7-90e1-a8dc79a919d8\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"Content-Type\": [\"application/json\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"09d15fc6-6c8e-4784-845a-952eadfe9277\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"SWITZERLANDNORTH:20260403T193832Z:ea03dd36-95b8-47c7-90e1-a8dc79a919d8\",\n                    ],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: EFBD4EB481774F60AB6EA1EDCD5ECFAF Ref B: CHI261140508025 Ref C: 2026-04-03T19:38:32Z\",\n                    ],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"80\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/switzerlandnorth/6e183501-6da1-4585-8100-e8772f21fd65\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"Date\": [\n                        \"Fri, 03 Apr 2026 19:38:32 GMT\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc0004e0000\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                     ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:104]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000ecca20\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-kfr69r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-kfr69r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc0012090e0\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"c90c7b13-2d84-49c6-9e50-7c4041ceed5c\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"Date\": [\n                        \"Fri, 03 Apr 2026 19:32:05 GMT\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"151\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"SWITZERLANDNORTH:20260403T193206Z:209e4973-2412-46de-aeb0-3a5184e5b37d\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/switzerlandnorth/1c2454d9-3f36-4903-bf4d-7d6a91e76c4f\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: DA3E32013CFF4BB48925BAB35C81373F Ref B: CHI261140504060 Ref C: 2026-04-03T19:32:05Z\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"209e4973-2412-46de-aeb0-3a5184e5b37d\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc000ecc960\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                     ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:104]: Unexpected error:\n    \u003c*fmt.wrapError | 0xc000ecca20\u003e: \n    failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-kfr69r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/requestAdminCredential\n    --------------------------------------------------------------------------------\n    RESPONSE 500: 500 Internal Server Error\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"Internal server error.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    {\n        msg: \"failed to start credential request: POST https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/clusternp128-kfr69r/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/hcp-cluster-np-128/requestAdminCredential\\n--------------------------------------------------------------------------------\\nRESPONSE 500: 500 Internal Server Error\\nERROR CODE: InternalServerError\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InternalServerError\\\",\\n    \\\"message\\\": \\\"Internal server error.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\",\n        err: \u003c*exported.ResponseError | 0xc0012090e0\u003e{\n            ErrorCode: \"InternalServerError\",\n            StatusCode: 500,\n            RawResponse: {\n                Status: \"500 Internal Server Error\",\n                StatusCode: 500,\n                Proto: \"HTTP/2.0\",\n                ProtoMajor: 2,\n                ProtoMinor: 0,\n                Header: {\n                    \"Content-Type\": [\"application/json\"],\n                    \"X-Ms-Error-Code\": [\n                        \"InternalServerError\",\n                    ],\n                    \"X-Ms-Request-Id\": [\n                        \"c90c7b13-2d84-49c6-9e50-7c4041ceed5c\",\n                    ],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Writes\": [\"199\"],\n                    \"Date\": [\n                        \"Fri, 03 Apr 2026 19:32:05 GMT\",\n                    ],\n                    \"Cache-Control\": [\"no-cache\"],\n                    \"Content-Length\": [\"107\"],\n                    \"X-Envoy-Upstream-Service-Time\": [\"151\"],\n                    \"X-Ms-Routing-Request-Id\": [\n                        \"SWITZERLANDNORTH:20260403T193206Z:209e4973-2412-46de-aeb0-3a5184e5b37d\",\n                    ],\n                    \"Strict-Transport-Security\": [\n                        \"max-age=31536000; includeSubDomains\",\n                    ],\n                    \"X-Content-Type-Options\": [\"nosniff\"],\n                    \"X-Cache\": [\"CONFIG_NOCACHE\"],\n                    \"X-Ms-Operation-Identifier\": [\n                        \"appId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,tenantId=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX,objectId=0e96dbb7-17a2-4393-b2d3-d84524e39c32/switzerlandnorth/1c2454d9-3f36-4903-bf4d-7d6a91e76c4f\",\n                    ],\n                    \"X-Msedge-Ref\": [\n                        \"Ref A: DA3E32013CFF4BB48925BAB35C81373F Ref B: CHI261140504060 Ref C: 2026-04-03T19:32:05Z\",\n                    ],\n                    \"Pragma\": [\"no-cache\"],\n                    \"Expires\": [\"-1\"],\n                    \"X-Ms-Failure-Cause\": [\"service\"],\n                    \"X-Ms-Ratelimit-Remaining-Subscription-Global-Writes\": [\"2999\"],\n                    \"X-Ms-Correlation-Request-Id\": [\n                        \"209e4973-2412-46de-aeb0-3a5184e5b37d\",\n                    ],\n                },\n                Body: \u003c*exported.nopClosingBytesReader | 0xc000ecc960\u003e{\n                    s: [123, 10, 32, 32, 32, 32, 34, 101, 114, 114, 111, 114, 34, 58, 32, 123, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 99, 111, 100, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 83, 101, 114, 118, 101, 114, 69, 114, 114, 111, 114, 34, 44, 10, 32, 32, 32, 32, 32, 32, 32, 32, 34, 109, 101, 115, 115, 97, 103, 101, 34, 58, 32, 34, 73, 110, 116, 101, 114, 110, 97, 108, 32, 115, 101, 114, 118, 101, 114, 32, 101, 114, 114, 111, 114, 46, 34, 10, 32, 32, 32, 32, 125, 10, 125],\n                    i: 0,\n                },\n                ContentLength: 107,\n                TransferEncoding: nil,\n                Close: false,\n                Uncompressed: false,\n                Trailer: nil,\n                Request: {\n                    Method: \"POST\",\n                    URL: {\n                        Scheme: \"https\",\n                        Opaque: \"\",\n                        User: nil,\n                     ...\n\nGomega truncated this representation as it exceeds 'format.MaxLength'.\nConsider having the object provide a custom 'GomegaStringer' representation\nor adjust the parameters in Gomega's 'format' package.\n\nLearn more here: https://onsi.github.io/gomega/#adjusting-output\n\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"c193d6ca0bf762e1335e6248bbfaedb1513e9e907031106da6506fe9e3ae11b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"c225061e4e181c2491304d3cbd154d4d968c63217496bd6f9e38d71d74b4ea92","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"c56aa4a27cb369c16d605912ab2f8d6cfd883cef06344a11b5dd1d06a710ba17","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040090253817024512","occurred_at":"2026-04-03T15:33:14Z","signature_id":"00a576e27987d92bd608669bdce0f6021d525d41d8ac0bd32cc3d4b74b822638","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040090253817024512","occurred_at":"2026-04-03T15:33:14Z","signature_id":"0592517f8993dd48a7d28ecb3a66d91e3cacbcb2d1ba137fadd1f4b1499b1c5d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","signature_id":"7575214c43750b29ac611602bb7070d16644923e94908c9a24d6a80a30d55228","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951538536845312","occurred_at":"2026-04-03T06:22:01Z","signature_id":"848faab63125a0547ef1d80afb4f94d0f38adeebf4bc99e16f7dcb85fa48d55a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"fa8b536e8650ce97ae7570bef6054027cbd99a52276b9ac4f3e9e80e8ecb21e4","failure_pattern":"Deserializaion Error: no output from command","search_query":"Deserializaion Error: no output from command","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":6,"runs_affected":3,"failed_runs":3,"run_impact_percent":60,"anchor_occurrences":6,"after_last_push_count":29,"trend_counts":[0,0,0,0,0,18,6],"trend_range":"2026-03-28..2026-04-03 UTC","prior_windows_present":3,"prior_window_starts":["2026-03-02","2026-03-16","2026-03-23"],"prior_runs_affected":19,"prior_last_seen_at":"2026-03-26T14:57:05Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest","occurrences":3}],"full_error_samples":["\"=\"0\" \"beta.kubernetes.io/arch\"=\"amd64\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"beta.kubernetes.io/os\"=\"linux\" \"hypershift.openshift.io/managed\"=\"true\" \"hypershift.openshift.io/nodePool\"=\"b0t7x8b0g1n2w3n-np-1\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"key1\"=\"value1\" \"kubernetes.io/os\"=\"linux\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.disk.csi.azure.com/zone\"=\"\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"failure-domain.beta.kubernetes.io/region\"=\"switzerlandnorth\" \"kubernetes.io/hostname\"=\"b0t7x8b0g1n2w3n-np-1-wqrsn-fsz9k\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"topology.kubernetes.io/region\"=\"switzerlandnorth\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n  \"ts\"=\"2026-04-03 20:34:41.587768\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"b0t7x8b0g1n2w3n-np-1-wqrsn-hzcv6\" \"labels\"={\"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/region\"=\"switzerlandnorth\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"kubernetes.io/os\"=\"linux\" \"node.openshift.io/os_id\"=\"rhel\" \"hypershift.openshift.io/nodePool\"=\"b0t7x8b0g1n2w3n-np-1\" \"topology.kubernetes.io/zone\"=\"1\" \"failure-domain.beta.kubernetes.io/zone\"=\"1\" \"key1\"=\"value1\" \"kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/hostname\"=\"b0t7x8b0g1n2w3n-np-1-wqrsn-hzcv6\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"hypershift.openshift.io/managed\"=\"true\" \"node-role.kubernetes.io/worker\"=\"\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/region\"=\"switzerlandnorth\" \"beta.kubernetes.io/arch\"=\"amd64\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n  \"ts\"=\"2026-04-03 20:34:41.587818\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"b0t7x8b0g1n2w3n-np-1-wqrsn-k6pvw\" \"labels\"={\"beta.kubernetes.io/arch\"=\"amd64\" \"failure-domain.beta.kubernetes.io/region\"=\"switzerlandnorth\" \"kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/hostname\"=\"b0t7x8b0g1n2w3n-np-1-wqrsn-k6pvw\" \"node-role.kubernetes.io/worker\"=\"\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"topology.kubernetes.io/zone\"=\"0\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/os\"=\"linux\" \"hypershift.openshift.io/managed\"=\"true\" \"hypershift.openshift.io/nodePool\"=\"b0t7x8b0g1n2w3n-np-1\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"kubernetes.io/os\"=\"linux\" \"key1\"=\"value1\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/region\"=\"switzerlandnorth\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\"} \"taints\"=[{\"key\"=\"key2\" \"value\"=\"value2\" \"effect\"=\"PreferNoSchedule\"}]\n    STEP: verifying original labels persist on 3 nodes @ 04/03/26 20:34:41.587\n    STEP: verifying original taints persist on 2 nodes @ 04/03/26 20:34:41.587\n  \"ts\"=\"2026-04-03 20:34:41.588122\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-03 20:34:41.588264\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-44fz9p\" \"resourceGroup\"=\"rg-np-labels-taints-4z7w6h\"\n  \"ts\"=\"2026-04-03 20:34:41.767633\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-np-labels-taints-44fz9p\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-03 20:34:43.091201\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n  \"ts\"=\"2026-04-03 20:34:43.091299\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n  \"ts\"=\"2026-04-03 20:34:43.091358\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-03 20:34:43.092891\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-96\"\n  \"ts\"=\"2026-04-03 20:34:43.092910\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-labels-taints-4z7w6h\"\n  \"ts\"=\"2026-04-03 20:34:43.282140\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-4z7w6h\" \"deployment\"=\"pooledNonMsiScopedAssignments-r6cuakilyx67k\"\n  \"ts\"=\"2026-04-03 20:34:43.445177\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-96\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-03 20:34:43.456328\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-4z7w6h\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-44fz9p\"\n  \"ts\"=\"2026-04-03 20:34:43.747143\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-96\" \"deployment\"=\"msi-deployment-96\"\n  \"ts\"=\"2026-04-03 20:34:44.098708\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-03 20:34:44.098863\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-03 20:34:44.098996\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-labels-taints-4z7w6h\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\n\"=\"0\" \"beta.kubernetes.io/arch\"=\"amd64\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\" \"beta.kubernetes.io/os\"=\"linux\" \"hypershift.openshift.io/managed\"=\"true\" \"hypershift.openshift.io/nodePool\"=\"b0t7x8b0g1n2w3n-np-1\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"key1\"=\"value1\" \"kubernetes.io/os\"=\"linux\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.disk.csi.azure.com/zone\"=\"\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"failure-domain.beta.kubernetes.io/region\"=\"switzerlandnorth\" \"kubernetes.io/hostname\"=\"b0t7x8b0g1n2w3n-np-1-wqrsn-fsz9k\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"topology.kubernetes.io/region\"=\"switzerlandnorth\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n  \"ts\"=\"2026-04-03 20:34:41.587768\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"b0t7x8b0g1n2w3n-np-1-wqrsn-hzcv6\" \"labels\"={\"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/os\"=\"linux\" \"failure-domain.beta.kubernetes.io/region\"=\"switzerlandnorth\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"kubernetes.io/os\"=\"linux\" \"node.openshift.io/os_id\"=\"rhel\" \"hypershift.openshift.io/nodePool\"=\"b0t7x8b0g1n2w3n-np-1\" \"topology.kubernetes.io/zone\"=\"1\" \"failure-domain.beta.kubernetes.io/zone\"=\"1\" \"key1\"=\"value1\" \"kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/hostname\"=\"b0t7x8b0g1n2w3n-np-1-wqrsn-hzcv6\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"hypershift.openshift.io/managed\"=\"true\" \"node-role.kubernetes.io/worker\"=\"\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/region\"=\"switzerlandnorth\" \"beta.kubernetes.io/arch\"=\"amd64\"} \"taints\"=[{\"key\"=\"key1\" \"value\"=\"value1\" \"effect\"=\"NoSchedule\"}]\n  \"ts\"=\"2026-04-03 20:34:41.587818\" \"level\"=0 \"msg\"=\"Node\" \"name\"=\"b0t7x8b0g1n2w3n-np-1-wqrsn-k6pvw\" \"labels\"={\"beta.kubernetes.io/arch\"=\"amd64\" \"failure-domain.beta.kubernetes.io/region\"=\"switzerlandnorth\" \"kubernetes.io/arch\"=\"amd64\" \"kubernetes.io/hostname\"=\"b0t7x8b0g1n2w3n-np-1-wqrsn-k6pvw\" \"node-role.kubernetes.io/worker\"=\"\" \"node.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"topology.kubernetes.io/zone\"=\"0\" \"beta.kubernetes.io/instance-type\"=\"Standard_D4s_v3\" \"beta.kubernetes.io/os\"=\"linux\" \"hypershift.openshift.io/managed\"=\"true\" \"hypershift.openshift.io/nodePool\"=\"b0t7x8b0g1n2w3n-np-1\" \"hypershift.openshift.io/nodepool-globalps-enabled\"=\"true\" \"kubernetes.io/os\"=\"linux\" \"key1\"=\"value1\" \"node.openshift.io/os_id\"=\"rhel\" \"topology.disk.csi.azure.com/zone\"=\"\" \"topology.kubernetes.io/region\"=\"switzerlandnorth\" \"failure-domain.beta.kubernetes.io/zone\"=\"0\"} \"taints\"=[{\"key\"=\"key2\" \"value\"=\"value2\" \"effect\"=\"PreferNoSchedule\"}]\n    STEP: verifying original labels persist on 3 nodes @ 04/03/26 20:34:41.587\n    STEP: verifying original taints persist on 2 nodes @ 04/03/26 20:34:41.587\n  \"ts\"=\"2026-04-03 20:34:41.588122\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-03 20:34:41.588264\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-44fz9p\" \"resourceGroup\"=\"rg-np-labels-taints-4z7w6h\"\n  \"ts\"=\"2026-04-03 20:34:41.767633\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-cluster-np-labels-taints-44fz9p\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-03 20:34:43.091201\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n  \"ts\"=\"2026-04-03 20:34:43.091299\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-11d9063b8cbda82463f6839234612e7ec58e097fdac2676470a369504689ddfe.yaml.gz\"\n  \"ts\"=\"2026-04-03 20:34:43.091358\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-03 20:34:43.092891\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-96\"\n  \"ts\"=\"2026-04-03 20:34:43.092910\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-labels-taints-4z7w6h\"\n  \"ts\"=\"2026-04-03 20:34:43.282140\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-4z7w6h\" \"deployment\"=\"pooledNonMsiScopedAssignments-r6cuakilyx67k\"\n  \"ts\"=\"2026-04-03 20:34:43.445177\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-96\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-03 20:34:43.456328\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-labels-taints-4z7w6h\" \"deployment\"=\"customer-infra-cluster-np-labels-taints-44fz9p\"\n  \"ts\"=\"2026-04-03 20:34:43.747143\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-96\" \"deployment\"=\"msi-deployment-96\"\n  \"ts\"=\"2026-04-03 20:34:44.098708\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-03 20:34:44.098863\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-03 20:34:44.098996\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-labels-taints-4z7w6h\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775245500 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n    STEP: creating resource group @ 04/03/26 19:45:00.935\n  \"ts\"=\"2026-04-03 19:45:00.935972\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\"\n    STEP: creating cluster parameters at control plane version @ 04/03/26 19:45:03.992\n    STEP: creating customer resources @ 04/03/26 19:45:04.081\n    STEP: creating the HCP cluster with version 4.21.7 @ 04/03/26 19:46:06.079\n  \"ts\"=\"2026-04-03 19:46:06.079283\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-pldnh4\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\"\n    STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/03/26 19:59:38.878\n    STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/03/26 20:09:14.928\n    STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/03/26 20:11:13.489\n    STEP: capturing node release images before upgrade @ 04/03/26 20:11:13.637\n    STEP: triggering nodepool upgrade to version 4.21.7 @ 04/03/26 20:11:13.841\n    STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/03/26 20:12:07.744\n    STEP: verifying node pool GET still reflects the new version @ 04/03/26 20:40:16.114\n  \"ts\"=\"2026-04-03 20:40:17.269118\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-03 20:40:17.269207\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-pldnh4-8gpfcs\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\"\n  \"ts\"=\"2026-04-03 20:40:17.673955\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-pldnh4-8gpfcs\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-03 20:40:19.163768\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n  \"ts\"=\"2026-04-03 20:40:19.163906\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n  \"ts\"=\"2026-04-03 20:40:19.163979\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-03 20:40:19.165684\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-47\"\n  \"ts\"=\"2026-04-03 20:40:19.165685\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\"\n  \"ts\"=\"2026-04-03 20:40:19.357542\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\" \"deployment\"=\"pooledNonMsiScopedAssignments-7onhmt3j3ybeg\"\n  \"ts\"=\"2026-04-03 20:40:19.452634\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-47\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-03 20:40:19.512189\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-pldnh4-8gpfcs\"\n  \"ts\"=\"2026-04-03 20:40:19.735871\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-47\" \"deployment\"=\"msi-deployment-47\"\n  \"ts\"=\"2026-04-03 20:40:19.871923\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-03 20:40:19.872029\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-03 20:40:19.872148\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775245500 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should upgrade a nodepool from 4.20.z to 4.21.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:231\n    STEP: creating resource group @ 04/03/26 19:45:00.935\n  \"ts\"=\"2026-04-03 19:45:00.935972\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\"\n    STEP: creating cluster parameters at control plane version @ 04/03/26 19:45:03.992\n    STEP: creating customer resources @ 04/03/26 19:45:04.081\n    STEP: creating the HCP cluster with version 4.21.7 @ 04/03/26 19:46:06.079\n  \"ts\"=\"2026-04-03 19:46:06.079283\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-pldnh4\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\"\n    STEP: creating nodepool with version 4.20.17 (behind control plane; upgrade path validated via Cincinnati) @ 04/03/26 19:59:38.878\n    STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/03/26 20:09:14.928\n    STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/03/26 20:11:13.489\n    STEP: capturing node release images before upgrade @ 04/03/26 20:11:13.637\n    STEP: triggering nodepool upgrade to version 4.21.7 @ 04/03/26 20:11:13.841\n    STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/03/26 20:12:07.744\n    STEP: verifying node pool GET still reflects the new version @ 04/03/26 20:40:16.114\n  \"ts\"=\"2026-04-03 20:40:17.269118\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-03 20:40:17.269207\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-pldnh4-8gpfcs\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\"\n  \"ts\"=\"2026-04-03 20:40:17.673955\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-pldnh4-8gpfcs\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-03 20:40:19.163768\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n  \"ts\"=\"2026-04-03 20:40:19.163906\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-d75bd3b573d86e38666f5431fc1f04dc7901ed57b6ee602c711bd58b56307a0b.yaml.gz\"\n  \"ts\"=\"2026-04-03 20:40:19.163979\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-03 20:40:19.165684\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-47\"\n  \"ts\"=\"2026-04-03 20:40:19.165685\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\"\n  \"ts\"=\"2026-04-03 20:40:19.357542\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\" \"deployment\"=\"pooledNonMsiScopedAssignments-7onhmt3j3ybeg\"\n  \"ts\"=\"2026-04-03 20:40:19.452634\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-47\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-03 20:40:19.512189\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-pldnh4-8gpfcs\"\n  \"ts\"=\"2026-04-03 20:40:19.735871\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-47\" \"deployment\"=\"msi-deployment-47\"\n  \"ts\"=\"2026-04-03 20:40:19.871923\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-03 20:40:19.872029\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-03 20:40:19.872148\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-pldnh4-hs5cqs\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command","Running Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775245707 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:237\n    STEP: creating resource group @ 04/03/26 19:48:28.242\n  \"ts\"=\"2026-04-03 19:48:28.242687\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\"\n    STEP: creating cluster parameters at control plane version @ 04/03/26 19:48:32.172\n    STEP: creating customer resources @ 04/03/26 19:48:32.273\n    STEP: creating the HCP cluster with version 4.20.17 @ 04/03/26 19:49:34.027\n  \"ts\"=\"2026-04-03 19:49:34.027707\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-gh68jv\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\"\n    STEP: creating nodepool with version 4.20.16 (behind control plane; upgrade path validated via Cincinnati) @ 04/03/26 20:01:59.339\n    STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/03/26 20:11:15.755\n    STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/03/26 20:13:15.571\n    STEP: capturing node release images before upgrade @ 04/03/26 20:13:15.748\n    STEP: triggering nodepool upgrade to version 4.20.17 @ 04/03/26 20:13:15.951\n    STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/03/26 20:14:02.797\n    STEP: verifying node pool GET still reflects the new version @ 04/03/26 20:42:11.233\n  \"ts\"=\"2026-04-03 20:42:12.161810\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-03 20:42:12.161939\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-gh68jv-rmkr2d\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\"\n  \"ts\"=\"2026-04-03 20:42:12.456525\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-gh68jv-rmkr2d\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-03 20:42:14.379525\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n  \"ts\"=\"2026-04-03 20:42:14.379643\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n  \"ts\"=\"2026-04-03 20:42:14.379720\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-03 20:42:14.381290\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-45\"\n  \"ts\"=\"2026-04-03 20:42:14.381308\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\"\n  \"ts\"=\"2026-04-03 20:42:14.597243\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\" \"deployment\"=\"pooledNonMsiScopedAssignments-gyhigscniinga\"\n  \"ts\"=\"2026-04-03 20:42:14.870794\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-45\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-03 20:42:14.912330\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-gh68jv-rmkr2d\"\n  \"ts\"=\"2026-04-03 20:42:15.466293\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-45\" \"deployment\"=\"msi-deployment-45\"\n  \"ts\"=\"2026-04-03 20:42:15.973131\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-03 20:42:15.973254\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-03 20:42:15.973386\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command\nRunning Suite:  - /opt/app-root/src/github.com/Azure/ARO-HCP\n  ============================================================\n  Random Seed: 1775245707 - will randomize all specs\n\n  Will run 1 of 1 specs\n  ------------------------------\n  Customer should upgrade a nodepool from 4.20.z to 4.20.zLatest [PreLaunchSetup:None, Importance:Critical, Positivity:Positive, ARO-HCP-RP-API-Compatible]\n  /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:237\n    STEP: creating resource group @ 04/03/26 19:48:28.242\n  \"ts\"=\"2026-04-03 19:48:28.242687\" \"level\"=0 \"msg\"=\"creating resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\"\n    STEP: creating cluster parameters at control plane version @ 04/03/26 19:48:32.172\n    STEP: creating customer resources @ 04/03/26 19:48:32.273\n    STEP: creating the HCP cluster with version 4.20.17 @ 04/03/26 19:49:34.027\n  \"ts\"=\"2026-04-03 19:49:34.027707\" \"level\"=0 \"msg\"=\"Starting HCP cluster creation\" \"clusterName\"=\"np-version-upgrade-cluster-gh68jv\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\"\n    STEP: creating nodepool with version 4.20.16 (behind control plane; upgrade path validated via Cincinnati) @ 04/03/26 20:01:59.339\n    STEP: getting admin credentials and lowest control plane version from OpenShift version history @ 04/03/26 20:11:15.755\n    STEP: computing nodepool desired version from Cincinnati (lowest \u003c= lowest control plane version with upgrade path from nodepool) @ 04/03/26 20:13:15.571\n    STEP: capturing node release images before upgrade @ 04/03/26 20:13:15.748\n    STEP: triggering nodepool upgrade to version 4.20.17 @ 04/03/26 20:13:15.951\n    STEP: verifying nodes are ready, updated to expected version, and release images differ from pre-upgrade @ 04/03/26 20:14:02.797\n    STEP: verifying node pool GET still reflects the new version @ 04/03/26 20:42:11.233\n  \"ts\"=\"2026-04-03 20:42:12.161810\" \"level\"=0 \"msg\"=\"Commiting timing metadata.\"\n  \"ts\"=\"2026-04-03 20:42:12.161939\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-gh68jv-rmkr2d\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\"\n  \"ts\"=\"2026-04-03 20:42:12.456525\" \"level\"=0 \"msg\"=\"Dumping deployment operations.\" \"deployment\"=\"mi-np-version-upgrade-cluster-gh68jv-rmkr2d\" \"resourceGroup\"=\"\"\n  \"ts\"=\"2026-04-03 20:42:14.379525\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/tmp/secret/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n  \"ts\"=\"2026-04-03 20:42:14.379643\" \"level\"=0 \"msg\"=\"Wrote timing metadata\" \"path\"=\"/logs/artifacts/test-timing/timing-metadata-06e5d303cf312917255f4ef7fc1bf912c96515354295cfec0e753053c32d9eb0.yaml.gz\"\n  \"ts\"=\"2026-04-03 20:42:14.379720\" \"level\"=0 \"msg\"=\"collecting debug info\"\n  \"ts\"=\"2026-04-03 20:42:14.381290\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-45\"\n  \"ts\"=\"2026-04-03 20:42:14.381308\" \"level\"=0 \"msg\"=\"collecting deployments\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\"\n  \"ts\"=\"2026-04-03 20:42:14.597243\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\" \"deployment\"=\"pooledNonMsiScopedAssignments-gyhigscniinga\"\n  \"ts\"=\"2026-04-03 20:42:14.870794\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-45\" \"deployment\"=\"pooledMsiScopedAssignments\"\n  \"ts\"=\"2026-04-03 20:42:14.912330\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\" \"deployment\"=\"customer-infra-np-version-upgrade-cluster-gh68jv-rmkr2d\"\n  \"ts\"=\"2026-04-03 20:42:15.466293\" \"level\"=0 \"msg\"=\"collecting operations\" \"resourceGroup\"=\"aro-hcp-test-msi-containers-prod-45\" \"deployment\"=\"msi-deployment-45\"\n  \"ts\"=\"2026-04-03 20:42:15.973131\" \"level\"=0 \"msg\"=\"finished collecting debug info\"\n  \"ts\"=\"2026-04-03 20:42:15.973254\" \"level\"=0 \"msg\"=\"deleting created resources\"\n  \"ts\"=\"2026-04-03 20:42:15.973386\" \"level\"=0 \"msg\"=\"deleting all hcp clusters in resource group\" \"resourceGroup\"=\"rg-np-version-upgrade-gh68jv-8kf8pf\"\nCommand Error: signal: killed\nDeserializaion Error: no output from command"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"520eafa06c89c4752bee7811e891867fb3257bfcb33fdc483d9269e770f97de2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"6359997a37ab65572d65f95e29e72f2054254ddf3a4cf4ac0c8c2fc46f0a1cda","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"a8d31a96b11e8a03efdf422a533fe81c28e318cf4353c55b406a4488dbb92f45","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040090253817024512","occurred_at":"2026-04-03T15:33:14Z","signature_id":"1dc9576b0674e0507d7233876890b3bd2e9f22bf38043e90a256f509b7f8b620","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040090253817024512","occurred_at":"2026-04-03T15:33:14Z","signature_id":"39b876f33ac69f73f9d738dea3e0601e180c615ee792257845cc5cbaeca39648","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"f7bc02877084b7a6fc5ec919ec45196d2dced18cbc2816a4647afaedf0634f72","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"f45cbb1539040a56252aede43b93b142cda480af738df1f97f79f169f427e4cc","failure_pattern":"Process did not finish before 2h0m0s timeout","search_query":"Process did not finish before 2h0m0s timeout","failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":40,"anchor_occurrences":2,"after_last_push_count":6,"trend_counts":[0,0,0,0,0,2,2],"trend_range":"2026-03-28..2026-04-03 UTC","prior_windows_present":1,"prior_window_starts":["2026-03-23"],"prior_runs_affected":6,"prior_last_seen_at":"2026-03-26T14:57:05Z","contributing_tests":[{"failed_at":"unknown","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Job run should complete before timeout","occurrences":2}],"full_error_samples":["{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-03T20:48:28Z\"}","{\"component\":\"entrypoint\",\"file\":\"sigs.k8s.io/prow/pkg/entrypoint/run.go:169\",\"func\":\"sigs.k8s.io/prow/pkg/entrypoint.Options.ExecuteProcess\",\"level\":\"error\",\"msg\":\"Process did not finish before 2h0m0s timeout\",\"severity\":\"error\",\"time\":\"2026-04-03T08:22:36Z\"}"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2040138651320455168","occurred_at":"2026-04-03T18:45:33Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2039951072490950656","occurred_at":"2026-04-03T06:20:10Z","signature_id":"f3f2dce38b62b61774227ec4cdba4a64cf5ed1c49ed0a90b558e06d8d5ca9096","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}