{"meta":{"start_date":"2026-09-24","end_date":"2026-09-24","start_at":"2026-09-24T00:00:00Z","end_at":"2026-09-25T00:00:00Z","timezone":"UTC","generated_at":"2026-10-14T10:00:17Z","environments":["dev","int","prod","stg"]},"environments":[{"environment":"dev","summary":{"total_runs":123,"failed_runs":94,"raw_occurrences":204,"matched_occurrences":201,"runs_affected":91},"rows":[{"environment":"dev","failure_pattern_id":"6d8dc45d23506f3fc347e3787dba7f7a8cf67b11b35625521b2874f80ca5a9ae","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":45,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":45,"after_last_push_count":123,"also_in":["STG"],"trend_counts":[0,0,0,29,56,34,45],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":7,"prior_last_seen_at":"2026-09-17T00:56:54Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to rotate KMS key for a cluster with version \u003e= 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should be able to retrieve serial console logs for a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"SRE should return 409 when boot diagnostics is disabled on a VM","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000687d40\u003e: \n    failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-jl6ktm48b4wh\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a0fce08-88e2-4170-ac88-340cc76559e9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a0fce08-88e2-4170-ac88-340cc76559e9\",\n      \"name\": \"1a0fce08-88e2-4170-ac88-340cc76559e9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T07:41:08.812973457Z\",\n      \"endTime\": \"2026-09-24T08:10:12.41059037Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/admin_api.go:308]: failed to create HCP cluster \"sre-hcp-cluster-sc\" for serial console test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000687d40\u003e: \n    failed to create HCP cluster sre-hcp-cluster-sc: failed waiting for cluster=\"sre-hcp-cluster-sc\" in resourcegroup=\"admin-api-serialconsole-jl6ktm48b4wh\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a0fce08-88e2-4170-ac88-340cc76559e9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/1a0fce08-88e2-4170-ac88-340cc76559e9\",\n      \"name\": \"1a0fce08-88e2-4170-ac88-340cc76559e9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T07:41:08.812973457Z\",\n      \"endTime\": \"2026-09-24T08:10:12.41059037Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fced80\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-tmsknb268d6w\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03151659-c9c2-4800-86d8-c833276deaec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03151659-c9c2-4800-86d8-c833276deaec\",\n      \"name\": \"03151659-c9c2-4800-86d8-c833276deaec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T07:41:08.321205783Z\",\n      \"endTime\": \"2026-09-24T08:10:12.392342627Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fced80\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed waiting for cluster=\"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-tmsknb268d6w\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03151659-c9c2-4800-86d8-c833276deaec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/03151659-c9c2-4800-86d8-c833276deaec\",\n      \"name\": \"03151659-c9c2-4800-86d8-c833276deaec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T07:41:08.321205783Z\",\n      \"endTime\": \"2026-09-24T08:10:12.392342627Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-kzgnsh\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000be2600\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-kzgnsh: failed waiting for cluster=\"cp-ystream-upgrade-5-0-kzgnsh\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-nh8nv9nrhbfl\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e49bd5a2-59c3-457f-8f93-e961de30f26b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e49bd5a2-59c3-457f-8f93-e961de30f26b\",\n      \"name\": \"e49bd5a2-59c3-457f-8f93-e961de30f26b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T07:41:06.715026226Z\",\n      \"endTime\": \"2026-09-24T08:10:12.435086078Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-kzgnsh\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000be2600\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-kzgnsh: failed waiting for cluster=\"cp-ystream-upgrade-5-0-kzgnsh\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-nh8nv9nrhbfl\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-04.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e49bd5a2-59c3-457f-8f93-e961de30f26b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/e49bd5a2-59c3-457f-8f93-e961de30f26b\",\n      \"name\": \"e49bd5a2-59c3-457f-8f93-e961de30f26b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T07:41:06.715026226Z\",\n      \"endTime\": \"2026-09-24T08:10:12.435086078Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"038937131ff2f8c58cc268033cbe88327c0ecf6260a7273ef6763e38ee56eae3","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"0da4df177339bd718ad09c5a87da0bc13c7840ad3a67b75bdb0840e9e8d78aac","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"114c59c2bf6570c0eefc1b7b5e5a65d9ac897e0594f2a59218c2788db88a3400","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"1472d735a7d85501b8306e000dd5d8b6374ba7f340c2e4ceebc2265fef3c431f","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"1c0d63e4358fa1d03a508395235eb4276e75a2a3b1c57828b7c15799aa97cac3","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"1dbed21c30a383bd27b5dec9ae60753fd2bb2ee86f63bc372cce6f9506e12b4d","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"1eff726a97351513b4ec821562ed516277791ca031fbd1e1aa1de9c857212fba","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"35ebec740614a91af05434491fc4858f3e451d08e674d21eef134a37cf5d6508","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"3ae4d5da8c6399c11ae57e63b6c3ccd31dbb7427337ebb04cd7c8172656b71c1","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"429c76b6d09d4b68e8144f42b4ace52233d91d9e8ee5711e33046c0d5d3cacb0","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"4725982cb9ff6e3263bd7373f034c89fe01c51d46047009fa1ed4ed4ee62e38c","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"500f59df8165aef3ffa121de8a9c035e4de2dc3301c2c3ddf3ea43873d0b709c","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"502b4a1a6e0a45081dc28c470a10f7dc5d98fa3c12fe64214fb3ad5d8b1d6533","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"55c4b969417e538731e376035caad7977a5aca6f1f2c72913b4936de93c287e7","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"61d2a995fa7cf5627014299f798acd76615bfafedc8979ad3e36a0eb0c7a2662","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"61d69de89a82efb1426dba6f56799b44a42e7adce363871a9a6a5cea8d8616fc","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"6502c23f1e7ec0cf695a6476bed4b50f740a286a0b7fe2eed834a4003a061b59","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"6e4748bfac20ea730efd2ae4f326feca89e989b0d1171f412e6d519f9e1670e7","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"7418caf66506fff095427d27ff9a805df356854e0d321f2288eaa6d95d485b6b","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"7517cba222e621154182cc4ff18e45927f0a4901a789bfab867f7abd52b27ff1","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"79eab85227deaa5a504f26b49c3794ac1f2793daeeccfb151c2ebda50a94f291","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"7c3d27901cce4d1113cefa72f1c0a7197bf58cec686d39a74e26062de2c2d826","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"878621b3a630c1b8725c92562eb8fd6ff7e73ca262fd7d2d1a4d3c703bd7181b","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"8a6e31d88bf2e3e5d322a70ec29117555c04f8de5f296cdc0c68246bbf060ddf","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"8f058c62c69f94c523da555f854a97a9bc6956f04a0436c7a2ee43ab827dddb2","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"8fc3ce0aaf075ecc748035615ba580d85a206c822b6daabc625d94f5b3a03ac5","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"91bb81a2eca403f9cb3887462e7e35230600fa3a01fbacce487d699ab1f7ea14","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"91d967ad1e2051c89285c002d456dc98a5e8ae32206d5cccbef47bc1c326378e","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"943aa3a834a0c2cefb204a1c45c66cc889d4122f1d10900e7220b69be2bceb26","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"9efffb68dfea6877253ddef458a441c37343a3bffa28ae1e9b9b91339b87e767","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"9fd5e8c5641bc6751859b9a36c13659f53bb439f0e7a58e1cb5d575101bf0bb9","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"b5ccca774add3458c9bfae3e4c9d277eb98b1b593daf2a2bb7c71a1a8090c02f","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"bb3afbdbc42ca49133c243c7445c3bbf9eb564d2c828e805ba0ded344b330675","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"c445de35915f6d6808ebd93b49d8b192fef5dd052bad1e1a4994444e170535d0","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"c44a3318beb91865b30052009d15e7f5adae10dc8a6c1f345d1fd64c63b78b16","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"c5761bbbe7076b8cad854fc72af7546a6a999f18ed1f68b4bec3edea52bf3259","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"c92e73f72644876ddde62cf422ddb3b39fc8bfa8297fb43d32cbf01215c8b722","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"d92c446c7baf0ccb5ed48e5e01951aa4edeca26f9fddab2165a4455ffedb13f5","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"e1aa6d1a4b4a58d1ebe60d873db4901be2203876d71301a5e963bfa6dd923550","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"e4af5e82f7cb726f365dd5dbed7b27d574f194386873276eef8bf8579929b089","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"e538f2a0230f20027d8293b5a6c68382ea045c9e1651af9ed3dd5ac5b523af39","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"f1c7a583a370723cfe1f78b08a12ea6d5d2e3ceec2dcab1ff58a6c2cc20ec68f","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"f7d663f4b5351be1b0ab0e046173f64d43548235e11b8e908df07ebaa35514ad","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"fd7b575d29435c0f905dacf7124fa2c4be1abd2c35d8c3bd9219b47920c3bc47","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"ff19a7ca5677fbfdea92fb5506f95dcdd516adf5a368b611a7ee13809b4061f3","pr_number":6503,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"dc8bda8f389215048c33f72f37c409384f65d78546cee426dcd410044d8bf026","failure_pattern":"alert [svc] ServiceMemoryDrift fired","search_query":"alert [svc] ServiceMemoryDrift fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":28,"runs_affected":28,"failed_runs":28,"run_impact_percent":22.764227642276424,"anchor_occurrences":28,"after_last_push_count":44,"trend_counts":[1,3,3,17,31,18,28],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":107,"prior_last_seen_at":"2026-09-20T23:13:50Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceMemoryDrift does not fire","occurrences":28}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T18:08:30Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j4417536-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-2fbf2\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-2fbf2 (namespace arobit) on cluster ci01-j4417536-svc has averaged 125.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T17:41:02Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1914624-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-8vzwj\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-8vzwj (namespace arobit) on cluster ci01-j1914624-svc has averaged 126.8% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T17:49:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceMemoryDrift\", cluster=\"ci01-j1279232-svc\", component=\"service-memory\", container=\"fluentbit\", namespace=\"arobit\", pod=\"arobit-forwarder-5zn5h\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container fluentbit in pod arobit-forwarder-5zn5h (namespace arobit) on cluster ci01-j1279232-svc has averaged 125.1% of its memory request over the last 30 minutes.\nThis may indicate a memory leak or workload growth that requires right-sizing the request in config.yaml."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103154812324417536","occurred_at":"2026-09-24T16:09:16Z","signature_id":"66131251b5fecbeb3371e744344966e17e34b09c66d06812a5e0fe0811e83ed1","pr_number":6970,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146310981914624","occurred_at":"2026-09-24T15:35:16Z","signature_id":"4e2ec19ff9d069c5bb64045806ef329a6aca9db297b07736e4b201ead6847e07","pr_number":7045,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103145551821279232","occurred_at":"2026-09-24T15:32:28Z","signature_id":"8c94a4b03784c995aa979e22c654512f8c6707bcd9d456af76a36f2b9dd1e313","pr_number":6917,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103139552272060416","occurred_at":"2026-09-24T15:08:38Z","signature_id":"2d8869925bade65a292b27f5efb8dda6ad2b9c0f4a99be4072145040c21d0bd5","pr_number":7053,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103136210296770560","occurred_at":"2026-09-24T14:55:21Z","signature_id":"ec42ed0291695c354e24001557eb7c5d81b9f3854081433267b5026166de670e","pr_number":7135,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103114369016532992","occurred_at":"2026-09-24T13:28:34Z","signature_id":"017b8f139e649921e31c93332e3a78fedbb3f1194cc2d5d6b68bc998ac21643b","pr_number":6915,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7131/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103110260871925760","occurred_at":"2026-09-24T13:12:14Z","signature_id":"db67d276ff0371dcc22947883bb69cbbfdb27c1787f10ba7322cfaa3b73aa198","pr_number":7131,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7132/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103106328183443456","occurred_at":"2026-09-24T12:56:36Z","signature_id":"03aacfb93bf8d8b6783be4fea0e1b48e62869181b58bb5f7f4cdf2c172610e39","pr_number":7132,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103099181529829376","occurred_at":"2026-09-24T12:28:13Z","signature_id":"ae74f769fa6d33550badb8679083a5b953a0ef29b38b87aaf9b9ee5f82f844cd","pr_number":6917,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103099182641319936","occurred_at":"2026-09-24T12:28:13Z","signature_id":"598f96ff4cc7f38b9158674f984f82130937ad37ec6425753f1589dfdb918be5","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103089013568049152","occurred_at":"2026-09-24T11:47:48Z","signature_id":"ac2d659b6d8c225f448dfa59b70e7e1449c341473cabe70fbc8cfa0120d5269c","pr_number":7123,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103078971544440832","occurred_at":"2026-09-24T11:07:54Z","signature_id":"c3c8df3e62c73e7b4c4be31e24a070ec366f9237969b5d0bdd9fb689ac18eaa2","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055146278719488","occurred_at":"2026-09-24T09:33:14Z","signature_id":"a410a024ffc0f26ed41523481fe4553235341e68f41df89e12a0df8d70a45e2c","pr_number":6922,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055129497309184","occurred_at":"2026-09-24T09:33:10Z","signature_id":"fecdb95305d7912b34e85ac60e3a333a4105e66f05be7ed59027fc3150684b84","pr_number":6917,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055124225069056","occurred_at":"2026-09-24T09:33:09Z","signature_id":"94cc4e8f43733aad08243e0c9d7614fcd959bf692829e56a21d66324fe6568f8","pr_number":6226,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7127/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055116536909824","occurred_at":"2026-09-24T09:33:07Z","signature_id":"33cdaea8b871495d558c36e8c9add34f9b20b7366192a7363dd1ee244293bb4b","pr_number":7127,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103034733737807872","occurred_at":"2026-09-24T08:12:03Z","signature_id":"34737877aa3d834f2668fa205d349b5946f34479dc5d4c773b3f919ae25587c8","pr_number":7045,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103026030053691392","occurred_at":"2026-09-24T07:37:32Z","signature_id":"967796441e8b38c863ade49bdd491e7af8db03f90c559c543dbfafc769074a03","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103024914473684992","occurred_at":"2026-09-24T07:33:06Z","signature_id":"774298937099e086350da5d45672b54735082f9b7780d4691b1ecf885eb8f21d","pr_number":6995,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7126/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103010871180529664","occurred_at":"2026-09-24T06:37:18Z","signature_id":"10f8755438a83f36559eb45ee70fb70663d5582f7bec31700fca4ba57353d6f8","pr_number":7126,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7105/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103001507350712320","occurred_at":"2026-09-24T06:00:05Z","signature_id":"6fd93723b17ab024ac620ec98b9e4488b425d7110afc72440506d1c61f3eab4d","pr_number":7105,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102989394158817280","occurred_at":"2026-09-24T05:11:57Z","signature_id":"5417773a0fcbc7245cd61bb18c7acb92d76ae842a3f206a9a2b3bef948abce1f","pr_number":5562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102961799233015808","occurred_at":"2026-09-24T03:22:14Z","signature_id":"ce1c532ec2e46cd04d99a6539b8974a73f6fa6a9c1f96c6db3d23304ad9e593b","pr_number":7112,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102921426674651136","occurred_at":"2026-09-24T00:41:53Z","signature_id":"3797bbedc99fc8f57829a65247ea354a468bfe71a528c5bc92588f0d1aa0858f","pr_number":7115,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102916121228218368","occurred_at":"2026-09-24T00:20:48Z","signature_id":"87d3d8dba8e6534afffc2d80f759bd53c5872f8a0e40478e7a7531c5b2e7dae9","pr_number":7054,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102914094729269248","occurred_at":"2026-09-24T00:12:45Z","signature_id":"79341176e7e5e053b430da83424bffbe33d8cbf6966dd23ee9a8f0cd6624d271","pr_number":7123,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102913782756937728","occurred_at":"2026-09-24T00:11:30Z","signature_id":"739f026725e19a96d09fe6b2030f5896718a89453fad23e329ee1552ca883a94","pr_number":7112,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102912928209440768","occurred_at":"2026-09-24T00:08:06Z","signature_id":"6e5e5e7d518f60d91ecb09219367313b7d859bc18594e43d3b6d1d6226085c18","pr_number":6846,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"de6b3c5d1247b806936e7bea5768b9bf3e901126af8e87f638a9a85f983527dd","failure_pattern":"alert [svc] FrontendPathMedianLatency fired","search_query":"alert [svc] FrontendPathMedianLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":24,"runs_affected":24,"failed_runs":24,"run_impact_percent":19.51219512195122,"anchor_occurrences":24,"after_last_push_count":53,"trend_counts":[11,6,5,20,21,22,24],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":124,"prior_last_seen_at":"2026-09-20T23:13:50Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathMedianLatency does not fire","occurrences":24}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-25T00:29:48Z\n  Ended:       2026-09-25T01:02:49Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j6834176-svc\", component=\"frontend\", method=\"put\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j6834176-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-25T00:06:48Z\n  Ended:       2026-09-25T00:12:51Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j4729344-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j4729344-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-24T22:56:37Z\n  Ended:       2026-09-24T23:04:39Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathMedianLatency\", cluster=\"ci01-j9945984-svc\", component=\"frontend\", method=\"put\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}\", severity=\"warning\"\n  Description: The median (p50) of frontend request latency for put /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename} has exceeded 250ms over the past 30 minutes (cluster ci01-j9945984-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103269477926834176","occurred_at":"2026-09-24T23:44:55Z","signature_id":"6f2b6a144e3541517f2630051f3cefcfffdb5893b1112dd3f933dca064c7ef54","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7145/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103255589084729344","occurred_at":"2026-09-24T22:49:43Z","signature_id":"84c6dd2f539442a735ddad2dbf44c1744086e075f67b8f3edb677f60e9a862b7","pr_number":7145,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233926699945984","occurred_at":"2026-09-24T21:23:38Z","signature_id":"d8855120e86c8409ebf1202c522f0520dd727db9ff94a1dfff81d2c5c15fcc43","pr_number":7135,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233919481548800","occurred_at":"2026-09-24T21:23:37Z","signature_id":"d06adfd8f5d6c7ecc9b42d9b62223e573b7f16b0df6ea81be882241614b5334c","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233417079427072","occurred_at":"2026-09-24T21:21:37Z","signature_id":"cd75991f1a2d3f936234a24c188b3b95270f16a178b722a2c9be2086d6903383","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103218829420138496","occurred_at":"2026-09-24T20:23:36Z","signature_id":"09e1889769e13cbac0f19e0bdd92a9ef75986032637b4b9ddaaed3dbdd08afff","pr_number":7053,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192892754366464","occurred_at":"2026-09-24T18:40:35Z","signature_id":"8cd39d3a27a9983e4c03864ddd77a3a279fd3ba43a01731a97e5915930fd1c60","pr_number":7135,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192838828199936","occurred_at":"2026-09-24T18:40:22Z","signature_id":"3ecef63f876f1b80222fe16ba8dddbc997352451ab492a895459bf901b651a77","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103170613366493184","occurred_at":"2026-09-24T17:12:03Z","signature_id":"f850918927d721d1a9c993363a6f66eb1a0457832ced1c9c9d5463e320ffdb40","pr_number":7117,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103161541376348160","occurred_at":"2026-09-24T16:36:00Z","signature_id":"2c5be9a399728a0c355066d00beeef8ebab315261af0ce0681e717060c90e781","pr_number":7123,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103155003655983104","occurred_at":"2026-09-24T16:09:57Z","signature_id":"fe0efdc1006e609d9473efff132bf7955c32a6ea999c6a2e2fb575020cee0852","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103154812324417536","occurred_at":"2026-09-24T16:09:16Z","signature_id":"1d82549097eef2d75be7580f57f3df55a47f5833cfb899cc75ff5cc5acc415fc","pr_number":6970,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146310981914624","occurred_at":"2026-09-24T15:35:16Z","signature_id":"342114364ff6caa404513f6899c73460c984413773999113333d0dc8ff64a1da","pr_number":7045,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103136210296770560","occurred_at":"2026-09-24T14:55:21Z","signature_id":"95d616d62fd8b68affd2cc0528b1726d4f7bac8f448ad84828096223e0ed6a1a","pr_number":7135,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7131/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103110260871925760","occurred_at":"2026-09-24T13:12:14Z","signature_id":"0316ed16bae2b06726cffc3f37180891ba349c08b7cf18aa3d8d47e6021122fd","pr_number":7131,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6746/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103109521260941312","occurred_at":"2026-09-24T13:09:15Z","signature_id":"d8ab59ad81a765023a17dd370421f6eb9d9e6dcc676911949deb1b2a6ad616d7","pr_number":6746,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055129497309184","occurred_at":"2026-09-24T09:33:10Z","signature_id":"6dc0e8968ec60f0324b6d07caa91d4f95c59a07d57bde431637fa99a19424678","pr_number":6917,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055091593383936","occurred_at":"2026-09-24T09:33:01Z","signature_id":"f048e9c9961e95f1fc58ff19153a9940b10cb95928a512cbddafab4912d29e0d","pr_number":7112,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7126/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103010871180529664","occurred_at":"2026-09-24T06:37:18Z","signature_id":"a0b9e359a9ec67136305ac5041742a0b1b9f95678b9b1c144a344ebc7e1e7a01","pr_number":7126,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102989394158817280","occurred_at":"2026-09-24T05:11:57Z","signature_id":"bd015bfe1b7ee0302c28b53124982a3fc176057803cf96dcb50518f45695f0ed","pr_number":5562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102931134991044608","occurred_at":"2026-09-24T01:20:27Z","signature_id":"5d2d7a79a877fd36a7ce50a19b88b5be576cc7d6481c13905922189eaaa22c37","pr_number":6226,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102921426674651136","occurred_at":"2026-09-24T00:41:53Z","signature_id":"c9c5fd9f1bd379784e5f4e5bbdd029af5ef5b392fcb1f2d33857ec4a703e485d","pr_number":7115,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102915118240108544","occurred_at":"2026-09-24T00:16:20Z","signature_id":"30e0e523f1fde040099a61098a3eec7b89130d7155eceabe4360309fa9b4a812","pr_number":7119,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102913782756937728","occurred_at":"2026-09-24T00:11:30Z","signature_id":"c5508e299b7fba05140b4f25dbef14c99e81680c4220a923738b6a2265553a53","pr_number":7112,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"208637581f2b9518289c7d8add0680b427ee5c78f2a2204ead4b38991424a4ab","failure_pattern":"alert [svc] ServiceCPUDrift fired","search_query":"alert [svc] ServiceCPUDrift fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","occurrences":15,"runs_affected":15,"failed_runs":15,"run_impact_percent":12.195121951219512,"anchor_occurrences":15,"after_last_push_count":42,"trend_counts":[0,0,0,0,0,0,15],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":82,"prior_last_seen_at":"2026-09-17T07:22:21Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ServiceCPUDrift does not fire","occurrences":15}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-25T01:15:33Z\n  Ended:       2026-09-25T01:24:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j0596736-mgmt-2\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-646d8d7d48-fmq9z\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container kube-applier in pod kube-applier-646d8d7d48-fmq9z (namespace kube-applier) on cluster ci01-j0596736-mgmt-2 has averaged 129.5% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-25T00:08:21Z\n  Ended:       2026-09-25T00:18:21Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9877248-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-7487b8fc78-kvh8n\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container kube-applier in pod kube-applier-7487b8fc78-kvh8n (namespace kube-applier) on cluster ci01-j9877248-mgmt-1 has averaged 133.7% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-24T23:55:38Z\n  Ended:       2026-09-25T00:01:38Z\n  Severity:    Sev3\n  Labels:      alertname=\"ServiceCPUDrift\", cluster=\"ci01-j9156224-mgmt-1\", component=\"service-memory\", container=\"kube-applier\", namespace=\"kube-applier\", pod=\"kube-applier-75497856bf-g2n4k\", severity=\"warning\", team=\"hcp-sl\"\n  Description: Container kube-applier in pod kube-applier-75497856bf-g2n4k (namespace kube-applier) on cluster ci01-j9156224-mgmt-1 has averaged 122.6% of its CPU request over the last 30 minutes.\nThis may indicate workload growth that requires right-sizing the request in config.yaml."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6846/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103270655540596736","occurred_at":"2026-09-24T23:49:35Z","signature_id":"cbcc41fb87ceb239baf4c483d3ab5268cdfd05b5c1837f310e5d12738e55e904","pr_number":6846,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103252727629877248","occurred_at":"2026-09-24T22:38:21Z","signature_id":"00ca63290ee1dd8cd9cc17fb467e5abff9ca540cb77527300f636b61f37a158b","pr_number":7123,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103247783359156224","occurred_at":"2026-09-24T22:18:40Z","signature_id":"31141dc7fe43f6d2aa89337a0a716ccfb12d368d6474df9ae0a1733b03fcf72c","pr_number":7148,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7059/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103245132269621248","occurred_at":"2026-09-24T22:08:10Z","signature_id":"cdf5e1de46d904b5754f4fcc48d70aa65ef0855075ecd7f8c6dfc039233eac7c","pr_number":7059,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233926699945984","occurred_at":"2026-09-24T21:23:38Z","signature_id":"6a245a5382730698531a737265dac078b3a944a08cb0f737e3cfdc39f19615db","pr_number":7135,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233919481548800","occurred_at":"2026-09-24T21:23:37Z","signature_id":"8d1d5c509bd74b9cfd90fd52435b1d0a29f7a99c0e5bc532798559b82949daa7","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233417079427072","occurred_at":"2026-09-24T21:21:37Z","signature_id":"cedddb9dfe24b0472f0b69938e215469ad1fa1732808690eb588c2842038524b","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103218872113958912","occurred_at":"2026-09-24T20:23:49Z","signature_id":"f448f1107e13285139f08f0b5d64b030989f1fce9c96e0627368c8e075d775c1","pr_number":6993,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203513864556544","occurred_at":"2026-09-24T19:22:46Z","signature_id":"474887e104d72f6b5e1fe9db5422a8d481f6d2378182349a4ce3015e81b513f6","pr_number":6970,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6866/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203214361890816","occurred_at":"2026-09-24T19:21:25Z","signature_id":"855925e2df7033b23d85ecff7ede1ca6cb961907816303723c8e216cb000b37a","pr_number":6866,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103200191157899264","occurred_at":"2026-09-24T19:09:35Z","signature_id":"784e6484d3fc29d5f43a3850fcf318afb800acb38164e1b18952339bbb4b08b9","pr_number":7123,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103200143447691264","occurred_at":"2026-09-24T19:09:24Z","signature_id":"b7cba4cec6f5a81a201092312fdb1ca862649a7f4b0539eae14accd98ac8dba5","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192892754366464","occurred_at":"2026-09-24T18:40:35Z","signature_id":"c23483f9fa6966a89f6cf4d0f9d5416c765a770957ef669010321a77abc1e041","pr_number":7135,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7068/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192838828199936","occurred_at":"2026-09-24T18:40:22Z","signature_id":"78083eff97a88f0d621458e0dfb940e0f79aaa0a1700d409ae56663484ef417b","pr_number":7068,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7053/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103175794774773760","occurred_at":"2026-09-24T17:32:39Z","signature_id":"e7b9c2ed622a10fb2e558d1618704e5e291864f9724b4b5ab5eb019d5bcadf46","pr_number":7053,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"69844ebd91a7bea6a85a8e0d235efb3ec57fe0b5fb3b8dab78ed169e1b1fe4bc","failure_pattern":"ERROR CODE: DeploymentFailed; detail code Conflict; provider Microsoft.EventGrid","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":11,"runs_affected":11,"failed_runs":11,"run_impact_percent":8.94308943089431,"anchor_occurrences":11,"after_last_push_count":3,"trend_counts":[0,0,0,0,0,0,11],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":11,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/regional/infra","occurrences":11}],"full_error_samples":["time=2026-09-24T23:48:24.249Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T23:48:25.676Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T23:48:26.816Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=9fe92f2075d3a162115703f2a2dd0a40f461263d369dfb42681221c1b82cb479 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1648000%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9fe92f2075d3a162115703f2a2dd0a40f461263d369dfb42681221c1b82cb479\ntime=2026-09-24T23:50:57.493Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1648000/providers/Microsoft.Resources/deployments/9fe92f2075d3a162115703f2a2dd0a40f461263d369dfb42681221c1b82cb479/operationStatuses/08584113131791545522\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1648000/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1648000/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1648000\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report a6a8d0ae-c018-4639-a518-3ed1c7beb811:9/24/2026 11:49:54 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T23:52:57.493Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T23:52:58.386Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T23:52:59.129Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=cf6e37d6cff770de12dbdbffa8c4623b3a737efd76d033bcf339dec1a4fb04e1 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1648000%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fcf6e37d6cff770de12dbdbffa8c4623b3a737efd76d033bcf339dec1a4fb04e1\ntime=2026-09-25T00:09:04.051Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1648000/providers/Microsoft.Resources/deployments/cf6e37d6cff770de12dbdbffa8c4623b3a737efd76d033bcf339dec1a4fb04e1/operationStatuses/08584113129068388577\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1648000/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1648000/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1648000\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report fc9f1334-cb66-4a1b-a69a-b44d3f0d15c9:9/25/2026 12:08:37 AM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-25T00:11:04.059Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-25T00:11:04.960Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-25T00:11:05.763Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=a4eb99266d655f9a85d7de315dc769d2d5008a2127b29773f350744d3fa412d7 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1648000%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fa4eb99266d655f9a85d7de315dc769d2d5008a2127b29773f350744d3fa412d7\ntime=2026-09-25T00:18:24.009Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-09-24T23:03:28.056Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T23:03:29.473Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T23:03:30.301Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=12f2d669fa06dd4de2ba8409d1e9b7aa7f63bae2d0c78ba209c8d1082b4c957e portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1689984%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F12f2d669fa06dd4de2ba8409d1e9b7aa7f63bae2d0c78ba209c8d1082b4c957e\ntime=2026-09-24T23:06:00.848Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1689984/providers/Microsoft.Resources/deployments/12f2d669fa06dd4de2ba8409d1e9b7aa7f63bae2d0c78ba209c8d1082b4c957e/operationStatuses/08584113158756314414\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1689984/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1689984/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1689984\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 6bb9bafd-0fb4-4aa8-a6e2-b2e4391847ba:9/24/2026 11:04:31 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T23:08:00.850Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T23:08:01.497Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T23:08:02.303Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=4b6dbf996876bb5363df66ec3bbff19c39d2fbac30f092592bf67f3fff9b6f47 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1689984%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F4b6dbf996876bb5363df66ec3bbff19c39d2fbac30f092592bf67f3fff9b6f47\ntime=2026-09-24T23:24:36.647Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1689984/providers/Microsoft.Resources/deployments/4b6dbf996876bb5363df66ec3bbff19c39d2fbac30f092592bf67f3fff9b6f47/operationStatuses/08584113156036989890\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1689984/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1689984/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1689984\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 7e35e8ae-d5b1-4064-b001-fb471f9768fb:9/24/2026 11:23:59 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T23:26:36.652Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T23:26:37.173Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T23:26:37.799Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=1b5feed18dc2aa2598959e31b9190f5f385e888a1a154d00d07d2ff191153df4 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1689984%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F1b5feed18dc2aa2598959e31b9190f5f385e888a1a154d00d07d2ff191153df4\ntime=2026-09-24T23:33:27.754Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\"","time=2026-09-24T23:03:18.918Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T23:03:20.600Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T23:03:21.313Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=d81bee64ce02324faba35ebcfefbaa384d773cf4fc43c777c70f8e291440954b portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1382656%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fd81bee64ce02324faba35ebcfefbaa384d773cf4fc43c777c70f8e291440954b\ntime=2026-09-24T23:05:52.100Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1382656/providers/Microsoft.Resources/deployments/d81bee64ce02324faba35ebcfefbaa384d773cf4fc43c777c70f8e291440954b/operationStatuses/08584113158845641130\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1382656/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1382656/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1382656\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report 1a3e1a3d-0894-450c-b8f5-f4aae39286df:9/24/2026 11:04:34 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T23:07:52.108Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T23:07:53.036Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T23:07:53.967Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=e77ffa30236038ba6be6aa6728547670b7fd78c60b95d6556ff96ad801ee7bf4 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1382656%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Fe77ffa30236038ba6be6aa6728547670b7fd78c60b95d6556ff96ad801ee7bf4\ntime=2026-09-24T23:24:01.808Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra step=infra error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1382656/providers/Microsoft.Resources/deployments/e77ffa30236038ba6be6aa6728547670b7fd78c60b95d6556ff96ad801ee7bf4/operationStatuses/08584113156121186022\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1382656/providers/Microsoft.Resources/deployments/maestro-infra-deployment\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j1382656/providers/Microsoft.EventGrid/namespaces/arohcp-ci01-maestro-j1382656\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"Internal error\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The operation failed due to an internal server error. The initial state of the impacted resources (if any) are restored. Please try again in few minutes. If error still persists, report f12586db-dfe7-4f9a-a86d-256d3a24ca8b:9/24/2026 11:23:57 PM (UTC) to our forums for assistance or raise a support ticket .\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-09-24T23:26:01.815Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra description=\"Step infra\\n  Kind: ARM\\n  Template: templates/region.bicep\\n  Parameters: configurations/region.tmpl.bicepparam\"\ntime=2026-09-24T23:26:02.592Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra\ntime=2026-09-24T23:26:03.555Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra deployment=8cd82016b5e13f611cfcadf8f8916eda701f48b5be5c35226e79dda3dd59a43e portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j1382656%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F8cd82016b5e13f611cfcadf8f8916eda701f48b5be5c35226e79dda3dd59a43e\ntime=2026-09-24T23:33:18.520Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=regional step=infra err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7118/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103269397211648000","occurred_at":"2026-09-24T23:44:35Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7118,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7150/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103255953611689984","occurred_at":"2026-09-24T22:51:10Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7150,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103252721661382656","occurred_at":"2026-09-24T22:38:19Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6821,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7149/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103246251251208192","occurred_at":"2026-09-24T22:12:37Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7149,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6359/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103232602474287104","occurred_at":"2026-09-24T21:18:23Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6359,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6969/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103213571209957376","occurred_at":"2026-09-24T20:02:45Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6969,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7142/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103205737139277824","occurred_at":"2026-09-24T19:31:38Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7142,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103198517097926656","occurred_at":"2026-09-24T19:02:55Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6915,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6920/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103186015765139456","occurred_at":"2026-09-24T18:13:16Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6920,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103181727441620992","occurred_at":"2026-09-24T17:56:13Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":6915,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7054/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103178709123993600","occurred_at":"2026-09-24T17:44:13Z","signature_id":"76d1e1acef0b5860ba8119210c110d72b0e9484abeea5273e251a840e1ba48f1","pr_number":7054,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a4f3c24f075e1dad558636095f209b5b0ee737bcb81bd0c18823a283d1255124","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available.; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":7,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.6260162601626016,"anchor_occurrences":7,"after_last_push_count":8,"trend_counts":[2,0,0,0,0,1,7],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":12,"prior_last_seen_at":"2026-09-18T03:01:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a31f40\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-dlxzmd\" in resourcegroup=\"rg-np-skip-minor-dlxzmd-8tb8jspnqx48\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2e06418b-9c11-492d-8bf1-bc7db3b3afdf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2e06418b-9c11-492d-8bf1-bc7db3b3afdf\",\n      \"name\": \"2e06418b-9c11-492d-8bf1-bc7db3b3afdf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T10:30:31.115048125Z\",\n      \"endTime\": \"2026-09-24T10:31:10.228268414Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a31f40\u003e: \n    failed to create NodePool nps-420-422: failed waiting for nodepool=\"nps-420-422\" for cluster \"np-skip-minor-dlxzmd\" in resourcegroup=\"rg-np-skip-minor-dlxzmd-8tb8jspnqx48\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2e06418b-9c11-492d-8bf1-bc7db3b3afdf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/2e06418b-9c11-492d-8bf1-bc7db3b3afdf\",\n      \"name\": \"2e06418b-9c11-492d-8bf1-bc7db3b3afdf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T10:30:31.115048125Z\",\n      \"endTime\": \"2026-09-24T10:31:10.228268414Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-d4v522\" for cluster \"cluster-candidate-4-22-d4v522\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008ad540\u003e: \n    failed to create NodePool np-d4v522: failed waiting for nodepool=\"np-d4v522\" for cluster \"cluster-candidate-4-22-d4v522\" in resourcegroup=\"rg-candidate-4-22-q2jqvkjfll66\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a57495d-192b-47f3-8f08-3dbd939c2ccc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a57495d-192b-47f3-8f08-3dbd939c2ccc\",\n      \"name\": \"9a57495d-192b-47f3-8f08-3dbd939c2ccc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T10:30:51.31445342Z\",\n      \"endTime\": \"2026-09-24T10:31:30.134898766Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-d4v522\" for cluster \"cluster-candidate-4-22-d4v522\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0008ad540\u003e: \n    failed to create NodePool np-d4v522: failed waiting for nodepool=\"np-d4v522\" for cluster \"cluster-candidate-4-22-d4v522\" in resourcegroup=\"rg-candidate-4-22-q2jqvkjfll66\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a57495d-192b-47f3-8f08-3dbd939c2ccc\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/9a57495d-192b-47f3-8f08-3dbd939c2ccc\",\n      \"name\": \"9a57495d-192b-47f3-8f08-3dbd939c2ccc\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T10:30:51.31445342Z\",\n      \"endTime\": \"2026-09-24T10:31:30.134898766Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f01380\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-bpr946thzk52\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/62555631-7ec6-4b02-b164-698c4d6f90b4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/62555631-7ec6-4b02-b164-698c4d6f90b4\",\n      \"name\": \"62555631-7ec6-4b02-b164-698c4d6f90b4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T10:24:31.78038551Z\",\n      \"endTime\": \"2026-09-24T10:25:09.920364245Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f01380\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-bpr946thzk52\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/62555631-7ec6-4b02-b164-698c4d6f90b4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/62555631-7ec6-4b02-b164-698c4d6f90b4\",\n      \"name\": \"62555631-7ec6-4b02-b164-698c4d6f90b4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T10:24:31.78038551Z\",\n      \"endTime\": \"2026-09-24T10:25:09.920364245Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055129497309184","occurred_at":"2026-09-24T09:33:10Z","signature_id":"648b322dd2b99b65149b365370cd6cb7208f3487446dfc081728eb5b19b547de","pr_number":6917,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055129497309184","occurred_at":"2026-09-24T09:33:10Z","signature_id":"c83a023c6700a1a72267fb7de4ac02824e0ac5dfe0406b81242145d8b6fc24f6","pr_number":6917,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055129497309184","occurred_at":"2026-09-24T09:33:10Z","signature_id":"dd690b63f5f18c2dbee25e78c6edf3eeb01364dc7ce7e0d6fe2be002e3e85b36","pr_number":6917,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102915118240108544","occurred_at":"2026-09-24T00:16:20Z","signature_id":"88da56dc9cbaf5632d307314826460b6bc87c0605368c5a80838d8aa0648f7a2","pr_number":7119,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102915118240108544","occurred_at":"2026-09-24T00:16:20Z","signature_id":"8bf81576a6cc6f8137c7015977c9d4e64ecd2342fb685d20f3e351e64bbf3b0b","pr_number":7119,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102915118240108544","occurred_at":"2026-09-24T00:16:20Z","signature_id":"e6dd7dea20380c21745a379f6c7147226e56bf52ffe41815559c0e6f0dbfefbd","pr_number":7119,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102915118240108544","occurred_at":"2026-09-24T00:16:20Z","signature_id":"f9b1fc76540f4c53a5a55de6e48304b8e30d158e9f8f3534a80d2ae685ba6b26","pr_number":7119,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7932344330c95787cfae1d106219b0ef01a6cb7298b5bee090ad8716b954e4b0","failure_pattern":"alert [svc] FrontendPathLatency fired","search_query":"alert [svc] FrontendPathLatency fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":5,"runs_affected":5,"failed_runs":5,"run_impact_percent":4.065040650406504,"anchor_occurrences":5,"after_last_push_count":20,"trend_counts":[3,3,5,4,13,3,5],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":36,"prior_last_seen_at":"2026-09-20T19:55:27Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert FrontendPathLatency does not fire","occurrences":5}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-24T17:31:32Z\n  Ended:       2026-09-24T17:39:31Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j1914624-svc\", component=\"frontend\", method=\"get\", region=\"canadacentral\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j1914624-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-24T12:40:11Z\n  Ended:       2026-09-24T12:46:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j4440832-svc\", component=\"frontend\", method=\"post\", region=\"centralus\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/requestadmincredential\", severity=\"warning\"\n  Description: More than 5% of frontend requests for post /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/requestadmincredential exceeded 1 second over the past 30 minutes (cluster ci01-j4440832-svc).","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-24T08:24:07Z\n  Ended:       2026-09-24T08:31:08Z\n  Severity:    Sev3\n  Labels:      alertname=\"FrontendPathLatency\", cluster=\"ci01-j2442112-svc\", component=\"frontend\", method=\"get\", region=\"westus3\", route=\"/subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname}\", severity=\"warning\"\n  Description: More than 5% of frontend requests for get /subscriptions/{subscriptionid}/resourcegroups/{resourcegroupname}/providers/microsoft.redhatopenshift/hcpopenshiftclusters/{resourcename}/nodepools/{nodepoolname} exceeded 1 second over the past 30 minutes (cluster ci01-j2442112-svc)."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146310981914624","occurred_at":"2026-09-24T15:35:16Z","signature_id":"0e8f57e37d73e9d51e134e1e0ceb298fcf33a07521e65f7fbe44784034a97653","pr_number":7045,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103078971544440832","occurred_at":"2026-09-24T11:07:54Z","signature_id":"52a5a38b47cc6e69825210c368c8f32d8cf5f29e6283dbea51644a315c099800","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6884/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103015428132442112","occurred_at":"2026-09-24T06:55:24Z","signature_id":"609ef47bc200be1ff06713d78b2540ac62b150a8d0e17b016ed8c4fd89fa13fe","pr_number":6884,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/5562/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102989394158817280","occurred_at":"2026-09-24T05:11:57Z","signature_id":"e02ac96b476af77a18c8814f0b86746bd97b987ea683b9a1aa418df78ed20adc","pr_number":5562,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7115/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102921426674651136","occurred_at":"2026-09-24T00:41:53Z","signature_id":"7715e65904649de36554c3fa389bc722801fea292dc922d83d7ddf72b37313e4","pr_number":7115,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":3.252032520325203,"anchor_occurrences":4,"after_last_push_count":46,"trend_counts":[15,4,4,2,2,2,4],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":94,"prior_last_seen_at":"2026-09-20T23:42:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","occurrences":4}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-24T20:33:02Z\n  Ended:       2026-09-24T21:02:59Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4556544-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t1jgr6a3uevc16juiqil4ndk5set1ce-o5o5w3j3g5c4j6m\", pod=\"router-66cb8fd48d-wgc7r\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t1jgr6a3uevc16juiqil4ndk5set1ce-o5o5w3j3g5c4j6m/router-66cb8fd48d-wgc7r has been in a non-ready state for longer than 5 minutes.","alert fired 12 time(s) (1 unknown, 11 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-24T19:41:45Z\n  Ended:       2026-09-24T20:10:44Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8866176-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t1io3vu7p3gua60c6258horequmjsus-f3a9j8b5h7f1o0j\", pod=\"router-5cc57c6448-t4p5n\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t1io3vu7p3gua60c6258horequmjsus-f3a9j8b5h7f1o0j/router-5cc57c6448-t4p5n has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-24T15:02:27Z\n  Ended:       2026-09-24T15:08:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j3943808-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2t1ecumio2t0so7i2erb9l3i7dm6lu3m-sre-hcp-cluster\", pod=\"kube-apiserver-5977bd7447-4zvrr\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2t1ecumio2t0so7i2erb9l3i7dm6lu3m-sre-hcp-cluster/kube-apiserver-5977bd7447-4zvrr has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203513864556544","occurred_at":"2026-09-24T19:22:46Z","signature_id":"9148700f82fd5b370a512210ca6a6cb677852f9f2d8abcc81bf3649ab5047f43","pr_number":6970,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103191956178866176","occurred_at":"2026-09-24T18:36:45Z","signature_id":"7fbc78ba550f54a61f9bb8680a644c7d2c72d2a80281632662dd5cddec3c05d7","pr_number":7141,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7130/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103111106493943808","occurred_at":"2026-09-24T13:15:33Z","signature_id":"71628d4204794d2bf5a1e409f017e03485e53450d5f8ef544bb7d4efc8d8ed17","pr_number":7130,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102919814468079616","occurred_at":"2026-09-24T00:35:28Z","signature_id":"eb34a8b64fb9114676f2191bea2c8d39236f5170336f8b7d13fb0795cc5edc83","pr_number":6917,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"7ce686769ff043136690244623d4e41638e076307605ca4f958fb70e7e2b2051","failure_pattern":"alert [svc] KubePodNotReady fired","search_query":"alert [svc] KubePodNotReady fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":3.252032520325203,"anchor_occurrences":4,"after_last_push_count":4,"trend_counts":[0,0,1,1,3,2,4],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":17,"prior_last_seen_at":"2026-09-20T23:13:50Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubePodNotReady does not fire","occurrences":4}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T17:24:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4604416-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-fw4cp\", region=\"westus3\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-fw4cp has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T17:24:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4604416-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-dfd7c\", region=\"westus3\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-dfd7c has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T17:24:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4604416-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-vflzc\", region=\"westus3\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-vflzc has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T17:24:03Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j4604416-svc\", component=\"kubernetes-infrastructure\", namespace=\"arobit\", pod=\"arobit-forwarder-fn4cr\", region=\"westus3\", severity=\"warning\"\n  Description: Pod arobit/arobit-forwarder-fn4cr has been in a non-ready state for longer than 5 minutes.","alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T16:38:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1695232-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"eraser-aks-system-40199610-vmss000000-j9kbh\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod kube-system/eraser-aks-system-40199610-vmss000000-j9kbh has been in a non-ready state for longer than 5 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T16:38:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1695232-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"aks-secrets-store-csi-driver-nffxs\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod kube-system/aks-secrets-store-csi-driver-nffxs has been in a non-ready state for longer than 5 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T16:38:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1695232-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"kube-system\", pod=\"ama-metrics-node-2szd2\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod kube-system/ama-metrics-node-2szd2 has been in a non-ready state for longer than 5 minutes.\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T16:40:04Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1695232-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"swift-recorder\", pod=\"swift-recorder-tw7kk\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod swift-recorder/swift-recorder-tw7kk has been in a non-ready state for longer than 5 minutes.","alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T07:42:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"open-cluster-management-agent-addon\", pod=\"klusterlet-addon-workmgr-749558849-6dgkt\", region=\"westus3\", severity=\"warning\"\n  Description: Pod open-cluster-management-agent-addon/klusterlet-addon-workmgr-749558849-6dgkt has been in a non-ready state for longer than 5 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103161107324604416","occurred_at":"2026-09-24T16:34:17Z","signature_id":"39ce01c9a16cf6ef63990abb5b52786f61c0e8bb85c12b3c8bff45bc21ee3246","pr_number":6820,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146302601695232","occurred_at":"2026-09-24T15:35:15Z","signature_id":"49174f0842b090006dcda4660be00bccd297316410781deb14da7700d135010d","pr_number":7117,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"8a9c66eda94e766b0ef899f88f4ac1a07026419074e6b27f42b8d0218d54c88e","pr_number":6503,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7090/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102921738877669376","occurred_at":"2026-09-24T00:43:07Z","signature_id":"3b853c2ad740c6b27854bb7b28c66560f90ec20a918beecff56ae64e308ded54","pr_number":7090,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"0421c66c980bab7883ea258cf0dfcba450cd116eeb9b42d78247faa93a3a1288","failure_pattern":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","search_query":"certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":3.252032520325203,"anchor_occurrences":4,"after_last_push_count":8,"trend_counts":[2,0,2,0,4,1,4],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":16,"prior_last_seen_at":"2026-09-20T12:02:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.003s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001ea9b70\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.003s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001ea9b70\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001cb6930\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.002s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc001cb6930\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.004s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000bbd950\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:389]: Timed out after 600.004s.\nCertificate should be issued by Let's Encrypt staging via ACME DNS-01\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xc000bbd950\u003e: \n    certificate not ready: reason=DoesNotExist message=Issuing certificate as Secret does not exist\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6364/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103224244262408192","occurred_at":"2026-09-24T20:45:10Z","signature_id":"ecbec1ab9f9617f6b0594d4fbd470c36978a735ccd4e20c469402d74bd7943c7","pr_number":6364,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103078389232439296","occurred_at":"2026-09-24T11:05:35Z","signature_id":"90dd6825e45727c7dbdbd1c4c8cd0b287a1d641727762971db09d4ac1b8c9c2a","pr_number":7045,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7127/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103015078914691072","occurred_at":"2026-09-24T06:54:00Z","signature_id":"423b6e69577f5d18ef882789710f57eff74dad24bb4c0c9c09ca525154f2add8","pr_number":7127,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7126/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103010871180529664","occurred_at":"2026-09-24T06:37:18Z","signature_id":"3f39acc6e073ba27c21beb42a91e286c5d52972e441b176f4734975709089b1e","pr_number":7126,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d91796eeda5752e6e2b49b398ec435f90a4ab6d3394ea2d0cd4ce5b2c2f30a11","failure_pattern":"not all KSM metrics appeared in Azure Monitor","search_query":"not all KSM metrics appeared in Azure Monitor","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics metrics should be present in HCP Azure Monitoring Workspace","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":3.252032520325203,"anchor_occurrences":4,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,1,4],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":5,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"KSM HCP Metrics metrics should be present in HCP Azure Monitoring Workspace","occurrences":4}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:185]: Timed out after 1500.005s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:183 with:\nexpected kube_node_status_condition from kube-state-metrics; kube_node_info from kube-state-metrics but got no results\nExpected\n    \u003c[]string | len:2, cap:2\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:185]: Timed out after 1500.005s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:183 with:\nexpected kube_node_status_condition from kube-state-metrics; kube_node_info from kube-state-metrics but got no results\nExpected\n    \u003c[]string | len:2, cap:2\u003e: ...\nto be empty","fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:185]: Timed out after 1500.000s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:183 with:\nexpected kube_node_status_condition from kube-state-metrics; kube_node_info from kube-state-metrics but got no results\nExpected\n    \u003c[]string | len:2, cap:2\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:185]: Timed out after 1500.000s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:183 with:\nexpected kube_node_status_condition from kube-state-metrics; kube_node_info from kube-state-metrics but got no results\nExpected\n    \u003c[]string | len:2, cap:2\u003e: ...\nto be empty","fail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:147]: Timed out after 900.000s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:143 with:\nexpected ingresscontroller_info metrics from kube-state-metrics but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:147]: Timed out after 900.000s.\nnot all KSM metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/ksm_hcp_metrics.go:143 with:\nexpected ingresscontroller_info metrics from kube-state-metrics but got no results\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: ...\nnot to be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233919481548800","occurred_at":"2026-09-24T21:23:37Z","signature_id":"9783002e2eb58bbef4bfb807305e4c98f54648869ece93a50ed71011664015e3","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192810386624512","occurred_at":"2026-09-24T18:40:16Z","signature_id":"b935a8a8e1ec9f7c77dd3ec089fb77afdc37f7a2e866b79eb951cee11d3e6a47","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103078971544440832","occurred_at":"2026-09-24T11:07:54Z","signature_id":"b935a8a8e1ec9f7c77dd3ec089fb77afdc37f7a2e866b79eb951cee11d3e6a47","pr_number":7009,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103026030053691392","occurred_at":"2026-09-24T07:37:32Z","signature_id":"b9e992e4dedaba952b861c4393e5971bfc00933bdfd510734b71bce7ea9324b7","pr_number":7009,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"36601573516c72370b5de3f2f5199840f1883a8e5ad5f6d84f833bef7eb8e57a","failure_pattern":"Command failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:\u003chex\u003e arohcpsvcdev.azurecr.io/nshneor/cs:\u003chex\u003e --from-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json --to-registry-config /tmp/tmp.\u003cid\u003e/containers/auth.json","search_query":"Command failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 arohcpsvcdev.azurecr.io/nshneor/cs:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 --from-registry-config /tmp/tmp.bxHO86cpwB/containers/auth.json --to-registry-config /tmp/tmp.bxHO86cpwB/containers/auth.json","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/global/mirror-image","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.6260162601626016,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.ClusterService/global/mirror-image","occurrences":2}],"full_error_samples":["time=2026-09-24T12:31:58.312Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e clustersService.image.registry \u003cnil\u003e}:{\u003cnil\u003e clustersService.image.repository \u003cnil\u003e}@{\u003cnil\u003e clustersService.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-09-24T12:32:36.173Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nFetch pull secret for source registry arohcpsvcdev from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image arohcpsvcdev/nshneor/cs@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 to arohcpsvcdev.azurecr.io/nshneor/cs:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6.\\nThe image will still be available under it's original digest sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 in the target registry.\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 arohcpsvcdev.azurecr.io/nshneor/cs:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 --from-registry-config /tmp/tmp.wiDcX7URTm/containers/auth.json --to-registry-config /tmp/tmp.wiDcX7URTm/containers/auth.json\\n exit status 1\"","time=2026-09-24T10:30:31.503Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image description=\"Step mirror-image\\n  Kind: ImageMirror\\n  From {\u003cnil\u003e clustersService.image.registry \u003cnil\u003e}:{\u003cnil\u003e clustersService.image.repository \u003cnil\u003e}@{\u003cnil\u003e clustersService.image.digest \u003cnil\u003e} to {\u003cnil\u003e acr.svc.name \u003cnil\u003e}\\n\"\ntime=2026-09-24T10:31:04.673Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.ClusterService resourceGroup=global step=mirror-image err=\"error running Image Mirror Step, failed to execute shell command: Checking USE_OC_LOGIN_REGISTRIES: registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org registry.build11.ci.openshift.org\\nFetch pull secret for source registry arohcpsvcdev from arohcpdev-global KV.\\nLogging into target ACR arohcpsvcdev.\\nLogin Succeeded\\nMirroring image arohcpsvcdev/nshneor/cs@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 to arohcpsvcdev.azurecr.io/nshneor/cs:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6.\\nThe image will still be available under it's original digest sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 in the target registry.\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 1/5). Retrying in 2s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 2/5). Retrying in 4s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 3/5). Retrying in 8s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed (attempt 4/5). Retrying in 16s...\\nError: failed to resolve sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6: Get \\\"https://arohcpsvcdev/v2/nshneor/cs/manifests/sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6\\\": dial tcp: lookup arohcpsvcdev on 172.30.0.10:53: no such host\\nCommand failed after 5 attempts: oras cp arohcpsvcdev/nshneor/cs@sha256:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 arohcpsvcdev.azurecr.io/nshneor/cs:299c08f05b5b37da0e9c8b727bd0f57e519da49d0bb6daf96b57d0744493e5a6 --from-registry-config /tmp/tmp.bxHO86cpwB/containers/auth.json --to-registry-config /tmp/tmp.bxHO86cpwB/containers/auth.json\\n exit status 1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103088386418937856","occurred_at":"2026-09-24T11:45:17Z","signature_id":"062e807a8092da1711f0e05e391e8b306673f16936facbff876328c71872d5ee","pr_number":6889,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6889/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103060333542248448","occurred_at":"2026-09-24T09:53:51Z","signature_id":"062e807a8092da1711f0e05e391e8b306673f16936facbff876328c71872d5ee","pr_number":6889,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"a0a59479c62dfd2c7f3e63aae1a5a2204589ae2e696b0c89f2a905f43af2559f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [router]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.6260162601626016,"anchor_occurrences":2,"after_last_push_count":63,"trend_counts":[19,4,0,0,1,2,2],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":83,"prior_last_seen_at":"2026-09-19T20:01:44Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-c44pd8c2jtgp/cluster-candidate-4-21-k4rdfg should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010bc2e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-k4rdfg: failed waiting for cluster=\"cluster-candidate-4-21-k4rdfg\" in resourcegroup=\"rg-candidate-4-21-c44pd8c2jtgp\" to finish creating: GET https://rp.j4556544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c4682523-7a01-4560-b35e-4fc4d15613ae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c4682523-7a01-4560-b35e-4fc4d15613ae\",\n      \"name\": \"c4682523-7a01-4560-b35e-4fc4d15613ae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T20:22:03.886308121Z\",\n      \"endTime\": \"2026-09-24T20:51:09.533160535Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 24m24s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-21-c44pd8c2jtgp/cluster-candidate-4-21-k4rdfg should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010bc2e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-21-k4rdfg: failed waiting for cluster=\"cluster-candidate-4-21-k4rdfg\" in resourcegroup=\"rg-candidate-4-21-c44pd8c2jtgp\" to finish creating: GET https://rp.j4556544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c4682523-7a01-4560-b35e-4fc4d15613ae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c4682523-7a01-4560-b35e-4fc4d15613ae\",\n      \"name\": \"c4682523-7a01-4560-b35e-4fc4d15613ae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T20:22:03.886308121Z\",\n      \"endTime\": \"2026-09-24T20:51:09.533160535Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 24m24s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-c7xkx7wln57r/cluster-candidate-5-0-87wr9p should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010cc280\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-87wr9p: failed waiting for cluster=\"cluster-candidate-5-0-87wr9p\" in resourcegroup=\"rg-candidate-5-0-c7xkx7wln57r\" to finish creating: GET https://rp.j8866176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d9d1e9a3-4f45-45e2-a7a8-d0765c81f904\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d9d1e9a3-4f45-45e2-a7a8-d0765c81f904\",\n      \"name\": \"d9d1e9a3-4f45-45e2-a7a8-d0765c81f904\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T19:29:18.072972108Z\",\n      \"endTime\": \"2026-09-24T19:58:19.977586305Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.3 is Partial (want Completed), started 22m42s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-5-0-c7xkx7wln57r/cluster-candidate-5-0-87wr9p should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010cc280\u003e: \n    failed to create HCP cluster cluster-candidate-5-0-87wr9p: failed waiting for cluster=\"cluster-candidate-5-0-87wr9p\" in resourcegroup=\"rg-candidate-5-0-c7xkx7wln57r\" to finish creating: GET https://rp.j8866176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d9d1e9a3-4f45-45e2-a7a8-d0765c81f904\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d9d1e9a3-4f45-45e2-a7a8-d0765c81f904\",\n      \"name\": \"d9d1e9a3-4f45-45e2-a7a8-d0765c81f904\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T19:29:18.072972108Z\",\n      \"endTime\": \"2026-09-24T19:58:19.977586305Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 5.0.0-rc.3 is Partial (want Completed), started 22m42s ago; hosted cluster degraded: UnavailableReplicas: router deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203513864556544","occurred_at":"2026-09-24T19:22:46Z","signature_id":"2dd1d4adf03c4cf1ef2152b091a8d8db32bd11c08e0192a8877e14758cab2f4d","pr_number":6970,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103191956178866176","occurred_at":"2026-09-24T18:36:45Z","signature_id":"c29dc181d38e6ceb97183fd208302b380dff64e5d9a1ab9ca671c1309bb44d49","pr_number":7141,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"b105225ce4af9b235b53d80e208abf25ff1fafd2782079fae61606d8309203af","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.6260162601626016,"anchor_occurrences":2,"after_last_push_count":21,"also_in":["PROD","STG"],"trend_counts":[4,1,5,3,8,7,2],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":40,"prior_last_seen_at":"2026-09-20T23:13:50Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000721c20\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-8nk2s8\" in resourcegroup=\"rg-np-noedge-8nk2s8-pbf4w4lpxflj\" to finish creating: GET https://rp.j9067520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf579354-82a6-4511-b470-a74098b9d7f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf579354-82a6-4511-b470-a74098b9d7f0\",\n      \"name\": \"bf579354-82a6-4511-b470-a74098b9d7f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T13:38:51.627589072Z\",\n      \"endTime\": \"2026-09-24T13:57:59.764961372Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000721c20\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-8nk2s8\" in resourcegroup=\"rg-np-noedge-8nk2s8-pbf4w4lpxflj\" to finish creating: GET https://rp.j9067520.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf579354-82a6-4511-b470-a74098b9d7f0\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/bf579354-82a6-4511-b470-a74098b9d7f0\",\n      \"name\": \"bf579354-82a6-4511-b470-a74098b9d7f0\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T13:38:51.627589072Z\",\n      \"endTime\": \"2026-09-24T13:57:59.764961372Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00100c160\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-wjm85h57dqlz\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/deca3434-569c-4feb-abcd-1a0dae06f921\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/deca3434-569c-4feb-abcd-1a0dae06f921\",\n      \"name\": \"deca3434-569c-4feb-abcd-1a0dae06f921\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T08:38:16.231035723Z\",\n      \"endTime\": \"2026-09-24T08:57:18.997473969Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kv.go:138]: failed to create node pool \"np-1\" for private keyvault cluster \"private-kv-cluster\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00100c160\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"private-kv-cluster\" in resourcegroup=\"private-keyvault-wjm85h57dqlz\" to finish creating: GET https://rp.aro-hcp-dev-shard2-slot-00.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/deca3434-569c-4feb-abcd-1a0dae06f921\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/deca3434-569c-4feb-abcd-1a0dae06f921\",\n      \"name\": \"deca3434-569c-4feb-abcd-1a0dae06f921\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T08:38:16.231035723Z\",\n      \"endTime\": \"2026-09-24T08:57:18.997473969Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103098516569067520","occurred_at":"2026-09-24T12:25:34Z","signature_id":"ed02ea736a12b453c1efc5a8d4a86dc6bf8f47885f359a94f77192dc2ef4318b","pr_number":7065,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103024914473684992","occurred_at":"2026-09-24T07:33:06Z","signature_id":"569706336b0bb772a90b6de148018cee0d7066e56c9777657f620d7bb7c75db6","pr_number":6995,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"0905a6f0720326490c59375e0404554c58e1e0996cc236e2a705a956863de47f","failure_pattern":"ERROR CODE: InvalidTemplate; detail message Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'maestroMqttEventHubId'. The parameters defined in the template are: 'databaseName, eventHubId, eventhubEnabled, kustoConsumerGroupName, kustoDataConnectionName, kustoEnabled, kustoName, location'.","search_query":"ERROR CODE: InvalidTemplate","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/kusto/maestroMqttDataConnection","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.6260162601626016,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Region/kusto/maestroMqttDataConnection","occurrences":2}],"full_error_samples":["time=2026-09-24T23:55:56.171Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection description=\"Step maestroMqttDataConnection\\n  Kind: ARM\\n  Template: templates/maestro-mqtt-data-connection.bicep\\n  Parameters: configurations/maestro-mqtt-data-connection.tmpl.bicepparam\"\ntime=2026-09-24T23:55:56.789Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection\ntime=2026-09-24T23:55:57.168Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection err=\"failed to run ARM step: failed to create deployment: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-kusto-us/providers/Microsoft.Resources/deployments/23db75d3c4a6f5fe32de7df35567f808137cc8acac2d401192c396cc43f1121e\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidTemplate\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidTemplate\\\",\\n    \\\"message\\\": \\\"Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'maestroMqttEventHubId'. The parameters defined in the template are: 'databaseName, eventHubId, eventhubEnabled, kustoConsumerGroupName, kustoDataConnectionName, kustoEnabled, kustoName, location'. Please see https://aka.ms/arm-pass-parameter-values for usage details.'.\\\",\\n    \\\"additionalInfo\\\": [\\n      {\\n        \\\"type\\\": \\\"TemplateViolation\\\",\\n        \\\"info\\\": {\\n          \\\"lineNumber\\\": 0,\\n          \\\"linePosition\\\": 0,\\n          \\\"path\\\": \\\"\\\"\\n        }\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","time=2026-09-24T22:47:58.457Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection description=\"Step maestroMqttDataConnection\\n  Kind: ARM\\n  Template: templates/maestro-mqtt-data-connection.bicep\\n  Parameters: configurations/maestro-mqtt-data-connection.tmpl.bicepparam\"\ntime=2026-09-24T22:47:59.030Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection\ntime=2026-09-24T22:47:59.240Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Region resourceGroup=kusto step=maestroMqttDataConnection err=\"failed to run ARM step: failed to create deployment: PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-kusto-us/providers/Microsoft.Resources/deployments/754de7cea577bd27c73251066594765f388be9580303d6f77fd3975785c55d9b\\n--------------------------------------------------------------------------------\\nRESPONSE 400: 400 Bad Request\\nERROR CODE: InvalidTemplate\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"InvalidTemplate\\\",\\n    \\\"message\\\": \\\"Deployment template validation failed: 'The following parameters were supplied, but do not correspond to any parameters defined in the template: 'maestroMqttEventHubId'. The parameters defined in the template are: 'databaseName, eventHubId, eventhubEnabled, kustoConsumerGroupName, kustoDataConnectionName, kustoEnabled, kustoName, location'. Please see https://aka.ms/arm-pass-parameter-values for usage details.'.\\\",\\n    \\\"additionalInfo\\\": [\\n      {\\n        \\\"type\\\": \\\"TemplateViolation\\\",\\n        \\\"info\\\": {\\n          \\\"lineNumber\\\": 0,\\n          \\\"linePosition\\\": 0,\\n          \\\"path\\\": \\\"\\\"\\n        }\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7146/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103270905017798656","occurred_at":"2026-09-24T23:50:35Z","signature_id":"8b0f84d3fd13e52b773c2d5762e6e413e0d4ff59cb390c1fe435499d6e4b2b92","pr_number":7146,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7146/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103249876442681344","occurred_at":"2026-09-24T22:27:01Z","signature_id":"8b0f84d3fd13e52b773c2d5762e6e413e0d4ff59cb390c1fe435499d6e4b2b92","pr_number":7146,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"750410b055b83de5666e72039e788f64cc42e64b3191a432c4cf3efab4dc693f","failure_pattern":"Operation cannot be fulfilled on nodes \"\u003cnode\u003e\": the object has been modified; please apply your changes to the latest version and try again","search_query":"occurred","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.6260162601626016,"anchor_occurrences":2,"after_last_push_count":3,"also_in":["PROD"],"trend_counts":[0,0,1,0,0,1,2],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-20T09:53:04Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node k8w2j3t8n4o7w6y-cordon-np-fqrz9-2zs4k\nUnexpected error:\n    \u003c*errors.StatusError | 0xc0012d1400\u003e: \n    Operation cannot be fulfilled on nodes \"k8w2j3t8n4o7w6y-cordon-np-fqrz9-2zs4k\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node k8w2j3t8n4o7w6y-cordon-np-fqrz9-2zs4k\nUnexpected error:\n    \u003c*errors.StatusError | 0xc0012d1400\u003e: \n    Operation cannot be fulfilled on nodes \"k8w2j3t8n4o7w6y-cordon-np-fqrz9-2zs4k\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:214]: failed to cordon node a2u7a3d0r9z7j0c-cordon-np-tvtlh-5pbth\nUnexpected error:\n    \u003c*errors.StatusError | 0xc000a01cc0\u003e: \n    Operation cannot be fulfilled on nodes \"a2u7a3d0r9z7j0c-cordon-np-tvtlh-5pbth\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:214]: failed to cordon node a2u7a3d0r9z7j0c-cordon-np-tvtlh-5pbth\nUnexpected error:\n    \u003c*errors.StatusError | 0xc000a01cc0\u003e: \n    Operation cannot be fulfilled on nodes \"a2u7a3d0r9z7j0c-cordon-np-tvtlh-5pbth\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6226/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103099155617419264","occurred_at":"2026-09-24T12:28:06Z","signature_id":"d8b50df50945eaa6be746f392a278fe85b30b5ed045a7c3106d19fdebdab862c","pr_number":6226,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6995/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103024914473684992","occurred_at":"2026-09-24T07:33:06Z","signature_id":"28954fd4b540505310c8d2b6c0bea86f4588ec03f99fb14067897e28bef073a8","pr_number":6995,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"24a47081b4324640c960819b2257133d98c31587850da06feb9220df68f66df2","failure_pattern":"alert [svc] BackendControllerQueueDepthHigh fired","search_query":"alert [svc] BackendControllerQueueDepthHigh fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.6260162601626016,"anchor_occurrences":2,"after_last_push_count":8,"trend_counts":[4,1,1,3,6,4,2],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":25,"prior_last_seen_at":"2026-09-20T16:10:57Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerQueueDepthHigh does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-24T16:50:03Z\n  Ended:       2026-09-24T16:57:05Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j1914624-svc\", component=\"backend\", name=\"clusterresources\", region=\"canadacentral\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-24T04:25:52Z\n  Ended:       2026-09-24T04:43:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerQueueDepthHigh\", cluster=\"ci01-j3233536-svc\", component=\"backend\", name=\"clusterresources\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue clusterresources has had a depth \u003e 10 for more than 15 minutes, indicating work is accumulating faster than it can be processed."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146310981914624","occurred_at":"2026-09-24T15:35:16Z","signature_id":"5b1692b0736093952d02a0be6a075ce32dc0d7ca1d5d24abfc74fcf89047b522","pr_number":7045,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102965797063233536","occurred_at":"2026-09-24T03:38:11Z","signature_id":"663938495541e0fb2e27706ec8f49f136079b5f0b456ecbeb22d95b137d6d08b","pr_number":6917,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"abc571fde08fbdcc334f3c43a6d9bd7fa845989e30588363eb40a96e3e4293b9","failure_pattern":"alert [svc] MaestroAgentMetricsProxyDown fired","search_query":"alert [svc] MaestroAgentMetricsProxyDown fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.6260162601626016,"anchor_occurrences":2,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-17T20:52:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert MaestroAgentMetricsProxyDown does not fire","occurrences":2}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T19:45:14Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j6380032-mgmt-1\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing.","alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-24T16:29:24Z\n  Ended:       2026-09-24T16:38:26Z\n  Severity:    Sev3\n  Labels:      alertname=\"MaestroAgentMetricsProxyDown\", cluster=\"ci01-j1695232-mgmt-1\", component=\"monitoring-infrastructure\", region=\"canadacentral\", severity=\"3\"\n  Description: Prometheus has not scraped maestro-agent /metrics through the metrics-proxy sidecar (namespace=maestro, endpoint=metrics) for 10 minutes, while a kube-state-metrics scrape series still exists for this management cluster. Maestro agent metrics and any alerts that depend on them may be missing."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192859166380032","occurred_at":"2026-09-24T18:40:27Z","signature_id":"c8d2c94c46ea41bb6616321fc23a840d9b2ec094985319abefcb6af50ae87597","pr_number":7045,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146302601695232","occurred_at":"2026-09-24T15:35:15Z","signature_id":"7d7d5f484ae9cf25602860243ff6931ef9c6211bb6bd55469a3f901426372f60","pr_number":7117,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-2tslxd58w56h\" to finish creating: GET https://rp","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":2,"runs_affected":2,"failed_runs":2,"run_impact_percent":1.6260162601626016,"anchor_occurrences":2,"after_last_push_count":0,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,0,2],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":9,"prior_last_seen_at":"2026-09-19T10:22:26Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":2}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc003790c60\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-2tslxd58w56h\" to finish creating: GET https://rp.j9945984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3bbdcadb-3a21-4369-a3b9-263a4eb93a49\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3bbdcadb-3a21-4369-a3b9-263a4eb93a49\",\n      \"name\": \"3bbdcadb-3a21-4369-a3b9-263a4eb93a49\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T22:33:47.939426596Z\",\n      \"endTime\": \"2026-09-24T22:52:50.218624914Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j9h7b2t4y3p0g4b-np-delete-main-vngst-vfmpn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-2tslxd58w56h--managed/providers/Microsoft.Compute/virtualMachines/j9h7b2t4y3p0g4b-np-delete-main-vngst-vfmpn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j9h7b2t4y3p0g4b-np-delete-main-vngst-vfmpn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-2tslxd58w56h--managed/providers/Microsoft.Compute/virtualMachines/j9h7b2t4y3p0g4b-np-delete-main-vngst-vfmpn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc003790c60\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-2tslxd58w56h\" to finish creating: GET https://rp.j9945984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3bbdcadb-3a21-4369-a3b9-263a4eb93a49\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/3bbdcadb-3a21-4369-a3b9-263a4eb93a49\",\n      \"name\": \"3bbdcadb-3a21-4369-a3b9-263a4eb93a49\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T22:33:47.939426596Z\",\n      \"endTime\": \"2026-09-24T22:52:50.218624914Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j9h7b2t4y3p0g4b-np-delete-main-vngst-vfmpn: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-2tslxd58w56h--managed/providers/Microsoft.Compute/virtualMachines/j9h7b2t4y3p0g4b-np-delete-main-vngst-vfmpn to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j9h7b2t4y3p0g4b-np-delete-main-vngst-vfmpn: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-2tslxd58w56h--managed/providers/Microsoft.Compute/virtualMachines/j9h7b2t4y3p0g4b-np-delete-main-vngst-vfmpn to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc003143460\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dtj57jh59wlh\" to finish creating: GET https://rp.j6624512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7eb325fa-7f6c-4e10-8c9b-3bebf9248544\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7eb325fa-7f6c-4e10-8c9b-3bebf9248544\",\n      \"name\": \"7eb325fa-7f6c-4e10-8c9b-3bebf9248544\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T19:46:17.078177969Z\",\n      \"endTime\": \"2026-09-24T20:05:23.632105826Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l6n2u5f4n0l6i5m-np-delete-main-lqq79-j492z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dtj57jh59wlh--managed/providers/Microsoft.Compute/virtualMachines/l6n2u5f4n0l6i5m-np-delete-main-lqq79-j492z to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l6n2u5f4n0l6i5m-np-delete-main-lqq79-j492z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dtj57jh59wlh--managed/providers/Microsoft.Compute/virtualMachines/l6n2u5f4n0l6i5m-np-delete-main-lqq79-j492z to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc003143460\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-dtj57jh59wlh\" to finish creating: GET https://rp.j6624512.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7eb325fa-7f6c-4e10-8c9b-3bebf9248544\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/7eb325fa-7f6c-4e10-8c9b-3bebf9248544\",\n      \"name\": \"7eb325fa-7f6c-4e10-8c9b-3bebf9248544\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T19:46:17.078177969Z\",\n      \"endTime\": \"2026-09-24T20:05:23.632105826Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l6n2u5f4n0l6i5m-np-delete-main-lqq79-j492z: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dtj57jh59wlh--managed/providers/Microsoft.Compute/virtualMachines/l6n2u5f4n0l6i5m-np-delete-main-lqq79-j492z to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine l6n2u5f4n0l6i5m-np-delete-main-lqq79-j492z: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/nodepool-delete-dtj57jh59wlh--managed/providers/Microsoft.Compute/virtualMachines/l6n2u5f4n0l6i5m-np-delete-main-lqq79-j492z to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233926699945984","occurred_at":"2026-09-24T21:23:38Z","signature_id":"84b235bf896d01e3deaa66856a2e9786768f1d93481789804dd05b50eee4738a","pr_number":7135,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192810386624512","occurred_at":"2026-09-24T18:40:16Z","signature_id":"4de39a43d9c45741b51ae3bace1a381fb4260155eb8ea95a416b6fbf37a19a98","pr_number":7009,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ccf8ebc7f99dd3758df5a50af86c29fba7755ee8f0d641da7630a416917e5d64","failure_pattern":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","search_query":"VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":2,"after_last_push_count":0,"trend_counts":[0,0,0,2,0,0,2],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00141d0c8\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.33) failed: no version in clusterversion/version status.history is greater than initial \"4.21.33\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.000s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00141d0c8\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.21.33) failed: no version in clusterversion/version status.history is greater than initial \"4.21.33\"\n    ...","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000e85578\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.22.14) failed: no version in clusterversion/version status.history is greater than initial \"4.22.14\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:153]: Timed out after 2700.001s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000e85578\u003e: \n    VerifyHostedControlPlaneZStreamUpgradeOnly(initial=4.22.14) failed: no version in clusterversion/version status.history is greater than initial \"4.22.14\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103126923872833536","occurred_at":"2026-09-24T14:18:18Z","signature_id":"001bfebb218903b1b7737fc9b09415cf50b63b5c1c904d8a608f8a8de2c5cb07","pr_number":6577,"after_last_push_of_merged_pr":false},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103126923872833536","occurred_at":"2026-09-24T14:18:18Z","signature_id":"e31b9c3e430438e0c3a30a2dbdd5700e039ffeab1240ef87b983e42b9cff5173","pr_number":6577,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"ddf1d2dccbf60132f42b17a2e35e1ecb5877bb9ed00a6e3bd0211a12c42985f5","failure_pattern":"--------------------------------------------------------------------------------","search_query":"--------------------------------------------------------------------------------","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":2,"trend_counts":[0,0,0,0,1,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-16T20:00:02Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gqsdx8vgs4cm\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/020af38d-4caa-4310-a595-881bec9e9405\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/020af38d-4caa-4310-a595-881bec9e9405\",\n  \"name\": \"020af38d-4caa-4310-a595-881bec9e9405\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-24T10:30:43.520242825Z\",\n  \"endTime\": \"2026-09-24T10:31:20.235327414Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gqsdx8vgs4cm\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edfe1588-77de-4f6d-a724-94f5c5b56313\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edfe1588-77de-4f6d-a724-94f5c5b56313\",\n  \"name\": \"edfe1588-77de-4f6d-a724-94f5c5b56313\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-24T10:30:43.575931644Z\",\n  \"endTime\": \"2026-09-24T10:31:20.247783059Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:2, cap:2\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gqsdx8vgs4cm\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/020af38d-4caa-4310-a595-881bec9e9405\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/020af38d-4caa-4310-a595-881bec9e9405\",\n  \"name\": \"020af38d-4caa-4310-a595-881bec9e9405\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-24T10:30:43.520242825Z\",\n  \"endTime\": \"2026-09-24T10:31:20.235327414Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-gqsdx8vgs4cm\" to finish creating: GET https://rp.j7309184.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edfe1588-77de-4f6d-a724-94f5c5b56313\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/edfe1588-77de-4f6d-a724-94f5c5b56313\",\n  \"name\": \"edfe1588-77de-4f6d-a724-94f5c5b56313\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-24T10:30:43.575931644Z\",\n  \"endTime\": \"2026-09-24T10:31:20.247783059Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n]\nExpected\n    \u003c[]error | len:2, cap:2\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103055129497309184","occurred_at":"2026-09-24T09:33:10Z","signature_id":"12c0bbf57d474e0cd6cf2451ff2cf882646f3b861a1cce2aae346ddfcc9bb6c4","pr_number":6917,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"df97d77d6b4cee54a445e0885955d907238beaed81721d9f40fecefbca8f021c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: dial udp \u003cip\u003e: i/o timeout): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":11,"prior_last_seen_at":"2026-09-20T19:59:59Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fec560\u003e: \n    DNS for route host agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:121]: failed to verify simple web app runs on v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000fec560\u003e: \n    DNS for route host agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-f5rnl.apps.aro.v20260901.t5ah.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: dial udp 172.30.0.10:53: i/o timeout): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103012742733500416","occurred_at":"2026-09-24T06:44:44Z","signature_id":"03a9333f9dd2535bd3ce94a401581227e326295231899b72504aa1c180c57341","pr_number":6915,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c16862eb3a0b5f7f0945b9a402b5b1bee591652a6c51f39ea011104fcd65aa5c","failure_pattern":"DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"DNS for route host agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":7,"prior_last_seen_at":"2026-09-16T15:56:55Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-bxncxk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b16880\u003e: \n    DNS for route host agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:189]: failed to verify simple web app runs on cluster \"cluster-candidate-4-21-bxncxk\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b16880\u003e: \n    DNS for route host agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-lvch8.apps.aro.n2m9k8u9s3b0p8r.wpls.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6915/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103012742733500416","occurred_at":"2026-09-24T06:44:44Z","signature_id":"c66a01613e5fe6442ebdb12c9ea63dd7c9f1f53d810adbdc9b51b498b061280c","pr_number":6915,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"9220c54af3cdcf82280969ee7ca39e91829e564fcd5ac622742ec149542a29fd","failure_pattern":"ERROR CODE: DeploymentFailed; detail code HttpRequestException; detail message Error while processing network security perimeter association: An error occurred while sending the request.; provider Microsoft.Network","search_query":"ERROR CODE: DeploymentFailed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/nsp (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-24T17:25:13.342Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 description=\"Step nsp\\n  Kind: ARM\\n  Template: templates/mgmt-nsp.bicep\\n  Parameters: configurations/mgmt-nsp.tmpl.bicepparam\"\ntime=2026-09-24T17:25:13.641Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2\ntime=2026-09-24T17:25:14.167Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 deployment=9d3f2ed9b958df750e3b7f101250ebe891c1a35bc943187871975b34cdf31ee3 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j4687104-mgmt-2%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F9d3f2ed9b958df750e3b7f101250ebe891c1a35bc943187871975b34cdf31ee3\ntime=2026-09-24T17:25:44.283Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=nsp stamp=2 err=\"stamp 2: failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4687104-mgmt-2/providers/Microsoft.Resources/deployments/9d3f2ed9b958df750e3b7f101250ebe891c1a35bc943187871975b34cdf31ee3/operationStatuses/08584113361715956682\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4687104-mgmt-2/providers/Microsoft.Resources/deployments/profile-bmfsiknuehg4q\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4687104-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j4687104-mgmt-2/resourceAssociations/nsp-j4687104-mgmt-2-ra-d68332b2-0c36-56b4-8794-de05fd33fea0\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"HttpRequestException\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"Error while processing request for association /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j4687104-mgmt-2/providers/Microsoft.Network/networkSecurityPerimeters/nsp-j4687104-mgmt-2/resourceAssociations/nsp-j4687104-mgmt-2-ra-d68332b2-0c36-56b4-8794-de05fd33fea0, Error message: An error occurred while sending the request.\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7009/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103162898384687104","occurred_at":"2026-09-24T16:41:24Z","signature_id":"d465f96a6ebaf98458f369563694ed7b4f0235597cd9af66af75425f966c9780","pr_number":7009,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c62a41b1ffa48cd7eb5519e7f4bdc548f260009543ddaf09220fa0719901910a","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceede... message truncated; provider Microsoft.Network","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d1d080\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-dkvsbz\" in resourcegroup=\"rg-np-version-upgrade-dkvsbz-sql98dwv9hds\" to finish creating: GET https://rp.j8866176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa5258ae-6ac0-4393-ba90-fd7fff99d1fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa5258ae-6ac0-4393-ba90-fd7fff99d1fd\",\n      \"name\": \"aa5258ae-6ac0-4393-ba90-fd7fff99d1fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T19:41:58.103375269Z\",\n      \"endTime\": \"2026-09-24T20:01:00.35851472Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c8k0y6n8u7a0y5r-npupgrade-4-20-ctsc2-95r6h: Failed: interfaces failed to create or update. err: failed to get existing resource rg-np-version-upgrade-dkvsbz-sql98dwv9hds-np-upgrade-dk-d348d3ab/c8k0y6n8u7a0y5r-npupgrade-4-20-ctsc2-95r6h-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-dkvsbz-sql98dwv9hds-np-upgrade-dk-d348d3ab/providers/Microsoft.Network/networkInterfaces/c8k0y6n8u7a0y5r-npupgrade-4-20-ctsc2-95r6h-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'a76148d4-cb11-4b93-9363-54ba8239b0b1' exceede... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.39\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d1d080\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-dkvsbz\" in resourcegroup=\"rg-np-version-upgrade-dkvsbz-sql98dwv9hds\" to finish creating: GET https://rp.j8866176.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa5258ae-6ac0-4393-ba90-fd7fff99d1fd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa5258ae-6ac0-4393-ba90-fd7fff99d1fd\",\n      \"name\": \"aa5258ae-6ac0-4393-ba90-fd7fff99d1fd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T19:41:58.103375269Z\",\n      \"endTime\": \"2026-09-24T20:01:00.35851472Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine c8k0y6n8u7a0y5r-npupgrade-4-20-ctsc2-95r6h: Failed: interfaces failed to create or update. err: failed to get existing resource rg-np-version-upgrade-dkvsbz-sql98dwv9hds-np-upgrade-dk-d348d3ab/c8k0y6n8u7a0y5r-npupgrade-4-20-ctsc2-95r6h-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-dkvsbz-sql98dwv9hds-np-upgrade-dk-d348d3ab/providers/Microsoft.Network/networkInterfaces/c8k0y6n8u7a0y5r-npupgrade-4-20-ctsc2-95r6h-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'a76148d4-cb11-4b93-9363-54ba8239b0b1' exceede... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7141/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103191956178866176","occurred_at":"2026-09-24T18:36:45Z","signature_id":"43e7b49a53b9f05fe2662927a58e42b643a0bf4af76dedddd2bff94bc094d954","pr_number":7141,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"372d0c16d6516fbc2516124c1f8eb533277ffb462595208eacf857f749c90520","failure_pattern":"ERROR CODE: InternalServerError; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '\u003cseconds\u003e' seconds after additional tokens are available. ... message truncated; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00145fb18\u003e: \n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-4bnpzclpb4vn\" to finish creating: GET https://rp.j9945984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ae2ff5d-353f-4924-8074-8541fe3fffdf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ae2ff5d-353f-4924-8074-8541fe3fffdf\",\n      \"name\": \"1ae2ff5d-353f-4924-8074-8541fe3fffdf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T22:34:11.896891667Z\",\n      \"endTime\": \"2026-09-24T22:53:20.096362323Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-vbqjx-klzb2: Failed: virtualmachine failed to create or update. err: failed to get existing resource feature-aggregation-4bnpzclpb4vn--managed/agg-cluster-agg-np-a-vbqjx-klzb2 (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-4bnpzclpb4vn--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-vbqjx-klzb2\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '07aa0e83-353e-444c-9f1d-d023ac3c5396' exceeded. Please try again after '10' seconds after additional tokens are available. ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc00145fb18\u003e: \n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-4bnpzclpb4vn\" to finish creating: GET https://rp.j9945984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ae2ff5d-353f-4924-8074-8541fe3fffdf\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1ae2ff5d-353f-4924-8074-8541fe3fffdf\",\n      \"name\": \"1ae2ff5d-353f-4924-8074-8541fe3fffdf\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T22:34:11.896891667Z\",\n      \"endTime\": \"2026-09-24T22:53:20.096362323Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want \\u003e= 2 (autoscaling min); node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine agg-cluster-agg-np-a-vbqjx-klzb2: Failed: virtualmachine failed to create or update. err: failed to get existing resource feature-aggregation-4bnpzclpb4vn--managed/agg-cluster-agg-np-a-vbqjx-klzb2 (service: virtualmachine): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-4bnpzclpb4vn--managed/providers/Microsoft.Compute/virtualMachines/agg-cluster-agg-np-a-vbqjx-klzb2\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '07aa0e83-353e-444c-9f1d-d023ac3c5396' exceeded. Please try again after '10' seconds after additional tokens are available. ... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103233926699945984","occurred_at":"2026-09-24T21:23:38Z","signature_id":"142775d0e7b84904771b9c9baeca0e991ed25c62693ee1753f67f0e8192808e1","pr_number":7135,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"0e33bf9c876122b5f27225338e1ee7dd193ea167c268da2c880fb16fb083cfb4","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator, openshift-apiserver]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005d4e40\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-mdj8lgwtvhhv\" to finish creating: GET https://rp.j8049152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1e1d081b-a194-4d7e-ad25-c5f0f75190c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1e1d081b-a194-4d7e-ad25-c5f0f75190c7\",\n      \"name\": \"1e1d081b-a194-4d7e-ad25-c5f0f75190c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T12:47:20.580135273Z\",\n      \"endTime\": \"2026-09-24T13:16:21.048177334Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 1m5s ago; version 4.21.34 is Partial (want Completed), started 3m10s ago; version 4.21.34 is Partial (want Completed), started 6m2s ago; version 4.21.34 is Partial (want Completed), started 6m38s ago; version 4.21.34 is Partial (want Completed), started 10m34s ago; version 4.21.34 is Partial (want Completed), started 16m16s ago; version 4.21.34 is Partial (want Completed), started 20m17s ago; version 4.21.34 is Partial (want Completed), started 23m52s ago; version 4.21.34 is Partial (want Completed), started 24m53s ago; version 4.21.34 is Partial (want Completed), started 25m36s ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:111]: failed to create HCP cluster \"cidr-connectivity-test\" with authorized CIDRs\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0005d4e40\u003e: \n    failed to create HCP cluster cidr-connectivity-test: failed waiting for cluster=\"cidr-connectivity-test\" in resourcegroup=\"e2e-cidr-connectivity-mdj8lgwtvhhv\" to finish creating: GET https://rp.j8049152.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1e1d081b-a194-4d7e-ad25-c5f0f75190c7\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/1e1d081b-a194-4d7e-ad25-c5f0f75190c7\",\n      \"name\": \"1e1d081b-a194-4d7e-ad25-c5f0f75190c7\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T12:47:20.580135273Z\",\n      \"endTime\": \"2026-09-24T13:16:21.048177334Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.34 is Partial (want Completed), started 1m5s ago; version 4.21.34 is Partial (want Completed), started 3m10s ago; version 4.21.34 is Partial (want Completed), started 6m2s ago; version 4.21.34 is Partial (want Completed), started 6m38s ago; version 4.21.34 is Partial (want Completed), started 10m34s ago; version 4.21.34 is Partial (want Completed), started 16m16s ago; version 4.21.34 is Partial (want Completed), started 20m17s ago; version 4.21.34 is Partial (want Completed), started 23m52s ago; version 4.21.34 is Partial (want Completed), started 24m53s ago; version 4.21.34 is Partial (want Completed), started 25m36s ago; hosted cluster degraded: UnavailableReplicas: [cluster-version-operator deployment has 1 unavailable replicas, openshift-apiserver deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103089013568049152","occurred_at":"2026-09-24T11:47:48Z","signature_id":"a1699dd27afd8a16f801afb5b59631643f3ad5b14b803812cf5d901f58fe2d87","pr_number":7123,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b4593ef0dd9fbe369bd9dc5fb00d11d789b0f9fff11fea0544e9abdf8738d8b1","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID Machine \u003cnode\u003e: InspectionFailed: Waiting for AzureMachine to report spec.providerID ; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready Machine \u003cnode\u003e: Creating: virtualmachine creating or updating Machine \u003cnode\u003e: Creating: virtualmachine creating or updating; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b80880\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-cpn7mwhqrdqw\" to finish creating: GET https://rp.j4556544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2d56b67b-43e0-4cd7-a969-5552b2f1c302\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2d56b67b-43e0-4cd7-a969-5552b2f1c302\",\n      \"name\": \"2d56b67b-43e0-4cd7-a969-5552b2f1c302\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T20:50:39.494369799Z\",\n      \"endTime\": \"2026-09-24T21:09:39.1222123Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-7rv28: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-drpbc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-7rv28: Creating: virtualmachine creating or updating\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-drpbc: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/oidc_issuer_workload_identity.go:270]: failed to create node pool oidc-wi-np\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b80880\u003e: \n    failed to create NodePool oidc-wi-np: failed waiting for nodepool=\"oidc-wi-np\" for cluster \"oidc-wi-cluster\" in resourcegroup=\"oidc-wi-cpn7mwhqrdqw\" to finish creating: GET https://rp.j4556544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2d56b67b-43e0-4cd7-a969-5552b2f1c302\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/2d56b67b-43e0-4cd7-a969-5552b2f1c302\",\n      \"name\": \"2d56b67b-43e0-4cd7-a969-5552b2f1c302\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T20:50:39.494369799Z\",\n      \"endTime\": \"2026-09-24T21:09:39.1222123Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-7rv28: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-drpbc: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-7rv28: Creating: virtualmachine creating or updating\\nMachine oidc-wi-cluster-oidc-wi-np-bm6bt-drpbc: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203513864556544","occurred_at":"2026-09-24T19:22:46Z","signature_id":"073b0db0ceea3f28f404fe1b7904861cb57586c11302c87c1ccfd472e1f2a7a3","pr_number":6970,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"64bc5ac277db781b54ff950d9ea3c0c424d37517be1bdac9b31411f402f5258f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2nj474\" for cluster \"cluster-candidate-5-0-2nj474\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b7a380\u003e: \n    failed to create NodePool np-2nj474: failed waiting for nodepool=\"np-2nj474\" for cluster \"cluster-candidate-5-0-2nj474\" in resourcegroup=\"rg-candidate-5-0-vjxqrms5jclz\" to finish creating: GET https://rp.j9877248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/514bc0c8-a828-4772-95da-f95954e05701\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/514bc0c8-a828-4772-95da-f95954e05701\",\n      \"name\": \"514bc0c8-a828-4772-95da-f95954e05701\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T23:48:54.785691346Z\",\n      \"endTime\": \"2026-09-25T00:08:00.18158216Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np to exist\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np to exist\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-2nj474\" for cluster \"cluster-candidate-5-0-2nj474\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b7a380\u003e: \n    failed to create NodePool np-2nj474: failed waiting for nodepool=\"np-2nj474\" for cluster \"cluster-candidate-5-0-2nj474\" in resourcegroup=\"rg-candidate-5-0-vjxqrms5jclz\" to finish creating: GET https://rp.j9877248.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/514bc0c8-a828-4772-95da-f95954e05701\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/514bc0c8-a828-4772-95da-f95954e05701\",\n      \"name\": \"514bc0c8-a828-4772-95da-f95954e05701\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T23:48:54.785691346Z\",\n      \"endTime\": \"2026-09-25T00:08:00.18158216Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 2 of 2 machines are not healthy\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np to exist\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66 to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-s79np to exist\\nMachine m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-vjxqrms5jclz-candidate-2nj474--managed/providers/Microsoft.Compute/virtualMachines/m6g6a2k9x2n9s0x-np-2nj474-2qbdc-tcv66 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7123/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103252727629877248","occurred_at":"2026-09-24T22:38:21Z","signature_id":"1074b07740a45107502c1b1a3cd307b20f68100d4367f76d22ba40ec484f0d3d","pr_number":7123,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"eacd13fa6af2b06f56ce1295197593af50aee51808239f944fd6ca6273414a50","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cea980\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-dx85jrjs9gbf\" to finish creating: GET https://rp.j7862784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ddf0b59-a1c5-4f88-9e2a-09c2b8b441c9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ddf0b59-a1c5-4f88-9e2a-09c2b8b441c9\",\n      \"name\": \"9ddf0b59-a1c5-4f88-9e2a-09c2b8b441c9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T17:42:43.803875728Z\",\n      \"endTime\": \"2026-09-24T18:01:44.201124036Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j3g0e5m6b3n7n1m-np-1-45jtt-ld7k7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-dx85jrjs9gbf--managed/providers/Microsoft.Compute/virtualMachines/j3g0e5m6b3n7n1m-np-1-45jtt-ld7k7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j3g0e5m6b3n7n1m-np-1-45jtt-ld7k7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-dx85jrjs9gbf--managed/providers/Microsoft.Compute/virtualMachines/j3g0e5m6b3n7n1m-np-1-45jtt-ld7k7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:141]: failed to create node pool np-1 with initial labels and taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000cea980\u003e: \n    failed waiting for nodepool=\"np-1\" for cluster \"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-dx85jrjs9gbf\" to finish creating: GET https://rp.j7862784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ddf0b59-a1c5-4f88-9e2a-09c2b8b441c9\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/9ddf0b59-a1c5-4f88-9e2a-09c2b8b441c9\",\n      \"name\": \"9ddf0b59-a1c5-4f88-9e2a-09c2b8b441c9\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T17:42:43.803875728Z\",\n      \"endTime\": \"2026-09-24T18:01:44.201124036Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j3g0e5m6b3n7n1m-np-1-45jtt-ld7k7: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-dx85jrjs9gbf--managed/providers/Microsoft.Compute/virtualMachines/j3g0e5m6b3n7n1m-np-1-45jtt-ld7k7 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j3g0e5m6b3n7n1m-np-1-45jtt-ld7k7: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-labels-taints-dx85jrjs9gbf--managed/providers/Microsoft.Compute/virtualMachines/j3g0e5m6b3n7n1m-np-1-45jtt-ld7k7 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103162932127862784","occurred_at":"2026-09-24T16:41:32Z","signature_id":"71ad104d7c45702c68a9e44ecbe6eb8ff08e7116726d4744adcdde9ab6895dc5","pr_number":6821,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"faaf4b1f5915530cc590a6d965e51a07e2893bcc928bcf257ec4329e7f00397f","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-djkxv-\u003cid\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-djkxv-\u003cid\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002703dd0\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-x45qskfpvk4k\" to finish creating: GET https://rp.j7111680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50f10e36-adf3-41e9-a281-5a223249a720\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50f10e36-adf3-41e9-a281-5a223249a720\",\n      \"name\": \"50f10e36-adf3-41e9-a281-5a223249a720\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T18:57:03.302695229Z\",\n      \"endTime\": \"2026-09-24T19:16:10.982967273Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-djkxv-44p2x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-x45qskfpvk4k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-djkxv-44p2x to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-djkxv-44p2x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-x45qskfpvk4k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-djkxv-44p2x to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_cni_cilium.go:182]: failed to verify cilium is running and nodes are Ready for cluster \"cilium-cl\"\nUnexpected error:\n    \u003c*errors.joinError | 0xc002703dd0\u003e: \n    failed to create NodePool cilium-np: failed waiting for nodepool=\"cilium-np\" for cluster \"cilium-cl\" in resourcegroup=\"cni-cilium-x45qskfpvk4k\" to finish creating: GET https://rp.j7111680.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50f10e36-adf3-41e9-a281-5a223249a720\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/50f10e36-adf3-41e9-a281-5a223249a720\",\n      \"name\": \"50f10e36-adf3-41e9-a281-5a223249a720\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T18:57:03.302695229Z\",\n      \"endTime\": \"2026-09-24T19:16:10.982967273Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine cilium-cl-cilium-np-djkxv-44p2x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-x45qskfpvk4k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-djkxv-44p2x to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine cilium-cl-cilium-np-djkxv-44p2x: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/cni-cilium-x45qskfpvk4k--managed/providers/Microsoft.Compute/virtualMachines/cilium-cl-cilium-np-djkxv-44p2x to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6993/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103182792887111680","occurred_at":"2026-09-24T18:00:08Z","signature_id":"5bf349cb8ec9f251b4fddcd8eba4f881e8bb266c023ecc7561970e4ac0e37274","pr_number":6993,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"881c8a92b161acdd9b8088fd7670ee486a4176a53d8400a1bad38528667ae5f4","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftNodePool] hypershift NodePool status replicas is 2, want \u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist ; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.Compute","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0017d2100\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kmp4pw976cvp\" to finish creating: GET https://rp.j4556544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fe3f92f-008e-475a-a316-514fa9d56551\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fe3f92f-008e-475a-a316-514fa9d56551\",\n      \"name\": \"4fe3f92f-008e-475a-a316-514fa9d56551\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T20:45:04.211205802Z\",\n      \"endTime\": \"2026-09-24T21:04:09.101063234Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-kmp4pw976cvp--managed/providers/Microsoft.Compute/virtualMachines/l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-kmp4pw976cvp--managed/providers/Microsoft.Compute/virtualMachines/l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0017d2100\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-kmp4pw976cvp\" to finish creating: GET https://rp.j4556544.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fe3f92f-008e-475a-a316-514fa9d56551\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/4fe3f92f-008e-475a-a316-514fa9d56551\",\n      \"name\": \"4fe3f92f-008e-475a-a316-514fa9d56551\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T20:45:04.211205802Z\",\n      \"endTime\": \"2026-09-24T21:04:09.101063234Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-kmp4pw976cvp--managed/providers/Microsoft.Compute/virtualMachines/l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-kmp4pw976cvp--managed/providers/Microsoft.Compute/virtualMachines/l7c9r0y4z3t5x4s-autoscale-noaz-9blrl-6wcfm to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6970/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103203513864556544","occurred_at":"2026-09-24T19:22:46Z","signature_id":"b24161097af08c2ace32acbd8380e530f701f116d3593a7a6af06880e3f2dbd7","pr_number":6970,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4eb79c3efbc97c0c126107692df16c56c190e54e8a0ec9d1e2108b1c54f07feb","failure_pattern":"UpdateNodePoolAndWait timed out after \u003cminutes\u003e minutes; context deadline exceeded","search_query":"failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-jgw9qgklj7f5\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-jgw9qgklj7f5, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":5,"prior_last_seen_at":"2026-09-18T18:24:36Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002fcb410\u003e: \n    failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-jgw9qgklj7f5\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-jgw9qgklj7f5, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:154]: failed to scale up node pool np-update-NODES from 2 to 3 replicas\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc002fcb410\u003e: \n    failed waiting for nodepool=\"np-update-NODES\" in cluster=\"np-update-nodes-hcp-cluster\" resourcegroup=\"nodepool-update-nodes-jgw9qgklj7f5\" to finish updating, caused by: timeout '20.000000' minutes exceeded during UpdateNodePoolAndWait for nodepool np-update-NODES in cluster np-update-nodes-hcp-cluster in resource group nodepool-update-nodes-jgw9qgklj7f5, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192892754366464","occurred_at":"2026-09-24T18:40:35Z","signature_id":"bf45d3a1f9dc91a9af5a7119dd1b60337e9f49919a6a81afba2cb9b9a6885627","pr_number":7135,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"7a87df562eaa8a0bfda8c14840c36e7197e35a3b92f22e4f536b1e3d2ed289ff","failure_pattern":"VerifySimpleWebApp failed: DNS for route host \u003croute-host\u003e did not resolve: DNS for \u003croute-host\u003e did not resolve within \u003cduration\u003e (last error: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host): context deadline exceeded","search_query":"VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":2,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:209]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc00268e060\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_complex_cilium_kv.go:209]: failed to run simple web app and connectivity check with cilium CNI\nUnexpected error:\n    \u003c*errors.joinError | 0xc00268e060\u003e: \n    VerifySimpleWebApp failed: DNS for route host agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud did not resolve: DNS for agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud did not resolve within 10m0s (last error: lookup agnhost-e2e-sample-app-dr7hw.apps.aro.cilium-cluster.vzkq.j7691264.hcp.osadev.cloud on 172.30.0.10:53: no such host): context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6821/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103200143447691264","occurred_at":"2026-09-24T19:09:24Z","signature_id":"5b25ef7df46295d6b0322467710d77611928e860a415a5c20e227a626a8235c7","pr_number":6821,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":16,"trend_counts":[4,1,4,4,3,2,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":25,"prior_last_seen_at":"2026-09-20T23:42:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 4 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-24T01:36:59Z\n  Ended:       2026-09-24T01:43:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8079616/providers/microsoft.monitor/accounts/hcps-j8079616\", namespace=\"ocm-arohcpci01-2t12nvqr38rk88v2oib5stvkqbcsin5h-r3r9m3h6q3b6v7a\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-q4z8l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t12nvqr38rk88v2oib5stvkqbcsin5h-r3r9m3h6q3b6v7a/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-24T01:36:59Z\n  Ended:       2026-09-24T01:43:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8079616/providers/microsoft.monitor/accounts/hcps-j8079616\", namespace=\"ocm-arohcpci01-2t12o9jo7duvi4mjl85qtghmc0bu27i9-j6r7x7q1c9p4g7n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-q4z8l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t12o9jo7duvi4mjl85qtghmc0bu27i9-j6r7x7q1c9p4g7n/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-24T01:37:00Z\n  Ended:       2026-09-24T01:43:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8079616/providers/microsoft.monitor/accounts/hcps-j8079616\", namespace=\"ocm-arohcpci01-2t12nvj5ubvth7im4d53igd84r482cjj-n2f1o7y4s2y9x7u\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-q4z8l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t12nvj5ubvth7im4d53igd84r482cjj-n2f1o7y4s2y9x7u/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 4:\n  State:       Resolved\n  Started:     2026-09-24T01:37:00Z\n  Ended:       2026-09-24T01:43:01Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j8079616-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.147:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8079616/providers/microsoft.monitor/accounts/hcps-j8079616\", namespace=\"ocm-arohcpci01-2t12o9jo7duvi4mjl85qtghmc0bu27i9-j6r7x7q1c9p4g7n\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-q4z8l\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2t12o9jo7duvi4mjl85qtghmc0bu27i9-j6r7x7q1c9p4g7n/etcd has not matched the expected number of replicas for longer than 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6917/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102919814468079616","occurred_at":"2026-09-24T00:35:28Z","signature_id":"98313410c810095dec26c72ec7f4b43b7a09e0afa8d35824ba5fc5fec76fe165","pr_number":6917,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"5898285a3c5799ec36953b0ada4226d296964976e5282c81c2e32b12abcb98ec","failure_pattern":"alert [svc] ArobitForwarderJobUp fired","search_query":"alert [svc] ArobitForwarderJobUp fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,3,1,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":9,"prior_last_seen_at":"2026-09-16T21:13:51Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert ArobitForwarderJobUp does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T17:33:47Z\n  Severity:    Sev3\n  Labels:      alertname=\"ArobitForwarderJobUp\", cluster=\"ci01-j4604416-svc\", component=\"monitoring-infrastructure\", region=\"westus3\", severity=\"warning\"\n  Description: The Arobit forwarder metrics endpoint on cluster ci01-j4604416-svc has been unreachable for 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103161107324604416","occurred_at":"2026-09-24T16:34:17Z","signature_id":"679aa0117967b7ad541fd1b96fee91e26fcf0b44074834ec0f9774a9ce5ea7a4","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"98f1463348d97c51097a0ae12c49f4f35bae17bce77821066385171497e12544","failure_pattern":"alert [svc] BackendControllerRetryHotLoop fired","search_query":"alert [svc] BackendControllerRetryHotLoop fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,1,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-17T20:17:48Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert BackendControllerRetryHotLoop does not fire","occurrences":1}],"full_error_samples":["alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-09-24T15:19:32Z\n  Ended:       2026-09-24T16:14:33Z\n  Severity:    Sev3\n  Labels:      alertname=\"BackendControllerRetryHotLoop\", cluster=\"ci01-j2833536-svc\", component=\"backend\", name=\"controlplaneversionstatuscollector\", region=\"westus3\", severity=\"warning\"\n  Description: Backend controller workqueue controlplaneversionstatuscollector has a retry ratio of \u003e 50% sustained over 10 minutes, indicating most queue activity is failed retries rather than fresh work."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6577/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103126923872833536","occurred_at":"2026-09-24T14:18:18Z","signature_id":"42836fa06f25aa02442410f2c7556dd0019e006d1c588c77355505f2e0955532","pr_number":6577,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"c4e2bd93d0410f48ddfa390cc77604492607a58b2a4eaff204baa6c7bed3af07","failure_pattern":"alert [svc] KubeContainerOOMKilled fired","search_query":"alert [svc] KubeContainerOOMKilled fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-17T11:57:12Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeContainerOOMKilled does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T08:49:12Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"prometheus-collector\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"kube-system\", pod=\"ama-metrics-node-5rpz7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"OOMKilled\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8afb0369-75a6-47c9-a399-fdc0457fc23d\"\n  Description: Container prometheus-collector in pod kube-system/ama-metrics-node-5rpz7 on cluster ci01-j8969088-mgmt-1 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T08:50:13Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeContainerOOMKilled\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"prometheus-collector\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"kube-system\", pod=\"ama-metrics-node-5rpz7\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"OOMKilled\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8afb0369-75a6-47c9-a399-fdc0457fc23d\"\n  Description: Container prometheus-collector in pod kube-system/ama-metrics-node-5rpz7 on cluster ci01-j8969088-mgmt-1 has been OOMKilled. This indicates the container exceeded its memory limit and was terminated by the kernel."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"dd3366bd92c2564cb6f449817e8c4d513e034b81a60fe1b01c561f9570ac3115","pr_number":6503,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f5e947b6ed7bf96ad80534c042635c42eae546f22fd6012bbe8463c15537a70c","failure_pattern":"alert [svc] KubeDaemonSetRolloutStuck fired","search_query":"alert [svc] KubeDaemonSetRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,3,1,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":10,"prior_last_seen_at":"2026-09-16T21:13:51Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDaemonSetRolloutStuck does not fire","occurrences":1}],"full_error_samples":["alert fired 2 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T17:34:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j4604416-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.41:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4604416/providers/microsoft.monitor/accounts/services-j4604416\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-849cb69576-mmg7d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T17:34:06Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDaemonSetRolloutStuck\", cluster=\"ci01-j4604416-svc\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", daemonset=\"arobit-forwarder\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.41:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j4604416/providers/microsoft.monitor/accounts/services-j4604416\", namespace=\"arobit\", pod=\"arohcp-monitor-kube-state-metrics-849cb69576-mmg7d\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: DaemonSet arobit/arobit-forwarder has not finished or progressed for at least 15 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103161107324604416","occurred_at":"2026-09-24T16:34:17Z","signature_id":"7726a18972655e8be5469370e6d72d78ec561de245d068ff794d9ec202b33f3d","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"4033d8c230ccf6abe05525fad14dc9766fcbee12a05064b260dc31268b48600b","failure_pattern":"alert [svc] KubeDeploymentReplicasMismatch fired","search_query":"alert [svc] KubeDeploymentReplicasMismatch fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":9,"trend_counts":[1,0,2,3,2,2,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":15,"prior_last_seen_at":"2026-09-20T23:42:35Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentReplicasMismatch does not fire","occurrences":1}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T08:07:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-7jtls\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T08:19:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-7jtls\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes.\nFiring 3:\n  State:       Resolved\n  Started:     2026-09-24T08:19:22Z\n  Ended:       2026-09-24T08:37:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"klusterlet-2t18bqdbena2h0p1tg4m0lfs0js71a72\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-7jtls\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment klusterlet-2t18bqdbena2h0p1tg4m0lfs0js71a72/klusterlet-addon-workmgr has not matched the expected number of replicas for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"6ffa9e2af62696b854cbaa4af470e9d2a538a83db7cca9ca00946bce22eef1d8","pr_number":6503,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"cc58ab85ccf7f8901007262baa5b046324f5e8aea4fb20f9726e8407124603b0","failure_pattern":"alert [svc] KubeDeploymentRolloutStuck fired","search_query":"alert [svc] KubeDeploymentRolloutStuck fired","failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":4,"trend_counts":[0,0,0,1,0,2,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":5,"prior_last_seen_at":"2026-09-17T00:56:54Z","contributing_tests":[{"failed_at":"alert","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"[aro-hcp-observability] [svc] alert KubeDeploymentRolloutStuck does not fire","occurrences":1}],"full_error_samples":["alert fired 3 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T08:19:22Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-7jtls\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-09-24T08:23:25Z\n  Ended:       2026-09-24T08:37:24Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"klusterlet-2t18bqdbena2h0p1tg4m0lfs0js71a72\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-7jtls\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment klusterlet-2t18bqdbena2h0p1tg4m0lfs0js71a72/klusterlet-addon-workmgr is not progressing for longer than 30 minutes.\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-09-24T08:44:23Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentRolloutStuck\", cluster=\"ci01-j8969088-mgmt-1\", component=\"kubernetes-infrastructure\", condition=\"Progressing\", container=\"kube-state-metrics\", deployment=\"klusterlet-addon-workmgr\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8969088/providers/microsoft.monitor/accounts/services-j8969088\", namespace=\"open-cluster-management-agent-addon\", pod=\"arohcp-monitor-kube-state-metrics-7955558448-7jtls\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"progressdeadlineexceeded\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", status=\"false\"\n  Description: Rollout of deployment open-cluster-management-agent-addon/klusterlet-addon-workmgr is not progressing for longer than 30 minutes."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"db9539a235d0c98a40af1909968de9adb2c176e0ca9c6ae7df7436f7869eb782","pr_number":6503,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"f6ef2ac0f7328af3748e9146847857d1ee199899d686f90d717683741234c875","failure_pattern":"cluster entered terminal Failed state after role assignment deployment","search_query":"cluster entered terminal Failed state after role assignment deployment","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[2,1,0,0,2,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":13,"prior_last_seen_at":"2026-09-19T02:41:11Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:227]: Timed out after 2580.000s.\ncluster should eventually succeed after role assignments are created\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:223 with:\ncluster entered terminal Failed state after role assignment deployment\nExpected\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ...\nnot to equal\n    \u003carmredhatopenshifthcp.ProvisioningState\u003e: ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"b9b50758a1aa006036c7c1ee58af90cd509c7b1900722ef0229b46ed2f5e8563","pr_number":6503,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"dbba05f95aaac62366d47b6f23fc0ba2f14aa49eae9bd82309cadecd8332be34","failure_pattern":"error running Shell Step, failed to execute shell command: {\"level\":\"INFO\",\"msg\":\"starting cluster reconciliation\",\"component\":\"aks-cluster-create\",\"subscriptionID\":\"\u003credacted-id\u003e\",\"resourceGroup\":\"hcp-underlay-ci01-j6380032-mgmt-2\",\"clusterName\":\"ci01-j638003","search_query":"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-24T19:21:37.041921702Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6380032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6380032-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-24T19:21:38.603615087Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6380032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6380032-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-24T19:21:39.051190133Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6380032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6380032-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/cluster-create (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-24T19:21:37.035Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 description=\"Step cluster-create\\n  Kind: Shell\\n  Command: ./aks-cluster-create\\n\"\ntime=2026-09-24T19:51:36.670Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=cluster-create stamp=2 err=\"stamp 2: error running Shell Step, failed to execute shell command: {\\\"time\\\":\\\"2026-09-24T19:21:37.041921702Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"starting cluster reconciliation\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6380032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6380032-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\"}\\n{\\\"time\\\":\\\"2026-09-24T19:21:38.603615087Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"configured pools\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6380032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6380032-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"system\\\":1,\\\"worker\\\":3,\\\"infra\\\":2}\\n{\\\"time\\\":\\\"2026-09-24T19:21:39.051190133Z\\\",\\\"level\\\":\\\"INFO\\\",\\\"msg\\\":\\\"deploying configured resources\\\",\\\"component\\\":\\\"aks-cluster-create\\\",\\\"subscriptionID\\\":\\\"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\\\",\\\"resourceGroup\\\":\\\"hcp-underlay-ci01-j6380032-mgmt-2\\\",\\\"clusterName\\\":\\\"ci01-j6380032-mgmt-2\\\",\\\"region\\\":\\\"canadacentral\\\",\\\"deployment\\\":\\\"aks-cluster-create\\\",\\\"poolCount\\\":6,\\\"clusterFields\\\":null}\\n signal: killed\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7045/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103192859166380032","occurred_at":"2026-09-24T18:40:27Z","signature_id":"3d35abb788cea68e7144d2ec2e84c98ce2744861ebc8486e8349a5f051f0df62","pr_number":7045,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"58103f0bb0f23a6bcc7180fb82d4fd105caee89b2c2667152f6ec564d76dd64c","failure_pattern":"failed to create customer-infra: failed waiting for deployment \"customer-infra-autoscaling-hcp-cluster-\u003cid\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish: context deadline exceeded","search_query":"failed to create customer-infra: failed waiting for deployment \"customer-infra-autoscaling-hcp-cluster-pjz7p4\" in resourcegroup=\"autoscaling-cluster-7qnnh4q5h8fk\" to finish: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create customer resources for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ac8540\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-autoscaling-hcp-cluster-pjz7p4\" in resourcegroup=\"autoscaling-cluster-7qnnh4q5h8fk\" to finish: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:97]: failed to create customer resources for autoscaling cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ac8540\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-autoscaling-hcp-cluster-pjz7p4\" in resourcegroup=\"autoscaling-cluster-7qnnh4q5h8fk\" to finish: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7135/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103136210296770560","occurred_at":"2026-09-24T14:55:21Z","signature_id":"69e234e12ff1dfef8a8e49d1251b6e3da1b82338b95055a0f148ecb1c35fafd2","pr_number":7135,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"e86119606257db722a985501dec6db3353e9294ec584e08a2cf350342957ce78","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: context canceled]","search_query":"failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lqvvv444d6v8\" to finish creating: context canceled]","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-18T03:01:52Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to update nodepool replicas and autoscaling","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lqvvv444d6v8\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3af991c4-642e-4bf3-b698-91174beba451\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3af991c4-642e-4bf3-b698-91174beba451\",\n  \"name\": \"3af991c4-642e-4bf3-b698-91174beba451\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-24T01:19:13.956261131Z\",\n  \"endTime\": \"2026-09-24T01:19:51.914301535Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lqvvv444d6v8\" to finish creating: context canceled]\nExpected\n    \u003c[]error | len:2, cap:2\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_update_nodes.go:132]: nodepool creation errors: [failed to create NodePool np-update-NODES: failed waiting for nodepool=\"np-update-NODES\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lqvvv444d6v8\" to finish creating: GET https://rp.aro-hcp-dev-shard3-slot-01.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3af991c4-642e-4bf3-b698-91174beba451\n--------------------------------------------------------------------------------\nRESPONSE 200: 200 OK\nERROR CODE: InternalServerError\n--------------------------------------------------------------------------------\n{\n  \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/3af991c4-642e-4bf3-b698-91174beba451\",\n  \"name\": \"3af991c4-642e-4bf3-b698-91174beba451\",\n  \"status\": \"Failed\",\n  \"startTime\": \"2026-09-24T01:19:13.956261131Z\",\n  \"endTime\": \"2026-09-24T01:19:51.914301535Z\",\n  \"error\": {\n    \"code\": \"InternalServerError\",\n    \"message\": \"[clusterServiceNodePoolStatus] GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Compute/skus\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '47f69502-0065-4d9a-b19b-d403e183d2f4' exceeded. Please try again after '10' seconds after additional tokens are available. Refer to https://aka.ms/arm-throttling for additional information.\\\"\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\n  }\n}\n--------------------------------------------------------------------------------\n failed to create NodePool np-one-node: failed waiting for nodepool=\"np-one-node\" for cluster \"np-update-nodes-hcp-cluster\" in resourcegroup=\"nodepool-update-nodes-lqvvv444d6v8\" to finish creating: context canceled]\nExpected\n    \u003c[]error | len:2, cap:2\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7119/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102915118240108544","occurred_at":"2026-09-24T00:16:20Z","signature_id":"12c0bbf57d474e0cd6cf2451ff2cf882646f3b861a1cce2aae346ddfcc9bb6c4","pr_number":7119,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"2553fa7889a9af1fa3b8eb97a5f22ebd02ed4cb66f230b2442c0efe51892094d","failure_pattern":"failed to query public IP echo service: Get \"\u003curl\u003e\": net/http: TLS handshake timeout","search_query":"failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": net/http: TLS handshake timeout","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:172]: failed to resolve test runner public IP\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d70460\u003e: \n    failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": net/http: TLS handshake timeout\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:172]: failed to resolve test runner public IP\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d70460\u003e: \n    failed to query public IP echo service: Get \"https://checkip.amazonaws.com\": net/http: TLS handshake timeout\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7148/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103247783359156224","occurred_at":"2026-09-24T22:18:40Z","signature_id":"d790667a42bc853d57fab4f94768ab9d93901fef5701562ee2de998ed5c7396d","pr_number":7148,"after_last_push_of_merged_pr":true}]},{"environment":"dev","failure_pattern_id":"ddff750332e2b1ca3eb501c2c3d8de7459750a5c472ea03840787dfc45722643","failure_pattern":"failed to roll out Helm release: context deadline exceeded","search_query":"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,3,1,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":7,"prior_last_seen_at":"2026-09-16T09:56:19Z","contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/arobit","occurrences":1}],"full_error_samples":["time=2026-09-24T17:17:41.012Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-24T17:17:41.580Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j4604416-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j4604416-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:7af6cb650e29c367219bebbce4020e6d53088d22e07ab5040d4b4a03685e7176 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:4dc69aba-ed69-4d27-a425-df6d14df249a region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#4dc69aba-ed69-4d27-a425-df6d14df249a INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-24T17:17:41.580Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:17:41.794Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-24T17:17:42.103Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-24T17:17:42.103Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:17:42.232Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:27:46.168Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-24T17:27:46.168Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-24T17:28:46.169Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-24T17:28:47.119Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j4604416-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j4604416-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:7af6cb650e29c367219bebbce4020e6d53088d22e07ab5040d4b4a03685e7176 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:4dc69aba-ed69-4d27-a425-df6d14df249a region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#4dc69aba-ed69-4d27-a425-df6d14df249a INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-24T17:28:47.119Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:28:47.488Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-24T17:28:48.000Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:28:48.000Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:28:48.185Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-24T17:28:49.205Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:28:49.216Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:28:49.680Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:28:50.092Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-24T17:28:50.620Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-24T17:28:50.999Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:28:51.506Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:28:51.949Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:28:52.233Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-24T17:28:52.233Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:28:52.429Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=1\ntime=2026-09-24T17:38:55.132Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-24T17:38:55.133Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit step=arobit error=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/arobit/arobit-forwarder not ready. status: InProgress, message: Available: 0/4\\ncontext deadline exceeded\"\ntime=2026-09-24T17:39:55.134Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit description=\"Step arobit\\n Kind: Helm\\n\"\ntime=2026-09-24T17:39:56.017Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit values=\"map[forwarder:map[azureCloudEnvironmentName:AzurePublicCloud clusterName:ci01-j4604416-svc clusterType:svc dynamicConfig:false environment:dev fluentbit:map[image:map[digest:sha256:9f61c8da15b7578f04d809e2ee60266aca19fc9aab53f382123dafe0fee6f818 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:oss/v2/fluent/fluent-bit] resources:map[limits:map[memory:1248Mi] requests:map[cpu:250m memory:256Mi]]] geoShortId:us kusto:map[blob_uri_length:128 buffer_dir:/var/kusto buffer_file_delete_early:false buffering:true buffering_enabled:true delete_on_max_upload_error:true enabled:true hostedControlPlaneLogsDatabase:HostedControlPlaneLogs ingestionEndpoint:https://ingest-hcp-dev-us-2.eastus2.kusto.windows.net io_timeout:60s scheduler_max_retries:9 serviceLogsDatabase:ServiceLogs store_dir_limit_size:1.8GB unify_tag:true upload_file_size:16M upload_timeout:1m] mdsd:map[enabled:false geneva:map[clusterLogsAccountName:placeholder clusterLogsNamespace: clusterLogsSan:empty-sentinel clusterLogsSecretName:clusterlogs configVersion:1 environment:Test region:westus3 role:hcp-underlay-ci01-j4604416-svc rpAccountName:placeholder rpNamespace: rpSan:empty-sentinel rpSecretName:rplogs] image:map[digest:sha256:7af6cb650e29c367219bebbce4020e6d53088d22e07ab5040d4b4a03685e7176 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:geneva/distroless/mdsd]] msiClientId:4dc69aba-ed69-4d27-a425-df6d14df249a region:westus3 secretProvider:map[keyVault:aro-hcp-dev-svc-kv] securityContext:map[enabled:true runAsGroup:65532 runAsUser:65532] serviceAccountName:arobit-forwarder tenantId:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX watchdog:map[image:map[digest:sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:aks/command/runtime] intervalSeconds:15 timeoutSeconds:300]] watson:map[env:map[AUTHENTICATION_ID:MsiClientId#4dc69aba-ed69-4d27-a425-df6d14df249a INCOMING_QUEUE:/azure-watson/cores MDSD_ROLE_PREFIX:/var/run/mdsd/asa/ REPORT_QUEUE:/azure-watson/queue WA_CORE_PATTERN:/azure-watson/cores/%e.%p.%h.%t] image:map[digest:sha256:8e5ef3e76be33794cef7460712eb9ff0c73bce8b8e5ffc33c5c6bfe0762f5db0 pullPolicy:IfNotPresent registry:mcr.microsoft.com repository:azure-watson/agent/azlinux3] resources:map[limits:map[cpu:250m memory:512Mi] requests:map[cpu:50m memory:128Mi]] securityContext:map[privileged:true] volumes:map[crashdumpHostPath:/home/docker/azure-watson]]]\"\ntime=2026-09-24T17:39:56.017Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:39:56.414Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit namespace=arobit\ntime=2026-09-24T17:39:56.965Z level=INFO msg=\"Doing dry-run of Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:39:56.965Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:39:57.150Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-24T17:39:58.124Z level=INFO msg=\"Removing old field manager from objects in release manifest.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:39:58.124Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ServiceAccount\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:39:58.718Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=ConfigMap\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:39:59.129Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRole\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-24T17:39:59.937Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"rbac.authorization.k8s.io/v1, Kind=ClusterRoleBinding\" namespace=\"\" name=arobit-forwarder\ntime=2026-09-24T17:40:00.244Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"/v1, Kind=Service\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:40:00.730Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"apps/v1, Kind=DaemonSet\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:40:01.347Z level=INFO msg=\"Decoded resource from manifests.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit gvk=\"monitoring.coreos.com/v1, Kind=ServiceMonitor\" namespace=arobit name=arobit-forwarder\ntime=2026-09-24T17:40:01.716Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit dryRun=false\ntime=2026-09-24T17:40:01.716Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit\ntime=2026-09-24T17:40:01.972Z level=INFO msg=\"Found release history, upgrading...\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit numVersions=2\ntime=2026-09-24T17:47:41.098Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"context deadline exceeded\"\ntime=2026-09-24T17:47:41.098Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=arobit err=\"error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: context deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6820/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103161107324604416","occurred_at":"2026-09-24T16:34:17Z","signature_id":"09989fcafa95c4bca4519a71c6fa418fa3c6af4fbe8bd7b270ead898a41c8b5c","pr_number":6820,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"abb1429e70773e39316bd0948d4a9486cc702f42873918d630fe98006b70e184","failure_pattern":"failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Management.Infra/management/mitigations (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-24T23:38:34.810Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 description=\"Step mitigations\\n Kind: Helm\\n\"\ntime=2026-09-24T23:38:35.282Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 values=\"map[azureCnsCpuLimits:map[enabled:true] jobImage:mcr.microsoft.com/aks/command/runtime@sha256:161971cdd3bfe66be6a9b96d67fabf8c4b54a88b841add1aaced26b0a00358b4]\"\ntime=2026-09-24T23:38:35.282Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-09-24T23:38:35.513Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 namespace=aro-hcp-mitigations\ntime=2026-09-24T23:38:35.703Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 dryRun=false\ntime=2026-09-24T23:38:35.703Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-09-24T23:38:35.755Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2\ntime=2026-09-24T23:39:19.239Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\"\ntime=2026-09-24T23:39:19.239Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Management.Infra resourceGroup=management step=mitigations stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: failed post-install: resource Job/aro-hcp-mitigations/azure-cns-cpu-limits-patch not ready. status: Failed, message: Job Failed. failed: 1/1\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7133/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103259159142076416","occurred_at":"2026-09-24T23:03:54Z","signature_id":"36808247d658b9b22b59f418949b27fa87b1655933f39638ef6bec47bfa0327e","pr_number":7133,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d78f890a8b842ac794308d3c81fec10ddb32f68783d012d98fa9ac7172f3fde6","failure_pattern":"failed to roll out Helm release: resource DaemonSet/swift-recorder/swift-recorder not ready. status: InProgress, message: Available: 16/17 context deadline exceeded","search_query":"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/swift-recorder/swift-recorder not ready. status: InProgress, message: Available: 16/17\\ncontext deadline exceeded","failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SwiftRecorder/management/deploy (stamp=2)","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"provision","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.SwiftRecorder/management/deploy (stamp=2)","occurrences":1}],"full_error_samples":["time=2026-09-24T16:33:44.590Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 description=\"Step deploy\\n Kind: Helm\\n\"\ntime=2026-09-24T16:33:45.122Z level=INFO msg=\"Resolved input values.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 values=\"map[captureMode:all clusterName:ci01-j1695232-mgmt-2 enabled:true environment:ci01 image:map[digest:sha256:21861cc03f51cd5c5fcc5f37f79aa945bfe633b1fbdb5c662d6376edb84a8a45 registry:arohcpsvcdev.azurecr.io repository:ci-op-s7p7jyyz/pipeline] region:canadacentral resources:map[limits:map[memory:256Mi] requests:map[cpu:20m memory:64Mi]] serviceAccount:map[name:swift-recorder]]\"\ntime=2026-09-24T16:33:45.122Z level=INFO msg=\"Applying namespaces.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2\ntime=2026-09-24T16:33:45.293Z level=INFO msg=\"Namespace applied successfully.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 namespace=swift-recorder\ntime=2026-09-24T16:33:45.465Z level=INFO msg=\"Rolling out Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 dryRun=false\ntime=2026-09-24T16:33:45.465Z level=INFO msg=\"Searching for release history...\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2\ntime=2026-09-24T16:33:45.516Z level=INFO msg=\"No release history found, running the first release...\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2\ntime=2026-09-24T16:43:47.867Z level=ERROR msg=\"Failed to roll out the Helm release.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 err=\"resource DaemonSet/swift-recorder/swift-recorder not ready. status: InProgress, message: Available: 16/17\\ncontext deadline exceeded\"\ntime=2026-09-24T16:43:47.867Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.SwiftRecorder resourceGroup=management step=deploy stamp=2 err=\"stamp 2: error running Helm release deployment Step, failed to deploy helm release: failed to roll out Helm release: resource DaemonSet/swift-recorder/swift-recorder not ready. status: InProgress, message: Available: 16/17\\ncontext deadline exceeded\""],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7117/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103146302601695232","occurred_at":"2026-09-24T15:35:15Z","signature_id":"feaab49de56d93336deae269ed4dd6eb946386d142e5a85a65c80e15acb2b43d","pr_number":7117,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"36ebd23c937dc29d944a27853b2ad54818920906b2bd4877347b2a0825abe2fc","failure_pattern":"pod targeting cordoned node \u003cnode\u003e should not be assigned a node","search_query":"pod targeting cordoned node x6j5o0j5b0f4m6v-cordon-np-xzf6f-b82cp should not be assigned a node","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:273]: Failed after 0.060s.\npod on cordoned node x6j5o0j5b0f4m6v-cordon-np-xzf6f-b82cp should remain Pending\nThe function passed to Consistently failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:269 with:\npod targeting cordoned node x6j5o0j5b0f4m6v-cordon-np-xzf6f-b82cp should not be assigned a node\nExpected\n    \u003cstring\u003e: ...\nto be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:273]: Failed after 0.060s.\npod on cordoned node x6j5o0j5b0f4m6v-cordon-np-xzf6f-b82cp should remain Pending\nThe function passed to Consistently failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:269 with:\npod targeting cordoned node x6j5o0j5b0f4m6v-cordon-np-xzf6f-b82cp should not be assigned a node\nExpected\n    \u003cstring\u003e: ...\nto be empty"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6922/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103006860184391680","occurred_at":"2026-09-24T06:21:22Z","signature_id":"613a9b7adaccf1b621c4d00e5e9f881ac489c8f71ae2c82dddfa7345990fdc47","pr_number":6922,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"d47ff99fd1bcd56f3dd1a2d45fb345dc45ebaade86da8c2155a6cfa662ad0280","failure_pattern":"public ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"\u003curl\u003e\": dial tcp: lookup \u003chost\u003e on \u003cdns-server\u003e: no such host","search_query":"public ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":0,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:216]: Timed out after 600.001s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:214 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host\nUnexpected error:\n    \u003c*url.Error | 0xc001758f60\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:216]: Timed out after 600.001s.\npublic ingress should be reachable from outside the VNet despite private KAS\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_kas.go:214 with:\npublic ingress (console) should be reachable from outside the VNet for private KAS cluster, but got error: Get \"https://console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host\nUnexpected error:\n    \u003c*url.Error | 0xc001758f60\u003e: \n    Get \"https://console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud\": dial tcp: lookup console-openshift-console.apps.aro.private-kas.fpqv.aro-hcp-dev-shard2-slot-03.hcp.osadev.cloud on 172.30.0.10:53: no such host\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7112/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2102913782756937728","occurred_at":"2026-09-24T00:11:30Z","signature_id":"9fce1ec92cfb95fb8f8175f8912efb491fd5065485198557528e2813ccb986fe","pr_number":7112,"after_last_push_of_merged_pr":false}]},{"environment":"dev","failure_pattern_id":"f19e3fb43cdd0e92a9cf2e821f7ef2d5066fcf7d40e88477e1b3ef713f1e84f0","failure_pattern":"timeout during CreateHCPClusterAndWait; context deadline exceeded","search_query":"failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-mkjrz7qtzs5m\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-mkjrz7qtzs5m, error: context deadline exceeded","failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":0.8130081300813008,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":6,"prior_last_seen_at":"2026-09-17T08:03:06Z","contributing_tests":[{"failed_at":"e2e","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f4e930\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-mkjrz7qtzs5m\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-mkjrz7qtzs5m, error: context deadline exceeded\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_version_backlevel.go:152]: failed to create HCP cluster version 4.20\nUnexpected error:\n    \u003c*fmt.wrapErrors | 0xc000f4e930\u003e: \n    failed waiting for cluster=\"cluster-ver-4-20\" in resourcegroup=\"rg-cluster-back-version-mkjrz7qtzs5m\" to finish creating, caused by: timeout '30.000000' minutes exceeded during CreateHCPClusterAndWait for cluster cluster-ver-4-20 in resource group rg-cluster-back-version-mkjrz7qtzs5m, error: context deadline exceeded\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6503/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2103014767818969088","occurred_at":"2026-09-24T06:52:47Z","signature_id":"5940ace73d01ab95b0fb2644669c5a69b3f53304c116e4dc03542a7fde89df81","pr_number":6503,"after_last_push_of_merged_pr":true}]}]},{"environment":"int","summary":{"total_runs":6,"failed_runs":1,"raw_occurrences":1,"matched_occurrences":1,"runs_affected":1},"rows":[{"environment":"int","failure_pattern_id":"9bf4f3ad9b586932b28d6b547ac6a878bc489620fc6853c6e80a7ffdde8e88d1","failure_pattern":"VerifyBasicAccess failed: failed to list services: Get \"\u003curl\u003e","search_query":"VerifyBasicAccess failed: failed to list services: Get \"https://api","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":16.666666666666668,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:155]: cluster viability check failed\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00158c4c8\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": dial tcp: lookup api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.129.166.38:32939-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.129.166.38:32939-\u003e172.30.0.10:53: i/o timeout\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:155]: cluster viability check failed\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc00158c4c8\u003e: \n    VerifyBasicAccess failed: failed to list services: Get \"https://api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net:443/api/v1/namespaces/default/services\": dial tcp: lookup api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.129.166.38:32939-\u003e172.30.0.10:53: i/o timeout\n    VerifyAllAPIServicesAvailable failed: failed to list all APIServices: Get \"https://api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net:443/apis/apiregistration.k8s.io/v1/apiservices\": dial tcp: lookup api.ingress-cert-le.6r8h.westus3.aroapp-hcp.azure-test.net on 172.30.0.10:53: read udp 10.129.166.38:32939-\u003e172.30.0.10:53: i/o timeout\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-integration-e2e-parallel/2103136634483511296","occurred_at":"2026-09-24T14:57:02Z","signature_id":"c6250a5a334f195a565bf422264939eadf384e54a5927ab71926e7b2cd23c48c","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"prod","summary":{"total_runs":5,"failed_runs":5,"raw_occurrences":28,"matched_occurrences":28,"runs_affected":5},"rows":[{"environment":"prod","failure_pattern_id":"311cf34a4c4f3b566e5116064a5629e1e90748dd27e550507f3fd24ec3ea3f16","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Required value: field cannot be null; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":7,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":7,"after_last_push_count":21,"trend_counts":[0,0,0,0,0,14,7],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ab7ea0\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-r6szljgv5m4p\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-r6szljgv5m4p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_private_ingress.go:87]: failed to create HCP cluster \"private-ingress\" with private ingress\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ab7ea0\u003e: \n    failed to create HCP cluster private-ingress: failed starting cluster creation \"private-ingress\" in resourcegroup=\"private-ingress-r6szljgv5m4p\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/private-ingress-r6szljgv5m4p/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/private-ingress\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e90d80\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-pc4xbw2wvdgz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-pc4xbw2wvdgz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:113]: failed to create HCP cluster with ImageDigestMirrors\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e90d80\u003e: \n    failed to create HCP cluster idms-e2e-hcp-cluster: failed starting cluster creation \"idms-e2e-hcp-cluster\" in resourcegroup=\"idms-pc4xbw2wvdgz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-pc4xbw2wvdgz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0015fdec0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-gk9hkpv4f4bq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_delayed_role_assignments.go:140]: failed to start cluster creation\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0015fdec0\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/delayed-rbac-gk9hkpv4f4bq/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/delayed-rbac-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Required value: field cannot be null\",\n        \"target\": \"properties.platform.vnetIntegrationSubnetId\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","signature_id":"0f42ca5bdd5424ebd462017e995420f979246e19a7f68c1a45f2e182770b37a3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","signature_id":"3c9ac5911363cbb207f63f2d230928ca57009144d5f790aadca90edd77cd8e2e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","signature_id":"a043abfb4a137ba5c7fb99a1284be7cfe198204b214476f6fd57e00fee845d3a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","signature_id":"dfda133e06b8ddc58347c148dd004db522d35f830c18a3e53b5d3cde82d08ea0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","signature_id":"e328078175fb41c8dfbd4578e31c90a7c72998a5382576bff48a3e7cd16f791f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","signature_id":"f1db6949bdfa0c436485b175eee7c33e360c184472692055a39f14766a6685ef","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","signature_id":"fa688a48213278aa49f8f4617b7ea8d841d18133d8772e5ee0c28401f6d5f777","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"495b310385dd288529e197181c60811816aa0b5d5228548d89735eecd162621a","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":5,"after_last_push_count":43,"also_in":["STG"],"trend_counts":[5,5,5,5,5,4,5],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":9,"prior_last_seen_at":"2026-09-20T23:00:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0017055c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-vj7bqc7l746k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0017055c0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-vj7bqc7l746k/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0002b1230\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-cp45sss7fz6v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/idms_lifecycle.go:175]: failed to update cluster with second ImageDigestMirror set\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0002b1230\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/idms-cp45sss7fz6v/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/idms-e2e-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0010a2930\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-grftwltzcrgn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:275]: failed to begin cluster \"agg-cluster\" authorized CIDR update\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0010a2930\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/feature-aggregation-grftwltzcrgn/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/agg-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"0856a5b7612ceaf27aef57335208fdb96fec3578c31cded0af090414cbc459ac","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"163ba41eff7851369f9490bc51639528b5ab45ae39f71f38a3bbb8b825ec7e1d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"369b11784caf993a1266d4a2191af0129727f06deaa23b4c15d665538f183d95","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"39cc51f3ed9337761bd1c79d82ca52214f6889badaaab5d696c36fd31637d8b2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"78618845cda7ac86dc1a7ff167f19e01de1f527c7ec62a7cbacf618e8f8a10e3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a12061aaa350639620c074c15cdff3a4c2e9affb7d03ada099f48aad9dbbccef","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":4,"runs_affected":4,"failed_runs":4,"run_impact_percent":80,"anchor_occurrences":4,"after_last_push_count":4,"also_in":["STG"],"trend_counts":[0,0,0,0,0,0,4],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxjdxchm and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-lthncjjflrt5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc001150fb0\u003e: \n    storage account log verification failed\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxjdxchm and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-lthncjjflrt5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc001150fb0\u003e: \n    storage account log verification failed\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxdxtlkr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-zhq2pjd4clbv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc00066d200\u003e: \n    storage account log verification failed\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxdxtlkr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-zhq2pjd4clbv/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc00066d200\u003e: \n    storage account log verification failed\n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxj5dtdr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-v7bfrx7ddjpw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc000e01080\u003e: \n    storage account log verification failed\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoeboxj5dtdr and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-v7bfrx7ddjpw/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc000e01080\u003e: \n    storage account log verification failed\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103267802377883648","occurred_at":"2026-09-24T23:38:15Z","signature_id":"5159cce0088044d8bc192dbe0cedf4bcbebc146393b1f4ddad44f3a9d431c657","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"a0d8bdae7638bd5124c215d98805edbc939d87889da4c9f0c02457aecd6221df","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103224303930576896","occurred_at":"2026-09-24T20:45:24Z","signature_id":"5c46d71a0a4a0451e999a028441816085c5b759ae39c7276fb744cd76b6b57a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103171230415720448","occurred_at":"2026-09-24T17:14:30Z","signature_id":"849d9e776f46a941d1482cb6eef9a594d740471e89a56ba737d225b1ff968668","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"a62b8fbbaf9c627015e5dc3294140f5101cf61e54eb322e674013c60283fb8cc","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":4,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":4,"after_last_push_count":32,"also_in":["STG"],"trend_counts":[4,4,4,4,4,4,4],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":8,"prior_last_seen_at":"2026-09-20T23:00:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010c9720\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-r2rlkr\" resourcegroup=\"rg-np-dg-minor-r2rlkr-ngfwrz48zjxx\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-r2rlkr-ngfwrz48zjxx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-r2rlkr/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0010c9720\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-r2rlkr\" resourcegroup=\"rg-np-dg-minor-r2rlkr-ngfwrz48zjxx\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-r2rlkr-ngfwrz48zjxx/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-r2rlkr/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-d25jl2\" for cluster \"cluster-nightly-4-20-d25jl2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f0a140\u003e: \n    failed to create NodePool np-d25jl2: failed starting nodepool creation \"np-d25jl2\" for cluster \"cluster-nightly-4-20-d25jl2\" in resourcegroup=\"rg-nightly-4-20-kl7rft6vttsr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-kl7rft6vttsr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-d25jl2/nodePools/np-d25jl2\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-d25jl2\" for cluster \"cluster-nightly-4-20-d25jl2\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f0a140\u003e: \n    failed to create NodePool np-d25jl2: failed starting nodepool creation \"np-d25jl2\" for cluster \"cluster-nightly-4-20-d25jl2\" in resourcegroup=\"rg-nightly-4-20-kl7rft6vttsr\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-kl7rft6vttsr/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-d25jl2/nodePools/np-d25jl2\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d9eba0\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ftgp6q\" in resourcegroup=\"rg-np-version-upgrade-ftgp6q-8slz9vtsg7dc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ftgp6q-8slz9vtsg7dc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ftgp6q/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d9eba0\u003e: \n    failed to create NodePool npupgrade-4-20: failed starting nodepool creation \"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-ftgp6q\" in resourcegroup=\"rg-np-version-upgrade-ftgp6q-8slz9vtsg7dc\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-ftgp6q-8slz9vtsg7dc/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-version-upgrade-cluster-ftgp6q/nodePools/npupgrade-4-20\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"2c2448420f0585ced642c7a920d0fe02a87408a0b9d29f4ef0081d2ced376382","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"62770c4f0e621d8962a43577db753baffbfa217cfa5c51a5444cc20ba8dcbaa1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"6e1813acc449a6535ee059b4dc15d561d04689edaf2ac0927414e732fa390dd3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"7e29c61c5ec9623a1d7631cc8d8c2bf09662bec493f2b1e0c6761a8e6df9e943","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"33e461bde7d86bb5d4bfd94f97db799337aa5ac5cbedbb75cbe7f44a09ebce48","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":3,"runs_affected":3,"failed_runs":3,"run_impact_percent":60,"anchor_occurrences":3,"after_last_push_count":22,"also_in":["DEV","STG"],"trend_counts":[5,1,3,1,1,2,3],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":16,"prior_last_seen_at":"2026-09-20T02:00:37Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00088a9a0\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-6bxr9l\" in resourcegroup=\"rg-np-noedge-6bxr9l-s8q7zdxcxq2b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2b9dc29e-fe8f-45b4-b8b8-139dfe59a7f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2b9dc29e-fe8f-45b4-b8b8-139dfe59a7f6\",\n      \"name\": \"2b9dc29e-fe8f-45b4-b8b8-139dfe59a7f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T01:01:50.31869719Z\",\n      \"endTime\": \"2026-09-25T01:20:50.832464334Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:319]: failed to create nodepool npnoedge-4-20 at version 4.20.38\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00088a9a0\u003e: \n    failed to create NodePool npnoedge-4-20: failed waiting for nodepool=\"npnoedge-4-20\" for cluster \"np-noedge-6bxr9l\" in resourcegroup=\"rg-np-noedge-6bxr9l-s8q7zdxcxq2b\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2b9dc29e-fe8f-45b4-b8b8-139dfe59a7f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2b9dc29e-fe8f-45b4-b8b8-139dfe59a7f6\",\n      \"name\": \"2b9dc29e-fe8f-45b4-b8b8-139dfe59a7f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T01:01:50.31869719Z\",\n      \"endTime\": \"2026-09-25T01:20:50.832464334Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b24be0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-qvrt9xzb9h7m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/05a99209-7724-468c-8d11-789afc14d5d3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/05a99209-7724-468c-8d11-789afc14d5d3\",\n      \"name\": \"05a99209-7724-468c-8d11-789afc14d5d3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T23:58:35.632366587Z\",\n      \"endTime\": \"2026-09-25T00:17:38.767686221Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b24be0\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-qvrt9xzb9h7m\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/05a99209-7724-468c-8d11-789afc14d5d3\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/05a99209-7724-468c-8d11-789afc14d5d3\",\n      \"name\": \"05a99209-7724-468c-8d11-789afc14d5d3\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T23:58:35.632366587Z\",\n      \"endTime\": \"2026-09-25T00:17:38.767686221Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011137a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-vhxnnlnn4h55\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8121b572-5dd9-49c7-9e1e-bb37581297c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8121b572-5dd9-49c7-9e1e-bb37581297c6\",\n      \"name\": \"8121b572-5dd9-49c7-9e1e-bb37581297c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T20:58:20.871989951Z\",\n      \"endTime\": \"2026-09-24T21:17:23.599073745Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_v20260901preview.go:102]: failed to create node pool \"np-1\" for v20260901preview cluster \"v20260901\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0011137a0\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"v20260901\" in resourcegroup=\"v20260901preview-vhxnnlnn4h55\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8121b572-5dd9-49c7-9e1e-bb37581297c6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8121b572-5dd9-49c7-9e1e-bb37581297c6\",\n      \"name\": \"8121b572-5dd9-49c7-9e1e-bb37581297c6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T20:58:20.871989951Z\",\n      \"endTime\": \"2026-09-24T21:17:23.599073745Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103267802377883648","occurred_at":"2026-09-24T23:38:15Z","signature_id":"bd9b3b7f5ce7275bd4f32fb355609d9038fed94688be48bf618a3c2e0b9ca102","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"6d281a26d8ac50b58188d06190db9e64800fa423362a34399d5e8db878ac7e06","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103224303930576896","occurred_at":"2026-09-24T20:45:24Z","signature_id":"3112eb82261054cccf2006078302df6fc47e6d756350b2f95e0fd2f5e3283de9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"adac49aa0bb9d152b2ab9854289415882c692dc1116c20bca083f457ae5d7a62","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[0,0,0,0,1,1,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":3,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-854hq8s8lwp6/cluster-nightly-5-1-npws6l should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ed0ea0\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-npws6l: failed waiting for cluster=\"cluster-nightly-5-1-npws6l\" in resourcegroup=\"rg-nightly-5-1-854hq8s8lwp6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5427e4d8-cab2-43c9-ac23-7d670008e23b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5427e4d8-cab2-43c9-ac23-7d670008e23b\",\n      \"name\": \"5427e4d8-cab2-43c9-ac23-7d670008e23b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T23:33:11.266527055Z\",\n      \"endTime\": \"2026-09-25T00:02:18.550348976Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-854hq8s8lwp6/cluster-nightly-5-1-npws6l should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000ed0ea0\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-npws6l: failed waiting for cluster=\"cluster-nightly-5-1-npws6l\" in resourcegroup=\"rg-nightly-5-1-854hq8s8lwp6\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5427e4d8-cab2-43c9-ac23-7d670008e23b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/5427e4d8-cab2-43c9-ac23-7d670008e23b\",\n      \"name\": \"5427e4d8-cab2-43c9-ac23-7d670008e23b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T23:33:11.266527055Z\",\n      \"endTime\": \"2026-09-25T00:02:18.550348976Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [servingCABundle] ServingCABundle not yet populated; [clusterServiceClusterStatus] cluster service has not been successfully created; [hypershiftHostedCluster] hosted cluster state not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"6d276974ce3c3d3fd4a182577c36d577de093c2c2e4513abb2edf58915543439","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"e3e491019ecf3f140e87c882b7d45ae9d3071ed1df24cc639d33c7a9d0e29087","failure_pattern":"ERROR CODE: InternalServerError; detail message [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":3,"trend_counts":[2,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":3,"prior_last_seen_at":"2026-09-18T17:06:01Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-ml64wq with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013c02a0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-ml64wq: failed waiting for cluster=\"np-version-upgrade-cluster-ml64wq\" in resourcegroup=\"rg-np-version-upgrade-ml64wq-zkbzk7hvlvck\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\",\n      \"name\": \"2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T00:48:56.872501334Z\",\n      \"endTime\": \"2026-09-25T01:18:00.947451943Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-ml64wq with version 4.21.34\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0013c02a0\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-ml64wq: failed waiting for cluster=\"np-version-upgrade-cluster-ml64wq\" in resourcegroup=\"rg-np-version-upgrade-ml64wq-zkbzk7hvlvck\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\",\n      \"name\": \"2d3c2c93-508a-4ae2-aa7a-7a7d3546ac58\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-25T00:48:56.872501334Z\",\n      \"endTime\": \"2026-09-25T01:18:00.947451943Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103267802377883648","occurred_at":"2026-09-24T23:38:15Z","signature_id":"a7e14597b86188a7681e872f2938a0cc446f731377f16c3c6f83c2cc272f9397","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"1c2f7c1ada887c9974dadc757d0c5e8d6fb57702022a1a3603efe040da29e12d","failure_pattern":"Operation cannot be fulfilled on nodes \"\u003cnode\u003e\": the object has been modified; please apply your changes to the latest version and try again","search_query":"occurred","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node a8y5x1l0f9x5u8y-cordon-np-8kx7p-97f6x\nUnexpected error:\n    \u003c*errors.StatusError | 0xc00103ab40\u003e: \n    Operation cannot be fulfilled on nodes \"a8y5x1l0f9x5u8y-cordon-np-8kx7p-97f6x\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_cordon.go:346]: failed to uncordon node a8y5x1l0f9x5u8y-cordon-np-8kx7p-97f6x\nUnexpected error:\n    \u003c*errors.StatusError | 0xc00103ab40\u003e: \n    Operation cannot be fulfilled on nodes \"a8y5x1l0f9x5u8y-cordon-np-8kx7p-97f6x\": the object has been modified; please apply your changes to the latest version and try again\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-prod-e2e-parallel/2103171230415720448","occurred_at":"2026-09-24T17:14:30Z","signature_id":"0b0108efd360078eb545d9c8493397d07692133492d39ba4345c2659c856781d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"5c9f632c1c6b3de9afe3bf8d038c3be6318af9356b8de5a965d46f89ecd201a4","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":9,"also_in":["STG"],"trend_counts":[1,1,1,1,1,1,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":9,"prior_last_seen_at":"2026-09-20T23:00:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000c772d8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-24-142420, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc000c772d8\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-24-142420, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel-ocp-nightly/2103258234201575424","occurred_at":"2026-09-24T23:00:14Z","signature_id":"d8fdb12d282c5253e72136a3836928bc9db7e3e73c7eec5cc70b96bb18cad7d3","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"prod","failure_pattern_id":"23b4633e727195e3be494a8ed0c9fdf68f92a1aa3b342e2a14da0561d134af40","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-l2xrb7ls9tkm\" to finish creating: GET https://management","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":20,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-19T02:00:32Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001321160\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-l2xrb7ls9tkm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e3486a0-a8be-491f-b7ba-de40bdc6a4c5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e3486a0-a8be-491f-b7ba-de40bdc6a4c5\",\n      \"name\": \"4e3486a0-a8be-491f-b7ba-de40bdc6a4c5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T03:02:54.530721397Z\",\n      \"endTime\": \"2026-09-24T03:21:54.293711104Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xc001321160\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-l2xrb7ls9tkm\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e3486a0-a8be-491f-b7ba-de40bdc6a4c5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/4e3486a0-a8be-491f-b7ba-de40bdc6a4c5\",\n      \"name\": \"4e3486a0-a8be-491f-b7ba-de40bdc6a4c5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T03:02:54.530721397Z\",\n      \"endTime\": \"2026-09-24T03:21:54.293711104Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-prod-e2e-parallel/2102941141266599936","occurred_at":"2026-09-24T02:00:13Z","signature_id":"ae9cfa1cabd766fc0ba51cde7820c4aff0d9b5fe7146c215e79849948da08c5d","pr_number":0,"after_last_push_of_merged_pr":true}]}]},{"environment":"stg","summary":{"total_runs":8,"failed_runs":8,"raw_occurrences":420,"matched_occurrences":420,"runs_affected":8},"rows":[{"environment":"stg","failure_pattern_id":"82a0a484dde5bee42c4d52ec194321c8cace1de8ee70ecb15a127a59647d6c54","failure_pattern":"ERROR CODE: BadGatewayConnection; detail message The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: BadGatewayConnection","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":178,"runs_affected":6,"failed_runs":6,"run_impact_percent":75,"anchor_occurrences":178,"after_last_push_count":861,"trend_counts":[52,108,108,108,107,200,178],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":19,"prior_last_seen_at":"2026-09-20T23:00:39Z","contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","occurrences":5},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":3},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":3},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-p49qttrgxjfs/cluster-candidate-4-23-h8xv9w should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a0e9e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-h8xv9w: failed waiting for cluster=\"cluster-candidate-4-23-h8xv9w\" in resourcegroup=\"rg-candidate-4-23-p49qttrgxjfs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/775da561-b2d7-4a7a-bbc2-e2713e469817\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-candidate-4-23-p49qttrgxjfs/cluster-candidate-4-23-h8xv9w should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000a0e9e0\u003e: \n    failed to create HCP cluster cluster-candidate-4-23-h8xv9w: failed waiting for cluster=\"cluster-candidate-4-23-h8xv9w\" in resourcegroup=\"rg-candidate-4-23-p49qttrgxjfs\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/775da561-b2d7-4a7a-bbc2-e2713e469817\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-q9n7xf\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015a82e0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-q9n7xf: failed waiting for cluster=\"cp-ystream-upgrade-5-0-q9n7xf\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-mbh76rk9sprx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e50bc50d-ed3a-44e5-99bf-5d82436a9665\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:133]: failed to create HCP cluster \"cp-ystream-upgrade-5-0-q9n7xf\" at install version 4.22\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc0015a82e0\u003e: \n    failed to create HCP cluster cp-ystream-upgrade-5-0-q9n7xf: failed waiting for cluster=\"cp-ystream-upgrade-5-0-q9n7xf\" in resourcegroup=\"rg-cp-ystream-upgrade-5-0-mbh76rk9sprx\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/e50bc50d-ed3a-44e5-99bf-5d82436a9665\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000994380\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2v5hnvkbp6nz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ac014577-36dc-4208-bce8-f0e66ebed0a8\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_labels_taints.go:92]: failed to create HCP cluster cluster-np-labels-taints\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000994380\u003e: \n    failed to create HCP cluster cluster-np-labels-taints: failed waiting for cluster=\"cluster-np-labels-taints\" in resourcegroup=\"rg-np-labels-taints-2v5hnvkbp6nz\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/ac014577-36dc-4208-bce8-f0e66ebed0a8\n    --------------------------------------------------------------------------------\n    RESPONSE 502: 502 Bad Gateway\n    ERROR CODE: BadGatewayConnection\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"BadGatewayConnection\",\n        \"message\": \"The network connectivity issue encountered for 'Microsoft.RedHatOpenShift'; cannot fulfill the request.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"02065b62056fdb1be37e1f143fa9e3989f4c00365df7604d5fc5f67b38deeb74","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"07f72b351d9885024483bcdda053ed042e3ef275bc57e4fe920a64d613ecaef3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"0886a5c6a659b6005a58f499f6c69484cfe98d4e6289c7f072c5c6904bf7a54b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"08a625905fbc7734eab24c2da8e1f2f3c7e204001ec83f60fdbbd150f3f95d10","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"0948e3f0cfdd80324d14616eaa43a0f4cb0a32ab76564cb78669d9d2dcf1f472","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"1cec2a79be4975a9653bfd8882733838f7e123313a78520fb3323f7cd2a476f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"1d4510859afcc6e6a951c71249a3c54a4d1b3907f437de943b3dbcb9f98099bd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"2491fcd2eac3fe129ef5af4e2165dbf276bb34dd6d7b915bcd09f12fdd5df0f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"2496e95a86576d0de268af2d478e259fba5936be455d300c7e3b637d4c9c0436","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"25a79014eed6d04b3f486b4bd113cebd8ddda98d4daec17f489c251260e84d81","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"260a8c3ed58b404fe205841beee67492d697ec8ff77258801ee95054dcda2df3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"28f63e4ea877a5226e030abfc09e558054018cbba5fc549e1976e79199d95ace","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"344978cc9fbf73459cd931f815705d7f83f04f19bb1486d43bed4ef863a846f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"36eeb6a1e1b48cf8e706db53d0be8d91860b5935eb3db33b0ac89092c3fe1e98","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"3b2acae45c01d2462c31e39a66ceda7d89be3fb818bc996fd0c77eb9f0f34519","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"40a33c47440e8587f6d078af5f5c16c35ffdc39ca033c2eee6186595c1a87db3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"4825f3fea6b1d5d7b75547790c7d34a8a40c3c76a119b4bc51a6e1c1b00426cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"4c79d1369b373f888a3b45bc9980f8607e90b91cf48ef5a98e4523e05cd34677","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"4de0b73b423ac48667ae452b214274712219be45eefe629403a4f7ca9649d73b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"59208ccacb71632d7b6b4858b15d2c7fc131d39ca9231040ecd620b898b8bc2e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"5fa3292ca69ac2b8f8914d64f6c6e71c41533069457c029bc02a6af50ebe3d12","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"60ecda7dfb77cb74a1b276ceee25e1a751452462d2eb558d731953e82366126d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"6551a9274a31031d7c246caf9f38b3803753d5519917aa5228c3ef2caa0fdd99","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"6a1c116e54ff19de705618b83b20d85085d2d56b50e18f8214b3986e60f160cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"6cc2e47633c842a292f67376ce88252879c7a32f0fa0c8b61c35f7e7b1a49ce3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"720afddf7bab9bd3106da1f01488f44f6c8d6565ba8f839cfcc4eb64d06a9278","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"7c07a109a7f8eeb6001f12064984f5de864be533ff4a294d3a61fa4068710abb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"853ae8067a45c4c5af2c0b037c1dc51d90a61b2a9935e489015743fa89a40ce2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"92f971cf89077bcf6f7c95c75f6548c5f63e238aa46a1aae576b8a17cd5e41f3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"967b4f667dc4e0c254b60f79b36f69e3e0cba4279cc5c924e4597a63565d1a7e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"967f8bf89265c03055c4a80ded5359ab4d74d96b7b327c590c07ba6380770600","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"a098cb5165f51c38edade056cc88516c0f0a3f03634689a03285045f95820440","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"a742fb03b3de74be1d0b58aad6a2950c30af0f017f641245d24c851cefe749af","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"ac2713765a7efcac04a7835d38ada2e943f7084564106cb6c2933a048a1e3802","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"af40d913adbdce8e6103e8528992e78d1b771a4afc5a2f9de4c2c5f4a5305fe0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"bdf59d33ef572133b373eccc869cfcbe6143231f3e4511de63e7d2a518123c1f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"befcba4a92c1063abce85eaebfd5f49c658485ca00a56047126b556f822a3702","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"c243f446d5b36205395c8a9d49707a8bdd732debf06f3c70362d814c54919a5e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"c356e51b9b454f478643d1102b07e3cc98f9d26f83f8834c798c0a3d0e6db18e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"c6bd5ae45fe994223d884c6fd019d9bbc980c09700d74c7c6ec6d11950bf3641","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"c70f585d6ebd874ebdf87f09a6f5335bbd8ffbbe2510b6afa7a3cc5775812ae0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"d1479bc8ab6f8ba7adcc3af3628e73e527d7fb7d97afcf0aa9f15d21d77cfba4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"da45a36375afba2b38843fbb63997abffdb70fe444c87c2a54f106bc6f210096","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"dca63c35bedaaf7995439199f45e3ba19c7e03db5fa96015f6372c55929ccc8d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"e1d895a054a411be98f3dd14706c4b6811158995094d211b7e4fe2cf75726156","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"e2176f9fd669f072305456d519992c756a61ee3a64949c662685d7e2dec03a2f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"e4898a44c2d0dd7c9bad670935c424400fc52e32d0af7e2fff6aa931d5cdaf60","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"eaed15a6a2f8b355906a22e9b7c33ce97a5593a10c5d7448cb923809e315d678","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"f0a12fd2d92b9a02183a192dbf54b969f015ef75889c772832ec9cf6d3be6b18","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"f0d90df73e0ad52f9f8a704c64e5d1c3c05332de051b7b56f846352299f4f2e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"f44007f9ca5ba215de70c688ccbfb0a29ff777eb5bd506af4f556c69185fea90","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"fa4f63d6822358edacc49b065a20427c02d8e8a557b09adc67f44ddfbe510a96","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"fa74be0ea5ab5dc482d53b08882b7452407715e934f79592c346c3604704b150","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"fc7b68991da4719c194ace05c0d2d561215021a3d370ce936079026d3369b37e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103121232214888448","occurred_at":"2026-09-24T13:55:50Z","signature_id":"ff83937b404baac989bbd19b80b0cf998ad2d8f2250a11efd067cd3656f1c535","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"0de87cbcf5c9877b7a4237a0f07ba9b9a2ebc8fcc8d43dd86eb431dfe3ce5bfe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"11b0056d5f9eb91a6078381a70f2fad5d7118409110b509dab70404db8f3a474","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"146147e4c288d084c17ebda4302106fa26b576d46636bfe7aac69b5bd259555c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"19bb3f1582062678b47c57215798e7ed69eae547e6905ca14e129c211f2fd95f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"1c03a99aa561093294de0b7f438653c656628e4496548c3d82bdad67cc2948ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"1f713ffd36c3707139930bf34758bcbcddc571154f3158c1cdc2ef4e6eb94f99","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"33a53d9e10e046f64807c0665d9a96bdf32e483709eca3a34cf7d6405c07a06a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"390f99fccf43a9fdffa950872ef3818be3f84574f56671796e24f00762856e39","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"3acb3552748b9fd13e54ddeaf5b07bebbcfe389a5c337c06b8aca75747b6ddba","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"3da6d511cfdfcc9a966ff17298c613d692f9d8b2b2ff41d8fa714b027d68841e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"43e7f1565c600241e1717dd21fe3ce831088fb47b5bcc65de9a0851b50e2a95d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"4586bf9af4491d16af0f6a47fa3ff73e6e59cc51a18f4f7bc5b6a0748206dfd0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"46731606f59a9c0624f9396b197db65e833127d4c0d807797d4eb23355d2fef5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"46f505820c3047e75eaa0fdee320e69bd3f3b2fb1616f3230268704c75a50fa8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"48577c445231793eaba83204969b16f58df261f371ff2fcff7713d00ec5ff487","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"5a72b9f3b17f460b30a9c29aa4a08aa0bffaf0211205623d232115f9e61c81d5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"5aeade5b036df850b438dd9d19f71214c80198c23e9c7a18159685d5d6127efb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"5bf9671d8b88883fbca8a3384f2c419073f4736cf8d87ba15eda4718b7e70b3a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"60061ef754525bd1335a2775069174d5f92f29de7213ad5df21582aa86897763","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"6708cf6358c16ce63138d022c53fea4bcf335fd1b118e1a4b673daa90384d30b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"6aedc299554b7d252bc79ff70f6fb948566ab1c2514ae26ec861e15fb35fabb5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"6c26b8862baf61fb815a9bcc378adad747c42890538b5734c6866f9b99fd3e4b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"6d40f168b82b944aab38b5963703fbbf90d9e0d6a85bc07a462e258b20489400","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"6fd0b24e06192fccaa00e3b1c3b94dc37d332c9b2cef1818ca04f0a13ee89195","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"78a9fc61dc31966c8f5382a6adefb305e57ed7ce933c7353322e5f5babeac4ae","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"7beda16600f383a39e0d3cc113f1193f43bcc9b23fc90352dae1caee67106339","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"7c6822e43843c1ef4b10f112f34e70d1c8fbbbade14e202dbf3988631790bee2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"8389b7d6027c2f391604e97713773076785e864f55a45ec1f88e9f79114dda44","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"86cd7d2264f1df006b68597c7229c5b754c8bda3b5ed006204597c09ea46185d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"8d7cf3325e3f51dc9dd535c7d9b929725b10a46074af389b09950cf2fb1da226","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"8ec6de64c81fb8d075d11e6ffaa54baf82b7ba35b1232f6b902d46f2c56ae52f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"9276c54a5fbffccb202a50473f6fe048ba2a9c847ec78aa22b393009137283eb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"a6d19019536f0ffdf7b8d102ecd1f15f3a7e0e6fbe23d1a367e4af2586c60298","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"a7962d5682a436d220883c1d5f6842915a2a574c423b72fe7acf314cec931af4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"ad0100265512dd64b07c8991367895c07931a266bb11587312fb99858cd39dd8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"ad63ed6b9b2818b36afda8be9a47960e810863d76780c0fffb31e4de99ffdc65","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"af55c67ce41cf5118af08bb66b3fe1ad5ba797a697ead874cb5b7babd4f9361e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"b107cfaa0d2663af2e4815be747bec7cac7f76a98c014b790eba42d70dd84d23","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"b24096ca5ffba57fc98933a2488e339ea00b6584d3690b683f4ba35aeac04579","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"b3c2fd359c06e85b6c30642e7db5c4f3164f911439b26a9ebe32dbbc216aba1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"b69976e1243e6773fa8a60a6bf57877ebfa4d1090f0f4bf4996c7fd78024e054","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"bc73ce6f20fa67a9ec8483aefda2a2bd30ac78637b62659bb7edd381600f8d94","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"bf050428c36d99a435724d4cc497547088fd3c8b62f9ae651e02fb9c333b72cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"c2585ec4226888f6f576e96e437f1b2775fc3ebb2fc2462aa71665f8f2c6de32","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"c28773c54c9830f7b9e29005190ee26d6777b293c6a190ba25bd70d0a8a59efb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"c32d7fdb1bc89858a63402558b43f56a1300afac08a052f62e1c545d91b7df40","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"d173739cb5ea2347d8714d542da4da098196c8a0c7005c4787d2d2146a3dfab0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"dc40facbe59320069107fd1a5edbe83f0018e18aca83e4dbb8e3245a4a984b31","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"dcd5e0fbecd4cda0aa2fa67497226a4350e6689b2cc3138bafbad6f78f1ff4e1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"dea6b577a82026d7021eb80143d812f0e84b8b2c556ecfee6f5490a8a7dcd1a3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"e70b708fe9e564e7f0383d82d6551c4cd363a16cb4041d76a97df34265bce1bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"ec8594b18716573125d007b01c1d0d6dddb69ebe6a0ac57f7d8b0911bc05b4d1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"f38f22878ff4dc7e201880848f8c0291b74fc749dbdcde36131160cad489c060","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"f46adce6b1993d0da3f46389c99620d37df456802b56bd77fce4645c6ff1537e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"f55bcf54c5d67b37e04f8a536611e03e87fb64780bc1a8305f99107bf6029933","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"f750b8949c6048df3f06cc60730c23f17eed1205b46d7b1bef29f0143eec2ad1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"fda77eca2b5ca7a3f170a1eb6c18547b563d0dfccae707a6948d12a1c6e551f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103058579819859968","occurred_at":"2026-09-24T09:46:15Z","signature_id":"ff4468a0a3f2435884308ecd6b5b5e9484371fe0029dd446ac35d4ff6a062092","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"015a62d796a14aafc93ac8fd9fcd0192bc1f4daaf775afe7d5216b4fbed2d422","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"028ea5601862fa590d5e2201ae315bf0b1513b6769b08ed07c16e2de0ae99250","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"0567435825b2f1f46e7c8c0c92ccdf5c233255559fbb4ecd19ff369114130a56","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"08753c85308ae68cb595376a3e6a02505c40bc1c3f231edf2037c7c84ce54390","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"0b322092770ddc32a8a6514dee884547809162ca71778488279ef0d524e10934","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"1088ef782bb08563b7f5eedd4e1fa2ad693b60b2a406cedcb3466c7edeefd7d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"14576de538e43a898ea0afaebbf5578a0ad679c60e4b538e2c28a0d08595fffb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"195e663d42f52c6c17db4a06e43816ce2ca15b96d6fe36189b86067a43dc4c68","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"1ad44de4f84225657a0f5171167d0ef345449d251f077c5d71045f9eb0644d77","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"20d5cc742c1cee73f1b04cd10f6ea72323a6cb2809aa9f9667747252b4035ef5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"29ca14863816cf66991a82f9b46c9729fedc201d00186088b1efce79233b37e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"30cacbef53a9519a12502d8baa9dbf2e4ec518774dc9498d9b70b0e3f05f5dfb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"3c66cf27a6f188c03e99e0e620fe8a8673cd9acbed4dbb1db198db2a90216cd0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"3f517ece69506cbfef521fda194e3fbb39f7991afddffc45f05e2fdb1b402cc9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"423808405226253bf2780a731584c5c00ca80e8ac8b89cbe298d4b9bfaa20ef2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"42dab5794581704633e8583d39e306fe7eaa69761c0b3bdde3ae6b578beaad85","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"457829d398eaa8002f3a2d02b33f53ab433c3ee054e691177cd91348662c5085","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"48d0bbbeff9ebd85c0d242660d3ec069eb2a39489e5e738e5a0af642de48e47a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"4f8bced1b86c8983e52fabef07413b07a835bc6c67a4fb7b1ec887b5f4487e37","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"5343a8025b83b6578ac81a81ccd0c2c8f82d4d10cd53595fab1fde76e7b1139c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"587a41d8cddc604041bb53b0856b1237afdb02c7190fbbfe6c6a0aad7d417448","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"63c131654901b385f7d641263b5a6f2bdaa7a6105cd00a04562f380454869986","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"6a3887e150d79b2470eac482314e5825c603dbdf1cd847403c74e9971b98c5f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"6b345d5ab6014d402a2d652c0c5909b61c6a4242f0cc2ffe16076dc13b9b7b17","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"6eb87e754a0c8e7de25c379f966fd9286c4d705599dac33f311b2d71c14b420b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"6fe4ceadf290c9cd17620e6045688aae8219b7ba271ace3ac6aa8d59bd4dc942","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"70fa408a82d01d5e0229300a03628a7b7c697abcb7a4b9d83bddeb0c17eeaedd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"75547c088779acedcc0f688ec015777e98c9c620b93369dbad684de8c96d5068","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"87764a3a1c887346ee5ef8611cc6a8377a04492e4975331324104e456976820e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"8a7efe49c03007d1fdf090e4785ee39ae4747a5c5e30553be13dcc9aafd36922","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"8d03287f7ed3f0349f7714b3bb9f7d51bb8a6961412a9db10097515ba650bd89","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"8d9b406fbbb8cea6732da0cd609f796ba30974cc05d728b0633597b50f8c0abc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"8f3b0599281e22f58c3defe5f0d147b488c1b6a0f16143880bf51ef7355d85d2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"91bfbb27fa9e016442e0811e84d9a5a5a7c3beeb9aefb272528d99cbbfde1693","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"945c3b0b90dd70635719bf5c4663085b88eaf4bfe2ed27a8f72db49c7e4b63c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"95d33091b2e3ed0ead5eb1ceae0e4c3fd0d888e8ace45c0efcb0c25533a58f9e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"98bfd3289c6b69608510ac8f99b14ab0bffa05824f201bcb73f80da461684588","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"9c4d03f3479a5c70388413a70424168b5e2b17ad421c1bf6766436364b979d1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"a85e78a9cdf1147b1107f8d145242f52a53fb32fc15b96f67ddb5c9c38894042","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"abc370c7132d3dd0cad4c01b1df33504e2a5ee15cd985778c8ebeb08c7378217","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"b9ad954e8151aa5b0826d0d74bf5d49508ba660587f9fe7ab54350a7e715d196","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"be867ed293f062a9bd769325496bdaf3731f033dfbaf096811cf5f767ffb8479","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"c37bda4304af7e53e58a3919e84b1477a3a46695576de8ff29d4640d43029591","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"ca41ca40c2a7ea09a51dc74552627e7f098ec66a21f33ac2a37277b8e785ba2c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"cbb84a2251ba6846685f78b090898c0a3ac4b5ed2f079675620d0a5b6d6c16e4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"ce49b331dea25dcc6be08ddaf7d9a319f6c26e31fbd36c370bcef547c757521b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"cf518dbc290a9e0538e3f3c154869ccf678a1dcbe5e6134efa2facaa72d7ed91","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"d8cfcee194e8bef7aebd6061716b0c8a42cdcab9130e4a23056c9e64bf846076","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"de2fd51098d4fccf5d1bb093ad14ac97bee13c527a30b84d2038a84112fb873e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"e205db983efa4c7fe30a3098f99c7b28db976345b998b688245438d48a09e9bf","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"e259ff9177d1f12706b36fd539e24cc7a728c76256da9b5c37b773690fcf29fa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"e5cd62f25356431728e737caac9f39e8423781573072667b098d01ae01dd194b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"e9a3aecc6dfb1296c32f694be25ac887dd48af1f7379a8a0c02b86cd85efca8a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"ee72d14d762ba3e9d0e27f0a7cef6b5bdcbf2c73ce4814de75d5572f0890fae6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"f8ae1811368ec12293281df96c6cee170ccc9b5c6f2f7f431f5e0b46e83d6d68","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"f8f1d68ea61706af805ffd1d510750baa305613520982e0bba79e29979c78292","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"f9c44b3e72b1127fae5b53f6d7a080bc8f4bc6b26d9f291fb7dba9911f6078c6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103041133276827648","occurred_at":"2026-09-24T08:37:33Z","signature_id":"fac67d5451426d0f28252a0fbe031705ee4850720c8d46822b1614d3f7cd79da","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"087c5753d272a2ff86285ff6f45823005418b1e6f89d4b2bb703f0d6deb2094c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"fa62e44b14b31ae7655734e567ec10b60e7231167b3936203b6ca0bfe48dd179","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"4c47478a4a085b6edffed2a7105111b2464ffaf9972aae8beea63b50843450d3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"52b1633f73d41b43932bc93f25195e310e768881694919964ac5f253eaea7842","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"1353cd7fc2e7eb95af2a2957b1ec2d2b9416cd1f12ca6cfeab46ea5f033a2da7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"3c01497d2687bfb43b9288af1b54486f516b205e9813d8dce28ff93101312372","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"8b9341db534e17f1c007a11f07836dbec6a14166f62a28d78b3b65efe63e43a7","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"20732d2a60a1d4875fcdf4af593e7b2e3155e50c68b4f8ea2d8932aba7696897","failure_pattern":"ERROR CODE: DeploymentFailed; detail code RoleAssignmentLimitExceeded; detail message No more role assignments can be created.","search_query":"ERROR CODE: DeploymentFailed","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":170,"runs_affected":3,"failed_runs":3,"run_impact_percent":37.5,"anchor_occurrences":170,"after_last_push_count":194,"trend_counts":[0,0,0,0,0,24,170],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":4,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":2},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:73]: failed to create customer resources for v20261001preview cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106f760\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-v20261001-x6gm79\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-v20261001-x6gm79/operationStatuses/08584113797462734569\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-w8qdl2mz4kg9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-7e767rdphst3w\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-12/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_nodepool_active_versions.go:73]: failed to create customer resources for v20261001preview cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00106f760\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-v20261001-x6gm79\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-v20261001-x6gm79/operationStatuses/08584113797462734569\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/v20261001preview-w8qdl2mz4kg9/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-7e767rdphst3w\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-12/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:114]: failed to create customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b13100\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ingress-cert-le-5tmv7b\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ingress-cert-le-5tmv7b/operationStatuses/08584113797516953541\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-qzpjtfdnhg6c/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-nhimtfs3fueps\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:114]: failed to create customer resources\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b13100\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-ingress-cert-le-5tmv7b\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-ingress-cert-le-5tmv7b/operationStatuses/08584113797516953541\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-qzpjtfdnhg6c/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-nhimtfs3fueps\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:283]: failed to create cluster customer resources for node-limit test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e7dae0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-4lkc6c\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-4lkc6c/operationStatuses/08584113794966440147\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-48zl586s8mmx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:283]: failed to create cluster customer resources for node-limit test\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e7dae0\u003e: \n    failed to create managed identities: failed to create managed identities: failed waiting for deployment \"mi-node-limit-cluster-4lkc6c\" at subscription scope to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.Resources/deployments/mi-node-limit-cluster-4lkc6c/operationStatuses/08584113794966440147\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/node-limit-test-48zl586s8mmx/providers/Microsoft.Resources/deployments/pooledNonMsiScopedAssignments-j6nw3ohx7bzo4\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          },\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"status\\\": \\\"Failed\\\",\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ResourceDeploymentFailure\\\",\\r\\n    \\\"message\\\": \\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\",\\r\\n    \\\"details\\\": [\\r\\n      {\\r\\n        \\\"code\\\": \\\"DeploymentFailed\\\",\\r\\n        \\\"target\\\": \\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/aro-hcp-msi-container-stg-shard0-00-24/providers/Microsoft.Resources/deployments/pooledMsiScopedAssignments\\\",\\r\\n        \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\r\\n        \\\"details\\\": [\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          },\\r\\n          {\\r\\n            \\\"code\\\": \\\"RoleAssignmentLimitExceeded\\\",\\r\\n            \\\"message\\\": \\\"No more role assignments can be created.\\\"\\r\\n          }\\r\\n        ]\\r\\n      }\\r\\n    ]\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"00a20048fc932b0229905148fcba92613e4cec7d16f2f63fce5ced5f3389d702","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"029ca1a09e038a2fa4b3875ee6b1d8faaee67b0d1c74d5a1f35e3c471583ada0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"032d1d7a3e0d95f0228ee1c11b1d6af914ebb5716a5fa59eb161c8b7fed27753","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"08547dd081b44323301158c280d0f3434cd05b357d34cc1bffd49f97c29ef513","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"0ef524766d9390776ad48ecbf26703ecb351c6ee97b9799357d914d374e4f02f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"11b0e5a286f5fd7f972ed8af20e7424d0a69e619e29ef40e658a0fe7ce3e2fc3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"121a5f9e10fa93bb1cb0ceb5ae519962392d2b7e9720f3e5ec5a5c3456f46de3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"1264aa04996713c2ad0664fc180fefec085befb80e6a3d7d8c615745c0cafacb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"1275667544a960574ea11de5b15d9cc89597bc9e741df1ceb42178fc733c8b11","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"128bfc4ebf3e4f4ac7c72ef37dce1d6f1d2972dc28ac3084ac0eea3f1a935364","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"18b554797c4bf7fcf142d9ba43f2d0e3c7d0231faa68c4e122ac2697af20af1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"2ea82ff5e0105ddc39162bc3a1fc69443f8429775e71a60d26178a993e537cf9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"3202f84c16b137e078b9c7ecb07b12a42284442eaadf1a57d9b80732965d76d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"33d5d93bef7910045cddabd91ed8d0f16ffb3abd049de20101412caa4dbdbefa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"469958b60afe99ebd686b471f988b1e0ca0f744e31bbcdce7e52c5bf5c7f1549","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"4834d7a7ce86611f500d1d72078b83248491ffcd1d930005f197e68c04a38653","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"515b93d5e37ef059a66fe272396b5e973c12ddc43056fe961348a3e80afb8c97","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"538131ec214de2b12316575147e1e84a6f41ede3f84a3e93afdd963b3cfcafcc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"54f0aecb47d52922007a03672ff611741b8fc5034419c1f438b22e2214da6935","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"586af51ee2e7cbcdde4edd35a4a1a29ef19ee26309fb4a112ed256320c18c4de","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"5c5c223fc846287b6e5184b8ab2b2f4e1d090891bd7357cecbc12498cfaf9579","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"61ee045800b644443347ae3b2331e9a65e1d479e962d730dcbc98c53d281f3f5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"66b6a8c8ed850f996ab5708c37e3c8119f622e5f7661cee41c168026b7225a35","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"6c9e3e00c4943e8c6ae844a720ee3bd59a661f127a0c9f3e1eb2ce1bbec51ae6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"704eb4f5538f36e002f0ab88d0c2ebd5288bd5bee60926b525641b1c59219a62","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"70890c7ae52a1db455deec8b3eb7490070c782e63310b6e7b685353def12711a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"74310ad621787aad91f122d4ddf1a6955a3bd453e42f4f9419b9b40e9ac8483b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"759c7aff752bc2da8c0bbcbb05976cff2e0320d40d2167426639c5b03ea648f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"7d08c2187cf3f7a85081d3d94aa55ae175cd4f81026e76db422229194afedcc3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"80d153140ed2a6cc68ab892fa4a4c45411338bbd76b2c74ca7b9855bd7a64595","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"88bf0468667696d126054af4a76866d6e508bf841aaa491e564e9b9dc090b9d1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"8a7dd97750d4afe253dcc7f912d067e5dc02351e57c7ebdd90777aedf3f19660","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"9b1e9bfdfcae58a6d83f60788f5cda0dd05af07f85b2fd0f136917dfeee0ab3c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"9b2aa2b4206e53932073fbfb64d37b9c7758a12a200a2900f869b67ac71a624d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"a6bcd28d73459594e929894c498d1e701b6c725c8fc5e8572366838164afdef9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"b3cd8ed1378cd179b5adc6dba47cf4e17d2d259a48359faed9ab53bfebd698ab","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"b6d7774589a62ddba3343d513d47008b24ca7189da168c615a3350b70c6cea72","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"bcb1f165d9dbb4bacc1f9a57572c8fb656f3e7dfb80956cca46af347257d19e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"beadaf19cf2680a9bc17ebe12a30ff4c469faa490d5bcc9d5ea6546c43f5c8a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"c07a630ddfa4bf5e5c0aa0512829f6419194f24b58766d4a80c2eb68ccd03946","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"c5ed5a71619d53dc86b567736741d4b55c60b49cc02e82c98980eff5d0786438","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"c680824be3d2257cbbfa3f9069712276e088e630af553b971685d6c2d068c191","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"c83af701810277995349dcb52023292d5025daa4c73897a842f01114b802c577","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"c9904e37a7c79299aecd2c19efee7467df44886a34a6426023ae1408ffae1aa7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"ca6a37a8d8bd66ca90ec1156ff2a36ff215e8cc3cee04729bc0351a1eaa5db8e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"cb62db27ca09cf506870028f97396da42cf6ed7737e04d83b76d35e31b696eee","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"ce3b2648ff9b027b217801bdf840547004f2d0850e7343af604b6b1c4b1c31d0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"d95ee625f921273f4468efed93cdbe7bf7598dbc0e5fd1356a3bf91649094680","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"edbb12cee0cdc493ca116717026c262450dbb9303cb1deb1d5bae75c5bd6380f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"ee95ed63c942a4fe05405208f698b779fd419a59453da921efa0d030f65c38da","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"ef6b465c319e2089bf73fd85a6ea8152d3ab980148dac6a645125758ec975f33","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"f47fe5731873797ce03d1d26c02b76cb7c541a6c274c295b42b92e5642fd89a1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"f5544d328bdba7722d736b6d0f52967b438c0c26d6553943b71ca1a3207e4a31","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"f78aa7804dffd9c91eb7a06ca983c1ab58ba6e852310a8faab7a08106948a7f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"f929f4bbedfac90e88e7ca84f91f3733417298186cf3e83f4d4be17bfe2e51d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"fb5ce0354606977e7e3791538a4b663a81b113db149cb566343e57faaf207eb8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102988159166976000","occurred_at":"2026-09-24T05:07:03Z","signature_id":"fd8dce044fe3c3ec2ed6837b879a1f543587662baefa69190a60f6b8acd23377","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"0304d759fd7f723fc6fce46557757270276c6ef816162705b4c141fd03f33c2d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"037c80fef56df03fab8fd20ad0d7c86fd678553fe25a7b65681038b32e94955c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"07068c99aba746939ca0e7fd85f8d0268d89c19b415bcc1ee9c1c9a86fbe9b05","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"091b2e0d12e6d8ebf4284168ae4c1d8f0085c61f2ad326ef5833e98ccf402f45","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"0b5092b1753b51d12ff4c29870adbc656c13e6976d861905264ee7910638f41f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"0bd173944720937f7045f39d5686ebbee16cf55a1935292dc7a1905cc174def2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"0be8a0f9e257eed68f104074868b8b17f7d7098c3a69d645d20758aa0235f203","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"12dca558bd0b511fc4a5df715c750446668edb1880da33c19093890f8b547639","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"1623bf2f430076fdc11544face0d86cf5da4f75f4a70fb76b1fee7e3b47f839f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"23b8a93a781e18d248ec90141ae1a827a89330a7e7a6023ebb81f712b1aa55d8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"26253298960cf5b479585bcc88d9b58ed6d2ef483db264c7c7260d823ea27662","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"2d4c586a289252410c5432cc2b9b0821bd3ea82888527dba0a2f593c88ba237a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"390eb6735fe63cb426b4f5ffec27b7b5786fa5222502d31c5b03c65ff6af65f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"3c862361541ceae5c062b4ba0dc064b68cae0b5d598e706f6762f085e0607306","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"3d478be5b9189d62ed2c66e9b103f81f7b86937b57c08e9d67e6ca3c30555f01","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"4d8344447dfa277aaac9f86488fc2fba58be5beec2d8d53a92b1f78394fdde32","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"4e29cfc3d96fb63ef07b9b320d9429e79b35c452ece26e453a45510107fb8132","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"502c008e6a6776d9a96bd10fd01cc6e2363833b568e55b82bb09d9627e16887e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"50fa57f55c832fa17b6f7248a05fabef16d409042f554f1463f8a8292757b10c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"5b26974e1da059929c65b663e256e8e0b0cadfd7ed25c9ebdf8345a70426d39a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"5bd96944e5c33f0e4a58b08b961f8094f9e02ddbb7628a6b1e92dab670f4873d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"5ca763c04ba811fe44fa262d90c103eac78fc982660138b2fee9f970c2edb86f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"5ef6cd0378df1b88f9a1e441ce1513797d9143870e2e548690d877816da13ed4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"635d55a9178bb2083d65e2a4ad10093076f6eec948e15b30f0eb5b0f72168cb5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"63d34142a504e05f9036dd1c69be15d31757de56dafb77a0d0f0177274375c52","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"641eb4af1997048f835df858f7f7bef584d41aee6bb04fe645ddcf1e44aa00e3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"6722686da3723d8b0bf2c0c56e808e2f4cf683cdb005fcd9744c5aaecd3e702c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"712f9dd3b637bec2b5625a0186e7395b21baa4927eda84c10d89e02f5b9396d5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"725f05032bea5bd0ce516b91dba1d1346b457790f2cf4f5d22ff28e0e5fca8e2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"7a55e2a07f0fa2912503eae7151cf0970e3f6b2d6668fab56021d109adc9d5ad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"7d875d3f62b6b594e123ead8b904ca4dd939eeb32cbb39316567bbccc9207e3a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"83f41dede741122effea9bed86f31cb70cc79e3fe60190c74a81ccf095575a1e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"843855ae1f0f4f6cd0e66370447e4b9beddc6fb7e68d7e5e02492021371b9652","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"890e1a56439cee473baa172f8e4a9c46c0855344ecca13d7f575a0200da7b939","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"91478fa5c60e4c1a9e847cc7a2a672128b7fd9e86bf7c5aea99592fde71e73bb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"925e74b7b1612e5eeea6fcbb8dbc566263b823de593165702c918d865fb5320d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"96b896ebd4bd97e76579fe89b0bf3e2bf9f53a8625a52c8127ac1a1e8e6b3b60","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"982fb021ef82674968423b11c2470423c0daeb157256c664d1becb93e8210a75","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"994e65619d43c6d0e316a7b351700514908589ec59750bc0f6395e6be91c7836","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"9bb8908c4059fc50132ad3029a7c2b0cbf42b2c3896b45ab04df63c56500a523","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"a0382390c3701f1ad5bf1bd591fd3915695d3a07b9d0cc99388388748d139457","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"a1e87d9c7e66aaed64c7f8954d9753bc0574003ab4b1683459ff823c1d03d7c7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"a70700610a1b0fb554b19357597279649c0cb3ed6f5aee8c998069ad0ae49628","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"a7ea24781db537110c35917d98b4f72ecdedac8d30c9c33371781202312a2f05","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"aaca3b66cd93923710b64ada020932cb93aacc5dd79493313bc4c64ce07c0aad","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"b3fbbe913167b23f7cddbcc5beb175723e00dfbc86894399f39245e5c30ccad8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"b890ac9bd99e079a6b5aca1038203032bcdb627c5eafb16e539f64e5b257093a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"c17a5e36a9b108e239e9cbfe37ad374080446eb38001b2f19c480dc30ff0f28d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"cab3a7cd895f69adf001c052761d92c86ca6c64727e980c568920541d66b6693","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"d6768673dc1158caaa55d77db95d18d312042e5c93f5efa86e3fda846977c9a9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"d762de026942b3816437197f33b8846d8d2d9b251a4615d40901dd13cdf8719a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"dce118f0d6bf88fb6228496d672d2f34dcd0097ae26939c9a39426a75b29b854","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"e4854e50e27a97b30d706a8853005f6b6d013f9d8ed46f6cafc31a004843dc2f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"e720cadf1c011d298dc69fa164a357fd8c779a17a439688ca3a04d3526fe32d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"e7f4fee1c466f0459af35e40872aa4b567ea48855febfe78971983ee7706ff97","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"ea1a9664879960e3782bac1bab4881ef33ed58c68b27f3b3b3ea9ee942bc8f8e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2102983195996721152","occurred_at":"2026-09-24T04:47:20Z","signature_id":"fc6f49312f9cd176b6a5c0c05d229dbd5b8a414551493a3fe3a7e14f32ebb667","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"004b42a6843d9c599be4103c0b3a7bbc53c9a341d8d2c16ff587dddcdca93fea","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"02474ed7bffe3f2f3efb832206e4ee8bb2657d3eb50cde44d64f37e4e025c8d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"0a134035c835ada31f01131908200d3c87d62d1fac56b860bc5699a1bf6f2546","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"0b7b3b363d12dcde6dbe7dcaf8d1bfff05c68c4872ac68b624b4b5dd24798a34","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"21af537e81ae2811dae2782f1541dea999e6b6e005eb5ba0438334b0110de505","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"24a84c9669e3bf30f2efd1876022065618eeea5fd4fc89076c6ac044a71d108a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"292ecbb651b87ea834cc7055016245c2d8aa052ab0fd931987019f543f6e4d6e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"29363b364d248244b8cab05224e499f828ca4b3558f6b271121ac89221e31f80","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"29738fe680afcd7bb868f6b4e7bf0e48f78b9de7c821820e7ce46f1b77273631","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"303e71f56a7321ff7781f70fdf21e43e15eafba5a112fb795a421560bf8687d4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"378c2c4f5f003f5f14a5587c4fc6d847ccbc184c6e7fcf305a9df629726e8dce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"493df6fcec5673d5971d28b2495e7b60a93b79fc07aeb785fb4063e1017f859d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"49eac6d4a36a30efa10c1b8b1b8d2d65876ce6ea80106d791d42cf6c03bbbe8d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"4b1f37cc80923b8d11e34699631a1958ecd1c54c87c846691565df3c140d290b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"5275008bf6e2cd955a92ff0c773682149b95d29566406f66e1fed2199df22e64","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"58e4dfcaa596cd88fda970ca6914db4000b0185d3538f68ac836493847042863","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"590a7ce9c34dea2488d7bcaa818aa04f1e7333c2c32aa265eb01f529e4c09168","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"5e81e681dd240666f7f73c328e1314295bc989aa39e4424d393b9191c79a5847","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"63857a171b9f09d8d921021f794a90f8ab826e76cf935d7a69f3776cf78381b3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"63ff74a61a4695146a80e07d486ea919d166ac0907ae451cd218aaed86a6db14","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"6664222cdd56fe98e8670b4874f17c78c7d2168808f807394b24f384709545e8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"6d4c6f30d7a0118a761afb53daa3e256cfba084c934cfa3569d0253ab1d5a261","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"6efac1d441daa5ff54b4d89012d20f08600df5d158a441ec102e24943fc428e9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"8168feff532830edf3ac3497f6ce1e0b657479305b2853c1a48cbb283d165e76","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"8254913197285306ce0204858e98e4a4547494803895048d378c8049c527c44a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"84c63d590be5b6794793c8c510fecef7965137dfae7568e99ea9e7ba5cffbfc9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"93efcc50d250cefb160b3d3164d61008ab5df04f873fdfcbbf699d728ee56461","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"9d3778e9e95acd335ee5ab579b70317fa3b0f3a32b64bfc784b8aab8910d2702","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"a4a7b00e0cf6167aca09a6c39407324ab2cbdf83285b14e600ca3cbebdae2762","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"a6fdde63eb30cf8fde25cd7079bb5bd9247a18fcc4fc428d6930daff70e0c43f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"a9db7c4a29702a102e7f5ecd1187fb046e212b54e149b8fd5e9d352e54ffbcd4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"ac85942acdcf9dd62304e20814465a73dab2c6546d5dfd1ec23a0709cce01819","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"b0ee5240012c5afec47ab2c288045073ea38119a5b50f900847fb95919697b1a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"b27d8e1b8d937857f4f4398e505b9f826ce2b0dbdcdb3fdae66e91447431a872","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"b49424c3e5a898d2542be3de8a944c781a1c54008483164cde2612601cc567b6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"b60748e01d3770081b9092311833c353d24a585d648763d5987389842fb13c7f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"b7d1a1e20d148974b57db9b0bf46b2d32a4fe75a72b18053adfcd3ffc10008fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"b9b552cfce8aa134690435adce391de22a7d10909a7f0b0b344d1de5d5f13741","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"bfb111b44b68e6d49553ea155dabf55537d9068f181b21a0605c2d49bbf1a5f0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"c3a66ee84a59916e6ac4929f00304fe7deb56ae962c7df382745b1e42339ff55","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"c3c4a6ccbac232171b2b65ce786cdfef92bbef99d093fd8e6cdd95fc35731f03","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"c80bf9057eb592b0d78f55fa548e2f6ce83aaf5494a5e05657657b2e3f4c11d9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"ca4be0c2537a5986ad337e9dd7b0c174dbbfd555b40b995690b3ea0bae599488","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"cc5beb5e38150b98e6a39e86bd07ab29baf902b0225bf3a00a1113d79a08d467","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"d32366df33e5f54aeb4a5cac0b58617ef9795c05332cf16b4e3b9f7d2f335ba2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"d71fff82c2dfe656568e32c9670f076351175cfc25591cc171d7af2c71095862","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"d949d00c9bd448d0c2054b0f46eb2a81e5f314ec80d1542febb5b972b95b51fc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"dc85a4e926dccafcd7a84d59a6cd891fae237d24dda05b7aef3fa9e97dec9da2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"de91715b5519c3e9c48e9f9db31ee57baa410e66aff582268d9803787a051a40","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"df7c02235ae5a45c83397b743a331f854e72bcbbb98b42830177e7f9d038928e","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"eb699857478bc226c6dc8aaf326a1110ec3eb808736fd153bc787c847ef8fd08","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"ebaee311f476e862bdc8a9ed9fb42966e18b1240f73ae004a1a47556a07ef340","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"ed72ed9221365984a17f27e8ed0e35e5508168087855a2c0749774e962bc35bc","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"f473bdd0d73590343184ba03445fe9c69d83d507ba5dec3b6434fd8b6dd23c83","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"f4c7d50db1341281f9bbc735079e7958656096dd31c71f4c0842b77c630f5f87","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel/2102941141295960064","occurred_at":"2026-09-24T02:00:13Z","signature_id":"f939e6af64ed35d589b26c26d6589830f2545bdd650fd5e4cd474c0aca8fe905","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"7a57b92d72c0794062f69c3dfbe3099a539bc804b68f62feac466d03db614cae","failure_pattern":"ERROR CODE: LocationNotAvailableForResourceType; detail message The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: LocationNotAvailableForResourceType","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":58,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":58,"after_last_push_count":58,"trend_counts":[0,0,0,0,0,0,58],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO HCP Service should successfully provision cluster when MI role assignments are added after cluster creation","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Create HCPOpenShiftCluster with Private KeyVault should create a cluster with private keyvault using v20251223preview API","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to cordon, drain, and uncordon a node in an HCP cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a HCP cluster and use cilium CNI plugin","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create a no-CNI private cluster with a private key vault, a nodepool and install cilium CNI successfully","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster and manage pull secrets","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster then delete it by deleting the customer resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with back-level version 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create node pools with ARM64-based VMs","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to create several HCP clusters in their customer resource group, but not in the same managed resource group","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete a nodepool","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to delete an HCP cluster whose managed identities were deleted first","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to lifecycle and confirm external auth on a cluster","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to retrieve cluster and nodepool status active versions","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.20 minor to 4.21 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.21 minor to 4.22 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 4.23 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to successfully upgrade control plane minor version from 4.22 minor to 5.0 minor","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to test admin credentials before cluster ready, then full admin credential lifecycle","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to update node pool labels and taints","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to use workload identity via the cluster OIDC issuer URL","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster using v20260901preview API and verify cluster health","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private KAS and verify API server is only reachable from VNet while ingress remains public","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create a cluster with private ingress using v20260901preview and verify the ingress is internal","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should create an HCP cluster and validate TLS certificates","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should downgrade a nodepool version z-stream downgrade from 4.21.zLatest to 4.21.zPrevious","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to perform various invalid operations on cluster resources","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should not be able to reuse subnets and NSGs between clusters","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should respect cluster-wide node limits with nodepool autoscaling","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade a nodepool to a version without Cincinnati upgrade edge z-stream upgrade without Cincinnati edge in 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should upgrade and update a nodepool from 4.21.z to 4.21.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"FIPS Mode Support with v2026-09-01 API should create an HCP cluster with FIPS mode enabled via cryptoRestrictions property","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Nodepool Ephemeral OS Disk should create a nodepool with ephemeral OS disk when autoRepair is enabled","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.21","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.22","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 5.0","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Negative creates a cluster and fails to update its name with a PATCH request","occurrences":1},{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b7e680\u003e: \n    failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-d5k7vz9fgmp5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-d5k7vz9fgmp5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: LocationNotAvailableForResourceType\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LocationNotAvailableForResourceType\",\n        \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_ephemeral_osdisk.go:84]: failed to create HCP cluster ephemeral-disk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000b7e680\u003e: \n    failed to create HCP cluster ephemeral-disk: failed starting cluster creation \"ephemeral-disk\" in resourcegroup=\"ephemeral-osdisk-d5k7vz9fgmp5\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ephemeral-osdisk-d5k7vz9fgmp5/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/ephemeral-disk\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: LocationNotAvailableForResourceType\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LocationNotAvailableForResourceType\",\n        \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-5zxmn6\" with version 4.20.38 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e7afc0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-5zxmn6: failed starting cluster creation \"cluster-zstream-4-20-5zxmn6\" in resourcegroup=\"rg-zstream-upgrade-4-20-87qdz9hglgx4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-87qdz9hglgx4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-5zxmn6\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: LocationNotAvailableForResourceType\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LocationNotAvailableForResourceType\",\n        \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:123]: failed to create HCP cluster \"cluster-zstream-4-20-5zxmn6\" with version 4.20.38 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000e7afc0\u003e: \n    failed to create HCP cluster cluster-zstream-4-20-5zxmn6: failed starting cluster creation \"cluster-zstream-4-20-5zxmn6\" in resourcegroup=\"rg-zstream-upgrade-4-20-87qdz9hglgx4\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-zstream-upgrade-4-20-87qdz9hglgx4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-zstream-4-20-5zxmn6\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: LocationNotAvailableForResourceType\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LocationNotAvailableForResourceType\",\n        \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-25fcqw\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f12e80\u003e: \n    failed to create HCP cluster np-noedge-25fcqw: failed starting cluster creation \"np-noedge-25fcqw\" in resourcegroup=\"rg-np-noedge-25fcqw-mkk44mbtzfcl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-25fcqw-mkk44mbtzfcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-25fcqw\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: LocationNotAvailableForResourceType\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LocationNotAvailableForResourceType\",\n        \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:301]: failed to create HCP cluster np-noedge-25fcqw\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f12e80\u003e: \n    failed to create HCP cluster np-noedge-25fcqw: failed starting cluster creation \"np-noedge-25fcqw\" in resourcegroup=\"rg-np-noedge-25fcqw-mkk44mbtzfcl\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-noedge-25fcqw-mkk44mbtzfcl/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-noedge-25fcqw\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: LocationNotAvailableForResourceType\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"LocationNotAvailableForResourceType\",\n        \"message\": \"The provided location 'westus3' is not available for resource type 'Microsoft.RedHatOpenShift/HCPOpenShiftClusters'. List of available regions for the resource type is 'switzerlandnorth,canadacentral,australiaeast,westeurope,centralindia,brazilsouth,eastus2,uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"01df788dfb171c0668bb7074e3f88aca94749f84bfc4fdbd1a7fd91e08a7cfa0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"04637e18bfcb3f9e724928ed6c4c1cf581f36315f1bfeccdf3f80802e6356d9d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"08024cf849a325cbc5a336512e48936583c3f85cb4a3a2c148f103673d901cfe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"09ed45becaf29978462b1c3b2ec63dbc0323dfba597e299f51b743ca27d93962","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"0d104c19c86ac40a63eda038290eacd0119503471444454de0cfa1dd5d007ef7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"12e68ce9f4a1569a39aba64a40ff879ce989a6ff69156a6105233cf4609da0d5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"15f5f6ae828a46047c858aa69ce37554fa6dd71efe1a2cb7544916905c1ac2ff","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"15fba99f85a72c2b953f1221bc696b9bf82753423dac0791fefd94f24e82a3db","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"1d9102a2f82649aa9433fd7ca07f7f8568768752d9d21e09bf205c33862544e5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"1efd0a59df6c4ca49a656f7bba9764b2b191d6d3446eb06a4a269c70541d699f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"21ec5f782bbbcebf9c1bb08d8f3860277c036fb88121f67bd7df545279ec079f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"27e7bc710c289477425f6c73f134a06ae089c56c9d0c553da97d5810dc3c16dd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"27faee0e8e7daace9345589564265c839002d698d5b6bbaadbbd0fa8335b0ea3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"30e39a8546827489b79b43d722c71489114e6cee327dd845c522ff10e2a9e7f1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"3111af3121a7744fec69a5d201dca82460664234a0796e6e4967a0c4f1b2fb94","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"3557fb0d5e8fd6771781276cd1bd6b8e55cef0ac95abd5a7179bcf77ad589c58","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"39d2e978a6cb835985b2a92f5546d92872a269f4e9257aaf79741af5c03d26c5","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"53ef031d8e131f3373054e208b7c3fe7af1f963c377b7a456e7885180b60a0d6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"5603a2702aef8190ce731f49e3f6da6f4c67aef7c8a2f54bf3011d9751ff4929","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"5cedb6ab199d625ac70604831620fff130147fe82ac076113745bb3622573301","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"623139f8f571d2f5ae27e6d75e9b76c6e49ad61954e8cbea14bd347a34071a46","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"708386bc40e3cb953a5fa1b68320e189ab8f616758c7e5e8216987b789b6a323","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"736869058c088afa7ee0e36a8c8df936c6ebd0d3051c6c1de9701a4b9d0d1faa","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"7eec216fe53eb861071172ca28122a7fdf1495ace014b5cf90792e298e12b594","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"8d6a29106f6d7f3260c9698ad13eebf0c73b422c9bdeb8162463a6c655d9d521","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"8eab1b097f117f5fa387ae53997393440b33b00e85edf8ebaaec122145cf76b9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"90bf1a4bb0f55805455582cc5075cd286abac466f6ba8106c516801ad7b9f73f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"91e6b9daa46d3913f91a3381fa2d7d1c8603487001fbb4805d640eea2c9816bd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"97c7d0cc26d3fed4b372c2504b2ba57618874ad5b3a6f3eeea32900b0bdf71cb","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"996575218bcbe29d31c4ada2dd5e23243f75e74c9955fbef49e8df81a1d3822d","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"9baa846c847b6340711d575cf15d5a87dff938dfbaeea48c84709a07d2be7a00","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"9bb7d0520b881857aa31248f17345ac080812b7c304ff8cc09d37fcd8e3025d0","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"a322a676fef23495954c29cefc197a2d50b5612cdc27cedbae8d3022650db26a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"a83f70703c99a3e6b99485097810f7a5ce10697c954f41db95a0c6d9c746e8f3","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"aac0a4fadaa81917b45017334cfcee4d41249c967b1a67dc637c0244f20f2dce","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"aaef8153c4cd154e06971faa50b02660bd9e7ea2bbcc29a2f34394cf6867b48c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"ad047b313cd46ff403890ab91105d7e24c9d73411af2aa2c234da58252bd96f6","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"af5f6dc1f9242795b07b341d7cdab3660235226d8b0ef15223cbd5e68cdb26c1","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"b24288af3ae287714be949e08bf57141464dd56c19ed7d6236a3c80a3dac299a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"b5ec5970e9c7614d527f5257f02b1a83f847618cea746349aa1317c1475e49f9","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"b97260f6b219bb2a5cecd3306c8357f6c318ddfbf92f98f909285fe3dad5b88b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"bf64f485c1cc87388187d9c3b48f24e4866df71b94091b1f848430665c4c6977","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"c1f6def3d468e1d53dc22a82731f7fdf79b4f1e68b13db8097950ff0725b0301","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"c1fb319f0d065f046fccee3d75086273838a8ab728564590b0d0354f67fbb476","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"cc0de1274b9ceffba6b657f32cb1765d623a4c8525822e22472a4854ffdbdfd7","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"d030cec66b2abe74848996b1135963ec2f1a5deaa837b7c5222ce37dd6decfbe","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"d52eea2a457782a8ea36566d49d84e7651cc963ab39843912377b9d8c44a04c8","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"d8bdf92b2e9204714596bc6a2a6ce29e52d30033a7f33ed1bebc44c552ce7396","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"dfc4c8893a466ebc947358160114e3f5e45b2ef1a49b11beb24e101d649532cd","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"e10cca4cfd9139f298f73a8d53b94d84b35a5fdc8d47d507968b47ecc585cc9c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"e2c9151c9266bd39737c0145a83b25202e796e3680f387fe62de4718df836011","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"e4596221e8bb2095fe0e124bbbec22ca6138f5070eaf0a73880264e0ed679646","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"e56a54c25506fef4fd4c8605a192ca2458a0dba96b5fecb9ab73009654f3cea2","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"e62df2a6c20e4789165daee30b92b9ff0d162ba6cc3b030a9068c6492677d781","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"e6f223f3f936617d0a8b98f8caf439c033dc69e26b23715f31b40c276bc5de9f","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"eb955423d40836c1dcb045a85bcda0cb03498e55747b2de9e2048325db028477","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"f4de3b4a6d6871ebd4e70e0b4bd0fa3064ac875b32d2edd0ec307ea339df236b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"fd1596bb61848aaa7a93806f7b61359b36fdd12bacf8c47ddff014580b73c416","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"646a05af00a4f1877b65268791a98bf80e9b7eef40c1969d84b6dd766cce2436","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"4.21\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \"nightly\"; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":5,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":5,"after_last_push_count":10,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,0,5],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-16T23:00:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Authorized CIDRs Connectivity should allow API access only from authorized VM IP","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and manage ImageDigestMirrors","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster with custom autoscaling and update the autoscaling configuration","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Update HCPOpenShiftCluster Positive creates a cluster and updates tags with a PATCH request","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0019711d0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7dp4spk72w92/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_autoscaling.go:173]: failed to update cluster \"autoscaling-hcp-cluster\" autoscaling configuration\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0019711d0\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/autoscaling-cluster-7dp4spk72w92/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/autoscaling-hcp-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001566b10\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-z76xpv9lv7mg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_authorized_cidrs_connectivity.go:408]: failed to begin updating cluster \"cidr-connectivity-test\" authorized CIDRs\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc001566b10\u003e: \n    PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/e2e-cidr-connectivity-z76xpv9lv7mg/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cidr-connectivity-test\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0013a3320\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-cgk7n47ncfl4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_update.go:193]: failed to update HCP cluster tags via PATCH\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0013a3320\u003e: \n    PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/patch-tags-cgk7n47ncfl4/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/patch-tags-cluster\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.21\\\": must be specified as MAJOR.MINOR.PATCH (optionally with a pre-release, e.g. a nightly build suffix) when channelGroup is \\\"nightly\\\"\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"27740ed35f7486949fc1f199355bb1882c9f8cb9081d779fb41653a80527acd4","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"691dfc6d9fb2291ffc32616264d74d1f43851ef85e826708db94455617cad251","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"7fbe71528d20c91eedbc5d9a15147ed84d81599aec91f81260d0bbd325e1866b","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"9c871760b90c87c480ead77cf16197e08922039d3cb3658d834ce5592bb2917a","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"c5e7a15d5897753355cc4005fa9bf61ec537390259186723bdfc09da6d61808d","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"b962eb3a7ab867613434e2a4e89b0ddc6682535eed6ffd7e1cc734b6b5f6464f","failure_pattern":"ERROR CODE: InvalidRequestContent; detail message Invalid value: \"\u003cocp-nightly\u003e\": must be at least 4.20.8; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InvalidRequestContent","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":3,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":3,"after_last_push_count":7,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,0,3],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-16T23:00:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 4.20","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should downgrade a nodepool to a lower minor version from 4.22.zLatest to 4.20.zLatest","occurrences":1},{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade a nodepool skipping one minor version (+2) from 4.20.z to 4.22.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f5c1c0\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-k5htqv\" in resourcegroup=\"rg-np-skip-minor-k5htqv-87tsfgngkbxz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-k5htqv-87tsfgngkbxz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-k5htqv/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:459]: failed to create nodepool nps-420-422 at version 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000f5c1c0\u003e: \n    failed to create NodePool nps-420-422: failed starting nodepool creation \"nps-420-422\" for cluster \"np-skip-minor-k5htqv\" in resourcegroup=\"rg-np-skip-minor-k5htqv-87tsfgngkbxz\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-skip-minor-k5htqv-87tsfgngkbxz/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-skip-minor-k5htqv/nodePools/nps-420-422\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-g6wkqd\" for cluster \"cluster-nightly-4-20-g6wkqd\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000926600\u003e: \n    failed to create NodePool np-g6wkqd: failed starting nodepool creation \"np-g6wkqd\" for cluster \"cluster-nightly-4-20-g6wkqd\" in resourcegroup=\"rg-nightly-4-20-2j2bp6ng4gdp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-2j2bp6ng4gdp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-g6wkqd/nodePools/np-g6wkqd\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:176]: failed to create node pool \"np-g6wkqd\" for cluster \"cluster-nightly-4-20-g6wkqd\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000926600\u003e: \n    failed to create NodePool np-g6wkqd: failed starting nodepool creation \"np-g6wkqd\" for cluster \"cluster-nightly-4-20-g6wkqd\" in resourcegroup=\"rg-nightly-4-20-2j2bp6ng4gdp\": PUT https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-nightly-4-20-2j2bp6ng4gdp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/cluster-nightly-4-20-g6wkqd/nodePools/np-g6wkqd\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred","fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00151a140\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-4pqnxp\" resourcegroup=\"rg-np-dg-minor-4pqnxp-4lnmhzl52tvp\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-4pqnxp-4lnmhzl52tvp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-4pqnxp/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:766]: failed to update nodepool npdg-422-420 to downgrade target 4.20.0-0.nightly-multi-2026-09-23-090319\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00151a140\u003e: \n    failed to start nodepool \"npdg-422-420\" update in cluster \"np-dg-minor-4pqnxp\" resourcegroup=\"rg-np-dg-minor-4pqnxp-4lnmhzl52tvp\": PATCH https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-dg-minor-4pqnxp-4lnmhzl52tvp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/np-dg-minor-4pqnxp/nodePools/npdg-422-420\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: InvalidRequestContent\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"InvalidRequestContent\",\n        \"message\": \"Invalid value: \\\"4.20.0-0.nightly-multi-2026-09-23-090319\\\": must be at least 4.20.8\",\n        \"target\": \"properties.version.id\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"1b2eae294949b5b68cbe342df9ea515c496f1ce1130bf8c19dff2578413bca2c","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"2499142f7ad8e903e0f48a80ca5cf2771b399a595df8551871acaf3736c67691","pr_number":0,"after_last_push_of_merged_pr":true},{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"3b28ac3f4fe19a90ee317ea1737bd43f097cfc518ad408fd1fd876a861e94ad9","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d523a0d2817fdbba88731dd1892c07aa03075f35ab21021c384553c8aa80da9e","failure_pattern":"ERROR CODE: InternalServerError; detail message [cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP nightly channel for 5.1","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-wllchtq6tspw/cluster-nightly-5-1-xqrkdk should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000614980\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-xqrkdk: failed waiting for cluster=\"cluster-nightly-5-1-xqrkdk\" in resourcegroup=\"rg-nightly-5-1-wllchtq6tspw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8d90a29e-3743-4f19-9012-64f6e3adf3af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8d90a29e-3743-4f19-9012-64f6e3adf3af\",\n      \"name\": \"8d90a29e-3743-4f19-9012-64f6e3adf3af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T23:29:29.214562248Z\",\n      \"endTime\": \"2026-09-24T23:58:29.739929622Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:123]: HCP cluster rg-nightly-5-1-wllchtq6tspw/cluster-nightly-5-1-xqrkdk should provision\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000614980\u003e: \n    failed to create HCP cluster cluster-nightly-5-1-xqrkdk: failed waiting for cluster=\"cluster-nightly-5-1-xqrkdk\" in resourcegroup=\"rg-nightly-5-1-wllchtq6tspw\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8d90a29e-3743-4f19-9012-64f6e3adf3af\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/8d90a29e-3743-4f19-9012-64f6e3adf3af\",\n      \"name\": \"8d90a29e-3743-4f19-9012-64f6e3adf3af\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T23:29:29.214562248Z\",\n      \"endTime\": \"2026-09-24T23:58:29.739929622Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"[cosmosCluster] .api.url is empty; [hypershiftHostedCluster] ReadDesire has no kube content; [servingCABundle] ServingCABundle not yet populated\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"ccb025795b93e958b7aae9d388bb9cdb57b9a396064e500aff59a625a9f0e777","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"41dadc374b6fe2625e6ee7acdf8fbc9a991cbd05173905d55b61bd73224d1bb8","failure_pattern":"ERROR CODE: InternalServerError; detail message cluster creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["DEV"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-cf4l7z with version 4.21.0-0.nightly-multi-2026-09-24-033657\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00142c760\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-cf4l7z: failed waiting for cluster=\"np-version-upgrade-cluster-cf4l7z\" in resourcegroup=\"rg-np-version-upgrade-cf4l7z-c8gcs9n4rck7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/58578399-7a04-4d7e-9fff-65b3ebeb1ba4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/58578399-7a04-4d7e-9fff-65b3ebeb1ba4\",\n      \"name\": \"58578399-7a04-4d7e-9fff-65b3ebeb1ba4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T23:42:40.568397466Z\",\n      \"endTime\": \"2026-09-25T00:11:49.734192333Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:146]: failed to create HCP cluster np-version-upgrade-cluster-cf4l7z with version 4.21.0-0.nightly-multi-2026-09-24-033657\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc00142c760\u003e: \n    failed to create HCP cluster np-version-upgrade-cluster-cf4l7z: failed waiting for cluster=\"np-version-upgrade-cluster-cf4l7z\" in resourcegroup=\"rg-np-version-upgrade-cf4l7z-c8gcs9n4rck7\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/58578399-7a04-4d7e-9fff-65b3ebeb1ba4\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/58578399-7a04-4d7e-9fff-65b3ebeb1ba4\",\n      \"name\": \"58578399-7a04-4d7e-9fff-65b3ebeb1ba4\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T23:42:40.568397466Z\",\n      \"endTime\": \"2026-09-25T00:11:49.734192333Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"cluster creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"2c832c99908ab7e4af98af64b3ea848748c2d5da97c25a86fe752084f15d42f0","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"d5a27e23f87f5c18e786f32d10682561abfa5723896eb83d080ad6377beec74f","failure_pattern":"ERROR CODE: InternalServerError; detail message node pool creation did not complete before the deadline; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: InternalServerError","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":6,"also_in":["DEV","PROD"],"trend_counts":[1,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":4,"prior_last_seen_at":"2026-09-18T02:00:17Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to create an HCP cluster and custom node pool osDisk size","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d07ae0\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-xxwmzws6kxxc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85216ecc-6f15-4ee4-9411-ab10183b2cec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85216ecc-6f15-4ee4-9411-ab10183b2cec\",\n      \"name\": \"85216ecc-6f15-4ee4-9411-ab10183b2cec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T23:13:56.752561651Z\",\n      \"endTime\": \"2026-09-24T23:32:59.664565124Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_nodepool_osdisk.go:96]: failed to create node pool \"nodepool-128GiB\" with 128GiB osDisk\nUnexpected error:\n    \u003c*fmt.wrapError | 0xc000d07ae0\u003e: \n    failed to create NodePool nodepool-128GiB: failed waiting for nodepool=\"nodepool-128GiB\" for cluster \"hcp-cluster-np-128\" in resourcegroup=\"clusternp128-xxwmzws6kxxc\" to finish creating: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85216ecc-6f15-4ee4-9411-ab10183b2cec\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: InternalServerError\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/uksouth/hcpOperationStatuses/85216ecc-6f15-4ee4-9411-ab10183b2cec\",\n      \"name\": \"85216ecc-6f15-4ee4-9411-ab10183b2cec\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-09-24T23:13:56.752561651Z\",\n      \"endTime\": \"2026-09-24T23:32:59.664565124Z\",\n      \"error\": {\n        \"code\": \"InternalServerError\",\n        \"message\": \"node pool creation did not complete before the deadline\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"b1a2325b06cdb9a45a78b9f23c355fd6276e541cbba1065d34722c4530f70a01","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"68cf65637733da6389b66f0d98a7fd646026fde9e15cc697ce29959bae80ec43","failure_pattern":"ERROR CODE: NoRegisteredProviderFound; detail message No registered resource provider found for location 'westus3' and API version '2024-06-10-preview' for type 'locations/hcpOpenShiftVersions'. The supported api-versions are '2026-09-01-preview, 2026-06-30-preview, 2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2, uksouth'.; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: NoRegisteredProviderFound","failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":1,"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel","test_name":"Customer should be able to list available HCP OpenShift versions and validate response content","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0017589f0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOpenShiftVersions\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'westus3' and API version '2024-06-10-preview' for type 'locations/hcpOpenShiftVersions'. The supported api-versions are '2026-09-01-preview, 2026-06-30-preview, 2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2, uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_versions.go:43]: failed to list HCP OpenShift versions\nUnexpected error:\n    \u003c*exported.ResponseError | 0xc0017589f0\u003e: \n    GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOpenShiftVersions\n    --------------------------------------------------------------------------------\n    RESPONSE 400: 400 Bad Request\n    ERROR CODE: NoRegisteredProviderFound\n    --------------------------------------------------------------------------------\n    {\n      \"error\": {\n        \"code\": \"NoRegisteredProviderFound\",\n        \"message\": \"No registered resource provider found for location 'westus3' and API version '2024-06-10-preview' for type 'locations/hcpOpenShiftVersions'. The supported api-versions are '2026-09-01-preview, 2026-06-30-preview, 2025-12-23-preview, 2024-06-10-preview'. The supported locations are 'switzerlandnorth, canadacentral, australiaeast, westeurope, centralindia, brazilsouth, eastus2, uksouth'.\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/branch-ci-Azure-ARO-HCP-main-e2e-stage-e2e-parallel/2103214845615673344","occurred_at":"2026-09-24T20:07:48Z","signature_id":"847f7b17102dc7090d588a9d72db0f68b89cbbd60c61d629ca5d29b7d31645cd","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"28f998d2a4e958e0aea0f8902f23d28d43ccd641d96f3287420acefcd1caa571","failure_pattern":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","search_query":"VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":2,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":1,"prior_window_starts":["2026-09-14"],"prior_runs_affected":2,"prior_last_seen_at":"2026-09-16T23:00:16Z","contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to successfully upgrade control plane minor version from 5.0 minor to 5.1 minor","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0016823c0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-24-142420, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ...\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_minor_upgrade.go:176]: Timed out after 2700.002s.\nExpected success, but got an error:\n    \u003c*errors.joinError | 0xc0016823c0\u003e: \n    VerifyKubeAPIServerServerVersionUpgraded failed: kube-apiserver ServerVersion not updated (unchanged from pre-upgrade)\n    VerifyHostedControlPlaneYStreamUpgrade(previousMinor=5.0.0-0.nightly-multi-2026-09-24-142420, targetMinor=5.1.0-0.nightly-multi-2026-08-10-113053) failed: clusterversion status.history has no version in target minor \"5.1.0-0.nightly-multi-2026-08-10-113053\"\n    ..."],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"bb492d03ce36d043a7737e16613de6f3e8eaf5e4ffe34f36ac441cff010efa9a","pr_number":0,"after_last_push_of_merged_pr":true}]},{"environment":"stg","failure_pattern_id":"88c34ce3eb4dd39781e8ddf92764a71a4749619137d6e0e581fc858207a4e653","failure_pattern":"storage account log verification failed","search_query":"storage account log verification failed","failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1,"runs_affected":1,"failed_runs":1,"run_impact_percent":12.5,"anchor_occurrences":1,"after_last_push_count":1,"also_in":["PROD"],"trend_counts":[0,0,0,0,0,0,1],"trend_range":"2026-09-18..2026-09-24 UTC","prior_windows_present":0,"prior_runs_affected":1,"contributing_tests":[{"failed_at":"e2e","job_name":"periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly","test_name":"Customer should be able to forward control plane logs to a storage account and Event Hub via shoebox diagnostic settings","occurrences":1}],"full_error_samples":["fail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoebox6npjq8 and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-g4qtcfdblkfs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc000d18d20\u003e: \n    storage account log verification failed\n    ...\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/shoebox_logs.go:353]: failed to verify shoebox logs reached storage account shoebox6npjq8 and Event Hub shoebox-eh for cluster /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/shoebox-cluster-g4qtcfdblkfs/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/shoebox-hcp-cluster\nUnexpected error:\n    \u003c*errors.errorString | 0xc000d18d20\u003e: \n    storage account log verification failed\n    ...\noccurred"],"affected_runs":[{"run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/logs/periodic-ci-Azure-ARO-HCP-main-periodic-stage-e2e-parallel-ocp-nightly/2103258234264489985","occurred_at":"2026-09-24T23:00:14Z","signature_id":"2138dc55899dce5641331077f3e289aee19252fb625b5f07c0d419ec286f1599","pr_number":0,"after_last_push_of_merged_pr":true}]}]}]}