{"meta":{"date":"2026-10-26","timezone":"UTC","generated_at":"2026-10-29T00:40:54Z","environments":["dev"]},"environments":[{"environment":"dev","summary":{"total_runs":48,"failed_runs":22,"runs_with_occurrences":22,"runs_with_failure_pattern_matches":21,"runs_with_unmatched_occurrences":1,"runs_without_occurrence_rows":26,"failed_runs_without_occurrence_rows":0},"runs":[{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7766/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114861755761954816","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7766,"pr_state":"open","pr_sha":"aff22595609b9a130e727d24128b535ea8e7c2b3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-26T23:28:29Z","started_at":"2026-10-26T23:28:29Z","completed_at":"2026-10-27T01:30:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T01:47:10.231956791Z","timing_metadata_checked_at":"2026-10-27T01:47:10.231956791Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T01:47:10.23536538Z","region_metadata_checked_at":"2026-10-27T01:47:10.23536538Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1575127cc3e2611fdadf6e32b43a46269b9e9cab49b14012f82246bffa2794dc","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7766/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114861755761954816","occurred_at":"2026-10-26T23:28:29Z","failed_at":"alert","signature_id":"8a6cfe1f3b994f21ecf195a4cfd956a5b5ecbe8f556e74114048743a0dc4001d","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-27T00:54:17Z\n  Ended:       2026-10-27T01:00:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j1954816-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tmpuqe3u54c0c1kt3dhd472afk621tp-agg-cluster\", pod=\"ignition-server-proxy-cccb6f6bd-9jw8t\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tmpuqe3u54c0c1kt3dhd472afk621tp-agg-cluster/ignition-server-proxy-cccb6f6bd-9jw8t has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"ab9ade507672175ce763cf4285a35796d2830ca8d09bd4ff990b2265dedf9143","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7766/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114861755761954816","occurred_at":"2026-10-26T23:28:29Z","failed_at":"e2e","signature_id":"13be5e1341477c52eade788cb598f1cc4e1ab24ef4f65e65ff9ddcdba8af23bb","test_name":"SRE can pause schedules to stop backup execution for an HCP cluster","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.006s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true\nfail [github.com/Azure/ARO-HCP/test/e2e/hcp_backups.go:166]: Timed out after 660.006s.\nat least one schedule should have a LastBackupTime\nExpected\n    \u003cbool\u003e: false\nto be true","failure_pattern_match":{"status":"clustered","failure_pattern_id":"2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","failure_pattern":"at least one schedule should have a LastBackupTime","search_query":"at least one schedule should have a LastBackupTime"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["2375ce0c6cbf3a0f4e69c2abf000f7678072279ec079cc32ba518f83671ab5ba","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7232/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114852499935465472","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7232,"pr_state":"open","pr_sha":"9e25f85c60a7ff1275341208c42043e762b2d968","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-26T22:51:42Z","started_at":"2026-10-26T22:51:42Z","completed_at":"2026-10-27T01:08:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T01:26:15.501915323Z","timing_metadata_checked_at":"2026-10-27T01:26:15.501915323Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T01:26:15.505064613Z","region_metadata_checked_at":"2026-10-27T01:26:15.505064613Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7765/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114852453152198656","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7765,"pr_state":"open","pr_sha":"f613cd22bf0a5c2cac2222f48f31c80392f0da5b","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-26T22:51:31Z","started_at":"2026-10-26T22:51:31Z","completed_at":"2026-10-27T01:01:22Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T01:15:36.967049231Z","timing_metadata_checked_at":"2026-10-27T01:15:36.967049231Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T01:15:36.970724919Z","region_metadata_checked_at":"2026-10-27T01:15:36.970724919Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114852418565967872","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"bf8f8b8faa9aac6683c45c148f142ae9ebfaf703","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-26T22:51:23Z","started_at":"2026-10-26T22:51:23Z","completed_at":"2026-10-26T23:30:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T23:41:27.556881074Z","timing_metadata_checked_at":"2026-10-26T23:41:27.556881074Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T23:41:27.560784661Z","region_metadata_checked_at":"2026-10-26T23:41:27.560784661Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"0028736bb07de82400d19b2f728f3e50068bdd91c95053d0e9745be32045525a","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114852418565967872","occurred_at":"2026-10-26T22:51:23Z","failed_at":"provision","signature_id":"6285fb223b68ae5cc023810cff82298fb985932f6960b14b63ec7a0298c49add","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"templatize-pipeline","failure_text":"time=2026-10-26T23:19:14.439Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-26T23:19:15.908Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-26T23:19:17.082Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=facd493a02be1ad67e57b07e2fbd173b77c816ca0e9fe44bbdba2fb459fbeb85 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5967872-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2Ffacd493a02be1ad67e57b07e2fbd173b77c816ca0e9fe44bbdba2fb459fbeb85\ntime=2026-10-26T23:21:47.533Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster step=cluster error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5967872-svc/providers/Microsoft.Resources/deployments/facd493a02be1ad67e57b07e2fbd173b77c816ca0e9fe44bbdba2fb459fbeb85/operationStatuses/08584085501288714898\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5967872-svc/providers/Microsoft.Resources/deployments/cluster-wsenpawidiq6k\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j5967872-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j5967872-svc\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\\\\\\\\\"https://ah-ci01-se-j5967872-1.vault.azure.net\\\\\\\\\\\\\\\"\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-10-26T23:23:47.535Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-26T23:23:48.327Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-26T23:23:49.402Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=40b45e9bebcc2cf533d290687c03a069a8f711a71e9ba5f2836d47dbdaf9f362 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j5967872-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F40b45e9bebcc2cf533d290687c03a069a8f711a71e9ba5f2836d47dbdaf9f362\ntime=2026-10-26T23:26:19.774Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j5967872-svc/providers/Microsoft.Resources/deployments/40b45e9bebcc2cf533d290687c03a069a8f711a71e9ba5f2836d47dbdaf9f362/operationStatuses/08584085498565391842\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Failed to obtain the resource body, because the resource is currently in a 'Failed' provisioning state. If this resource has not been modified by a prior step in this deployment, you may need to address this before retrying the deployment.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"470629bcf987e8ae3a847a29e3d9d8ae832b84884a6d81ff75a201bcfd4c5ea1","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AzureKeyVaultKmsValidateIdentityPermissionCustomerError; detail message The identity does not have keys encrypt/decrypt permission on key vault \"\u003curl\u003e\"; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["470629bcf987e8ae3a847a29e3d9d8ae832b84884a6d81ff75a201bcfd4c5ea1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114843878455513088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7615,"pr_state":"closed","pr_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","final_merged_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-26T22:17:27Z","started_at":"2026-10-26T22:17:27Z","completed_at":"2026-10-27T00:24:15Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T00:44:13.136192904Z","timing_metadata_checked_at":"2026-10-27T00:44:13.136192904Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T00:44:13.140337191Z","region_metadata_checked_at":"2026-10-27T00:44:13.140337191Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"b696aed206abecb93981ee0da7edb01b209dc747bbf101e3ce47e855cf32edca","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114843878455513088","occurred_at":"2026-10-26T22:17:27Z","failed_at":"e2e","signature_id":"f6e115c9c6b9508390354502456ea7c5a9362dfff4b8c4d0e262e286ea1ff0bb","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ffmm65nf5h4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x37dc0b3be620\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ffmm65nf5h4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j5513088-mgmt-1/ocm-arohcpci01-2tmouldaggce1qoqe6i1onkn6rsmlu83-n4n8m4n5t5x8i6x router=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001: expected matching summary after test start and within 8m; summaryUID=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001 pass=0c2770ec-0dde-47a1-b3d7-288aa5e5e851 at=2026-10-26T23:34:57Z\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ffmm65nf5h4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j5513088-mgmt-1/ocm-arohcpci01-2tmouldaggce1qoqe6i1onkn6rsmlu83-n4n8m4n5t5x8i6x router=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001: expected matching summary after test start and within 8m; summaryUID=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001 pass=0c2770ec-0dde-47a1-b3d7-288aa5e5e851 at=2026-10-26T23:34:57Z\",\n        err: \u003c*errors.errorString | 0x37dc0b40dbb0\u003e{\n            s: \"dev/centralus/ci01-j5513088-mgmt-1/ocm-arohcpci01-2tmouldaggce1qoqe6i1onkn6rsmlu83-n4n8m4n5t5x8i6x router=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001: expected matching summary after test start and within 8m; summaryUID=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001 pass=0c2770ec-0dde-47a1-b3d7-288aa5e5e851 at=2026-10-26T23:34:57Z\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ffmm65nf5h4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x37dc0b3be620\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ffmm65nf5h4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j5513088-mgmt-1/ocm-arohcpci01-2tmouldaggce1qoqe6i1onkn6rsmlu83-n4n8m4n5t5x8i6x router=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001: expected matching summary after test start and within 8m; summaryUID=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001 pass=0c2770ec-0dde-47a1-b3d7-288aa5e5e851 at=2026-10-26T23:34:57Z\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ffmm65nf5h4s/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j5513088-mgmt-1/ocm-arohcpci01-2tmouldaggce1qoqe6i1onkn6rsmlu83-n4n8m4n5t5x8i6x router=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001: expected matching summary after test start and within 8m; summaryUID=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001 pass=0c2770ec-0dde-47a1-b3d7-288aa5e5e851 at=2026-10-26T23:34:57Z\",\n        err: \u003c*errors.errorString | 0x37dc0b40dbb0\u003e{\n            s: \"dev/centralus/ci01-j5513088-mgmt-1/ocm-arohcpci01-2tmouldaggce1qoqe6i1onkn6rsmlu83-n4n8m4n5t5x8i6x router=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001: expected matching summary after test start and within 8m; summaryUID=03348f59-5de2-40fa-9a2e-5c7b8f5ae9bc node=aks-userswft3-23912472-vmss000001 pass=0c2770ec-0dde-47a1-b3d7-288aa5e5e851 at=2026-10-26T23:34:57Z\",\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"4ffb59127307569342e99497be1f78f494e95d4f80f4e55bcb60b61f19b7e0d4","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.006s: dev/centralus/ci01-j5513088-mgmt-1/ocm-arohcpci01-\u003cid\u003e","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["4ffb59127307569342e99497be1f78f494e95d4f80f4e55bcb60b61f19b7e0d4"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7065/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114840265222000640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7065,"pr_state":"open","pr_sha":"735287ff3d2295b09d5aa4ed7235fac2f8c309b2","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-26T22:03:05Z","started_at":"2026-10-26T22:03:05Z","completed_at":"2026-10-27T00:16:27Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T00:33:44.954869273Z","timing_metadata_checked_at":"2026-10-27T00:33:44.954869273Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T00:33:44.958021963Z","region_metadata_checked_at":"2026-10-27T00:33:44.958021963Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114834707903418368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6661,"pr_state":"closed","pr_sha":"7c831308453ca033cf79b5aa7fd5635c2c1ed392","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-26T21:41:00Z","started_at":"2026-10-26T21:41:00Z","completed_at":"2026-10-26T23:57:57Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-27T00:12:24.781564342Z","timing_metadata_checked_at":"2026-10-27T00:12:24.781564342Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-27T00:12:24.78495693Z","region_metadata_checked_at":"2026-10-27T00:12:24.78495693Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"14ff8fe3d50a519152727824532951e5dd320c637e6970f9d58059ee328ad94f","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6661/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114834707903418368","occurred_at":"2026-10-26T21:41:00Z","failed_at":"e2e","signature_id":"f156ee19e1bea684566a9ff95c7087780488a5460193b9635d1b6c6e9093f7f5","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: failed to create node pool \"np-csnz6d\" for cluster \"cluster-candidate-5-0-csnz6d\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1a5d7d0bada0\u003e: \n    failed to create NodePool np-csnz6d: failed waiting for nodepool=\"np-csnz6d\" for cluster \"cluster-candidate-5-0-csnz6d\" in resourcegroup=\"rg-candidate-5-0-x5kw7d69w7fz\" to finish creating: GET https://rp.j3418368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/27744955-3d23-4917-a3e3-fea7a42353f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/27744955-3d23-4917-a3e3-fea7a42353f6\",\n      \"name\": \"27744955-3d23-4917-a3e3-fea7a42353f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T22:49:05.815309572Z\",\n      \"endTime\": \"2026-10-26T23:08:05.399599048Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l5s5h7x3f2f3n4c-np-csnz6d-9wh5b-gfx8x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-x5kw7d69w7fz-candidate-csnz6d--managed/providers/Microsoft.Compute/virtualMachines/l5s5h7x3f2f3n4c-np-csnz6d-9wh5b-gfx8x to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l5s5h7x3f2f3n4c-np-csnz6d-9wh5b-gfx8x: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-5-0-x5kw7d69w7fz-candidate-csnz6d--managed/l5s5h7x3f2f3n4c-np-csnz6d-9wh5b-gfx8x-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-x5kw7d69w7fz-candidate-csnz6d--managed/providers/Microsoft.Network/networkInterfaces/l5s5h7x3f2f3n4c-np-csnz6d-9wh5b-gfx8x-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'fd35ac5f-6493-40cb-9131-d669a70114c3' exceeded. Please try again after '1' sec... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3418368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/27744955-3d23-4917-a3e3-fea7a42353f6\n            correlation ID: aca41f08-bc3a-4eb0-aeee-dbec04407cb6\n            client request ID: 503f9ad7-c50a-4e8b-9147-306bbfbcb5a4\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:220]: failed to create node pool \"np-csnz6d\" for cluster \"cluster-candidate-5-0-csnz6d\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1a5d7d0bada0\u003e: \n    failed to create NodePool np-csnz6d: failed waiting for nodepool=\"np-csnz6d\" for cluster \"cluster-candidate-5-0-csnz6d\" in resourcegroup=\"rg-candidate-5-0-x5kw7d69w7fz\" to finish creating: GET https://rp.j3418368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/27744955-3d23-4917-a3e3-fea7a42353f6\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/27744955-3d23-4917-a3e3-fea7a42353f6\",\n      \"name\": \"27744955-3d23-4917-a3e3-fea7a42353f6\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T22:49:05.815309572Z\",\n      \"endTime\": \"2026-10-26T23:08:05.399599048Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 0, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine l5s5h7x3f2f3n4c-np-csnz6d-9wh5b-gfx8x: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-x5kw7d69w7fz-candidate-csnz6d--managed/providers/Microsoft.Compute/virtualMachines/l5s5h7x3f2f3n4c-np-csnz6d-9wh5b-gfx8x to exist\\n; node pool condition AllMachinesReady is False: 2 of 2 machines are not ready\\nMachine l5s5h7x3f2f3n4c-np-csnz6d-9wh5b-gfx8x: Failed: interfaces failed to create or update. err: failed to get existing resource rg-candidate-5-0-x5kw7d69w7fz-candidate-csnz6d--managed/l5s5h7x3f2f3n4c-np-csnz6d-9wh5b-gfx8x-nic (service: interfaces): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-candidate-5-0-x5kw7d69w7fz-candidate-csnz6d--managed/providers/Microsoft.Network/networkInterfaces/l5s5h7x3f2f3n4c-np-csnz6d-9wh5b-gfx8x-nic\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor 'fd35ac5f-6493-40cb-9131-d669a70114c3' exceeded. Please try again after '1' sec... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3418368.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/27744955-3d23-4917-a3e3-fea7a42353f6\n            correlation ID: aca41f08-bc3a-4eb0-aeee-dbec04407cb6\n            client request ID: 503f9ad7-c50a-4e8b-9147-306bbfbcb5a4\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"bf2604f0a57cc83f42180ea2d978fdb012e4de2c6d11e90a5b15c16b2e73342d","failure_pattern":"ERROR CODE: DeadlineExceeded; detail code SubscriptionRequestsThrottled; detail message Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '\u003cuuid\u003e' exceeded. Please try again after '1' sec... message truncated; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["bf2604f0a57cc83f42180ea2d978fdb012e4de2c6d11e90a5b15c16b2e73342d"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7727/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114824913121447936","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7727,"pr_state":"open","pr_sha":"a37be9689955179d7d10d530e2021e6edd24375a","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-26T21:02:05Z","started_at":"2026-10-26T21:02:05Z","completed_at":"2026-10-26T23:23:39Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T23:41:27.681570176Z","timing_metadata_checked_at":"2026-10-26T23:41:27.681570176Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T23:41:27.684643467Z","region_metadata_checked_at":"2026-10-26T23:41:27.684643467Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114823017199570944","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7177,"pr_state":"closed","pr_sha":"27fe8fe46cb154c4b4ff583dea5a4d9ad0f93dd9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-26T20:54:33Z","started_at":"2026-10-26T20:54:33Z","completed_at":"2026-10-26T23:00:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T23:20:47.175038353Z","timing_metadata_checked_at":"2026-10-26T23:20:47.175038353Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T23:20:47.178327943Z","region_metadata_checked_at":"2026-10-26T23:20:47.178327943Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"252e62a8964fe4ad215bb7baea4aea4dd04104d5e6b94f4e3042acea4263dd08","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7177/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114823017199570944","occurred_at":"2026-10-26T20:54:33Z","failed_at":"e2e","signature_id":"e7ac52e3e916ffb1ddc5bfb303098335d5fc083fc3a4ba8e96d544894507de5c","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plbw7phvf9sp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2148d2b87580\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plbw7phvf9sp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plbw7phvf9sp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*verifiers.pollObservationError | 0x2148d2b87560\u003e{\n            observation: \u003c*errors.errorString | 0x2148d2b63740\u003e{\n                s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2148d2b87540\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"canadacentral\\\"; let cl = \\\"ci01-j9570944-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tmnl5tm8ds97r0ebee6bni26dkgbogo-q6u2e1q1r9e8l2c\\\"; let start = datetime(2026-10-26T21:38:32.1507932Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x2148d2b84b90\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x2148d2b87500\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"canadacentral\\\"; let cl = \\\"ci01-j9570944-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tmnl5tm8ds97r0ebee6bni26dkgbogo-q6u2e1q1r9e8l2c\\\"; let start = datetime(2026-10-26T21:38:32.1507932Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x2148d332b860\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plbw7phvf9sp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2148d2b87580\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plbw7phvf9sp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-plbw7phvf9sp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n        err: \u003c*verifiers.pollObservationError | 0x2148d2b87560\u003e{\n            observation: \u003c*errors.errorString | 0x2148d2b63740\u003e{\n                s: \"no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e fleet management cluster mapping in Kusto\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2148d2b87540\u003e{\n                msg: \"query SWIFT checks in https://hcp-dev-us-2.eastus2.kusto.windows.net/ServiceLogs: Op(OpQuery): Kind(KHTTPError): With query: let env = \\\"dev\\\"; let rgn = \\\"canadacentral\\\"; let cl = \\\"ci01-j9570944-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tmnl5tm8ds97r0ebee6bni26dkgbogo-q6u2e1q1r9e8l2c\\\"; let start = datetime(2026-10-26T21:38:32.1507932Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                err: \u003c*errors.Error | 0x2148d2b84b90\u003e{\n                    Op: 1,\n                    Kind: 7,\n                    Err: \u003c*fmt.wrapError | 0x2148d2b87500\u003e{\n                        msg: \"With query: let env = \\\"dev\\\"; let rgn = \\\"canadacentral\\\"; let cl = \\\"ci01-j9570944-mgmt-1\\\"; let ns = \\\"ocm-arohcpci01-2tmnl5tm8ds97r0ebee6bni26dkgbogo-q6u2e1q1r9e8l2c\\\"; let start = datetime(2026-10-26T21:38:32.1507932Z);\\nlet scope = strcat(env, \\\"/\\\", rgn, \\\"/\\\", cl, \\\"/\\\", ns);\\nlet inventory = materialize(kubernetesResourceSnapshots\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl and namespace == ns\\n| where objectKind in (\\\"Pod\\\", \\\"Deployment\\\")\\n| summarize arg_max(timestamp, *) by objectKind, name\\n| where event != \\\"Delete\\\" and isempty(object.metadata.deletionTimestamp));\\nlet pods = inventory\\n| where objectKind == \\\"Pod\\\" and object.metadata.labels.app == \\\"private-router\\\" and object.status.phase == \\\"Running\\\"\\n| project UID = uid, Node = tostring(object.spec.nodeName);\\nlet desired = toscalar(inventory\\n| where objectKind == \\\"Deployment\\\" and name == \\\"router\\\"\\n| project tolong(object.spec.replicas));\\nlet current = toscalar(pods | count);\\nlet summaries = containerLogs\\n| where timestamp \u003e= start and environment == env and region == rgn and cluster == cl\\n| where container_name == \\\"swift-recorder\\\"\\n| where log.controller_name == \\\"swift-router-check\\\" and log.field == \\\"summary\\\"\\n| where tostring(log.pod_namespace) == ns\\n| extend UID = tostring(log.pod_uid)\\n| summarize arg_max(timestamp, *) by UID\\n| project UID, SummaryUID = tostring(log.pod_uid),\\n    SummaryNode = tostring(log.node_name), Pass = tostring(log.pass_id), At = timestamp, Summary = tostring(log.record);\\npods\\n| join kind=leftouter summaries on UID\\n| extend Scope = scope, Desired = coalesce(desired, tolong(0)), Current = coalesce(current, tolong(0))\\n| project Scope, Desired, Current, UID, Node, SummaryUID, SummaryNode, Pass, At, Summary\\n| order by Scope asc, UID asc\\n| take 101, Post \\\"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\\\": context deadline exceeded\",\n                        err: \u003c*url.Error | 0x2148d332b860\u003e{\n                            Op: \"Post\",\n                            URL: \"https://hcp-dev-us-2.eastus2.kusto.windows.net/v2/rest/query\",\n                            Err: \u003ccontext.deadlineExceededError\u003e{},\n                        },\n                    },\n                    restErrMsg: nil,\n                    decoded: nil,\n                    permanent: false,\n                    inner: nil,\n                },\n            },\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"9cd80d6b45c8eb9b1b33eb30b789a71289e43bb10694295e955e5ee50ed8fa41","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: no customer resource ID -\u003e live ReadDesire HostedCluster -\u003e","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["9cd80d6b45c8eb9b1b33eb30b789a71289e43bb10694295e955e5ee50ed8fa41"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6928/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114786352468856832","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6928,"pr_state":"closed","pr_sha":"afd3417980c4b5052b54fdf377e648bf6df383fc","final_merged_sha":"afd3417980c4b5052b54fdf377e648bf6df383fc","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-26T18:28:45Z","started_at":"2026-10-26T18:28:45Z","completed_at":"2026-10-26T20:37:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T20:44:55.30538416Z","timing_metadata_checked_at":"2026-10-26T20:44:55.30538416Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T20:44:55.309094148Z","region_metadata_checked_at":"2026-10-26T20:44:55.309094148Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114783564531765248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7598,"pr_state":"closed","pr_sha":"abff6048ddb84ee866a5561f09c473ae10d09d8a","final_merged_sha":"be965db2bf14d58e471ed21952ec3e210e01ecba","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-26T18:17:38Z","started_at":"2026-10-26T18:17:38Z","completed_at":"2026-10-26T18:45:30Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T18:49:39.840131288Z","timing_metadata_checked_at":"2026-10-26T18:49:39.840131288Z","region_metadata_state":"missing","region_metadata_first_checked_at":"2026-10-26T18:49:39.845549668Z","region_metadata_checked_at":"2026-10-26T19:52:57.63047026Z"},"failed_at":["unknown"],"failed_test_count":1,"occurrences":[{"row_id":"aea34322e8315e1a8b801651c489a0a66fef04ed3614d4276407b55a8ae8ea58","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7598/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114783564531765248","occurred_at":"2026-10-26T18:17:38Z","failed_at":"unknown","signature_id":"08d65f2a32fc2d3601bceb3a0f5f910617daaa0297f21afaf54c12f8dd10fb06","test_name":"unknown","test_suite":"unknown","failure_text":"non-artifact-backed failure (no junit artifacts)","non_artifact_backed":true,"failure_pattern_match":{"status":"unmatched"}}],"failure_pattern_summary":{"signature_count":1,"matched_occurrences":0,"unmatched_occurrences":1,"match_summary":"unmatched_only"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114776506176638976","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7615,"pr_state":"closed","pr_sha":"371a1d9c97b9d83ddeaea18ca4f6892b872c913a","final_merged_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-26T17:49:44Z","started_at":"2026-10-26T17:49:44Z","completed_at":"2026-10-26T20:03:44Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T20:13:15.585080344Z","timing_metadata_checked_at":"2026-10-26T20:13:15.585080344Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T20:13:15.588920731Z","region_metadata_checked_at":"2026-10-26T20:13:15.588920731Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"07598e31557d92af90c0583827c63c4edd00ea34dd1c5f53ecbcca8d2224d07b","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114776506176638976","occurred_at":"2026-10-26T17:49:44Z","failed_at":"e2e","signature_id":"f783a774c26e0e408f8fd53ae4f7628289675d69eccb5955b7816986c2ccbbe6","test_name":"ARO-HCP should be able to perform a control plane and node pool install with OCP candidate channel for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:248]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-99qq4z\"\nUnexpected error:\n    \u003c*errors.joinError | 0x39eb4f22870\u003e: \n    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\": nodes \"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x39eb302e180\u003e{\n                msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\\\": nodes \\\"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                err: \u003c*fmt.wrapError | 0x39eb302e120\u003e{\n                    msg: \"failed to label node x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\\\": nodes \\\"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                    err: \u003c*fmt.wrapError | 0x39eb302e100\u003e{\n                        msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\\\": nodes \\\"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                        err: \u003c*errors.StatusError | 0x39eb3896140\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"nodes \\\"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                                Reason: \"Forbidden\",\n                                Details: {\n                                    Name: \"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\",\n                                    Group: \"\",\n                                    Kind: \"nodes\",\n                                    UID: \"\",\n                                    Causes: nil,\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 403,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/complete_cluster_create_multiversion.go:248]: failed to verify cluster operators on cluster \"cluster-candidate-4-23-99qq4z\"\nUnexpected error:\n    \u003c*errors.joinError | 0x39eb4f22870\u003e: \n    VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\": nodes \"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\n    {\n        errs: [\n            \u003c*fmt.wrapError | 0x39eb302e180\u003e{\n                msg: \"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\\\": nodes \\\"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                err: \u003c*fmt.wrapError | 0x39eb302e120\u003e{\n                    msg: \"failed to label node x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \\\"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\\\": nodes \\\"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                    err: \u003c*fmt.wrapError | 0x39eb302e100\u003e{\n                        msg: \"failed to apply k8s.ovn.org/egress-assignable label to node \\\"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\\\": nodes \\\"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                        err: \u003c*errors.StatusError | 0x39eb3896140\u003e{\n                            ErrStatus: {\n                                TypeMeta: {Kind: \"\", APIVersion: \"\"},\n                                ListMeta: {\n                                    SelfLink: \"\",\n                                    ResourceVersion: \"\",\n                                    Continue: \"\",\n                                    RemainingItemCount: nil,\n                                    ShardInfo: nil,\n                                },\n                                Status: \"Failure\",\n                                Message: \"nodes \\\"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\\\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized\",\n                                Reason: \"Forbidden\",\n                                Details: {\n                                    Name: \"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\",\n                                    Group: \"\",\n                                    Kind: \"nodes\",\n                                    UID: \"\",\n                                    Causes: nil,\n                                    RetryAfterSeconds: 0,\n                                },\n                                Code: 403,\n                            },\n                        },\n                    },\n                },\n            },\n        ],\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527","failure_pattern":"VerifyCloudNetworkConfigEgressIP(\u003cip\u003e) failed: failed to label node \u003cnode\u003e egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"\u003cnode\u003e\": nodes \"\u003cnode\u003e\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not","search_query":"VerifyCloudNetworkConfigEgressIP(10.0.0.7) failed: failed to label node x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd egress-assignable: failed to apply k8s.ovn.org/egress-assignable label to node \"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\": nodes \"x7h3r3l6v2j3k1a-np-99qq4z-78ps4-4mpbd\" is forbidden: autoscaling.openshift.io/ManagedNode infra config cache not synchronized"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d4fcc16840f214747419d557674768b34dc7aa5f94595caea2a96e34f8c43527"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7721/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114776114332176384","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7721,"pr_state":"closed","pr_sha":"512407c6bf6f42ced2b9c7913b38ae4065c26e52","final_merged_sha":"512407c6bf6f42ced2b9c7913b38ae4065c26e52","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-26T17:48:10Z","started_at":"2026-10-26T17:48:10Z","completed_at":"2026-10-26T20:14:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T20:23:30.989338477Z","timing_metadata_checked_at":"2026-10-26T20:23:30.989338477Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T20:23:30.992758966Z","region_metadata_checked_at":"2026-10-26T20:23:30.992758966Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7758/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114755947703308288","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7758,"pr_state":"closed","pr_sha":"0ac694ac43832a4600a8ef824e6e4eefd23dd44c","final_merged_sha":"0ac694ac43832a4600a8ef824e6e4eefd23dd44c","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-26T16:28:02Z","started_at":"2026-10-26T16:28:02Z","completed_at":"2026-10-26T18:36:06Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T18:39:06.35497785Z","timing_metadata_checked_at":"2026-10-26T18:39:06.35497785Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T18:39:06.359357137Z","region_metadata_checked_at":"2026-10-26T18:39:06.359357137Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7693/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114753752853385216","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7693,"pr_state":"open","pr_sha":"7fda8f5d8dba29d8478822d1e380cf9f77345f35","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-26T16:19:19Z","started_at":"2026-10-26T16:19:19Z","completed_at":"2026-10-26T18:31:41Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T18:39:05.981852573Z","timing_metadata_checked_at":"2026-10-26T18:39:05.981852573Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T18:39:05.985068463Z","region_metadata_checked_at":"2026-10-26T18:39:05.985068463Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7754/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114752166043324416","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7754,"pr_state":"closed","pr_sha":"963a25104c8c5a5f3cc865ba13e8b03d18f16614","final_merged_sha":"963a25104c8c5a5f3cc865ba13e8b03d18f16614","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-26T16:13:01Z","started_at":"2026-10-26T16:13:01Z","completed_at":"2026-10-26T18:31:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T18:39:06.163435127Z","timing_metadata_checked_at":"2026-10-26T18:39:06.163435127Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T18:39:06.166709217Z","region_metadata_checked_at":"2026-10-26T18:39:06.166709217Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6982/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114751084755947520","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6982,"pr_state":"open","pr_sha":"d4992cb73d7ebdc483c820f322f1b4963d3e3dd6","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-26T16:08:31Z","started_at":"2026-10-26T16:08:31Z","completed_at":"2026-10-26T18:33:12Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T18:39:05.831110326Z","timing_metadata_checked_at":"2026-10-26T18:39:05.831110326Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T18:39:05.843634088Z","region_metadata_checked_at":"2026-10-26T18:39:05.843634088Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7725/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114739280365490176","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7725,"pr_state":"closed","pr_sha":"8862a4408f78ce78ac6f40b0600e9115c2e3ace8","final_merged_sha":"ddcdde4e996a0a299f14edd71f3fe9cfa025ce4a","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-26T15:21:47Z","started_at":"2026-10-26T15:21:47Z","completed_at":"2026-10-26T17:32:23Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T17:36:25.876207158Z","timing_metadata_checked_at":"2026-10-26T17:36:25.876207158Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T17:36:25.879543947Z","region_metadata_checked_at":"2026-10-26T17:36:25.879543947Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7763/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114736843655221248","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7763,"pr_state":"closed","pr_sha":"7e4f15985695edd82b80171a8292778bdd25a431","final_merged_sha":"7e4f15985695edd82b80171a8292778bdd25a431","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-26T15:12:08Z","started_at":"2026-10-26T15:12:08Z","completed_at":"2026-10-26T17:24:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T17:26:12.310990211Z","timing_metadata_checked_at":"2026-10-26T17:26:12.310990211Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T17:26:12.313752302Z","region_metadata_checked_at":"2026-10-26T17:26:12.313752302Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114736549408018432","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7688,"pr_state":"open","pr_sha":"a0818a8a12dfdbd850bee76b608c672a59a7f3b4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-26T15:10:57Z","started_at":"2026-10-26T15:10:57Z","completed_at":"2026-10-26T17:16:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T17:26:12.166632054Z","timing_metadata_checked_at":"2026-10-26T17:26:12.166632054Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T17:26:12.169561245Z","region_metadata_checked_at":"2026-10-26T17:26:12.169561245Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"4cf8a86f9403951fb90a5d0d8ac6777e7c5b3a0d65c64501a757b04bc4060ed6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114736549408018432","occurred_at":"2026-10-26T15:10:57Z","failed_at":"e2e","signature_id":"6cd1d0f68d1e9c651ce9d74f373226e952ca947646d805fdb880bdd4112f9954","test_name":"Customer should upgrade and update a nodepool from 4.20.z to 4.21.zLatest","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.44\nUnexpected error:\n    \u003c*fmt.wrapError | 0x75e04e9f9a0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-lmxnc7\" in resourcegroup=\"rg-np-version-upgrade-lmxnc7-q9mf48fq597k\" to finish creating: GET https://rp.j8018432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d40573b8-7e51-417d-bd40-a08b0b1a57bd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d40573b8-7e51-417d-bd40-a08b0b1a57bd\",\n      \"name\": \"d40573b8-7e51-417d-bd40-a08b0b1a57bd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T16:25:47.925761101Z\",\n      \"endTime\": \"2026-10-26T16:44:52.193515957Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s7q2t8r2n3l4t7i-npupgrade-4-20-29l2x-hnzm5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lmxnc7-q9mf48fq597k-np-upgrade-lm-08336457/providers/Microsoft.Compute/virtualMachines/s7q2t8r2n3l4t7i-npupgrade-4-20-29l2x-hnzm5 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s7q2t8r2n3l4t7i-npupgrade-4-20-29l2x-hnzm5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lmxnc7-q9mf48fq597k-np-upgrade-lm-08336457/providers/Microsoft.Compute/virtualMachines/s7q2t8r2n3l4t7i-npupgrade-4-20-29l2x-hnzm5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8018432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d40573b8-7e51-417d-bd40-a08b0b1a57bd\n            correlation ID: f7b1fc07-103a-435d-9acb-75f17e0334fb\n            client request ID: 9e0fce66-a04c-49ec-96a9-4c1b9377e626\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_version_upgrade.go:165]: failed to create node pool npupgrade-4-20 with version 4.20.44\nUnexpected error:\n    \u003c*fmt.wrapError | 0x75e04e9f9a0\u003e: \n    failed to create NodePool npupgrade-4-20: failed waiting for nodepool=\"npupgrade-4-20\" for cluster \"np-version-upgrade-cluster-lmxnc7\" in resourcegroup=\"rg-np-version-upgrade-lmxnc7-q9mf48fq597k\" to finish creating: GET https://rp.j8018432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d40573b8-7e51-417d-bd40-a08b0b1a57bd\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d40573b8-7e51-417d-bd40-a08b0b1a57bd\",\n      \"name\": \"d40573b8-7e51-417d-bd40-a08b0b1a57bd\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T16:25:47.925761101Z\",\n      \"endTime\": \"2026-10-26T16:44:52.193515957Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine s7q2t8r2n3l4t7i-npupgrade-4-20-29l2x-hnzm5: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lmxnc7-q9mf48fq597k-np-upgrade-lm-08336457/providers/Microsoft.Compute/virtualMachines/s7q2t8r2n3l4t7i-npupgrade-4-20-29l2x-hnzm5 to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine s7q2t8r2n3l4t7i-npupgrade-4-20-29l2x-hnzm5: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/rg-np-version-upgrade-lmxnc7-q9mf48fq597k-np-upgrade-lm-08336457/providers/Microsoft.Compute/virtualMachines/s7q2t8r2n3l4t7i-npupgrade-4-20-29l2x-hnzm5 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8018432.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/d40573b8-7e51-417d-bd40-a08b0b1a57bd\n            correlation ID: f7b1fc07-103a-435d-9acb-75f17e0334fb\n            client request ID: 9e0fce66-a04c-49ec-96a9-4c1b9377e626\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003ccount\u003e; node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d778aa42bcddfee24e777ab96568f19ff3b498fff9280528158778526e86eb53"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114732780888788992","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"ec6bc0e1e05f027eb4a5cd4688d78aeb7012f915","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-26T14:55:59Z","started_at":"2026-10-26T14:55:59Z","completed_at":"2026-10-26T16:57:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T17:04:59.263842655Z","timing_metadata_checked_at":"2026-10-26T17:04:59.263842655Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T17:04:59.274385621Z","region_metadata_checked_at":"2026-10-26T17:04:59.274385621Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"603a345044f393c44cc1303d423ad0c207d5d8bcbd14bb03058deef47fc5c21d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114732780888788992","occurred_at":"2026-10-26T14:55:59Z","failed_at":"alert","signature_id":"cece8a7f4a2f9a33b54d6b23adca1a5ac146b55ffe47ba10a70c28b7a7652d75","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-26T15:55:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8788992-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.51:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8788992/providers/microsoft.monitor/accounts/services-j8788992\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-n8rf5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d66701f9-547c-4900-999a-9cc3801125e4\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-n8rf5 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-26T15:55:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8788992-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.51:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8788992/providers/microsoft.monitor/accounts/services-j8788992\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-n8rf5\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d66701f9-547c-4900-999a-9cc3801125e4\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-n8rf5 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-26T15:55:42Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8788992-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.52:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8788992/providers/microsoft.monitor/accounts/services-j8788992\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-r6g8v\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d6ff7b03-cd37-4dcb-b713-5184927d807f\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-r6g8v (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-26T15:57:43Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j8788992-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.52:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j8788992/providers/microsoft.monitor/accounts/services-j8788992\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-r6g8v\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d6ff7b03-cd37-4dcb-b713-5184927d807f\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-r6g8v (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}},{"row_id":"9de0945db37ad747e85e5c83a81c1d4f9ff65edba64341ab08016b600b337095","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114732780888788992","occurred_at":"2026-10-26T14:55:59Z","failed_at":"e2e","signature_id":"56a5235eada3f8c8a737b876653b1f4d48462800a44eefb3574680f288886ed9","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3952e9095e40\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=13580, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=13580, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.001s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x3952e9095e40\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=13580, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=13580, acm-agent=0\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"25bd34308f5a077a278630c24a7efd62cb72a50de20cc84690294e8ce77710e0","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=13580, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=13580, acm-agent=0"}},{"row_id":"aa6f1fa2f573fa113d4e6febd47940f3feafc2bafd9cd3ed084d13d11a3840f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114732780888788992","occurred_at":"2026-10-26T14:55:59Z","failed_at":"alert","signature_id":"d89e640561546ab0437c91630c89df761f8f4b3267b8d41e85d378d9d14fc187","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 1 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-26T16:08:39Z\n  Ended:       2026-10-26T16:14:40Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j8788992-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tmic1o03pdh91igeu5gq690e02l56kv-private-kas\", pod=\"kube-scheduler-84685ff5d-cz6cm\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tmic1o03pdh91igeu5gq690e02l56kv-private-kas/kube-scheduler-84685ff5d-cz6cm has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"ef99543ae99acf0acb8d037268a03df4feed9b06d569115193df21b199dcb6a8","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114732780888788992","occurred_at":"2026-10-26T14:55:59Z","failed_at":"e2e","signature_id":"71b4b6dc383e5a878ae2899edaf985dd95ce3a388ec2b470b38589174c5894ad","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xa75b49b000\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-c7rz5j8lqwzr\" to finish creating: GET https://rp.j8788992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/116d7ad9-9cc2-4c9e-8347-2623016e8d01\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/116d7ad9-9cc2-4c9e-8347-2623016e8d01\",\n      \"name\": \"116d7ad9-9cc2-4c9e-8347-2623016e8d01\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T16:04:30.855869275Z\",\n      \"endTime\": \"2026-10-26T16:23:34.187561349Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine o9a0o9o2f5k6s4q-autoscale-noaz-rb8z8-ffnt2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-c7rz5j8lqwzr--managed/providers/Microsoft.Compute/virtualMachines/o9a0o9o2f5k6s4q-autoscale-noaz-rb8z8-ffnt2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine o9a0o9o2f5k6s4q-autoscale-noaz-rb8z8-ffnt2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-c7rz5j8lqwzr--managed/providers/Microsoft.Compute/virtualMachines/o9a0o9o2f5k6s4q-autoscale-noaz-rb8z8-ffnt2 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8788992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/116d7ad9-9cc2-4c9e-8347-2623016e8d01\n            correlation ID: 0413d2ef-0809-4a88-a400-52222573fc14\n            client request ID: 2d020f95-fbad-4853-8497-2a3822d5b948\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:220]: failed to create no-AZ nodepool autoscale-noaz with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0xa75b49b000\u003e: \n    failed to create NodePool autoscale-noaz: failed waiting for nodepool=\"autoscale-noaz\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-c7rz5j8lqwzr\" to finish creating: GET https://rp.j8788992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/116d7ad9-9cc2-4c9e-8347-2623016e8d01\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/116d7ad9-9cc2-4c9e-8347-2623016e8d01\",\n      \"name\": \"116d7ad9-9cc2-4c9e-8347-2623016e8d01\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T16:04:30.855869275Z\",\n      \"endTime\": \"2026-10-26T16:23:34.187561349Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 2, want \\u003e= 3 (autoscaling min); node pool condition AllNodesHealthy is False: 1 of 3 machines are not healthy\\nMachine o9a0o9o2f5k6s4q-autoscale-noaz-rb8z8-ffnt2: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-c7rz5j8lqwzr--managed/providers/Microsoft.Compute/virtualMachines/o9a0o9o2f5k6s4q-autoscale-noaz-rb8z8-ffnt2 to exist\\n; node pool condition AllMachinesReady is False: 1 of 3 machines are not ready\\nMachine o9a0o9o2f5k6s4q-autoscale-noaz-rb8z8-ffnt2: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/np-autoscaling-c7rz5j8lqwzr--managed/providers/Microsoft.Compute/virtualMachines/o9a0o9o2f5k6s4q-autoscale-noaz-rb8z8-ffnt2 to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j8788992.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/116d7ad9-9cc2-4c9e-8347-2623016e8d01\n            correlation ID: 0413d2ef-0809-4a88-a400-52222573fc14\n            client request ID: 2d020f95-fbad-4853-8497-2a3822d5b948\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d82ea80612ae9d46dde4279310ffe224a3955ba4144d71e7257a94d2b7bd0e3c","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is \u003ccount\u003e, want \u003e= \u003ccount\u003e (autoscaling min); node pool condition AllNodesHealthy is False: \u003ccount\u003e of \u003ccount\u003e machines are not healthy Machine \u003cnode\u003e: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; node pool condition AllMachinesReady is False: \u003ccount\u003e of \u003ccount\u003e machines are not ready Machine \u003cnode\u003e: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.Compute/virtualMachines/\u003cnode\u003e to exist; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["25bd34308f5a077a278630c24a7efd62cb72a50de20cc84690294e8ce77710e0","8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","d82ea80612ae9d46dde4279310ffe224a3955ba4144d71e7257a94d2b7bd0e3c"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114729811074093056","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7665,"pr_state":"closed","pr_sha":"b40bc5cbb80a171bde8ca5e40c43b439603f5493","final_merged_sha":"b40bc5cbb80a171bde8ca5e40c43b439603f5493","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-26T14:44:11Z","started_at":"2026-10-26T14:44:11Z","completed_at":"2026-10-26T16:53:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T16:54:37.612507404Z","timing_metadata_checked_at":"2026-10-26T16:54:37.612507404Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T16:54:37.615603994Z","region_metadata_checked_at":"2026-10-26T16:54:37.615603994Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7758/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114715638197915648","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7758,"pr_state":"closed","pr_sha":"0ac694ac43832a4600a8ef824e6e4eefd23dd44c","final_merged_sha":"0ac694ac43832a4600a8ef824e6e4eefd23dd44c","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-26T13:47:52Z","started_at":"2026-10-26T13:47:52Z","completed_at":"2026-10-26T16:18:45Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T16:23:07.982620449Z","timing_metadata_checked_at":"2026-10-26T16:23:07.982620449Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T16:23:07.98544944Z","region_metadata_checked_at":"2026-10-26T16:23:07.98544944Z"},"failed_at":["alert","e2e"],"failed_test_count":5,"occurrences":[{"row_id":"70a8720bbd783a382011e9171eb26132d492812517b85d9a28c44fe14ed15a56","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7758/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114715638197915648","occurred_at":"2026-10-26T13:47:52Z","failed_at":"alert","signature_id":"6060a41a5ff89f5748174d141ab758dd9bfd296792310bd5f6e435f59c60e104","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-26T15:16:58Z\n  Ended:       2026-10-26T15:41:54Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j7915648-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7915648/providers/microsoft.monitor/accounts/hcps-j7915648\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tmhnl5hj1n99osq72ci08bcfpe0967o-l3q7d3m3j2t0i9x\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-9766k\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2tmhnl5hj1n99osq72ci08bcfpe0967o-l3q7d3m3j2t0i9x/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-26T15:16:58Z\n  Ended:       2026-10-26T15:41:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j7915648-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7915648/providers/microsoft.monitor/accounts/hcps-j7915648\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tmhnl5hj1n99osq72ci08bcfpe0967o-l3q7d3m3j2t0i9x\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-9766k\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2tmhnl5hj1n99osq72ci08bcfpe0967o-l3q7d3m3j2t0i9x/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"7d6fc5f93a4c4ac1204c02675dd3f16a1a87f01e1d992db8d4720f60ae863628","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7758/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114715638197915648","occurred_at":"2026-10-26T13:47:52Z","failed_at":"e2e","signature_id":"9e335c0a832d5ff50fcfdbb223c4a042675de44458234ab8586e50623b976b80","test_name":"Service Provider should upgrade the control plane z-stream automatically on behalf of the customer for 4.23","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-rd6g78\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2215eed744c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-23-rd6g78: failed waiting for cluster=\"cluster-zstream-4-23-rd6g78\" in resourcegroup=\"rg-zstream-upgrade-4-23-87rsp94v6tm5\" to finish creating: GET https://rp.j7915648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa59838b-ffc9-4995-bb35-97d4c44beaff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa59838b-ffc9-4995-bb35-97d4c44beaff\",\n      \"name\": \"aa59838b-ffc9-4995-bb35-97d4c44beaff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T14:55:15.93882507Z\",\n      \"endTime\": \"2026-10-26T15:24:15.95371054Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7915648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa59838b-ffc9-4995-bb35-97d4c44beaff\n            correlation ID: b0d46297-d5fe-44e3-bdff-447f2228b7af\n            client request ID: fb0cf278-59cc-4ae8-8936-61b404d6c765\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/control_plane_automated_z_stream_upgrade.go:125]: failed to create HCP cluster \"cluster-zstream-4-23-rd6g78\" with version 4.23.0-ec.0 on candidate channel\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2215eed744c0\u003e: \n    failed to create HCP cluster cluster-zstream-4-23-rd6g78: failed waiting for cluster=\"cluster-zstream-4-23-rd6g78\" in resourcegroup=\"rg-zstream-upgrade-4-23-87rsp94v6tm5\" to finish creating: GET https://rp.j7915648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa59838b-ffc9-4995-bb35-97d4c44beaff\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa59838b-ffc9-4995-bb35-97d4c44beaff\",\n      \"name\": \"aa59838b-ffc9-4995-bb35-97d4c44beaff\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T14:55:15.93882507Z\",\n      \"endTime\": \"2026-10-26T15:24:15.95371054Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j7915648.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/aa59838b-ffc9-4995-bb35-97d4c44beaff\n            correlation ID: b0d46297-d5fe-44e3-bdff-447f2228b7af\n            client request ID: fb0cf278-59cc-4ae8-8936-61b404d6c765\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"9594d86b9ce16ed2d132eed0aa757658bb97f7b9ff78a7106ce91e27f225b879","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7758/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114715638197915648","occurred_at":"2026-10-26T13:47:52Z","failed_at":"alert","signature_id":"1cdfef7346400f71d7654ce8017a3db8d59770ee09e2fd2f34ac9179bfc979ed","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-26T15:18:52Z\n  Ended:       2026-10-26T15:45:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7915648-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7915648/providers/microsoft.monitor/accounts/hcps-j7915648\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tmhnl5hj1n99osq72ci08bcfpe0967o-l3q7d3m3j2t0i9x\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9290d46d-5d55-4654-a54f-7d8738f6108c\"\n  Description: Pod ocm-arohcpci01-2tmhnl5hj1n99osq72ci08bcfpe0967o-l3q7d3m3j2t0i9x/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-26T15:19:52Z\n  Ended:       2026-10-26T15:45:52Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j7915648-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7915648/providers/microsoft.monitor/accounts/hcps-j7915648\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tmhnl5hj1n99osq72ci08bcfpe0967o-l3q7d3m3j2t0i9x\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"9290d46d-5d55-4654-a54f-7d8738f6108c\"\n  Description: Pod ocm-arohcpci01-2tmhnl5hj1n99osq72ci08bcfpe0967o-l3q7d3m3j2t0i9x/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"a6cce0c83c64452c8f7ba1482539eb3b451b65daae0050cf0fe2444286bbcee3","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7758/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114715638197915648","occurred_at":"2026-10-26T13:47:52Z","failed_at":"alert","signature_id":"090e7f849d686e279e1fdb0a19cd25c6885887022d3ccec9aff0064e35b748f0","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 8 time(s) (1 unknown, 7 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-26T15:34:53Z\n  Ended:       2026-10-26T15:40:53Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j7915648-mgmt-1\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tmhn1j616q7r7qo51i9d3pca09c16db-h2y9l6v1q7f6h4e\", pod=\"cluster-network-operator-5f67bbdd9d-4fb4z\", region=\"canadacentral\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tmhn1j616q7r7qo51i9d3pca09c16db-h2y9l6v1q7f6h4e/cluster-network-operator-5f67bbdd9d-4fb4z has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"d467391d5486c5289ccfb995e6d02b654f0ae0a0b00b800914cd9548c555bfbf","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7758/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114715638197915648","occurred_at":"2026-10-26T13:47:52Z","failed_at":"alert","signature_id":"ea43d16b0dcc97d1db198e586b0982e6b5bd3e77e0dbce7ea5b1108644f712fd","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-26T15:30:55Z\n  Ended:       2026-10-26T15:41:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7915648-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7915648/providers/microsoft.monitor/accounts/hcps-j7915648\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tmhnl5hj1n99osq72ci08bcfpe0967o-l3q7d3m3j2t0i9x\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-9766k\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tmhnl5hj1n99osq72ci08bcfpe0967o-l3q7d3m3j2t0i9x/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-26T15:30:55Z\n  Ended:       2026-10-26T15:41:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j7915648-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.56:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j7915648/providers/microsoft.monitor/accounts/hcps-j7915648\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tmhnl5hj1n99osq72ci08bcfpe0967o-l3q7d3m3j2t0i9x\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-9766k\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tmhnl5hj1n99osq72ci08bcfpe0967o-l3q7d3m3j2t0i9x/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":5,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":5,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7759/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114714540984766464","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7759,"pr_state":"closed","pr_sha":"03863f1e37165b56eaaa156efba0d57fc487c7ab","final_merged_sha":"03863f1e37165b56eaaa156efba0d57fc487c7ab","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-26T13:43:30Z","started_at":"2026-10-26T13:43:30Z","completed_at":"2026-10-26T16:11:02Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T16:12:45.623793438Z","timing_metadata_checked_at":"2026-10-26T16:12:45.623793438Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T16:12:45.627049228Z","region_metadata_checked_at":"2026-10-26T16:12:45.627049228Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7760/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114707312055685120","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7760,"pr_state":"open","pr_sha":"e1ee1f576fe518093a7fc6c2c12cc94bff645d74","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-26T13:14:47Z","started_at":"2026-10-26T13:14:47Z","completed_at":"2026-10-26T15:27:43Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T15:30:59.695679559Z","timing_metadata_checked_at":"2026-10-26T15:30:59.695679559Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T15:30:59.698540449Z","region_metadata_checked_at":"2026-10-26T15:30:59.698540449Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7757/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114700407203696640","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7757,"pr_state":"open","pr_sha":"890e4b7ad12612a754ecb680230c60623eddeac7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-26T12:47:20Z","started_at":"2026-10-26T12:47:20Z","completed_at":"2026-10-26T14:50:38Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T15:09:18.495409219Z","timing_metadata_checked_at":"2026-10-26T15:09:18.495409219Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T15:09:18.499410007Z","region_metadata_checked_at":"2026-10-26T15:09:18.499410007Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"372585924628da5a9e653f1bd4552472e9cff0f96b13f03cbc65cd49a008db79","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7757/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114700407203696640","occurred_at":"2026-10-26T12:47:20Z","failed_at":"e2e","signature_id":"02547a32e2a28fef0dfb7aef6df6795a8819c05a7a7e03db3b64156256e4e2aa","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ccfclw8d6p4m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22cffa244a80\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ccfclw8d6p4m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j3696640-mgmt-2/ocm-arohcpci01-2tmgf7c99mhu459g7mk2ud5fkb7rv19p-h8m0e6o0u0k8h3e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ccfclw8d6p4m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j3696640-mgmt-2/ocm-arohcpci01-2tmgf7c99mhu459g7mk2ud5fkb7rv19p-h8m0e6o0u0k8h3e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x22cffa418620\u003e{\n            s: \"dev/centralus/ci01-j3696640-mgmt-2/ocm-arohcpci01-2tmgf7c99mhu459g7mk2ud5fkb7rv19p-h8m0e6o0u0k8h3e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ccfclw8d6p4m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x22cffa244a80\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ccfclw8d6p4m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j3696640-mgmt-2/ocm-arohcpci01-2tmgf7c99mhu459g7mk2ud5fkb7rv19p-h8m0e6o0u0k8h3e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-ccfclw8d6p4m/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j3696640-mgmt-2/ocm-arohcpci01-2tmgf7c99mhu459g7mk2ud5fkb7rv19p-h8m0e6o0u0k8h3e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x22cffa418620\u003e{\n            s: \"dev/centralus/ci01-j3696640-mgmt-2/ocm-arohcpci01-2tmgf7c99mhu459g7mk2ud5fkb7rv19p-h8m0e6o0u0k8h3e: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d06537352cd3471c3b96668b882a7f97e44d69999f5ee5ae3c9e3cfa12840861","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0s: dev/centralus/ci01-j3696640-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected all","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["d06537352cd3471c3b96668b882a7f97e44d69999f5ee5ae3c9e3cfa12840861"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7754/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114696412376600576","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7754,"pr_state":"closed","pr_sha":"963a25104c8c5a5f3cc865ba13e8b03d18f16614","final_merged_sha":"963a25104c8c5a5f3cc865ba13e8b03d18f16614","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-26T12:31:28Z","started_at":"2026-10-26T12:31:28Z","completed_at":"2026-10-26T13:04:09Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T13:15:39.521887966Z","timing_metadata_checked_at":"2026-10-26T13:15:39.521887966Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T13:15:39.524873357Z","region_metadata_checked_at":"2026-10-26T13:15:39.524873357Z"},"failed_at":["provision"],"failed_test_count":1,"occurrences":[{"row_id":"8639bf983cfa79aa688aa9e1e05a8d51c5b8ee92b6e9582cc32ca20cd0615173","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7754/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114696412376600576","occurred_at":"2026-10-26T12:31:28Z","failed_at":"provision","signature_id":"6285fb223b68ae5cc023810cff82298fb985932f6960b14b63ec7a0298c49add","test_name":"Run pipeline step Microsoft.Azure.ARO.HCP.Service.Infra/service/cluster","test_suite":"templatize-pipeline","failure_text":"time=2026-10-26T12:51:59.398Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-26T12:52:00.891Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-26T12:52:02.109Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=255e1b4b8b6e4372d437a9506a320bc949b0cc72bb4ea9195aae5ed8cedae0ec portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6600576-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F255e1b4b8b6e4372d437a9506a320bc949b0cc72bb4ea9195aae5ed8cedae0ec\ntime=2026-10-26T12:55:02.788Z level=INFO msg=\"Retrying step\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster step=cluster error=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6600576-svc/providers/Microsoft.Resources/deployments/255e1b4b8b6e4372d437a9506a320bc949b0cc72bb4ea9195aae5ed8cedae0ec/operationStatuses/08584085877640227854\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n    \\\\\\\"details\\\\\\\": [\\\\r\\\\n      {\\\\r\\\\n        \\\\\\\"code\\\\\\\": \\\\\\\"DeploymentFailed\\\\\\\",\\\\r\\\\n        \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6600576-svc/providers/Microsoft.Resources/deployments/cluster-pzvsdmxdkwyhw\\\\\\\",\\\\r\\\\n        \\\\\\\"message\\\\\\\": \\\\\\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\\\\\",\\\\r\\\\n        \\\\\\\"details\\\\\\\": [\\\\r\\\\n          {\\\\r\\\\n            \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n            \\\\\\\"target\\\\\\\": \\\\\\\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/hcp-underlay-ci01-j6600576-svc/providers/Microsoft.ContainerService/managedClusters/ci01-j6600576-svc\\\\\\\",\\\\r\\\\n            \\\\\\\"message\\\\\\\": \\\\\\\"The resource write operation failed to complete successfully, because it reached terminal provisioning state 'Failed'.\\\\\\\",\\\\r\\\\n            \\\\\\\"details\\\\\\\": [\\\\r\\\\n              {\\\\r\\\\n                \\\\\\\"code\\\\\\\": \\\\\\\"AzureKeyVaultKmsValidateIdentityPermissionCustomerError\\\\\\\",\\\\r\\\\n                \\\\\\\"message\\\\\\\": \\\\\\\"The identity does not have keys encrypt/decrypt permission on key vault \\\\\\\\\\\\\\\"https://ah-ci01-se-j6600576-1.vault.azure.net\\\\\\\\\\\\\\\"\\\\\\\"\\\\r\\\\n              }\\\\r\\\\n            ]\\\\r\\\\n          }\\\\r\\\\n        ]\\\\r\\\\n      }\\\\r\\\\n    ]\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"\ntime=2026-10-26T12:57:02.789Z level=INFO msg=\"Running step.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster description=\"Step cluster\\n  Kind: ARM\\n  Template: templates/svc-cluster.bicep\\n  Parameters: configurations/svc-cluster.tmpl.bicepparam\"\ntime=2026-10-26T12:57:03.570Z level=DEBUG+2 msg=\"Starting ARM deployment\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster\ntime=2026-10-26T12:57:05.013Z level=DEBUG+3 msg=\"Deployment started\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster deployment=4a6c02a4a9cb64920875392e566fb8079c7d7a854d89e39f64dab8204f190275 portal=https://ms.portal.azure.com/#view/Microsoft_Azure_Resources/DeploymentDetails.MenuView/~/overview/id/%2Fsubscriptions%2FXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX%2FresourceGroups%2Fhcp-underlay-ci01-j6600576-svc%2Fproviders%2FMicrosoft.Resources%2Fdeployments%2F4a6c02a4a9cb64920875392e566fb8079c7d7a854d89e39f64dab8204f190275\ntime=2026-10-26T12:58:35.328Z level=ERROR msg=\"Step errored.\" serviceGroup=Microsoft.Azure.ARO.HCP.Service.Infra resourceGroup=service step=cluster err=\"failed to run ARM step: failed to poll deployment: failed to wait for deployment completion: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6600576-svc/providers/Microsoft.Resources/deployments/4a6c02a4a9cb64920875392e566fb8079c7d7a854d89e39f64dab8204f190275/operationStatuses/08584085874612065441\\n--------------------------------------------------------------------------------\\nRESPONSE 200: 200 OK\\nERROR CODE: DeploymentFailed\\n--------------------------------------------------------------------------------\\n{\\n  \\\"status\\\": \\\"Failed\\\",\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"DeploymentFailed\\\",\\n    \\\"message\\\": \\\"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\\\",\\n    \\\"details\\\": [\\n      {\\n        \\\"code\\\": \\\"Conflict\\\",\\n        \\\"message\\\": \\\"{\\\\r\\\\n  \\\\\\\"status\\\\\\\": \\\\\\\"Failed\\\\\\\",\\\\r\\\\n  \\\\\\\"error\\\\\\\": {\\\\r\\\\n    \\\\\\\"code\\\\\\\": \\\\\\\"ResourceDeploymentFailure\\\\\\\",\\\\r\\\\n    \\\\\\\"message\\\\\\\": \\\\\\\"Failed to obtain the resource body, because the resource is currently in a 'Failed' provisioning state. If this resource has not been modified by a prior step in this deployment, you may need to address this before retrying the deployment.\\\\\\\"\\\\r\\\\n  }\\\\r\\\\n}\\\"\\n      }\\n    ]\\n  }\\n}\\n--------------------------------------------------------------------------------\\n\"","failure_pattern_match":{"status":"clustered","failure_pattern_id":"470629bcf987e8ae3a847a29e3d9d8ae832b84884a6d81ff75a201bcfd4c5ea1","failure_pattern":"ERROR CODE: DeploymentFailed; detail code AzureKeyVaultKmsValidateIdentityPermissionCustomerError; detail message The identity does not have keys encrypt/decrypt permission on key vault \"\u003curl\u003e\"; provider Microsoft.ContainerService","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["470629bcf987e8ae3a847a29e3d9d8ae832b84884a6d81ff75a201bcfd4c5ea1"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7756/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114694757086138368","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7756,"pr_state":"closed","pr_sha":"9fb84e71c007f51cfa944e0614aaa1734d18059b","final_merged_sha":"9fb84e71c007f51cfa944e0614aaa1734d18059b","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-26T12:24:52Z","started_at":"2026-10-26T12:24:52Z","completed_at":"2026-10-26T15:15:25Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T15:20:00.813223415Z","timing_metadata_checked_at":"2026-10-26T15:20:00.813223415Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T15:20:00.816550904Z","region_metadata_checked_at":"2026-10-26T15:20:00.816550904Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114692516660908032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7573,"pr_state":"closed","pr_sha":"650d867afd4f18c6cbf7447647057f493673a7d2","final_merged_sha":"650d867afd4f18c6cbf7447647057f493673a7d2","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-26T12:15:59Z","started_at":"2026-10-26T12:15:59Z","completed_at":"2026-10-26T14:38:07Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T14:48:26.689450028Z","timing_metadata_checked_at":"2026-10-26T14:48:26.689450028Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T14:48:26.692757317Z","region_metadata_checked_at":"2026-10-26T14:48:26.692757317Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6688/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114671567479771136","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6688,"pr_state":"open","pr_sha":"a630f5d29ba754ca53a2c8994a802571baafa9d4","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-26T10:52:44Z","started_at":"2026-10-26T10:52:44Z","completed_at":"2026-10-26T13:10:58Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T13:15:39.500382035Z","timing_metadata_checked_at":"2026-10-26T13:15:39.500382035Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T13:15:39.503837424Z","region_metadata_checked_at":"2026-10-26T13:15:39.503837424Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114665780556599296","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7665,"pr_state":"closed","pr_sha":"b8a07043e1848c9516335171b8332903c9751c1a","final_merged_sha":"b40bc5cbb80a171bde8ca5e40c43b439603f5493","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-26T10:29:42Z","started_at":"2026-10-26T10:29:42Z","completed_at":"2026-10-26T12:37:46Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T12:44:02.019747323Z","timing_metadata_checked_at":"2026-10-26T12:44:02.019747323Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T12:44:02.022696213Z","region_metadata_checked_at":"2026-10-26T12:44:02.022696213Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"3c89729d80a7a089e99b68e7ecbf85b639daeea627c8816e5a9a04bee4bed3f6","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114665780556599296","occurred_at":"2026-10-26T10:29:42Z","failed_at":"e2e","signature_id":"7ce7ec004ca06c2f4f394ab96fd260d7f8d5523221c536dc3b5faf13439d18c9","test_name":"Engineering want metrics from SVC and MGMT clusters to be present in Azure Monitor WOrkspace","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:122]: Timed out after 900.000s.\nnot all expected service metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:51 with:\nexpected maestro build info (MGMT) metrics but got no results (query: maestro_build_info)\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: []\nnot to be empty\nfail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:122]: Timed out after 900.000s.\nnot all expected service metrics appeared in Azure Monitor\nThe function passed to Eventually failed at /opt/app-root/src/github.com/Azure/ARO-HCP/test/e2e/metrics_amw_svc.go:51 with:\nexpected maestro build info (MGMT) metrics but got no results (query: maestro_build_info)\nExpected\n    \u003c[]prometheus.Result | len:0, cap:0\u003e: []\nnot to be empty","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b1d68134ee1583cdf3cf29ccf5f52a00bbda8ef7bbf33a93cfb29ce106c7b3bb","failure_pattern":"not all expected service metrics appeared in Azure Monitor","search_query":"not all expected service metrics appeared in Azure Monitor"}},{"row_id":"8eb6ad82a6b17c926e667997063ad22ab9321ece26fd292b22a0f7f2de66cf06","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7665/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114665780556599296","occurred_at":"2026-10-26T10:29:42Z","failed_at":"e2e","signature_id":"4f64685c87a06c2999566f5fae5dd314d3059900665f264709b0778579c1b337","test_name":"Customer should be able to create a cluster with an external auth config and get the external auth config","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:131]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1b303bad2a00\u003e: \n    failed to create service principal: create service principal for appId \"a52f84ec-ecc1-43ca-9731-aac8f9f81ad5\": Post \"https://graph.microsoft.com/v1.0/servicePrincipals\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)\n    {\n        msg: \"failed to create service principal: create service principal for appId \\\"a52f84ec-ecc1-43ca-9731-aac8f9f81ad5\\\": Post \\\"https://graph.microsoft.com/v1.0/servicePrincipals\\\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)\",\n        err: \u003c*fmt.wrapError | 0x1b303bad2980\u003e{\n            msg: \"create service principal for appId \\\"a52f84ec-ecc1-43ca-9731-aac8f9f81ad5\\\": Post \\\"https://graph.microsoft.com/v1.0/servicePrincipals\\\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)\",\n            err: \u003c*url.Error | 0x1b303c6b42d0\u003e{\n                Op: \"Post\",\n                URL: \"https://graph.microsoft.com/v1.0/servicePrincipals\",\n                Err: \u003c*http.timeoutError | 0x1b303d466f50\u003e{\n                    err: \"net/http: request canceled (Client.Timeout exceeded while awaiting headers)\",\n                },\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/external_auth_create.go:131]: failed to create app registration with service principal\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1b303bad2a00\u003e: \n    failed to create service principal: create service principal for appId \"a52f84ec-ecc1-43ca-9731-aac8f9f81ad5\": Post \"https://graph.microsoft.com/v1.0/servicePrincipals\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)\n    {\n        msg: \"failed to create service principal: create service principal for appId \\\"a52f84ec-ecc1-43ca-9731-aac8f9f81ad5\\\": Post \\\"https://graph.microsoft.com/v1.0/servicePrincipals\\\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)\",\n        err: \u003c*fmt.wrapError | 0x1b303bad2980\u003e{\n            msg: \"create service principal for appId \\\"a52f84ec-ecc1-43ca-9731-aac8f9f81ad5\\\": Post \\\"https://graph.microsoft.com/v1.0/servicePrincipals\\\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)\",\n            err: \u003c*url.Error | 0x1b303c6b42d0\u003e{\n                Op: \"Post\",\n                URL: \"https://graph.microsoft.com/v1.0/servicePrincipals\",\n                Err: \u003c*http.timeoutError | 0x1b303d466f50\u003e{\n                    err: \"net/http: request canceled (Client.Timeout exceeded while awaiting headers)\",\n                },\n            },\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"04bb1c86673a7de744e4cf404843226187998893a7858cc6ec66a84dbfdf8c20","failure_pattern":"failed to create service principal: create service principal for appId \"\u003cuuid\u003e\": Post \"\u003curl\u003e\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)","search_query":"failed to create service principal: create service principal for appId \"a52f84ec-ecc1-43ca-9731-aac8f9f81ad5\": Post \"https://graph.microsoft.com/v1.0/servicePrincipals\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["04bb1c86673a7de744e4cf404843226187998893a7858cc6ec66a84dbfdf8c20","b1d68134ee1583cdf3cf29ccf5f52a00bbda8ef7bbf33a93cfb29ce106c7b3bb"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6380/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114663174882988032","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6380,"pr_state":"closed","pr_sha":"47c980639a5b6dfbb15e7531c0fed9dd7523bc89","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-26T10:19:24Z","started_at":"2026-10-26T10:19:24Z","completed_at":"2026-10-26T12:29:13Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T12:33:28.136494345Z","timing_metadata_checked_at":"2026-10-26T12:33:28.136494345Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T12:33:28.140112334Z","region_metadata_checked_at":"2026-10-26T12:33:28.140112334Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6823/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114653090849230848","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6823,"pr_state":"closed","pr_sha":"3af050c1ff537616c107e003e73274ce62500796","final_merged_sha":"71adb8ff03ea9f64e19ed8844a0cf652d6050233","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-26T09:39:19Z","started_at":"2026-10-26T09:39:19Z","completed_at":"2026-10-26T11:54:54Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T12:12:15.59359856Z","timing_metadata_checked_at":"2026-10-26T12:12:15.59359856Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T12:12:15.597379449Z","region_metadata_checked_at":"2026-10-26T12:12:15.597379449Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114650067662016512","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7712,"pr_state":"closed","pr_sha":"8a249974fa0aa6cbf4bfec88da953bf8f5659b68","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-26T09:27:19Z","started_at":"2026-10-26T09:27:19Z","completed_at":"2026-10-26T11:41:01Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T11:50:28.704986984Z","timing_metadata_checked_at":"2026-10-26T11:50:28.704986984Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T11:50:28.708959871Z","region_metadata_checked_at":"2026-10-26T11:50:28.708959871Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"01ec2f06bfc26cba88b0a36127e66ded24bf6868db405034db8888dbac37eaeb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114650067662016512","occurred_at":"2026-10-26T09:27:19Z","failed_at":"alert","signature_id":"615406ebab48e14110a4e9d8f4e52090829600500bb8dd3e92ffc884098b6345","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-26T10:39:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2016512-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.217:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2016512/providers/microsoft.monitor/accounts/services-j2016512\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-x7p5s\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6a5476df-33c8-40f9-ae9f-b4673c8f9f41\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-x7p5s (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-26T10:41:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2016512-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.238:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2016512/providers/microsoft.monitor/accounts/services-j2016512\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-r248f\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"df5123e3-25f8-4237-b3ae-f6ee9c3dc14a\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-r248f (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-26T10:41:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2016512-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.238:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2016512/providers/microsoft.monitor/accounts/services-j2016512\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-r248f\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"df5123e3-25f8-4237-b3ae-f6ee9c3dc14a\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-r248f (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-26T10:41:50Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j2016512-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.217:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j2016512/providers/microsoft.monitor/accounts/services-j2016512\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-x7p5s\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"6a5476df-33c8-40f9-ae9f-b4673c8f9f41\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-x7p5s (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}},{"row_id":"126632e4d62d41836b41a125035d95f695f475ce79a65e8fc3231a3022066a42","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114650067662016512","occurred_at":"2026-10-26T09:27:19Z","failed_at":"e2e","signature_id":"ba0c38e1eca9a292d0e25be503d8588cc9faffce90f396b1e444f3df42590094","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 603.481s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x12a5a347cf40\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8035, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8035, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 603.481s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x12a5a347cf40\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8035, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8035, acm-agent=0\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"d5faba6a3467bcc4356e02ff8b763d0cb27996ad0cf0c261ffce5c17c56051d3","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8035, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=8035, acm-agent=0"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","d5faba6a3467bcc4356e02ff8b763d0cb27996ad0cf0c261ffce5c17c56051d3"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114635485509849088","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7573,"pr_state":"closed","pr_sha":"650d867afd4f18c6cbf7447647057f493673a7d2","final_merged_sha":"650d867afd4f18c6cbf7447647057f493673a7d2","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-26T08:29:22Z","started_at":"2026-10-26T08:29:22Z","completed_at":"2026-10-26T10:47:35Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T10:58:10.622590009Z","timing_metadata_checked_at":"2026-10-26T10:58:10.622590009Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T10:58:10.626112198Z","region_metadata_checked_at":"2026-10-26T10:58:10.626112198Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"1bf46cf5bc0e55b41ccbb103bd444727e0205b43a648c4e83a5cc4a8ab4bdb56","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114635485509849088","occurred_at":"2026-10-26T08:29:22Z","failed_at":"e2e","signature_id":"2df52a6d277b5dfd6e04438bec8a6c80649d92df6621c452eb3aa83a82cb7d5f","test_name":"Customer should be able to create a cluster and node pools with aggregated advanced features","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2d970cc94090\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-rx9rnfvmj655\" to finish creating: GET https://rp.j9849088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8761660-38f6-4612-a201-a6ddec925f84\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8761660-38f6-4612-a201-a6ddec925f84\",\n      \"name\": \"c8761660-38f6-4612-a201-a6ddec925f84\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T09:38:13.907922394Z\",\n      \"endTime\": \"2026-10-26T09:57:20.458727782Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9849088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8761660-38f6-4612-a201-a6ddec925f84\n            correlation ID: 4c993536-8323-4301-a24e-f6918a189628\n            client request ID: e13d57f0-4312-4271-8e8e-02240879d813\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_feature_aggregation.go:404]: failed to verify cilium is running and nodes are Ready for cluster \"agg-cluster\"\nUnexpected error:\n    \u003c*errors.joinError | 0x2d970cc94090\u003e: \n    VerifyNodesReady failed: no nodes found in the cluster\n    VerifyCiliumOperational failed: not all pods in kube-system namespace are running: failed to list pods in kube-system namespace: context deadline exceeded\n    failed to create NodePool agg-np-a: failed waiting for nodepool=\"agg-np-a\" for cluster \"agg-cluster\" in resourcegroup=\"feature-aggregation-rx9rnfvmj655\" to finish creating: GET https://rp.j9849088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8761660-38f6-4612-a201-a6ddec925f84\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8761660-38f6-4612-a201-a6ddec925f84\",\n      \"name\": \"c8761660-38f6-4612-a201-a6ddec925f84\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T09:38:13.907922394Z\",\n      \"endTime\": \"2026-10-26T09:57:20.458727782Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9849088.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/canadacentral/hcpOperationStatuses/c8761660-38f6-4612-a201-a6ddec925f84\n            correlation ID: 4c993536-8323-4301-a24e-f6918a189628\n            client request ID: e13d57f0-4312-4271-8e8e-02240879d813\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"a2bcc7b04bc914cf25a323de950e6d5a997eab78247d447ec16910ac5dbd2d13","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114635485509849088","occurred_at":"2026-10-26T08:29:22Z","failed_at":"e2e","signature_id":"541b77824c3f3a313c9ef1c117ae9a8f71c935bf6deaab5e9c18791b44825d21","test_name":"Customer should be able to create node pools with ARM64-based VMs","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:78]: failed to create customer resources for ARM64 nodepool cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x240d16aa66e0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-arm64-vm-hcp-cluster-z9s25h\" in resourcegroup=\"arm64-vm-cluster-r2854rq2nbt4\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/arm64-vm-cluster-r2854rq2nbt4/providers/Microsoft.Resources/deployments/customer-infra-arm64-vm-hcp-cluster-z9s25h/operationStatuses/08584086002951648222\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ConflictError\\\",\\r\\n    \\\"message\\\": \\\"A conflict occurred that prevented the operation from completing. The operation failed because the Key Vault '' changed from the point the operation began. This can happen if parallel operations are being performed on the Key Vault. To prevent this error, serialize the operations so that only one operation is performed on the Key Vault at a time. Follow this link for more information: https://go.microsoft.com/fwlink/?linkid=2147741\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/arm64-vm-cluster-r2854rq2nbt4/providers/Microsoft.Resources/deployments/customer-infra-arm64-vm-hcp-cluster-z9s25h/operationStatuses/08584086002951648222\n            correlation ID: 23c397c2-e55d-434d-a806-434e6889bce6\n            client request ID: unavailable\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/arm64_nodepool.go:78]: failed to create customer resources for ARM64 nodepool cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x240d16aa66e0\u003e: \n    failed to create customer-infra: failed waiting for deployment \"customer-infra-arm64-vm-hcp-cluster-z9s25h\" in resourcegroup=\"arm64-vm-cluster-r2854rq2nbt4\" to finish: GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/arm64-vm-cluster-r2854rq2nbt4/providers/Microsoft.Resources/deployments/customer-infra-arm64-vm-hcp-cluster-z9s25h/operationStatuses/08584086002951648222\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeploymentFailed\n    --------------------------------------------------------------------------------\n    {\n      \"status\": \"Failed\",\n      \"error\": {\n        \"code\": \"DeploymentFailed\",\n        \"message\": \"At least one resource deployment operation failed. Please list deployment operations for details. Please see https://aka.ms/arm-deployment-operations for usage details.\",\n        \"details\": [\n          {\n            \"code\": \"Conflict\",\n            \"message\": \"{\\r\\n  \\\"error\\\": {\\r\\n    \\\"code\\\": \\\"ConflictError\\\",\\r\\n    \\\"message\\\": \\\"A conflict occurred that prevented the operation from completing. The operation failed because the Key Vault '' changed from the point the operation began. This can happen if parallel operations are being performed on the Key Vault. To prevent this error, serialize the operations so that only one operation is performed on the Key Vault at a time. Follow this link for more information: https://go.microsoft.com/fwlink/?linkid=2147741\\\"\\r\\n  }\\r\\n}\"\n          }\n        ]\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/arm64-vm-cluster-r2854rq2nbt4/providers/Microsoft.Resources/deployments/customer-infra-arm64-vm-hcp-cluster-z9s25h/operationStatuses/08584086002951648222\n            correlation ID: 23c397c2-e55d-434d-a806-434e6889bce6\n            client request ID: unavailable\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"cb3f1af3ae5c6870302b8493e6567a2cb5b04bdc76461d8531d6cdaf4f74aada","failure_pattern":"ERROR CODE: DeploymentFailed; detail code ConflictError; detail message A conflict occurred that prevented the operation from completing. The operation failed because the Key Vault '' changed from the point the operation began. This can happen if parallel operations are being performed on the Key Vault. To prevent this error, serialize the operations so that only one operation is performed on the Key Vault at a time. Follow this link for more information: \u003curl\u003e","search_query":"ERROR CODE: DeploymentFailed"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["cb3f1af3ae5c6870302b8493e6567a2cb5b04bdc76461d8531d6cdaf4f74aada","f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7514/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114632624264712192","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7514,"pr_state":"open","pr_sha":"eb3b15b6eba57ec61003502a12cd451237522079","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-26T08:18:00Z","started_at":"2026-10-26T08:18:00Z","completed_at":"2026-10-26T10:27:10Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T10:48:01.5280756Z","timing_metadata_checked_at":"2026-10-26T10:48:01.5280756Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T10:48:01.532155987Z","region_metadata_checked_at":"2026-10-26T10:48:01.532155987Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7583/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114623573241892864","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7583,"pr_state":"open","pr_sha":"a98dfac0707c665fb5cc7da7d2d57c7f1b0c01f3","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-26T07:42:02Z","started_at":"2026-10-26T07:42:02Z","completed_at":"2026-10-26T09:53:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T10:06:56.076543805Z","timing_metadata_checked_at":"2026-10-26T10:06:56.076543805Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T10:06:56.081077291Z","region_metadata_checked_at":"2026-10-26T10:06:56.081077291Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114613195741073408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7662,"pr_state":"open","pr_sha":"ca9c1593d7266c954da33cbe3aa07287505d22c9","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-26T07:00:36Z","started_at":"2026-10-26T07:00:36Z","completed_at":"2026-10-26T09:00:52Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T09:13:51.500562846Z","timing_metadata_checked_at":"2026-10-26T09:13:51.500562846Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T09:13:51.503822436Z","region_metadata_checked_at":"2026-10-26T09:13:51.503822436Z"},"failed_at":["alert","e2e"],"failed_test_count":2,"occurrences":[{"row_id":"0225e83f88fe1f5057db120338e1d663ed73e1bf188650fa6d620769ec237efb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114613195741073408","occurred_at":"2026-10-26T07:00:36Z","failed_at":"alert","signature_id":"875a87e1cceb054032906ab6545693da41bff75ce6526371ca7a798456a52fa2","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-26T08:00:55Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1073408-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1073408/providers/microsoft.monitor/accounts/services-j1073408\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-jdbj8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"cf5b240e-dae8-4a44-acae-3a125b10e344\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-jdbj8 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-26T08:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1073408-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.15:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1073408/providers/microsoft.monitor/accounts/services-j1073408\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-b6k74\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"259cbe08-9376-4204-b2ee-52499ae8c38f\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-b6k74 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-26T08:05:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1073408-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.15:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1073408/providers/microsoft.monitor/accounts/services-j1073408\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-b6k74\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"259cbe08-9376-4204-b2ee-52499ae8c38f\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-b6k74 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-26T08:05:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1073408-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.164:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1073408/providers/microsoft.monitor/accounts/services-j1073408\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-jdbj8\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"canadacentral\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"cf5b240e-dae8-4a44-acae-3a125b10e344\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-jdbj8 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}},{"row_id":"eb740606592d99ca236cb6c0ffc96c66222d9ea80691d66b2de4edc20e327290","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7662/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114613195741073408","occurred_at":"2026-10-26T07:00:36Z","failed_at":"e2e","signature_id":"51acba6d51b844246e9a06873b1fe5c6da4f4a9182dfef26ae27d3095e26d5a8","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xe6261437860\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11751, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11751, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.005s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0xe6261437860\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11751, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11751, acm-agent=0\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"471610528d575f0ca7dcf6b5555ac33936f790050d7e1c045a353856d7e30af3","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11751, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=11751, acm-agent=0"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["471610528d575f0ca7dcf6b5555ac33936f790050d7e1c045a353856d7e30af3","8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7752/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114606858877538304","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7752,"pr_state":"closed","pr_sha":"e06ac54850b99a459a3fbeaa5c9f3a95e41b7db3","final_merged_sha":"e06ac54850b99a459a3fbeaa5c9f3a95e41b7db3","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-26T06:35:37Z","started_at":"2026-10-26T06:35:37Z","completed_at":"2026-10-26T08:55:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T09:13:51.876994163Z","timing_metadata_checked_at":"2026-10-26T09:13:51.876994163Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T09:13:51.879969955Z","region_metadata_checked_at":"2026-10-26T09:13:51.879969955Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114600535943286784","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7573,"pr_state":"closed","pr_sha":"650d867afd4f18c6cbf7447647057f493673a7d2","final_merged_sha":"650d867afd4f18c6cbf7447647057f493673a7d2","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-26T06:10:29Z","started_at":"2026-10-26T06:10:29Z","completed_at":"2026-10-26T08:19:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T08:31:49.095073225Z","timing_metadata_checked_at":"2026-10-26T08:31:49.095073225Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T08:31:49.097891517Z","region_metadata_checked_at":"2026-10-26T08:31:49.097891517Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"16c5f4404ad68a223ee6d18cca51b6896722409801d06871ba15cd940edb4af1","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114600535943286784","occurred_at":"2026-10-26T06:10:29Z","failed_at":"e2e","signature_id":"37d6b1898665834c194cc9382c6cb7f22a57b414b03e07c41ba2411d31830fda","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1d17e587640\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-zlz47zdxrs5r\" to finish creating: GET https://rp.j3286784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c622a94-9f78-4e75-b786-0aff8f0a726c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c622a94-9f78-4e75-b786-0aff8f0a726c\",\n      \"name\": \"7c622a94-9f78-4e75-b786-0aff8f0a726c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T07:28:38.842864831Z\",\n      \"endTime\": \"2026-10-26T07:47:44.945885677Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j7b4w8j4d4p7e3f-np-deletable-82q27-k447r: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j7b4w8j4d4p7e3f-np-deletable-82q27-k447r: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3286784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c622a94-9f78-4e75-b786-0aff8f0a726c\n            correlation ID: 08c3cc6c-2bcb-4d71-83fb-d91ac0512763\n            client request ID: dc9b7dea-fdfa-46b9-bb12-bc31e259fdc0\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x1d17e587640\u003e: \n    failed to create NodePool np-deletable: failed waiting for nodepool=\"np-deletable\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-zlz47zdxrs5r\" to finish creating: GET https://rp.j3286784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c622a94-9f78-4e75-b786-0aff8f0a726c\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c622a94-9f78-4e75-b786-0aff8f0a726c\",\n      \"name\": \"7c622a94-9f78-4e75-b786-0aff8f0a726c\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T07:28:38.842864831Z\",\n      \"endTime\": \"2026-10-26T07:47:44.945885677Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine j7b4w8j4d4p7e3f-np-deletable-82q27-k447r: NodeNotHealthy: * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine j7b4w8j4d4p7e3f-np-deletable-82q27-k447r: NotReady: * NodeHealthy:\\n  * Node.Ready: container runtime network not ready: NetworkReady=false reason:NetworkPluginNotReady message:Network plugin returns error: no CNI configuration file in /etc/kubernetes/cni/net.d/. Has your network provider started?\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3286784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7c622a94-9f78-4e75-b786-0aff8f0a726c\n            correlation ID: 08c3cc6c-2bcb-4d71-83fb-d91ac0512763\n            client request ID: dc9b7dea-fdfa-46b9-bb12-bc31e259fdc0","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5r2ss57dw6dn\" to finish creating: GET https://rp"}},{"row_id":"acd344a3a9d6044d17237db449639b377d06cdd624e08f2c7f63d7d25e4930eb","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7573/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114600535943286784","occurred_at":"2026-10-26T06:10:29Z","failed_at":"e2e","signature_id":"e3ae56b421b472adb90d5bf2cff478a4dadb50c8f4068e47ee19cfa9e72139da","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:166]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2ca8d1751180\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5r2ss57dw6dn\" to finish creating: GET https://rp.j3286784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be36ffa-b81d-4c05-9801-1c0112e4103b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be36ffa-b81d-4c05-9801-1c0112e4103b\",\n      \"name\": \"7be36ffa-b81d-4c05-9801-1c0112e4103b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T07:10:55.24771548Z\",\n      \"endTime\": \"2026-10-26T07:29:55.206194815Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-xtq4l-l6wv8: Failed: availabilitysets failed to create or update. err: failed to get existing resource ingress-cert-le-5r2ss57dw6dn--managed/2tmaoumf1ok3jc7bcp78t6cc6g7ou37b_ingress-cert-le-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-5r2ss57dw6dn--managed/providers/Microsoft.Compute/availabilitySets/2tmaoumf1ok3jc7bcp78t6cc6g7ou37b_ingress-cert-le-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '8d1ff6d0-aadd-4331-9171-9443ac5f4337' exceeded. Please try again after '1' sec... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3286784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be36ffa-b81d-4c05-9801-1c0112e4103b\n            correlation ID: c6deec64-4edf-4f99-842e-fae9ee3bc0d5\n            client request ID: 7c4b8c2b-3bc2-4ab4-98b5-cee20bd9e4da\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:166]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x2ca8d1751180\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5r2ss57dw6dn\" to finish creating: GET https://rp.j3286784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be36ffa-b81d-4c05-9801-1c0112e4103b\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be36ffa-b81d-4c05-9801-1c0112e4103b\",\n      \"name\": \"7be36ffa-b81d-4c05-9801-1c0112e4103b\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T07:10:55.24771548Z\",\n      \"endTime\": \"2026-10-26T07:29:55.206194815Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-xtq4l-l6wv8: Failed: availabilitysets failed to create or update. err: failed to get existing resource ingress-cert-le-5r2ss57dw6dn--managed/2tmaoumf1ok3jc7bcp78t6cc6g7ou37b_ingress-cert-le-np-1-as (service: availabilitysets): GET https://management.azure.com/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-5r2ss57dw6dn--managed/providers/Microsoft.Compute/availabilitySets/2tmaoumf1ok3jc7bcp78t6cc6g7ou37b_ingress-cert-le-np-1-as\\n--------------------------------------------------------------------------------\\nRESPONSE 429: 429 Too Many Requests\\nERROR CODE: SubscriptionRequestsThrottled\\n--------------------------------------------------------------------------------\\n{\\n  \\\"error\\\": {\\n    \\\"code\\\": \\\"SubscriptionRequestsThrottled\\\",\\n    \\\"message\\\": \\\"Number of 'read' requests for subscription 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' actor '8d1ff6d0-aadd-4331-9171-9443ac5f4337' exceeded. Please try again after '1' sec... message truncated\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3286784.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/7be36ffa-b81d-4c05-9801-1c0112e4103b\n            correlation ID: c6deec64-4edf-4f99-842e-fae9ee3bc0d5\n            client request ID: 7c4b8c2b-3bc2-4ab4-98b5-cee20bd9e4da","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5r2ss57dw6dn\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7705/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114590570570584064","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7705,"pr_state":"closed","pr_sha":"78193d44c70069ac60e402c5ee2f646eadb201d4","final_merged_sha":"e2aa104a4154e9875791cd051472491971fa6720","merged_pr":true,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-26T05:30:53Z","started_at":"2026-10-26T05:30:53Z","completed_at":"2026-10-26T07:45:08Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T08:00:05.146705224Z","timing_metadata_checked_at":"2026-10-26T08:00:05.146705224Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T08:00:05.150705512Z","region_metadata_checked_at":"2026-10-26T08:00:05.150705512Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114590452421234688","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7712,"pr_state":"closed","pr_sha":"8a249974fa0aa6cbf4bfec88da953bf8f5659b68","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-26T05:30:11Z","started_at":"2026-10-26T05:30:11Z","completed_at":"2026-10-26T07:43:49Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T08:00:05.168354558Z","timing_metadata_checked_at":"2026-10-26T08:00:05.168354558Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T08:00:05.171203749Z","region_metadata_checked_at":"2026-10-26T08:00:05.171203749Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"303441474608cea32d1aabbfa8569c268df29c47118d67d71e28207f5114f16d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114590452421234688","occurred_at":"2026-10-26T05:30:11Z","failed_at":"alert","signature_id":"1d391c5e1f8d80343056e901525b045ccb36fa0af9d1f6801f1f896f98f95d70","test_name":"[aro-hcp-observability] [svc] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 4 time(s)\nFiring 1:\n  State:       Fired (not resolved)\n  Started:     2026-10-26T06:39:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1234688-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.55:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1234688/providers/microsoft.monitor/accounts/services-j1234688\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-79mdz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"b0331707-593d-4709-aa13-977fd3c2bf42\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-79mdz (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Fired (not resolved)\n  Started:     2026-10-26T06:39:56Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1234688-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.55:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1234688/providers/microsoft.monitor/accounts/services-j1234688\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-79mdz\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"b0331707-593d-4709-aa13-977fd3c2bf42\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-79mdz (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n  State:       Fired (not resolved)\n  Started:     2026-10-26T06:41:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1234688-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.65:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1234688/providers/microsoft.monitor/accounts/services-j1234688\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-mjbv2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ceb7c145-a1d3-4aa0-b581-6b16c4403ca0\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-mjbv2 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n  State:       Fired (not resolved)\n  Started:     2026-10-26T06:42:57Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j1234688-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"hypershift-addon-agent\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.65.65:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j1234688/providers/microsoft.monitor/accounts/services-j1234688\", microsoft.metrics_include_label=\"service\", namespace=\"open-cluster-management-agent-addon\", pod=\"hypershift-addon-agent-76ffb6fdb8-mjbv2\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"ceb7c145-a1d3-4aa0-b581-6b16c4403ca0\"\n  Description: Pod open-cluster-management-agent-addon/hypershift-addon-agent-76ffb6fdb8-mjbv2 (hypershift-addon-agent) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","failure_pattern":"alert [svc] KubePodCrashLooping fired","search_query":"alert [svc] KubePodCrashLooping fired"}},{"row_id":"69aaa06cadd9a5e2afd51f038d83dd3c7dd8b9b71797c3661ceabf0898efb6fe","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114590452421234688","occurred_at":"2026-10-26T05:30:11Z","failed_at":"e2e","signature_id":"153f0aaddc00f1ad7874f65ac4a270a70a9ea96971f5237b887bc92ff2778128","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x294c3a4e6f50\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=9795, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=9795, acm-agent=0\",\n    }\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:118]: Timed out after 600.003s.\nExpected success, but got an error:\n    \u003c*errors.errorString | 0x294c3a4e6f50\u003e: \n    missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=9795, acm-agent=0\n    {\n        s: \"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=9795, acm-agent=0\",\n    }","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e6a6ef637ffd01a8f5540827ca11eebe7cd606076b9508c9bc2e31657ba24b47","failure_pattern":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=9795, acm-agent=0","search_query":"missing expected log sources: []; detailedInfraOrchestrationLogs expected at least one row per required source, missing: [acm-agent]; observed counts: hypershift=9795, acm-agent=0"}},{"row_id":"6c3894b5725ca58a9c2853637c2b188fbc7c9ff11206b089b325083c151022f5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114590452421234688","occurred_at":"2026-10-26T05:30:11Z","failed_at":"e2e","signature_id":"5b1a358ec60ee2fc7fa42a8d358fb8f4ff780a44d4fe2ab5e89289cd2283efed","test_name":"Customer should be able to delete a nodepool","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xaf47cdc3b00\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-q2l2cwwbkpwv\" to finish creating: GET https://rp.j1234688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bb9e7887-b3a8-4bd3-8f18-28eec04446ae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bb9e7887-b3a8-4bd3-8f18-28eec04446ae\",\n      \"name\": \"bb9e7887-b3a8-4bd3-8f18-28eec04446ae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T06:47:58.232968793Z\",\n      \"endTime\": \"2026-10-26T07:07:05.709044225Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w7m0v1s2c9s1k0b-np-delete-main-2jlcl-5bwv9: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w7m0v1s2c9s1k0b-np-delete-main-2jlcl-5bwv9: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1234688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bb9e7887-b3a8-4bd3-8f18-28eec04446ae\n            correlation ID: d2413cb9-bbc4-41c5-8fb7-450d4d4c9295\n            client request ID: 98336484-cdbd-4ed3-ad65-670b231d6a10\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_delete.go:128]: failed to create node pools\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0xaf47cdc3b00\u003e: \n    failed to create NodePool np-delete-main: failed waiting for nodepool=\"np-delete-main\" for cluster \"np-delete-hcp-cluster\" in resourcegroup=\"nodepool-delete-q2l2cwwbkpwv\" to finish creating: GET https://rp.j1234688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bb9e7887-b3a8-4bd3-8f18-28eec04446ae\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bb9e7887-b3a8-4bd3-8f18-28eec04446ae\",\n      \"name\": \"bb9e7887-b3a8-4bd3-8f18-28eec04446ae\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T06:47:58.232968793Z\",\n      \"endTime\": \"2026-10-26T07:07:05.709044225Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine w7m0v1s2c9s1k0b-np-delete-main-2jlcl-5bwv9: InspectionFailed: Waiting for AzureMachine to report spec.providerID\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine w7m0v1s2c9s1k0b-np-delete-main-2jlcl-5bwv9: Creating: virtualmachine creating or updating\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1234688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/bb9e7887-b3a8-4bd3-8f18-28eec04446ae\n            correlation ID: d2413cb9-bbc4-41c5-8fb7-450d4d4c9295\n            client request ID: 98336484-cdbd-4ed3-ad65-670b231d6a10","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5r2ss57dw6dn\" to finish creating: GET https://rp"}},{"row_id":"d69185eb17816640d68436d00cc295de818931d7bd850a08059ded1502eb2331","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7712/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114590452421234688","occurred_at":"2026-10-26T05:30:11Z","failed_at":"e2e","signature_id":"2cc1fbf964c09132d4e1288a799526694dfa03c7ea3a5312bc375046fdc651c1","test_name":"Customer should be able to issue the default ingress serving certificate from Let's Encrypt using cert-manager with ACME DNS-01 challenges for cluster version 5.0","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:166]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3405dea41d00\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-mjjkc7tsg6pw\" to finish creating: GET https://rp.j1234688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/152da1d5-1d63-4b70-adbf-23b972dcdae5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/152da1d5-1d63-4b70-adbf-23b972dcdae5\",\n      \"name\": \"152da1d5-1d63-4b70-adbf-23b972dcdae5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T06:30:24.860566935Z\",\n      \"endTime\": \"2026-10-26T06:49:25.947090385Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-2h8pd-mlvbj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-mjjkc7tsg6pw--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2h8pd-mlvbj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-2h8pd-mlvbj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-mjjkc7tsg6pw--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2h8pd-mlvbj to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1234688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/152da1d5-1d63-4b70-adbf-23b972dcdae5\n            correlation ID: 4ec64207-73fc-4c1b-8973-f8f47f8d6a7e\n            client request ID: e0882a3c-3a9e-4bc0-8c16-c4da5403ce37\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_ingress_cert_letsencrypt.go:166]: failed to create node pool\nExpected success, but got an error:\n    \u003c*fmt.wrapError | 0x3405dea41d00\u003e: \n    failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-mjjkc7tsg6pw\" to finish creating: GET https://rp.j1234688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/152da1d5-1d63-4b70-adbf-23b972dcdae5\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/152da1d5-1d63-4b70-adbf-23b972dcdae5\",\n      \"name\": \"152da1d5-1d63-4b70-adbf-23b972dcdae5\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T06:30:24.860566935Z\",\n      \"endTime\": \"2026-10-26T06:49:25.947090385Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift NodePool status replicas is 1, want 2; node pool condition AllNodesHealthy is False: 1 of 2 machines are not healthy\\nMachine ingress-cert-le-np-1-2h8pd-mlvbj: InspectionFailed: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-mjjkc7tsg6pw--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2h8pd-mlvbj to exist\\n; node pool condition AllMachinesReady is False: 1 of 2 machines are not ready\\nMachine ingress-cert-le-np-1-2h8pd-mlvbj: ReadyUnknown: * NodeHealthy: Waiting for a Node with spec.providerID azure:///subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/ingress-cert-le-mjjkc7tsg6pw--managed/providers/Microsoft.Compute/virtualMachines/ingress-cert-le-np-1-2h8pd-mlvbj to exist\\n\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j1234688.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/152da1d5-1d63-4b70-adbf-23b972dcdae5\n            correlation ID: 4ec64207-73fc-4c1b-8973-f8f47f8d6a7e\n            client request ID: e0882a3c-3a9e-4bc0-8c16-c4da5403ce37","failure_pattern_match":{"status":"clustered","failure_pattern_id":"aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","failure_pattern":"failed to create nodepool \u003cnodepool\u003e: failed waiting for nodepool=\"\u003cnodepool\u003e\" for cluster \"\u003ccluster\u003e\" in resourcegroup=\"\u003cresource-group\u003e\" to finish creating: GET \u003curl\u003e","search_query":"failed to create NodePool np-1: failed waiting for nodepool=\"np-1\" for cluster \"ingress-cert-le\" in resourcegroup=\"ingress-cert-le-5r2ss57dw6dn\" to finish creating: GET https://rp"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["8e37e2ece486f09d5935786a71048a3faf57c9a28c6fb6885f56fde9a87c9650","aa550a810a38f54c9772fbc63049301daa6256b99df1cff0e63c66a189a03172","e6a6ef637ffd01a8f5540827ca11eebe7cd606076b9508c9bc2e31657ba24b47"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7727/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114590084358475776","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7727,"pr_state":"open","pr_sha":"ca08ced119bb479f5b000757cde7279a437dd0f7","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-26T05:28:57Z","started_at":"2026-10-26T05:28:57Z","completed_at":"2026-10-26T07:47:20Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T08:00:05.188284396Z","timing_metadata_checked_at":"2026-10-26T08:00:05.188284396Z","region":"canadacentral","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T08:00:05.191621586Z","region_metadata_checked_at":"2026-10-26T08:00:05.191621586Z"},"failed_at":["e2e"],"failed_test_count":1,"occurrences":[{"row_id":"38178f8f89e9bd48f2869b70121a170054bffc912a496ab3ab9e0200888d4219","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7727/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114590084358475776","occurred_at":"2026-10-26T05:28:57Z","failed_at":"e2e","signature_id":"cb667908309974df41a36846f12e4ef39e2292926044076fc3350fbe6a737f00","test_name":"ARO-HCP HyperShift Presubmit should create a cluster and nodepool to completion","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:122]: cluster \"cluster-hs-pre-546c5c\" should complete its version rollout\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2e4bde720b40\u003e: \n    VerifyClusterVersionRolloutCompleted timed out after 45m0s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\n    {\n        msg: \"VerifyClusterVersionRolloutCompleted timed out after 45m0s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\\\"Partial\\\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\",\n        err: \u003c*verifiers.pollObservationError | 0x2e4bde720ac0\u003e{\n            observation: \u003c*errors.errorString | 0x2e4be03290f0\u003e{\n                s: \"clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\\\"Partial\\\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2e4bde720aa0\u003e{\n                msg: \"failed to get clusterversion \\\"version\\\": context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/cluster_create_hypershift_presubmit.go:122]: cluster \"cluster-hs-pre-546c5c\" should complete its version rollout\nUnexpected error:\n    \u003c*fmt.wrapError | 0x2e4bde720b40\u003e: \n    VerifyClusterVersionRolloutCompleted timed out after 45m0s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\n    {\n        msg: \"VerifyClusterVersionRolloutCompleted timed out after 45m0s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\\\"Partial\\\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\",\n        err: \u003c*verifiers.pollObservationError | 0x2e4bde720ac0\u003e{\n            observation: \u003c*errors.errorString | 0x2e4be03290f0\u003e{\n                s: \"clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\\\"Partial\\\"; Failing=True: Cluster operator console is not available; Progressing=True: Unable to apply 4.21.39: the cluster operator console is not available\",\n            },\n            interruption: \u003c*fmt.wrapError | 0x2e4bde720aa0\u003e{\n                msg: \"failed to get clusterversion \\\"version\\\": context deadline exceeded\",\n                err: \u003ccontext.deadlineExceededError\u003e{},\n            },\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"ec19e62203173fac0d4f8045f354fdbd5a551b968f9125422e7f270d533b0922","failure_pattern":"VerifyClusterVersionRolloutCompleted timed out after 45m0s: clusterversion/version rollout not completed, expected every status.history entry to be Completed: history: 4.21.39=\"Partial\"; Failing=True: Cluster operator console is not available","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":1,"failure_pattern_ids":["ec19e62203173fac0d4f8045f354fdbd5a551b968f9125422e7f270d533b0922"],"matched_occurrences":1,"unmatched_occurrences":0,"match_summary":"single_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7739/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114589729923010560","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7739,"pr_state":"closed","pr_sha":"a38be30ac65f8313cef5f9fefaca1721c2f80784","final_merged_sha":"85f478eff30207b95d439e1d26791f75c7ff20f7","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-26T05:27:33Z","started_at":"2026-10-26T05:27:33Z","completed_at":"2026-10-26T07:42:29Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T08:00:05.209393332Z","timing_metadata_checked_at":"2026-10-26T08:00:05.209393332Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T08:00:05.212832421Z","region_metadata_checked_at":"2026-10-26T08:00:05.212832421Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"0e465b38da34c21d97b14e8d9bd9a68d080ea3c8b1c1ac462a2aba3d48731cb5","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7739/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114589729923010560","occurred_at":"2026-10-26T05:27:33Z","failed_at":"alert","signature_id":"aab1d38cc546b720a4e4328212fd7ca1d1fffb8160414a15cb499298469c7377","test_name":"[aro-hcp-observability] [hcp] alert KubeStatefulSetReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-26T06:44:17Z\n  Ended:       2026-10-26T07:09:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3010560-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.218:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3010560/providers/microsoft.monitor/accounts/hcps-j3010560\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tma82m9jgtb144qbrpk5j7np32jsq49-r5o4h3w2d4s3l7j\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-jbwn4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2tma82m9jgtb144qbrpk5j7np32jsq49-r5o4h3w2d4s3l7j/etcd has not matched the expected number of replicas for longer than 15 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-26T06:45:17Z\n  Ended:       2026-10-26T07:14:20Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeStatefulSetReplicasMismatch\", cluster=\"ci01-j3010560-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.218:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3010560/providers/microsoft.monitor/accounts/hcps-j3010560\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tma82m9jgtb144qbrpk5j7np32jsq49-r5o4h3w2d4s3l7j\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-jbwn4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", statefulset=\"etcd\"\n  Description: StatefulSet ocm-arohcpci01-2tma82m9jgtb144qbrpk5j7np32jsq49-r5o4h3w2d4s3l7j/etcd has not matched the expected number of replicas for longer than 15 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","failure_pattern":"alert [hcp] KubeStatefulSetReplicasMismatch fired","search_query":"alert [hcp] KubeStatefulSetReplicasMismatch fired"}},{"row_id":"a7e8e0d7f77fcc95737f814d434a9a7142e6b9504c67f6d5b471c6a2a787862d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7739/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114589729923010560","occurred_at":"2026-10-26T05:27:33Z","failed_at":"e2e","signature_id":"b5e430d12aac3714e721d5dd8023ef20e2860c7fe2979215316742856535fb70","test_name":"Customer should be able to create an HCP cluster with Image Registry not present","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3625ebf8a680\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-gnkr42cnrl9g\" to finish creating: GET https://rp.j3010560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3982b99-da94-4694-b894-3515f5f00549\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3982b99-da94-4694-b894-3515f5f00549\",\n      \"name\": \"c3982b99-da94-4694-b894-3515f5f00549\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T06:24:09.477795034Z\",\n      \"endTime\": \"2026-10-26T06:53:17.186342249Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3010560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3982b99-da94-4694-b894-3515f5f00549\n            correlation ID: 9f2b05a4-0a2f-41bc-8784-edb61a547f4b\n            client request ID: 3e8ce286-5388-4234-a7ff-6e068949d596\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/image_registry_cluster_create.go:89]: failed to create HCP cluster with image registry disabled\nUnexpected error:\n    \u003c*fmt.wrapError | 0x3625ebf8a680\u003e: \n    failed to create HCP cluster disabled-image-registry-hcp-cluster: failed waiting for cluster=\"disabled-image-registry-hcp-cluster\" in resourcegroup=\"disabled-image-registry-gnkr42cnrl9g\" to finish creating: GET https://rp.j3010560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3982b99-da94-4694-b894-3515f5f00549\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3982b99-da94-4694-b894-3515f5f00549\",\n      \"name\": \"c3982b99-da94-4694-b894-3515f5f00549\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T06:24:09.477795034Z\",\n      \"endTime\": \"2026-10-26T06:53:17.186342249Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: capi-provider deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j3010560.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/c3982b99-da94-4694-b894-3515f5f00549\n            correlation ID: 9f2b05a4-0a2f-41bc-8784-edb61a547f4b\n            client request ID: 3e8ce286-5388-4234-a7ff-6e068949d596\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: KubeconfigWaitingForCreate: Waiting for hosted control plane kubeconfig to be created; hosted cluster degraded: UnavailableReplicas: [capi-provider]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c00cf13a84838692a3063177baa3177f4f57b8b7d06f4ef6501a9e09785c25b4","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7739/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114589729923010560","occurred_at":"2026-10-26T05:27:33Z","failed_at":"alert","signature_id":"f7150ebe4442c37aa1818a3aec9c86f34135b9ac48a16e972bc9d66865dcbf28","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-26T06:47:16Z\n  Ended:       2026-10-26T07:14:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3010560-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.218:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3010560/providers/microsoft.monitor/accounts/hcps-j3010560\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tma82m9jgtb144qbrpk5j7np32jsq49-r5o4h3w2d4s3l7j\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e5b32bab-4c53-46e1-80dd-3f2bfd84e7a7\"\n  Description: Pod ocm-arohcpci01-2tma82m9jgtb144qbrpk5j7np32jsq49-r5o4h3w2d4s3l7j/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-26T06:47:16Z\n  Ended:       2026-10-26T07:14:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j3010560-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"etcd\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.218:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3010560/providers/microsoft.monitor/accounts/hcps-j3010560\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tma82m9jgtb144qbrpk5j7np32jsq49-r5o4h3w2d4s3l7j\", pod=\"etcd-1\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"e5b32bab-4c53-46e1-80dd-3f2bfd84e7a7\"\n  Description: Pod ocm-arohcpci01-2tma82m9jgtb144qbrpk5j7np32jsq49-r5o4h3w2d4s3l7j/etcd-1 (etcd) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}},{"row_id":"caf3f961acc5acc3df91bcd2ac7497b7be99b4fc94e8024a8636fdf14e595723","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7739/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114589729923010560","occurred_at":"2026-10-26T05:27:33Z","failed_at":"alert","signature_id":"09e52265a08fddd83906616cecc930c8b6e12730b4e3ecb6f278281f9c17c195","test_name":"[aro-hcp-observability] [hcp] alert KubeDeploymentReplicasMismatch does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-26T06:59:16Z\n  Ended:       2026-10-26T07:10:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3010560-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.218:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3010560/providers/microsoft.monitor/accounts/hcps-j3010560\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tma82m9jgtb144qbrpk5j7np32jsq49-r5o4h3w2d4s3l7j\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-jbwn4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tma82m9jgtb144qbrpk5j7np32jsq49-r5o4h3w2d4s3l7j/capi-provider has not matched the expected number of replicas for longer than 30 minutes.\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-26T06:59:17Z\n  Ended:       2026-10-26T07:10:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubeDeploymentReplicasMismatch\", cluster=\"ci01-j3010560-mgmt-1\", component=\"kubernetes-infrastructure\", container=\"kube-state-metrics\", deployment=\"capi-provider\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.218:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j3010560/providers/microsoft.monitor/accounts/hcps-j3010560\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tma82m9jgtb144qbrpk5j7np32jsq49-r5o4h3w2d4s3l7j\", pod=\"arohcp-monitor-kube-state-metrics-85d48f46ff-jbwn4\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", region=\"westus3\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\"\n  Description: Deployment ocm-arohcpci01-2tma82m9jgtb144qbrpk5j7np32jsq49-r5o4h3w2d4s3l7j/capi-provider has not matched the expected number of replicas for longer than 30 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","failure_pattern":"alert [hcp] KubeDeploymentReplicasMismatch fired","search_query":"alert [hcp] KubeDeploymentReplicasMismatch fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["13d50040cd37624093a3023213b55d91d618e215a17cc13c29536d176b30d3dc","22ddd99df75957078b407ab89556cd80e3f99b4b1a1b6e6e538a10b3757c8f66","b0fc35b4cee21d9fc77498821d7039bd11d47ea3dffe9b31d595e5aa8f3edf54","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7073/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114568374359953408","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7073,"pr_state":"closed","pr_sha":"9c4815df62c125480cb9a6a711a2a7ca92f94b47","final_merged_sha":"9c4815df62c125480cb9a6a711a2a7ca92f94b47","merged_pr":true,"post_good_commit":true,"failed":false,"occurred_at":"2026-10-26T04:02:41Z","started_at":"2026-10-26T04:02:41Z","completed_at":"2026-10-26T06:15:28Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T06:25:40.175078118Z","timing_metadata_checked_at":"2026-10-26T06:25:40.175078118Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T06:25:40.178288808Z","region_metadata_checked_at":"2026-10-26T06:25:40.178288808Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114552936569769984","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7615,"pr_state":"closed","pr_sha":"371a1d9c97b9d83ddeaea18ca4f6892b872c913a","final_merged_sha":"a5905aebe390a38b45d06e1bc7538db591e7ff19","merged_pr":true,"post_good_commit":false,"failed":true,"occurred_at":"2026-10-26T03:00:55Z","started_at":"2026-10-26T03:00:55Z","completed_at":"2026-10-26T05:12:40Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T05:22:15.792467595Z","timing_metadata_checked_at":"2026-10-26T05:22:15.792467595Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T05:22:15.795693385Z","region_metadata_checked_at":"2026-10-26T05:22:15.795693385Z"},"failed_at":["e2e"],"failed_test_count":2,"occurrences":[{"row_id":"397cf1836772ec697ad0ce376082abf846a1e412b74aa551c491cfdbb863ff30","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114552936569769984","occurred_at":"2026-10-26T03:00:55Z","failed_at":"e2e","signature_id":"02fba510a7bd5f759b6d77e3aaafa60e37757c068ea5f489e06ec884b3999289","test_name":"Customer should be able to create a cluster with default autoscaling and a nodepool with autoscaling enabled up to replica limits","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x13f2532896c0\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-n22957ncbgjt\" to finish creating: GET https://rp.j9769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ecac4fcd-9bd7-4a22-9510-51d365a0dee8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ecac4fcd-9bd7-4a22-9510-51d365a0dee8\",\n      \"name\": \"ecac4fcd-9bd7-4a22-9510-51d365a0dee8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T03:58:11.830961438Z\",\n      \"endTime\": \"2026-10-26T04:17:16.025648143Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ecac4fcd-9bd7-4a22-9510-51d365a0dee8\n            correlation ID: 5b299ccf-a19f-4517-91e6-0d46d8d87839\n            client request ID: 050413c1-cc10-4b65-a905-bd6edbbaa97f\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/nodepool_autoscaling.go:154]: failed to create AZ nodepool autoscale-az with autoscaling\nUnexpected error:\n    \u003c*fmt.wrapError | 0x13f2532896c0\u003e: \n    failed to create NodePool autoscale-az: failed waiting for nodepool=\"autoscale-az\" for cluster \"np-autoscale-cluster\" in resourcegroup=\"np-autoscaling-n22957ncbgjt\" to finish creating: GET https://rp.j9769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ecac4fcd-9bd7-4a22-9510-51d365a0dee8\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ecac4fcd-9bd7-4a22-9510-51d365a0dee8\",\n      \"name\": \"ecac4fcd-9bd7-4a22-9510-51d365a0dee8\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T03:58:11.830961438Z\",\n      \"endTime\": \"2026-10-26T04:17:16.025648143Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j9769984.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/westus3/hcpOperationStatuses/ecac4fcd-9bd7-4a22-9510-51d365a0dee8\n            correlation ID: 5b299ccf-a19f-4517-91e6-0d46d8d87839\n            client request ID: 050413c1-cc10-4b65-a905-bd6edbbaa97f\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message node pool creation did not complete before the deadline; [clusterServiceNodePoolStatus] cluster service node pool is installing; [hypershiftNodePool] hypershift node pool not cached yet; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"c1ddd53e88b4e5327d18e784af550477308b4d84698705ba70b21cca040e7774","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7615/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114552936569769984","occurred_at":"2026-10-26T03:00:55Z","failed_at":"e2e","signature_id":"eb67026a69fe8b9ad7c4d8d12e56665ef0ed78588553797d173a254e33ae781b","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-l578hzffxzwp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1623311283c0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-l578hzffxzwp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9769984-mgmt-2/ocm-arohcpci01-2tm7tusca27vplqd33uq4jnhru40s174-h2w4a5z0a2c9f5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-l578hzffxzwp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9769984-mgmt-2/ocm-arohcpci01-2tm7tusca27vplqd33uq4jnhru40s174-h2w4a5z0a2c9f5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x1623329ba440\u003e{\n            s: \"dev/westus3/ci01-j9769984-mgmt-2/ocm-arohcpci01-2tm7tusca27vplqd33uq4jnhru40s174-h2w4a5z0a2c9f5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:112]: expected complete healthy SWIFT router, ignition, KAS, DNS and worker checks for /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-l578hzffxzwp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1623311283c0\u003e: \n    VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-l578hzffxzwp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9769984-mgmt-2/ocm-arohcpci01-2tm7tusca27vplqd33uq4jnhru40s174-h2w4a5z0a2c9f5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\n    {\n        msg: \"VerifySwiftRouterChecks /subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourceGroups/kusto-logs-l578hzffxzwp/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9769984-mgmt-2/ocm-arohcpci01-2tm7tusca27vplqd33uq4jnhru40s174-h2w4a5z0a2c9f5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        err: \u003c*errors.errorString | 0x1623329ba440\u003e{\n            s: \"dev/westus3/ci01-j9769984-mgmt-2/ocm-arohcpci01-2tm7tusca27vplqd33uq4jnhru40s174-h2w4a5z0a2c9f5f: expected all current routers \u003e= desired replicas \u003e 0; desired=3 current=2 rows=2\",\n        },\n    }\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"6d30b2fd9fdf4368d644d37bdb47abf2d01de4b385c3864e9ebaa6fed07eb9cc","failure_pattern":"VerifySwiftRouterChecks /subscriptions/\u003csubscription\u003e/resourcegroups/\u003cresource-group\u003e/providers/Microsoft.RedHatOpenShift/hcpOpenShiftClusters/engineering-hcp-cluster timed out after 12m0.001s: dev/westus3/ci01-j9769984-mgmt-2/ocm-arohcpci01-\u003cid\u003e: expected","search_query":"occurred"}}],"failure_pattern_summary":{"signature_count":2,"failure_pattern_ids":["6d30b2fd9fdf4368d644d37bdb47abf2d01de4b385c3864e9ebaa6fed07eb9cc","f932975011e9c4d2237c45c19168983efcba382a218ddef102efde673d8367f0"],"matched_occurrences":2,"unmatched_occurrences":0,"match_summary":"multiple_clustered"},"pr_caused_score":3,"pr_caused_reasons":["post-good=0","only seen in DEV","only seen in one PR"]},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7721/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114550200386195456","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":7721,"pr_state":"closed","pr_sha":"512407c6bf6f42ced2b9c7913b38ae4065c26e52","final_merged_sha":"512407c6bf6f42ced2b9c7913b38ae4065c26e52","merged_pr":true,"post_good_commit":true,"failed":true,"occurred_at":"2026-10-26T02:50:28Z","started_at":"2026-10-26T02:50:28Z","completed_at":"2026-10-26T04:53:36Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T05:01:02.837595547Z","timing_metadata_checked_at":"2026-10-26T05:01:02.837595547Z","region":"centralus","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T05:01:02.842206933Z","region_metadata_checked_at":"2026-10-26T05:01:02.842206933Z"},"failed_at":["alert","e2e"],"failed_test_count":4,"occurrences":[{"row_id":"59e4062e4c8c97f41dc34775925bf79f6c7c45f0bb94a41958b930bbb8a8ab9d","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7721/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114550200386195456","occurred_at":"2026-10-26T02:50:28Z","failed_at":"e2e","signature_id":"f42b397030cfd80714e248b2e32faebf18a0825358e6552e7da3dfc682076e7f","test_name":"Engineering want metrics from HCP to be present in Azure Monitor Workspace","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1cab81a88220\u003e: \n    failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-kfvwp2t79pvw\" to finish creating: GET https://rp.j6195456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b86345e2-8212-42f1-a199-1a2d8b540403\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b86345e2-8212-42f1-a199-1a2d8b540403\",\n      \"name\": \"b86345e2-8212-42f1-a199-1a2d8b540403\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T03:36:43.851254184Z\",\n      \"endTime\": \"2026-10-26T04:05:45.382791214Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 24m27s ago; hosted cluster degraded: UnavailableReplicas: cluster-policy-controller deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6195456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b86345e2-8212-42f1-a199-1a2d8b540403\n            correlation ID: 63ad8909-3abd-481b-ad9d-99ee8f5ef24a\n            client request ID: 2dbc76d4-ed2f-44e9-8bf3-81fac3665977\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/metrics_amw_hcp_ksm.go:87]: failed to create HCP cluster \"ksm-hcp-metrics\"\nUnexpected error:\n    \u003c*fmt.wrapError | 0x1cab81a88220\u003e: \n    failed to create HCP cluster ksm-hcp-metrics: failed waiting for cluster=\"ksm-hcp-metrics\" in resourcegroup=\"hcp-metrics-kfvwp2t79pvw\" to finish creating: GET https://rp.j6195456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b86345e2-8212-42f1-a199-1a2d8b540403\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b86345e2-8212-42f1-a199-1a2d8b540403\",\n      \"name\": \"b86345e2-8212-42f1-a199-1a2d8b540403\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T03:36:43.851254184Z\",\n      \"endTime\": \"2026-10-26T04:05:45.382791214Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version 4.21.39 is Partial (want Completed), started 24m27s ago; hosted cluster degraded: UnavailableReplicas: cluster-policy-controller deployment has 1 unavailable replicas\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6195456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/b86345e2-8212-42f1-a199-1a2d8b540403\n            correlation ID: 63ad8909-3abd-481b-ad9d-99ee8f5ef24a\n            client request ID: 2dbc76d4-ed2f-44e9-8bf3-81fac3665977\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"0316cbd90cb1deb59d43692438e3753b878c1a8c6b4dfbaa4805ffd81627a594","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [hypershiftHostedCluster] hosted cluster control plane version not yet completed: version \u003cversion\u003e is Partial (want Completed), started \u003cduration\u003e ago; hosted cluster degraded: UnavailableReplicas: [cluster-policy-controller]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"86a992287c08d08a2bf2983ba2d8d6a1b6ac745972f513691d2e1c1fb31f29b7","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7721/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114550200386195456","occurred_at":"2026-10-26T02:50:28Z","failed_at":"alert","signature_id":"4c00445100490919aaaa60858ad67d7b978d873b864c379367d94dc07300f433","test_name":"[aro-hcp-observability] [hcp] alert KubePodNotReady does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 2 time(s) (1 unknown, 1 known)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-26T03:51:15Z\n  Ended:       2026-10-26T04:13:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodNotReady\", cluster=\"ci01-j6195456-mgmt-2\", component=\"kubernetes-infrastructure\", namespace=\"ocm-arohcpci01-2tm7pdravivuv2be1jlq3hjvbmiphefk-q7v3u6v3c8g7z4g\", pod=\"kube-scheduler-8557ff549-x2ncn\", region=\"centralus\", severity=\"warning\"\n  Description: Pod ocm-arohcpci01-2tm7pdravivuv2be1jlq3hjvbmiphefk-q7v3u6v3c8g7z4g/kube-scheduler-8557ff549-x2ncn has been in a non-ready state for longer than 5 minutes.","failure_pattern_match":{"status":"clustered","failure_pattern_id":"93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","failure_pattern":"alert [hcp] KubePodNotReady fired","search_query":"alert [hcp] KubePodNotReady fired"}},{"row_id":"f355edd7b15d928ffa9d5fcc16a8f601175f0b6baead5e92088606c4ecb36440","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7721/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114550200386195456","occurred_at":"2026-10-26T02:50:28Z","failed_at":"e2e","signature_id":"f5c9cc123de6b12a7bef437313c3b38c0c01be58bd0124c1453666401f1de8ac","test_name":"Engineering should be able to retrieve kusto logs for a cluster and services","test_suite":"rp-api-compat-all/parallel","failure_text":"fail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:93]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17d7d76a6b60\u003e: \n    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-n574rs4kgzmb\" to finish creating: GET https://rp.j6195456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0a9279ed-f9e2-4b6a-ac1a-28f39ba0ae7e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0a9279ed-f9e2-4b6a-ac1a-28f39ba0ae7e\",\n      \"name\": \"0a9279ed-f9e2-4b6a-ac1a-28f39ba0ae7e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T03:36:22.647294159Z\",\n      \"endTime\": \"2026-10-26T04:05:25.39660378Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, kube-scheduler, azure-cloud-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6195456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0a9279ed-f9e2-4b6a-ac1a-28f39ba0ae7e\n            correlation ID: 2461ce1f-a2e3-4c70-ae70-58c7bc6e13e3\n            client request ID: 87ad1faa-4a8b-48cb-888d-882999d849fb\noccurred\nfail [github.com/Azure/ARO-HCP/test/e2e/kusto_logs_present.go:93]: failed to create HCP cluster for kusto logs test\nUnexpected error:\n    \u003c*fmt.wrapError | 0x17d7d76a6b60\u003e: \n    failed to create HCP cluster engineering-hcp-cluster: failed waiting for cluster=\"engineering-hcp-cluster\" in resourcegroup=\"kusto-logs-n574rs4kgzmb\" to finish creating: GET https://rp.j6195456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0a9279ed-f9e2-4b6a-ac1a-28f39ba0ae7e\n    --------------------------------------------------------------------------------\n    RESPONSE 200: 200 OK\n    ERROR CODE: DeadlineExceeded\n    --------------------------------------------------------------------------------\n    {\n      \"id\": \"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0a9279ed-f9e2-4b6a-ac1a-28f39ba0ae7e\",\n      \"name\": \"0a9279ed-f9e2-4b6a-ac1a-28f39ba0ae7e\",\n      \"status\": \"Failed\",\n      \"startTime\": \"2026-10-26T03:36:22.647294159Z\",\n      \"endTime\": \"2026-10-26T04:05:25.39660378Z\",\n      \"error\": {\n        \"code\": \"DeadlineExceeded\",\n        \"message\": \"cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable: Waiting for components to be available: catalog-operator, kube-scheduler, azure-cloud-controller-manager; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager deployment has 1 unavailable replicas, catalog-operator deployment has 1 unavailable replicas, kube-scheduler deployment has 1 unavailable replicas]\"\n      }\n    }\n    --------------------------------------------------------------------------------\n    \n    \u003cAzure error internals omitted\u003e\n        Azure request IDs:\n          GET https://rp.j6195456.hcpsvc.osadev.cloud/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/providers/Microsoft.RedHatOpenShift/locations/centralus/hcpOperationStatuses/0a9279ed-f9e2-4b6a-ac1a-28f39ba0ae7e\n            correlation ID: 2461ce1f-a2e3-4c70-ae70-58c7bc6e13e3\n            client request ID: 87ad1faa-4a8b-48cb-888d-882999d849fb\noccurred","failure_pattern_match":{"status":"clustered","failure_pattern_id":"e0c9b2f5fa0da8e29aa067a0a42cc3a25fadc4fe77c282ebc769212a8b5a7957","failure_pattern":"ERROR CODE: DeadlineExceeded; detail message cluster creation did not complete before the deadline; [clusterServiceClusterStatus] cluster service is installing; [hypershiftHostedCluster] hosted cluster is not available: ComponentsNotAvailable; hosted cluster degraded: UnavailableReplicas: [azure-cloud-controller-manager, catalog-operator, kube-scheduler]; provider Microsoft.RedHatOpenShift","search_query":"ERROR CODE: DeadlineExceeded"}},{"row_id":"f959a6b7c4ed78b3f29d3b5568b02aaeeb86a397bc1e6204fc9844388c5810f9","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/7721/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114550200386195456","occurred_at":"2026-10-26T02:50:28Z","failed_at":"alert","signature_id":"b5098cc6e3c36d2fdc8d9c2477b1cb87d5baf078e5312b4abdd7b8111d538f8c","test_name":"[aro-hcp-observability] [hcp] alert KubePodCrashLooping does not fire","test_suite":"aro-hcp-tests","failure_text":"alert fired 9 time(s)\nFiring 1:\n  State:       Resolved\n  Started:     2026-10-26T04:06:15Z\n  Ended:       2026-10-26T04:15:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6195456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"cloud-controller-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.193:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6195456/providers/microsoft.monitor/accounts/hcps-j6195456\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tm7pdravivuv2be1jlq3hjvbmiphefk-q7v3u6v3c8g7z4g\", pod=\"azure-cloud-controller-manager-8447d8b694-4zr5b\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d02104f0-060b-4edb-8a6c-e45f411809b0\"\n  Description: Pod ocm-arohcpci01-2tm7pdravivuv2be1jlq3hjvbmiphefk-q7v3u6v3c8g7z4g/azure-cloud-controller-manager-8447d8b694-4zr5b (cloud-controller-manager) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 2:\n  State:       Resolved\n  Started:     2026-10-26T04:06:16Z\n  Ended:       2026-10-26T04:15:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6195456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"cloud-controller-manager\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.193:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6195456/providers/microsoft.monitor/accounts/hcps-j6195456\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tm7pdravivuv2be1jlq3hjvbmiphefk-q7v3u6v3c8g7z4g\", pod=\"azure-cloud-controller-manager-8447d8b694-4zr5b\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d02104f0-060b-4edb-8a6c-e45f411809b0\"\n  Description: Pod ocm-arohcpci01-2tm7pdravivuv2be1jlq3hjvbmiphefk-q7v3u6v3c8g7z4g/azure-cloud-controller-manager-8447d8b694-4zr5b (cloud-controller-manager) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 3:\n  State:       Resolved\n  Started:     2026-10-26T04:08:15Z\n  Ended:       2026-10-26T04:15:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6195456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.193:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6195456/providers/microsoft.monitor/accounts/hcps-j6195456\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tm7padnde7vvi5gn0v8i8h43umncmvl-cilium-cl\", pod=\"csi-snapshot-controller-84dc6bbd7d-68jr9\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"07c48f71-4da8-4382-849e-8180c459ebb1\"\n  Description: Pod ocm-arohcpci01-2tm7padnde7vvi5gn0v8i8h43umncmvl-cilium-cl/csi-snapshot-controller-84dc6bbd7d-68jr9 (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 4:\n  State:       Resolved\n  Started:     2026-10-26T04:08:15Z\n  Ended:       2026-10-26T04:20:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6195456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"multus-admission-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.193:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6195456/providers/microsoft.monitor/accounts/hcps-j6195456\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tm7pj2l0bb8pscrpmbg65ggbiohhe24-ksm-hcp-metrics\", pod=\"multus-admission-controller-6f7d476866-zcg5s\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8fa3adbb-8673-43f6-8f2c-cb4b4dc5a2ff\"\n  Description: Pod ocm-arohcpci01-2tm7pj2l0bb8pscrpmbg65ggbiohhe24-ksm-hcp-metrics/multus-admission-controller-6f7d476866-zcg5s (multus-admission-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 5:\n  State:       Resolved\n  Started:     2026-10-26T04:08:15Z\n  Ended:       2026-10-26T04:18:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6195456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"cluster-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.193:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6195456/providers/microsoft.monitor/accounts/hcps-j6195456\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tm7pj2l0bb8pscrpmbg65ggbiohhe24-ksm-hcp-metrics\", pod=\"cluster-policy-controller-7c486c777d-94qnj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d0f7a448-44aa-43fc-891a-29f853013cc6\"\n  Description: Pod ocm-arohcpci01-2tm7pj2l0bb8pscrpmbg65ggbiohhe24-ksm-hcp-metrics/cluster-policy-controller-7c486c777d-94qnj (cluster-policy-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 6:\n  State:       Resolved\n  Started:     2026-10-26T04:08:15Z\n  Ended:       2026-10-26T04:20:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6195456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"multus-admission-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.193:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6195456/providers/microsoft.monitor/accounts/hcps-j6195456\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tm7pj2l0bb8pscrpmbg65ggbiohhe24-ksm-hcp-metrics\", pod=\"multus-admission-controller-6f7d476866-zcg5s\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"8fa3adbb-8673-43f6-8f2c-cb4b4dc5a2ff\"\n  Description: Pod ocm-arohcpci01-2tm7pj2l0bb8pscrpmbg65ggbiohhe24-ksm-hcp-metrics/multus-admission-controller-6f7d476866-zcg5s (multus-admission-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 7:\n  State:       Resolved\n  Started:     2026-10-26T04:08:15Z\n  Ended:       2026-10-26T04:18:15Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6195456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"cluster-policy-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.193:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6195456/providers/microsoft.monitor/accounts/hcps-j6195456\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tm7pj2l0bb8pscrpmbg65ggbiohhe24-ksm-hcp-metrics\", pod=\"cluster-policy-controller-7c486c777d-94qnj\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"d0f7a448-44aa-43fc-891a-29f853013cc6\"\n  Description: Pod ocm-arohcpci01-2tm7pj2l0bb8pscrpmbg65ggbiohhe24-ksm-hcp-metrics/cluster-policy-controller-7c486c777d-94qnj (cluster-policy-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 8:\n  State:       Resolved\n  Started:     2026-10-26T04:09:15Z\n  Ended:       2026-10-26T04:20:17Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6195456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.193:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6195456/providers/microsoft.monitor/accounts/hcps-j6195456\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tm7pj2l0bb8pscrpmbg65ggbiohhe24-ksm-hcp-metrics\", pod=\"csi-snapshot-controller-56f74747dd-q2c9t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-1\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"3dba828e-0851-4048-b70e-3efa4c0ac3c5\"\n  Description: Pod ocm-arohcpci01-2tm7pj2l0bb8pscrpmbg65ggbiohhe24-ksm-hcp-metrics/csi-snapshot-controller-56f74747dd-q2c9t (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").\nFiring 9:\n  State:       Resolved\n  Started:     2026-10-26T04:09:15Z\n  Ended:       2026-10-26T04:20:16Z\n  Severity:    Sev3\n  Labels:      alertname=\"KubePodCrashLooping\", cluster=\"ci01-j6195456-mgmt-2\", component=\"kubernetes-infrastructure\", container=\"snapshot-controller\", endpoint=\"http\", environment=\"ci01\", instance=\"10.128.64.193:8080\", job=\"kube-state-metrics\", microsoft.amwresourceid=\"/subscriptions/XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX/resourcegroups/hcp-underlay-ci01-j6195456/providers/microsoft.monitor/accounts/hcps-j6195456\", microsoft.metrics_include_label=\"service\", namespace=\"ocm-arohcpci01-2tm7pj2l0bb8pscrpmbg65ggbiohhe24-ksm-hcp-metrics\", pod=\"csi-snapshot-controller-56f74747dd-q2c9t\", prometheus=\"prometheus/prometheus\", prometheus_replica=\"prom-agent-prometheus-0\", reason=\"CrashLoopBackOff\", region=\"centralus\", service=\"arohcp-monitor-kube-state-metrics\", severity=\"warning\", uid=\"3dba828e-0851-4048-b70e-3efa4c0ac3c5\"\n  Description: Pod ocm-arohcpci01-2tm7pj2l0bb8pscrpmbg65ggbiohhe24-ksm-hcp-metrics/csi-snapshot-controller-56f74747dd-q2c9t (snapshot-controller) is in waiting state (reason: \"CrashLoopBackOff\").","failure_pattern_match":{"status":"clustered","failure_pattern_id":"f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099","failure_pattern":"alert [hcp] KubePodCrashLooping fired","search_query":"alert [hcp] KubePodCrashLooping fired"}}],"failure_pattern_summary":{"signature_count":4,"failure_pattern_ids":["0316cbd90cb1deb59d43692438e3753b878c1a8c6b4dfbaa4805ffd81627a594","93913140481ce6ebcf71bb15db6e35a879a3f4d11071b479cf20b4df2eabf8a4","e0c9b2f5fa0da8e29aa067a0a42cc3a25fadc4fe77c282ebc769212a8b5a7957","f35494afbb9652c9cb9207e6f83ead17beabb80e65eae6b49c967e2103c48099"],"matched_occurrences":4,"unmatched_occurrences":0,"match_summary":"multiple_clustered"}},{"run":{"environment":"dev","run_url":"https://prow.ci.openshift.org/view/gs/test-platform-results-public/pr-logs/pull/Azure_ARO-HCP/6191/pull-ci-Azure-ARO-HCP-main-e2e-parallel/2114537157698785280","job_name":"pull-ci-Azure-ARO-HCP-main-e2e-parallel","pr_number":6191,"pr_state":"open","pr_sha":"0b660a1c1d53873760d8044d183b30ce239eb8fb","final_merged_sha":"","merged_pr":false,"post_good_commit":false,"failed":false,"occurred_at":"2026-10-26T01:58:39Z","started_at":"2026-10-26T01:58:39Z","completed_at":"2026-10-26T04:04:00Z","timing_metadata_state":"found","timing_metadata_first_checked_at":"2026-10-26T04:18:45.861622882Z","timing_metadata_checked_at":"2026-10-26T04:18:45.861622882Z","region":"westus3","region_metadata_state":"found","region_metadata_first_checked_at":"2026-10-26T04:18:45.864874871Z","region_metadata_checked_at":"2026-10-26T04:18:45.864874871Z"},"failed_test_count":0,"failure_pattern_summary":{"signature_count":0,"matched_occurrences":0,"unmatched_occurrences":0,"match_summary":"no_failures"}}]}]}